def testAddInstance(self):
LoadInfo.InitializeTable()
LoadInfo.AddInstance('test-instance')
self.assertEqual(['test-instance'],
memcache.get(LoadInfo.ALL_INSTANCES))
self.assertEqual({}, LoadInfo.GetAll())
self.assertIsNone(memcache.get('test-instance'))
self.assertIsNotNone(SingleInstance.GetByName('test-instance'))
self.assertRaises(ValueError,
SingleInstance.GetByName('test-instance').ip_address)
def testRegisterInstance(self):
LoadInfo.InitializeTable()
LoadInfo.AddInstance('test-instance')
LoadInfo.RegisterInstanceIpAddress('test-instance', '1.2.3.4')
self.assertEqual(['test-instance'],
memcache.get(LoadInfo.ALL_INSTANCES))
self.assertEqual({'test-instance': {
'ip_address': '1.2.3.4'
}}, LoadInfo.GetAll())
self.assertEqual({'ip_address': '1.2.3.4'},
memcache.get('test-instance'))
self.assertEqual('1.2.3.4',
SingleInstance.GetByName('test-instance').ip_address)
def get(self):
"""Starts up initial Compute Engine cluster."""
instances = LoadInfo.GetAllInstances()
if instances:
raise SystemError("Instances already loaded, teardown first.")
ComputeEngineController(decorator.credentials).StartUpCluster()
def get(self):
"""Returns an available server's IP address in JSON format."""
ip = LoadInfo.GetIdleInstanceIpAddress()
if not ip:
ip = ''
self.response.out.write(json.dumps({'ipaddress': ip}))
IpAddressRequestLog(client_ip=self.request.remote_addr,
server_ip=ip).put()
def _DeleteInstance(self, instance_name):
"""Stops and deletes the instance specified by the name."""
logging.info('Deleting instance %s', instance_name)
LoadInfo.RemoveInstance(instance_name)
result = self.compute_api.instances().delete(
project=self.PROJECT_ID,
zone=self.DEFAULT_ZONE,
instance=instance_name).execute()
logging.info(str(result))
def StartUpCluster(self):
"""Initializes and start up Compute Engine cluster.
Records user ID and use it later by Taskqueue, Cron job handlers
and other handlers which is initiated by Compute Engine (therefore
without log in), to retrieve credential from Datastore. It means
those tasks work under permission of the user who started the cluster.
"""
LoadInfo.InitializeTable()
self.IncreaseEngine(self.INITIAL_CLUSTER_SIZE)
def post(self):
# TODO: Secure this URL by using Cloud Endpoints.
grid = self.request.get('grid')
num = int(self.request.get('numPlayers'))
logging.info('Server update received: ' + str(grid) +
' with numPlayers: ' + str(num))
loadresp = LoadInfo.UpdateServerNumPlayers(grid, num)
self.response.out.write(json.dumps({"loadresp": loadresp}))
def post(self):
"""Adds the new instance to managed cluster by registering IP address."""
# TODO(user): Secure this URL by using Cloud Endpoints.
name = self.request.get('name')
instance = ComputeEngineController().GetInstanceInfo(name)
if not instance:
return
logging.info('Instance created: %s', str(instance))
external_ip = instance['networkInterfaces'][0]['accessConfigs'][0][
'natIP']
LoadInfo.RegisterInstanceIpAddress(name, external_ip)
def post(self):
# TODO(user): Secure this URL by using Cloud Endpoints.
grid = self.request.get('grid')
logging.info('Received server require for grid:' + grid)
loadresp = LoadInfo.GetServerLoadInfo(grid)
if not loadresp:
logging.info('No server so starting server for grid:' + str(grid))
ComputeEngineController().AddServer(grid)
loadresp = {LoadInfo.STATUS: LoadInfo.STATUS_LOADING}
self.response.out.write(json.dumps(loadresp))
def testAverageLoadExcludingImmatureInstance(self):
LoadInfo.InitializeTable()
LoadInfo.AddInstance('test-instance1')
LoadInfo.RegisterInstanceIpAddress('test-instance1', '1.2.3.4')
# test-instance1 hasn't yet reported load information.
LoadInfo.AddInstance('test-instance2')
LoadInfo.RegisterInstanceIpAddress('test-instance2', '5.6.7.8')
LoadInfo.UpdateLoadInfo('test-instance2', 33)
self.assertEqual((1, 33), LoadInfo.GetAverageLoad())
def testGetIdleInstanceExcludingImmatureInstance(self):
LoadInfo.InitializeTable()
LoadInfo.AddInstance('test-instance1')
LoadInfo.RegisterInstanceIpAddress('test-instance1', '1.2.3.4')
LoadInfo.UpdateLoadInfo('test-instance1', 55)
LoadInfo.AddInstance('test-instance2')
LoadInfo.RegisterInstanceIpAddress('test-instance2', '5.6.7.8')
# test-instance2 hasn't yet reported load information.
self.assertEqual('1.2.3.4', LoadInfo.GetIdleInstanceIpAddress())
def get(self):
user = users.get_current_user()
# TODO: Check to make sure game account is up to date/paid for/whatever.
# Determine what server to transfer player to
servertoload = ""
q = Entity.all()
q.filter("user_id =", user.user_id())
q.filter("__type =", "Player")
userplayers = []
for userplayer in q.run(limit=5):
userplayers.append(userplayer)
if len(userplayers) > 1:
logging.error("UserID has more than one player")
return
elif len(userplayers) == 0:
logging.info("No current players, attaching to root")
servertoload = "0,0"
else:
servertoload = userplayers[0].gridkey
logging.info("Loading server " + servertoload + " for " +
user.nickname())
info = LoadInfo.GetServerLoadInfo(servertoload)
if info:
if info[LoadInfo.STATUS] == LoadInfo.STATUS_UP:
ip = info[LoadInfo.IP_ADDRESS]
port = info[LoadInfo.PORT]
# Generate login token
token_key = uuid.uuid4().hex
expiration_time = datetime.datetime.now() + datetime.timedelta(
seconds=20)
LoginToken(key_name=token_key,
expiration=expiration_time,
user_id=user.user_id()).put()
# Send IP and token info
self.response.out.write(
json.dumps({
'status': 'up',
'ipaddress': ip,
'port': port,
'token': token_key
}))
IpAddressRequestLog(client_ip=self.request.remote_addr,
server_ip=ip).put()
else:
self.response.out.write(json.dumps({'status': 'loading'}))
else:
ComputeEngineController().AddServer(servertoload)
self.response.out.write(json.dumps({'status': 'loading'}))
def get(self):
"""Returns stats of game instances for non logged-in users."""
load_entries = []
all_load_info = LoadInfo.GetAll()
for name, info in all_load_info.items():
load_entries.append({
'host': name,
'ipaddress': info.get(LoadInfo.IP_ADDRESS, ''),
'load': info.get(LoadInfo.LOAD, 0),
})
self.response.out.write(json.dumps(load_entries))
def IncreaseEngine(self, increase_count):
"""Starts specified number of Compute Engine instances.
Args:
increase_count: Number of instances to increase.
"""
for _ in xrange(increase_count):
instance_name = self.WORKER_NAME_PREFIX + str(uuid.uuid4())
# Add instance name to load information before actually creating the
# instance to avoid, to make sure the instance is managed
# when it registers IP address.
LoadInfo.AddInstance(instance_name)
self._StartInstance(instance_name)
def AddServer(self, grid):
# If current instance has open room, use it
idle_instance = LoadInfo.GetIdleInstance()
if not (idle_instance == None):
ip_address = idle_instance[LoadInfo.IP_ADDRESS]
# Send request to server manager to create new server
h = httplib2.Http()
data = {
"grid": grid
}
resp, content = h.request(
"http://" + ip_address + ":" + self.INSTANCE_MANAGER_PORT +
"/" + self.INSTANCE_MANAGER_ADDSERVER, "POST", urlencode(data))
# Track new server
LoadInfo.AddServer(grid, resp)
# Re-assess load
self.assessLoad()
else: # Otherwise, start new instance
logging.error(
'Attempting to add server %s, but no idle instances!', grid)
def testRemoveInstanceFromTwo(self):
LoadInfo.InitializeTable()
LoadInfo.AddInstance('test-instance1')
LoadInfo.RegisterInstanceIpAddress('test-instance1', '1.2.3.4')
LoadInfo.UpdateLoadInfo('test-instance1', 55)
LoadInfo.AddInstance('test-instance2')
LoadInfo.RegisterInstanceIpAddress('test-instance2', '5.6.7.8')
LoadInfo.UpdateLoadInfo('test-instance2', 22)
LoadInfo.RemoveInstance('test-instance1')
self.assertEqual(
{
'test-instance2': {
'ip_address': '5.6.7.8',
'load': 22,
'force': False
}
}, LoadInfo.GetAll())
self.assertIsNone(memcache.get('test-instance1'))
self.assertIsNone(SingleInstance.GetByName('test-instance1'))
def post(self):
# TODO: Secure this URL by using Cloud Endpoints.
name = self.request.get('name')
# TODO: Figure out credentials issues with below, likely due to ComputeEngineController credentials expiring
#instance = ComputeEngineController().GetInstanceInfo(name)
#if not instance:
# return
load = int(self.request.get('load'))
logging.info('Instance update received: ' + str(name) +
' with load: ' + str(load))
loadresp = LoadInfo.UpdateInstanceLoadInfo(name, load)
self.response.out.write(json.dumps({"loadresp": loadresp}))
def testRemoveInstanceFromOne(self):
LoadInfo.InitializeTable()
LoadInfo.AddInstance('test-instance')
LoadInfo.RegisterInstanceIpAddress('test-instance', '1.2.3.4')
LoadInfo.UpdateLoadInfo('test-instance', 55)
LoadInfo.RemoveInstance('test-instance')
self.assertEqual({}, LoadInfo.GetAll())
self.assertEqual([], memcache.get(LoadInfo.ALL_INSTANCES))
self.assertIsNone(memcache.get('test-instance'))
self.assertIsNone(SingleInstance.GetByName('test-instance'))
def post(self):
"""Receives request from instance and updates load information."""
# TODO(user): Secure this URL by using Cloud Endpoints.
name = self.request.get('name')
load = self.request.get('load')
if not load:
load = 0
force = self.request.get('force')
force_set = None
if force:
if int(force):
force_set = True
else:
force_set = False
LoadInfo.UpdateLoadInfo(name, int(load), force_set)
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write('ok')
def post(self):
""" Adds the new server to server list by registering IP/port """
# TODO(user): Secure this URL by using Cloud Endpoints
name = self.request.get('instancename')
grid = self.request.get('grid')
port = self.request.get('port')
instance = ComputeEngineController().GetInstanceInfo(name)
if not instance:
logging.error("Instance name doesn't match existing instance: %s",
name)
return
logging.info('Instance created: %s', str(instance))
external_ip = instance['networkInterfaces'][0]['accessConfigs'][0][
'natIP']
LoadInfo.RegisterServerAddress(grid, external_ip, port)
resp = {'external_ip': external_ip, 'success': 1}
self.response.out.write(json.dumps(resp))
def get(self):
"""Checks average load level and adjusts cluster size if necessary.
If average load level of instances is more than upper threshold, increase
the number of instances by 20% of original size. If average load level is
less than lower threshold and the current cluster size is larger than
minimum size, decrease the number of instances by 10%. Since shortage
of server is more harmful than excessive instances, we increase more
rapidly than we decrease.
However, shutting down the instance is more complicated than adding
instances depending on game servers. If client is not capable to auto-
reconnect, the game server must be drained before shutting down the
instance. In this exsample, decrement of the instance is not implemented.
"""
cluster_size, average_load = LoadInfo.GetAverageLoad()
if cluster_size:
if average_load > self.UPPER_THRESHOLD:
ComputeEngineController().IncreaseEngine(cluster_size / 5 + 1)
elif (average_load < self.LOWER_THRESHOLD
and cluster_size > self.MIN_CLUSTER_SIZE):
ComputeEngineController().DecreaseEngine(cluster_size / 10 + 1)
def testMemcacheClear(self):
LoadInfo.InitializeTable()
LoadInfo.AddInstance('test-instance')
LoadInfo.RegisterInstanceIpAddress('test-instance', '1.2.3.4')
LoadInfo.UpdateLoadInfo('test-instance', 55)
# Simulate loss of all data in Memcache.
memcache.flush_all()
LoadInfo.UpdateLoadInfo('test-instance', 38)
self.assertEqual(
{
'test-instance': {
'ip_address': '1.2.3.4',
'load': 38,
'force': False
}
}, LoadInfo.GetAll())
def post(self):
"""Adds the new instance to managed cluster by registering IP address."""
# TODO(user): Secure this URL by using Cloud Endpoints.
name = self.request.get('name')
instance = ComputeEngineController().GetInstanceInfo(name)
if not instance:
return
logging.info('Instance created: %s', str(instance))
external_ip = instance['networkInterfaces'][0]['accessConfigs'][0][
'natIP']
LoadInfo.RegisterInstanceIpAddress(name, external_ip)
"""Returns script to set up overarching server manager."""
template = jinja_environment.get_template('setup-and-start-game.sh')
self.response.out.write(
template.render({
'apphostname':
app_identity.get_default_version_hostname(),
'ip_address':
self.request.remote_addr,
'name':
name
}))
def get(self):
"""Returns stats of managed Compute Engine instances for Admin UI."""
load_entries = []
instance_list = ComputeEngineController(
decorator.credentials).ListInstances()
all_load_info = LoadInfo.GetAll()
# First, list managed instances whose Compute Engine status is found.
for instance in instance_list:
instance_name = instance['name']
if instance_name in all_load_info:
info = all_load_info[instance_name]
load_entries.append({
'host':
instance_name,
'ipaddress':
info.get(LoadInfo.IP_ADDRESS, ''),
'status':
instance['status'],
'load':
info.get(LoadInfo.LOAD, 0),
'force_set':
info.get(LoadInfo.FORCE, False),
})
del all_load_info[instance_name]
# Then, list managed instances without Compute Engine status.
for name, info in all_load_info.items():
load_entries.append({
'host': name,
'ipaddress': info.get(LoadInfo.IP_ADDRESS, ''),
'status': 'NOT FOUND',
'load': info.get(LoadInfo.LOAD, 0),
'force_set': info.get(LoadInfo.FORCE, False),
})
self.response.out.write(json.dumps(load_entries))
def testUpdateLoadInfo(self):
LoadInfo.InitializeTable()
LoadInfo.AddInstance('test-instance')
LoadInfo.RegisterInstanceIpAddress('test-instance', '1.2.3.4')
LoadInfo.UpdateLoadInfo('test-instance', 55)
self.assertEqual(
{
'test-instance': {
'ip_address': '1.2.3.4',
'load': 55,
'force': False
}
}, LoadInfo.GetAll())
LoadInfo.UpdateLoadInfo('test-instance', 73)
self.assertEqual(
{
'test-instance': {
'ip_address': '1.2.3.4',
'load': 73,
'force': False
}
}, LoadInfo.GetAll())
def StartUpCluster(self):
LoadInfo.InitializeTable()
self.IncreaseEngine()
def RemoveServer(self, grid):
# TODO: Maybe confirm with server manager that actual process is ended in case servers need to be force-quit
LoadInfo.RemoveServer(grid)
def testUpdateLoadInfoForce(self):
LoadInfo.InitializeTable()
LoadInfo.AddInstance('test-instance')
LoadInfo.RegisterInstanceIpAddress('test-instance', '1.2.3.4')
LoadInfo.UpdateLoadInfo('test-instance', 55)
self.assertEqual(
{
'test-instance': {
'ip_address': '1.2.3.4',
'load': 55,
'force': False
}
}, LoadInfo.GetAll())
LoadInfo.UpdateLoadInfo('test-instance', 92, True)
self.assertEqual(
{
'test-instance': {
'ip_address': '1.2.3.4',
'load': 92,
'force': True
}
}, LoadInfo.GetAll())
# This update is ignored since force flag is set in data and this is not
# force update.
LoadInfo.UpdateLoadInfo('test-instance', 15)
self.assertEqual(
{
'test-instance': {
'ip_address': '1.2.3.4',
'load': 92,
'force': True
}
}, LoadInfo.GetAll())
# Updated because of force_set flag.
LoadInfo.UpdateLoadInfo('test-instance', 8, True)
self.assertEqual(
{
'test-instance': {
'ip_address': '1.2.3.4',
'load': 8,
'force': True
}
}, LoadInfo.GetAll())
LoadInfo.UpdateLoadInfo('test-instance', 41, False)
self.assertEqual(
{
'test-instance': {
'ip_address': '1.2.3.4',
'load': 41,
'force': False
}
}, LoadInfo.GetAll())
LoadInfo.UpdateLoadInfo('test-instance', 28)
self.assertEqual(
{
'test-instance': {
'ip_address': '1.2.3.4',
'load': 28,
'force': False
}
}, LoadInfo.GetAll())
def IncreaseEngine(self):
instance_name = self.WORKER_NAME_PREFIX + str(uuid.uuid4())
response = self._StartInstance(instance_name)
LoadInfo.AddInstance(instance_name, response)
def get(self):
"""Deletes Compute Engine cluster."""
ComputeEngineController(decorator.credentials).TearDownCluster()
LoadInfo.RemoveAllInstancesAndServers()