def reclaim_machine(machine_key, reclamation_ts):
"""Attempts to reclaim the given machine.
Args:
machine_key: ndb.Key for a model.CatalogMachineEntry instance.
reclamation_ts: datetime.datetime instance indicating when the machine was
reclaimed.
Returns:
True if the machine was reclaimed, else False.
"""
machine = machine_key.get()
if not machine:
logging.warning('CatalogMachineEntry not found: %s', machine_key)
return
logging.info('Attempting to reclaim CatalogMachineEntry:\n%s', machine)
if machine.lease_expiration_ts is None:
# This can reasonably happen if e.g. the lease was voluntarily given up.
logging.warning('CatalogMachineEntry no longer leased:\n%s', machine)
return False
if reclamation_ts < machine.lease_expiration_ts:
# This can reasonably happen if e.g. the lease duration was extended.
logging.warning('CatalogMachineEntry no longer overdue:\n%s', machine)
return False
logging.info('Reclaiming CatalogMachineEntry:\n%s', machine)
lease = models.LeaseRequest.get_by_id(machine.lease_id)
hostname = lease.response.hostname
lease.response.hostname = None
params = {
'hostname': hostname,
'machine_key': machine.key.urlsafe(),
'machine_subscription': machine.pubsub_subscription,
'machine_subscription_project': machine.pubsub_subscription_project,
'machine_topic': machine.pubsub_topic,
'machine_topic_project': machine.pubsub_topic_project,
'policies': protojson.encode_message(machine.policies),
'request_json': protojson.encode_message(lease.request),
'response_json': protojson.encode_message(lease.response),
}
backend_attributes = {}
for attribute in machine.policies.backend_attributes:
backend_attributes[attribute.key] = attribute.value
params['backend_attributes'] = utils.encode_to_json(backend_attributes)
if lease.request.pubsub_topic:
params['lessee_project'] = lease.request.pubsub_project
params['lessee_topic'] = lease.request.pubsub_topic
if not utils.enqueue_task(
'/internal/queues/reclaim-machine',
'reclaim-machine',
params=params,
transactional=True,
):
raise TaskEnqueuingError('reclaim-machine')
return True
def lease_machine(machine_key, lease):
"""Attempts to lease the given machine.
Args:
machine_key: ndb.Key for a model.CatalogMachineEntry instance.
lease: model.LeaseRequest instance.
Returns:
True if the machine was leased, otherwise False.
"""
machine = machine_key.get()
lease = lease.key.get()
logging.info('Attempting to lease matching CatalogMachineEntry:\n%s',
machine)
if not can_fulfill(machine, lease.request):
logging.warning('CatalogMachineEntry no longer matches:\n%s', machine)
return False
if machine.state != models.CatalogMachineEntryStates.AVAILABLE:
logging.warning('CatalogMachineEntry no longer available:\n%s',
machine)
return False
if lease.response.state != rpc_messages.LeaseRequestState.UNTRIAGED:
logging.warning('LeaseRequest no longer untriaged:\n%s', lease)
return False
if not machine.pubsub_subscription:
logging.warning('CatalogMachineEntry not subscribed to Pub/Sub yet')
return False
logging.info('Leasing CatalogMachineEntry:\n%s', machine)
lease.leased_ts = utils.utcnow()
lease_expiration_ts = lease.leased_ts + datetime.timedelta(
seconds=lease.request.duration, )
lease.machine_id = machine.key.id()
lease.response.hostname = machine.dimensions.hostname
# datetime_to_timestamp returns microseconds, which are too fine grain.
lease.response.lease_expiration_ts = utils.datetime_to_timestamp(
lease_expiration_ts) / 1000 / 1000
lease.response.state = rpc_messages.LeaseRequestState.FULFILLED
machine.lease_id = lease.key.id()
machine.lease_expiration_ts = lease_expiration_ts
machine.state = models.CatalogMachineEntryStates.LEASED
ndb.put_multi([lease, machine])
params = {
'policies': protojson.encode_message(machine.policies),
'request_json': protojson.encode_message(lease.request),
'response_json': protojson.encode_message(lease.response),
'machine_project': machine.pubsub_topic_project,
'machine_topic': machine.pubsub_topic,
}
if not utils.enqueue_task(
'/internal/queues/fulfill-lease-request',
'fulfill-lease-request',
params=params,
transactional=True,
):
raise TaskEnqueuingError('fulfill-lease-request')
return True
def lease_machine(machine_key, lease):
"""Attempts to lease the given machine.
Args:
machine_key: ndb.Key for a model.CatalogMachineEntry instance.
lease: model.LeaseRequest instance.
Returns:
True if the machine was leased, otherwise False.
"""
machine = machine_key.get()
lease = lease.key.get()
logging.info('Attempting to lease matching CatalogMachineEntry:\n%s', machine)
if not can_fulfill(machine, lease.request):
logging.warning('CatalogMachineEntry no longer matches:\n%s', machine)
return False
if machine.state != models.CatalogMachineEntryStates.AVAILABLE:
logging.warning('CatalogMachineEntry no longer available:\n%s', machine)
return False
if lease.response.state != rpc_messages.LeaseRequestState.UNTRIAGED:
logging.warning('LeaseRequest no longer untriaged:\n%s', lease)
return False
logging.info('Leasing CatalogMachineEntry:\n%s', machine)
lease.leased_ts = utils.utcnow()
lease_expiration_ts = lease.leased_ts + datetime.timedelta(
seconds=lease.request.duration,
)
lease.machine_id = machine.key.id()
lease.response.hostname = machine.dimensions.hostname
# datetime_to_timestamp returns microseconds, which are too fine grain.
lease.response.lease_expiration_ts = utils.datetime_to_timestamp(
lease_expiration_ts) / 1000 / 1000
lease.response.state = rpc_messages.LeaseRequestState.FULFILLED
machine.lease_id = lease.key.id()
machine.lease_expiration_ts = lease_expiration_ts
machine.state = models.CatalogMachineEntryStates.LEASED
ndb.put_multi([lease, machine])
params = {
'policies': protojson.encode_message(machine.policies),
'request_json': protojson.encode_message(lease.request),
'response_json': protojson.encode_message(lease.response),
'machine_project': machine.pubsub_topic_project,
'machine_topic': machine.pubsub_topic,
}
if not utils.enqueue_task(
'/internal/queues/fulfill-lease-request',
'fulfill-lease-request',
params=params,
transactional=True,
):
raise TaskEnqueuingError('fulfill-lease-request')
return True
def extract_dimensions(instance, instance_template_revision):
"""Extracts Machine Provider dimensions.
Args:
instance: models.Instance entity.
instance_template_revision: models.InstanceTemplateRevision entity.
Returns:
A dict of dimensions.
"""
if instance_template_revision.dimensions:
dimensions = json.loads(protojson.encode_message(
instance_template_revision.dimensions))
else:
dimensions = {}
dimensions['backend'] = 'GCE'
if instance_template_revision.disk_size_gb:
dimensions['disk_size_gb'] = instance_template_revision.disk_size_gb
if instance_template_revision.machine_type:
dimensions['memory_gb'] = gce.machine_type_to_memory(
instance_template_revision.machine_type)
dimensions['num_cpus'] = gce.machine_type_to_num_cpus(
instance_template_revision.machine_type)
dimensions['hostname'] = instance.key.id()
return dimensions
def extract_dimensions(instance, instance_template_revision):
"""Extracts Machine Provider dimensions.
Args:
instance: models.Instance entity.
instance_template_revision: models.InstanceTemplateRevision entity.
Returns:
A dict of dimensions.
"""
if instance_template_revision.dimensions:
dimensions = json.loads(
protojson.encode_message(instance_template_revision.dimensions))
else:
dimensions = {}
dimensions['backend'] = 'GCE'
if instance_template_revision.disk_size_gb:
dimensions['disk_size_gb'] = instance_template_revision.disk_size_gb
if instance_template_revision.machine_type:
dimensions['memory_gb'] = gce.machine_type_to_memory(
instance_template_revision.machine_type)
dimensions['num_cpus'] = gce.machine_type_to_num_cpus(
instance_template_revision.machine_type)
dimensions['hostname'] = instance.key.id()
return dimensions
def rpc_to_json(rpc_message):
"""Converts the given RPC message to a POSTable JSON dict.
Args:
rpc_message: A protorpc.message.Message instance.
Returns:
A string representing a JSON dict.
"""
return json.loads(protojson.encode_message(rpc_message))
def search(self, request):
""" Call search endpoint.
"""
response = self.testapp.post('/_ah/spi/SupplierService.search',
protojson.encode_message(request),
content_type='application/json')
self.assertEqual(response.status, '200 OK')
return protojson.decode_message(SupplierCollectionMessage, response.body)
def finalize(self):
"""Log the activity in Titan Files."""
titan_file = files.File(self.activity.activity_id, _internal=True)
titan_file.write(
content=protojson.encode_message(self.activity.to_message()),
meta=self.file_meta, created=self.activity.timestamp,
modified=self.activity.timestamp)
# Ensure that it gets written first.
super(FileActivityLogger, self).finalize()
def delete(self, id, expect_errors=False):
""" Call delete endpoint.
"""
response = self.testapp.post('/_ah/spi/SupplierService.delete',
protojson.encode_message(
SupplierKeyMessage(id=id)), content_type='application/json',
expect_errors=expect_errors)
if not expect_errors:
self.assertEqual(response.status, '200 OK')
def rpc_to_json(rpc_message):
"""Converts the given RPC message to a POSTable JSON dict.
Args:
rpc_message: A protorpc.message.Message instance.
Returns:
A string representing a JSON dict.
"""
return json.loads(protojson.encode_message(rpc_message))
def delete(self, id, expect_errors=False):
""" Call delete endpoint.
"""
response = self.testapp.post('/_ah/spi/SupplierService.delete',
protojson.encode_message(
SupplierKeyMessage(id=id)),
content_type='application/json',
expect_errors=expect_errors)
if not expect_errors:
self.assertEqual(response.status, '200 OK')
def search(self, request):
""" Call search endpoint.
"""
response = self.testapp.post('/_ah/spi/SupplierService.search',
protojson.encode_message(request),
content_type='application/json')
self.assertEqual(response.status, '200 OK')
return protojson.decode_message(SupplierCollectionMessage,
response.body)
def save(self, request):
""" Call save endpoint.
"""
response = self.testapp.post('/_ah/spi/CustomerService.save',
protojson.encode_message(request),
content_type='application/json')
self.assertEqual(response.status, '200 OK')
return protojson.decode_message(CustomerGetMessage, response.body)
def finalize(self):
"""Log the activity in Titan Files."""
titan_file = files.File(self.activity.activity_id, _internal=True)
titan_file.write(content=protojson.encode_message(
self.activity.to_message()),
meta=self.file_meta,
created=self.activity.timestamp,
modified=self.activity.timestamp)
# Ensure that it gets written first.
super(FileActivityLogger, self).finalize()
def save(self, request):
""" Call save endpoint.
"""
response = self.testapp.post(
'/_ah/spi/CustomerService.save',
protojson.encode_message(request),
content_type='application/json')
self.assertEqual(response.status, '200 OK')
return protojson.decode_message(CustomerGetMessage, response.body)
def to_json_encodable(data):
"""Converts data into json-compatible data."""
if isinstance(data, messages.Message):
# protojson.encode_message returns a string that is already encoded json.
# Load it back into a json-compatible representation of the data.
return json.loads(protojson.encode_message(data))
if isinstance(data, unicode) or data is None:
return data
if isinstance(data, str):
return data.decode('utf-8')
if isinstance(data, (int, float, long)):
# Note: overflowing is an issue with int and long.
return data
if isinstance(data, (list, set, tuple)):
return [to_json_encodable(i) for i in data]
if isinstance(data, dict):
assert all(isinstance(k, basestring) for k in data), data
return {
to_json_encodable(k): to_json_encodable(v)
for k, v in data.items()
}
if isinstance(data, datetime.datetime):
# Convert datetime objects into a string, stripping off milliseconds. Only
# accept naive objects.
if data.tzinfo is not None:
raise ValueError('Can only serialize naive datetime instance')
return data.strftime(DATETIME_FORMAT)
if isinstance(data, datetime.date):
return data.strftime(DATE_FORMAT)
if isinstance(data, datetime.timedelta):
# Convert timedelta into seconds, stripping off milliseconds.
return int(data.total_seconds())
if hasattr(data, 'to_dict') and callable(data.to_dict):
# This takes care of ndb.Model.
return to_json_encodable(data.to_dict())
if hasattr(data, 'urlsafe') and callable(data.urlsafe):
# This takes care of ndb.Key.
return to_json_encodable(data.urlsafe())
if inspect.isgenerator(data):
return [to_json_encodable(i) for i in data]
if sys.version_info.major == 2 and isinstance(data, xrange):
# Handle it like a list. Sadly, xrange is not a proper generator so it has
# to be checked manually.
return [to_json_encodable(i) for i in data]
assert False, 'Don\'t know how to handle %r' % data
return None
def testUrlfetch(self):
# response = urlfetch.fetch('http://www.google.com')
url = 'http://localhost:9000/_ah/api/conference/v1/conference'
# form_fields = {
# "name": "Albert"
# }
form_fields = ConferenceForm(name='steven')
form_data = protojson.encode_message(form_fields)
# form_data = urllib.urlencode(form_fields)
response = urlfetch.fetch(url=url, payload=form_data, method=urlfetch.POST,
headers={'Content-Type': 'application/json'})
print(dir(response))
print(response.content)
self.assertEquals(200, response.status_code)
def to_json_encodable(data):
"""Converts data into json-compatible data."""
if isinstance(data, messages.Message):
# protojson.encode_message returns a string that is already encoded json.
# Load it back into a json-compatible representation of the data.
return json.loads(protojson.encode_message(data))
if isinstance(data, unicode) or data is None:
return data
if isinstance(data, str):
return data.decode('utf-8')
if isinstance(data, (int, float, long)):
# Note: overflowing is an issue with int and long.
return data
if isinstance(data, (list, set, tuple)):
return [to_json_encodable(i) for i in data]
if isinstance(data, dict):
assert all(isinstance(k, basestring) for k in data), data
return {
to_json_encodable(k): to_json_encodable(v) for k, v in data.iteritems()
}
if isinstance(data, datetime.datetime):
# Convert datetime objects into a string, stripping off milliseconds. Only
# accept naive objects.
if data.tzinfo is not None:
raise ValueError('Can only serialize naive datetime instance')
return data.strftime(DATETIME_FORMAT)
if isinstance(data, datetime.date):
return data.strftime(DATE_FORMAT)
if isinstance(data, datetime.timedelta):
# Convert timedelta into seconds, stripping off milliseconds.
return int(data.total_seconds())
if hasattr(data, 'to_dict') and callable(data.to_dict):
# This takes care of ndb.Model.
return to_json_encodable(data.to_dict())
if hasattr(data, 'urlsafe') and callable(data.urlsafe):
# This takes care of ndb.Key.
return to_json_encodable(data.urlsafe())
if inspect.isgenerator(data) or isinstance(data, xrange):
# Handle it like a list. Sadly, xrange is not a proper generator so it has
# to be checked manually.
return [to_json_encodable(i) for i in data]
assert False, 'Don\'t know how to handle %r' % data
def client_create_task_isolated(self, properties=None, **kwargs):
"""Creates a TaskRequest via the Cloud Endpoints API."""
params = {
'dimensions': [
{'key': 'os', 'value': 'Amiga'},
],
'env': [],
'execution_timeout_secs': 3600,
'io_timeout_secs': 1200,
'inputs_ref': {
'isolated': '0123456789012345678901234567890123456789',
'isolatedserver': 'http://*****:*****@example.com', 'localhost')
try:
response = api.call_api(
'new', body=json.loads(protojson.encode_message(request)))
finally:
endpoints.get_current_user = old_get_current_user
response = response.json
return response, response['task_id']
def endpoint_call(self, service, name, args):
srv = test_case.Endpoints(service, source_ip=self.source_ip)
if not isinstance(args, dict):
args = json.loads(protojson.encode_message(args))
return srv.call_api(name, body=args).json
def msg_dict(request):
return json.loads(protojson.encode_message(request))
def reclaim_machine(machine_key, reclamation_ts):
"""Attempts to reclaim the given machine.
Args:
machine_key: ndb.Key for a model.CatalogMachineEntry instance.
reclamation_ts: datetime.datetime instance indicating when the machine was
reclaimed.
Returns:
True if the machine was reclaimed, else False.
"""
machine = machine_key.get()
logging.info('Attempting to reclaim CatalogMachineEntry:\n%s', machine)
if machine.lease_expiration_ts is None:
# This can reasonably happen if e.g. the lease was voluntarily given up.
logging.warning('CatalogMachineEntry no longer leased:\n%s', machine)
return False
if reclamation_ts < machine.lease_expiration_ts:
# This can reasonably happen if e.g. the lease duration was extended.
logging.warning('CatalogMachineEntry no longer overdue:\n%s', machine)
return False
logging.info('Reclaiming CatalogMachineEntry:\n%s', machine)
lease = models.LeaseRequest.get_by_id(machine.lease_id)
hostname = lease.response.hostname
lease.machine_id = None
lease.response.hostname = None
machine.lease_id = None
machine.lease_expiration_ts = None
policy = machine.policies.on_reclamation
if policy == rpc_messages.MachineReclamationPolicy.DELETE:
logging.info('Executing MachineReclamationPolicy: DELETE')
lease.put()
machine.key.delete()
else:
if policy == rpc_messages.MachineReclamationPolicy.MAKE_AVAILABLE:
logging.info('Executing MachineReclamationPolicy: MAKE_AVAILABLE')
machine.state = models.CatalogMachineEntryStates.AVAILABLE
else:
if policy != rpc_messages.MachineReclamationPolicy.RECLAIM:
# Something is awry. Log an error, but still reclaim the machine.
# Fall back on the RECLAIM policy because it notifies the backend and
# prevents the machine from being leased out again, but keeps it in
# the Catalog in case we want to examine it further.
logging.error(
'Unexpected MachineReclamationPolicy: %s\nDefaulting to RECLAIM',
policy,
)
else:
logging.info('Executing MachineReclamationPolicy: RECLAIM')
machine.state = models.CatalogMachineEntryStates.RECLAIMED
ndb.put_multi([lease, machine])
params = {
'hostname': hostname,
'policies': protojson.encode_message(machine.policies),
'request_json': protojson.encode_message(lease.request),
'response_json': protojson.encode_message(lease.response),
}
backend_attributes = {}
for attribute in machine.policies.backend_attributes:
backend_attributes[attribute.key] = attribute.value
params['backend_attributes'] = utils.encode_to_json(backend_attributes)
if lease.request.pubsub_topic:
params['lessee_project'] = lease.request.pubsub_project
params['lessee_topic'] = lease.request.pubsub_topic
if not utils.enqueue_task(
'/internal/queues/reclaim-machine',
'reclaim-machine',
params=params,
transactional=True,
):
raise TaskEnqueuingError('reclaim-machine')
return True
def endpoint_call(self, service, name, args):
body = json.loads(protojson.encode_message(args))
return test_case.Endpoints(service).call_api(name, body=body).json
def msg_dict(request): return json.loads(protojson.encode_message(request))
def message_to_dict(message): """Returns a JSON-ish dictionary corresponding to the RPC message.""" return json.loads(protojson.encode_message(message))
def endpoint_call(self, service, name, args): body = json.loads(protojson.encode_message(args)) return test_case.Endpoints(service).call_api(name, body=body).json
def reclaim_machine(machine_key, reclamation_ts):
"""Attempts to reclaim the given machine.
Args:
machine_key: ndb.Key for a model.CatalogMachineEntry instance.
reclamation_ts: datetime.datetime instance indicating when the machine was
reclaimed.
Returns:
True if the machine was reclaimed, else False.
"""
machine = machine_key.get()
logging.info('Attempting to reclaim CatalogMachineEntry:\n%s', machine)
if machine.lease_expiration_ts is None:
# This can reasonably happen if e.g. the lease was voluntarily given up.
logging.warning('CatalogMachineEntry no longer leased:\n%s', machine)
return False
if reclamation_ts < machine.lease_expiration_ts:
# This can reasonably happen if e.g. the lease duration was extended.
logging.warning('CatalogMachineEntry no longer overdue:\n%s', machine)
return False
logging.info('Reclaiming CatalogMachineEntry:\n%s', machine)
lease = models.LeaseRequest.get_by_id(machine.lease_id)
hostname = lease.response.hostname
lease.machine_id = None
lease.response.hostname = None
machine.lease_id = None
machine.lease_expiration_ts = None
policy = machine.policies.on_reclamation
if policy == rpc_messages.MachineReclamationPolicy.DELETE:
logging.info('Executing MachineReclamationPolicy: DELETE')
lease.put()
machine.key.delete()
else:
if policy == rpc_messages.MachineReclamationPolicy.MAKE_AVAILABLE:
logging.info('Executing MachineReclamationPolicy: MAKE_AVAILABLE')
machine.state = models.CatalogMachineEntryStates.AVAILABLE
else:
if policy != rpc_messages.MachineReclamationPolicy.RECLAIM:
# Something is awry. Log an error, but still reclaim the machine.
# Fall back on the RECLAIM policy because it notifies the backend and
# prevents the machine from being leased out again, but keeps it in
# the Catalog in case we want to examine it further.
logging.error(
'Unexpected MachineReclamationPolicy: %s\nDefaulting to RECLAIM',
policy,
)
else:
logging.info('Executing MachineReclamationPolicy: RECLAIM')
machine.state = models.CatalogMachineEntryStates.RECLAIMED
ndb.put_multi([lease, machine])
params = {
'hostname': hostname,
'policies': protojson.encode_message(machine.policies),
'request_json': protojson.encode_message(lease.request),
'response_json': protojson.encode_message(lease.response),
}
backend_attributes = {}
for attribute in machine.policies.backend_attributes:
backend_attributes[attribute.key] = attribute.value
params['backend_attributes'] = utils.encode_to_json(backend_attributes)
if lease.request.pubsub_topic:
params['lessee_project'] = lease.request.pubsub_project
params['lessee_topic'] = lease.request.pubsub_topic
if not utils.enqueue_task(
'/internal/queues/reclaim-machine',
'reclaim-machine',
params=params,
transactional=True,
):
raise TaskEnqueuingError('reclaim-machine')
return True
def message_to_dict(rpc_message):
return json.loads(protojson.encode_message(rpc_message))
def message_to_dict(rpc_message): return json.loads(protojson.encode_message(rpc_message))
