def unpack(data):
if data is None:
return None
elif isinstance(data, str):
# Just one instance
return ndb.ModelAdapter().pb_to_entity(entity_pb.EntityProto(data))
else:
return [ndb.ModelAdapter().pb_to_entity(entity_pb.EntityProto(x)) for x in data]
def deserialize_entities(data):
# precondition: model class must be imported
if data is None:
return None
elif isinstance(data, str):
# Just one instance
return ndb.ModelAdapter().pb_to_entity(entity_pb.EntityProto(data))
else:
return [ndb.ModelAdapter().pb_to_entity(entity_pb.EntityProto(x)) for x in data]
def serialize_entities(models):
if models is None:
return None
elif isinstance(models, ndb.Model):
# Just one instance
return ndb.ModelAdapter().entity_to_pb(models).Encode()
else:
# A list
return [ndb.ModelAdapter().entity_to_pb(x).Encode() for x in models]
def pack(data_models):
if data_models is None:
return None
elif isinstance(data_models, ndb.Model):
# Just one instance
return ndb.ModelAdapter().entity_to_pb(data_models).Encode()
else:
# A list
return [ndb.ModelAdapter().entity_to_pb(x).Encode() for x in data_models]
def ndb_entity_to_protobuf(e):
"""
Saves an entity to a protobuf.
"""
if e is None:
return ""
return ndb.ModelAdapter().entity_to_pb(e).Encode()
def protobuf_to_ndb_entity(pb):
"""
Reads an entity from a protobuf.
"""
# precondition: model class must be imported
if pb == "":
return None
return ndb.ModelAdapter().pb_to_entity(entity_pb.EntityProto(pb))
def BinaryProtobufToEntity(pb_str):
"""Converts a protobuf message in binary format to an ndb entity.
Args:
pb_str: Binary encoded protocol buffer which is encoded as text.
Returns:
A ndb Entity.
"""
message = model.entity_pb.EntityProto(base64.b64decode(pb_str))
return ndb.ModelAdapter().pb_to_entity(message)
def EntityToBinaryProtobuf(entity):
"""Converts an ndb entity to a protobuf message in binary format."""
# Encode in binary representation of the protocol buffer.
message = ndb.ModelAdapter().entity_to_pb(entity).Encode()
# Base64 encode the data to text format for json.dumps.
return base64.b64encode(message)
def protobuf_to_entity(pb):
return ndb.ModelAdapter().pb_to_entity(entity_pb.EntityProto(pb))
def entity_to_protobuf(e):
return ndb.ModelAdapter().entity_to_pb(e).Encode()
def post(self):
"""Loads the requested data from the production server."""
if 'Development' not in os.environ['SERVER_SOFTWARE']:
self.RenderHtml(
'result.html', {
'errors': [
'This should not be run in production, only on dev server.'
]
})
return
sheriff = self.request.get('sheriff')
test_path = self.request.get('test_path')
if test_path:
num_points = self.request.get('num_points')
end_rev = self.request.get('end_rev')
url = ('%s?test_path=%s&num_points=%s' %
(_PROD_DUMP_GRAPH_JSON_URL, urllib.quote(test_path),
num_points))
if end_rev:
url += '&end_rev=%s' % end_rev
elif sheriff:
sheriff_name = self.request.get('sheriff')
num_alerts = self.request.get('num_alerts')
num_points = self.request.get('num_points')
url = ('%s?sheriff=%s&num_alerts=%s&num_points=%s' %
(_PROD_DUMP_GRAPH_JSON_URL, urllib.quote(sheriff_name),
num_alerts, num_points))
else:
self.RenderHtml(
'result.html',
{'errors': ['Need to specify a test_path or sheriff.']})
return
# This takes a while.
response = urlfetch.fetch(url, deadline=60)
if response.status_code != 200:
self.RenderHtml('result.html',
{'errors': ['Could not fetch %s' % url]})
return
protos = json.loads(response.content)
kinds = ['Master', 'Bot', 'Test', 'Row', 'Sheriff', 'Anomaly']
entities = {k: [] for k in kinds}
for proto in protos:
pb = model.entity_pb.EntityProto(base64.b64decode(proto))
# App ID is set in the key and all the ReferenceProperty keys to
# 's~chromeperf'. It won't be found in queries unless we use the
# devserver app ID.
key = pb.mutable_key()
key.set_app(_DEV_APP_ID)
for prop in pb.property_list():
val = prop.mutable_value()
if val.has_referencevalue():
ref = val.mutable_referencevalue()
ref.set_app(_DEV_APP_ID)
entity = ndb.ModelAdapter().pb_to_entity(pb)
entities[entity.key.kind()].append(entity)
for kind in kinds:
ndb.put_multi(entities[kind])
update_test_suites.UpdateTestSuites(datastore_hooks.INTERNAL)
update_test_suites.UpdateTestSuites(datastore_hooks.EXTERNAL)
num_entities = sum(len(entities[kind]) for kind in kinds)
self.RenderHtml(
'result.html', {
'results': [{
'name': 'Added data',
'value': '%d entities' % num_entities
}]
})
def protobuf_to_ndb_entity(pb):
# precondition: model class must be imported
return ndb.ModelAdapter().pb_to_entity(entity_pb.EntityProto(pb))
