class ReportMetadata(Model):
"""Metadata associated with a crash report."""
# Job type from testcase.
job_type = StringProperty()
# Revision of build from report.
crash_revision = ndb.IntegerProperty(default=-1)
# Has this report been successfully uploaded?
is_uploaded = ndb.BooleanProperty(default=False)
# Product.
product = StringProperty(default='')
# Version.
version = TextProperty(default='')
# Key to minidump previously written to blobstore.
minidump_key = TextProperty(default='')
# Processed crash bytes.
serialized_crash_stack_frames = ndb.BlobProperty(default='', indexed=False)
# Id of the associated testcase.
testcase_id = StringProperty(default='')
# Id of the associated bot.
bot_id = TextProperty(default='')
# Optional upload params, stored as a JSON object.
optional_params = TextProperty(indexed=False)
# Report id from crash/.
crash_report_id = StringProperty()
class Program(ndb.Model):
"""A single program"""
# Parent is a Folder
# key is the program's name (unique for a folder)
description = ndb.StringProperty()
source = ndb.TextProperty()
screenshot = ndb.BlobProperty()
datetime = ndb.DateTimeProperty() # this is UTC date and time
class ProcessedFileHead(ndb.Model):
etag = ndb.BlobProperty(required=True)
# HTTP ETag on this server, generated by us as a hash of the contents
encoding = ndb.BlobProperty(required=True)
# Encoding, always matches the corresponding 'RawFileContent' object
modified = ndb.DateTimeProperty(indexed=False, auto_now=True)
# Time when this file was generated
numparts = ndb.IntegerProperty(indexed=False)
# Number of parts; there will be 'numparts - 1' objects of kind
# 'ProcessedFilePart' in the database. Processed files are split up into
# parts as required by datastore blob limitations (currently these can only
# be up to 1 MiB in size)
data0 = ndb.BlobProperty(required=True)
class SomeKind(ndb.Model):
blob = ndb.BlobProperty()
class SomeKind(ndb.Model):
foo = ndb.BlobProperty(compressed=True)
class OtherKind(ndb.Model):
data = ndb.BlobProperty(indexed=False)
class ProcessedFilePart(ndb.Model):
data = ndb.BlobProperty(required=True)
# Contents
etag = ndb.BlobProperty(required=True)
class RawFileContent(ndb.Model):
data = ndb.BlobProperty(required=True)
# The data
encoding = ndb.BlobProperty(required=True)
class RawFileInfo(ndb.Model):
git_sha = ndb.BlobProperty()
# 'sha' property returned by GitHub API (not populated for vim_faq.txt)
etag = ndb.BlobProperty()
class Session(ndb.Model):
"""A shell session. Stores the session's globals.
Each session globals is stored in one of two places:
If the global is picklable, it's stored in the parallel globals and
global_names list properties. (They're parallel lists to work around the
unfortunate fact that the datastore can't store dictionaries natively.)
If the global is not picklable (e.g. modules, classes, and functions), or if
it was created by the same statement that created an unpicklable global,
it's not stored directly. Instead, the statement is stored in the
unpicklables list property. On each request, before executing the current
statement, the unpicklable statements are evaluated to recreate the
unpicklable globals.
The unpicklable_names property stores all of the names of globals that were
added by unpicklable statements. When we pickle and store the globals after
executing a statement, we skip the ones in unpicklable_names.
Using Text instead of string is an optimization. We don't query on any of
these properties, so they don't need to be indexed.
"""
global_names = ndb.TextProperty(repeated=True)
globals = ndb.BlobProperty(repeated=True)
unpicklable_names = ndb.TextProperty(repeated=True)
unpicklables = ndb.TextProperty(repeated=True)
def set_global(self, name, value):
"""Adds a global, or updates it if it already exists.
Also removes the global from the list of unpicklable names.
Args:
name: the name of the global to remove
value: any picklable value
"""
# We need to disable the pickling optimization here in order to get the
# correct values out.
blob = self.fast_dumps(value, 1)
if name in self.global_names:
index = self.global_names.index(name)
self.globals[index] = blob
else:
self.global_names.append(name)
self.globals.append(blob)
self.remove_unpicklable_name(name)
def remove_global(self, name):
"""Removes a global, if it exists.
Args:
name: string, the name of the global to remove
"""
if name in self.global_names:
index = self.global_names.index(name)
del self.global_names[index]
del self.globals[index]
def globals_dict(self):
"""Returns a dictionary view of the globals.
"""
return dict((name, pickle.loads(val))
for name, val in zip(self.global_names, self.globals))
def add_unpicklable(self, statement, names):
"""Adds a statement and list of names to the unpicklables.
Also removes the names from the globals.
Args:
statement: string, the statement that created new unpicklable global(s).
names: list of strings; the names of the globals created by the statement.
"""
self.unpicklables.append(statement)
for name in names:
self.remove_global(name)
if name not in self.unpicklable_names:
self.unpicklable_names.append(name)
def remove_unpicklable_name(self, name):
"""Removes a name from the list of unpicklable names, if it exists.
Args:
name: string, the name of the unpicklable global to remove
"""
if name in self.unpicklable_names:
self.unpicklable_names.remove(name)
def fast_dumps(self, obj, protocol=None):
"""Performs the same function as pickle.dumps but with optimizations off.
Args:
obj: object, object to be pickled
protocol: int, optional protocol option to emulate pickle.dumps
Note: It is necessary to pickle SymPy values with the fast option in order
to get the correct assumptions when unpickling. See Issue 2587.
"""
file = StringIO()
p = pickle.Pickler(file, protocol)
p.fast = 1
p.dump(obj)
return file.getvalue()
