class Xml(ndb.Model):
# A row in the database.
xml_hash = ndb.IntegerProperty()
xml_content = ndb.TextProperty()
xml_author = ndb.TextProperty()
xml_date = ndb.TextProperty()
xml_desc = ndb.TextProperty()
class ReportMetadata(Model): """Metadata associated with a crash report.""" # Job type from testcase. job_type = ndb.StringProperty() # Revision of build from report. crash_revision = ndb.IntegerProperty(default=-1) # Has this report been successfully uploaded? is_uploaded = ndb.BooleanProperty(default=False) # Product. product = ndb.StringProperty(default='') # Version. version = ndb.TextProperty(default='') # Key to minidump previously written to blobstore. minidump_key = ndb.TextProperty(default='') # Processed crash bytes. serialized_crash_stack_frames = ndb.BlobProperty(default='', indexed=False) # Id of the associated testcase. testcase_id = ndb.StringProperty(default='') # Id of the associated bot. bot_id = ndb.TextProperty(default='') # Optional upload params, stored as a JSON object. optional_params = ndb.TextProperty(indexed=False) # Report id from crash/. crash_report_id = ndb.StringProperty()
class SportCenter(ndb.Model):
title = ndb.StringProperty()
email_address = ndb.StringProperty()
street = ndb.TextProperty()
city = ndb.TextProperty()
zip_number = ndb.StringProperty()
country = ndb.TextProperty()
created = ndb.DateTimeProperty(auto_now_add=True)
updated = ndb.DateTimeProperty(auto_now=True)
deleted = ndb.BooleanProperty(default=False)
@classmethod
def create(cls, text):
with client.context(
): # with client.context() is obligatory to use in the new ndb library
message = cls(text=text)
message.put()
return message
@classmethod
def fetch_all(cls):
with client.context():
messages = cls.query().fetch()
return messages
class Team(ndb.Model):
"""
Teams represent FIRST Robotics Competition teams.
key_name is like 'frc177'
"""
team_number = ndb.IntegerProperty(required=True)
name = ndb.TextProperty(indexed=False)
nickname = ndb.TextProperty(indexed=False)
school_name = ndb.TextProperty(indexed=False)
home_cmp = ndb.StringProperty()
# city, state_prov, country, and postalcode are from FIRST
city = ndb.StringProperty() # Equivalent to locality. From FRCAPI
state_prov = ndb.StringProperty() # Equivalent to region. From FRCAPI
country = ndb.StringProperty() # From FRCAPI
postalcode = (
ndb.StringProperty()
) # From ElasticSearch only. String because it can be like "95126-1215"
website = ndb.TextProperty(indexed=False)
first_tpid = (ndb.IntegerProperty()
) # from USFIRST. FIRST team ID number. -greg 5/20/2010
first_tpid_year = (
ndb.IntegerProperty()
) # from USFIRST. Year tpid is applicable for. -greg 9 Jan 2011
rookie_year = ndb.IntegerProperty()
motto = ndb.TextProperty(indexed=False)
created = ndb.DateTimeProperty(auto_now_add=True, indexed=False)
updated = ndb.DateTimeProperty(auto_now=True, indexed=False)
class MastodonAuth(BaseAuth):
"""An authenticated Mastodon user.
Provides methods that return information about this user and make OAuth-signed
requests to the Mastodon REST API. Stores OAuth credentials in the datastore.
See models.BaseAuth for usage details.
Key name is the fully qualified actor address, ie @[email protected].
Implements get() and post() but not urlopen() or api().
"""
app = ndb.KeyProperty()
access_token_str = ndb.TextProperty(required=True)
user_json = ndb.TextProperty()
def site_name(self):
return 'Mastodon'
def user_display_name(self):
"""Returns the user's full ActivityPub address, eg @[email protected]."""
return self.key.id()
def instance(self):
"""Returns the instance base URL, eg https://mastodon.social/."""
return self.app.get().instance
def username(self):
"""Returns the user's username, eg ryan."""
return json_loads(self.user_json).get('username')
def user_id(self):
"""Returns the user's id, eg 123."""
return json_loads(self.user_json).get('id')
def access_token(self):
"""Returns the OAuth access token string."""
return self.access_token_str
def get(self, *args, **kwargs):
"""Wraps requests.get() and adds instance base URL and Bearer token header."""
url = urljoin(self.instance(), args[0])
return self._requests_call(util.requests_get, url, *args[1:], **kwargs)
def post(self, *args, **kwargs):
"""Wraps requests.post() and adds the Bearer token header."""
return self._requests_call(util.requests_post, *args, **kwargs)
def _requests_call(self, fn, *args, **kwargs):
headers = kwargs.setdefault('headers', {})
headers['Authorization'] = 'Bearer ' + self.access_token_str
resp = fn(*args, **kwargs)
try:
resp.raise_for_status()
except BaseException as e:
util.interpret_http_exception(e)
raise
return resp
class MastodonApp(ndb.Model):
"""A Mastodon API OAuth2 app registered with a specific instance."""
instance = ndb.StringProperty(
required=True) # URL, eg https://mastodon.social/
data = ndb.TextProperty(required=True) # JSON; includes client id/secret
instance_info = ndb.TextProperty() # JSON; from /api/v1/instance
app_url = ndb.StringProperty()
app_name = ndb.StringProperty()
created_at = ndb.DateTimeProperty(auto_now_add=True, required=True)
class TestcaseUploadMetadata(Model):
"""Metadata associated with a user uploaded test case."""
# Timestamp.
timestamp = ndb.DateTimeProperty()
# Testcase filename.
filename = ndb.StringProperty()
# Current status of the testcase.
status = ndb.StringProperty()
# Uploader email address.
uploader_email = ndb.StringProperty()
# Name of the bot that ran analyze on this testcase.
bot_name = ndb.StringProperty()
# Id of the associated testcase.
testcase_id = ndb.IntegerProperty()
# Id of the testcase that this is marked as a duplicate of.
duplicate_of = ndb.IntegerProperty()
# Blobstore key for the testcase associated with this object.
blobstore_key = ndb.StringProperty()
# Testcase timeout.
timeout = ndb.IntegerProperty()
# Is this a single testcase bundled in an archive?
bundled = ndb.BooleanProperty()
# Path to the file in the archive.
path_in_archive = ndb.TextProperty()
# Original blobstore key for this object (used for archives).
original_blobstore_key = ndb.StringProperty()
# Security flag.
security_flag = ndb.BooleanProperty(default=False)
# Number of retries for this testcase.
retries = ndb.IntegerProperty()
# Flag to indicate where bug title should be updated or not.
bug_summary_update_flag = ndb.BooleanProperty()
# Flag to indicate if we are running in quiet mode (e.g. bug updates).
quiet_flag = ndb.BooleanProperty()
# Additional testcase metadata dict stored as a string.
additional_metadata_string = ndb.TextProperty(indexed=False)
# Specified issue id.
bug_information = ndb.StringProperty()
class Activity(StringIdModel): """An activity with responses to be propagated. The key name is the activity id as a tag URI. Currently only used for posts sent to us by the browser extension. """ source = ndb.KeyProperty() created = ndb.DateTimeProperty(auto_now_add=True, tzinfo=timezone.utc) updated = ndb.DateTimeProperty(auto_now=True, tzinfo=timezone.utc) activity_json = ndb.TextProperty() html = ndb.TextProperty()
class LoveLink(ndb.Model):
"""Models an instance of a Love Link."""
hash_key = ndb.StringProperty()
message = ndb.TextProperty()
recipient_list = ndb.TextProperty()
timestamp = ndb.DateTimeProperty(auto_now_add=True)
@property
def seconds_since_epoch(self):
return int(mktime(self.timestamp.timetuple()))
@property
def url(self):
return config.APP_BASE_URL + 'l/' + self.hash_key
class Feed(models.StringIdModel):
"""An HTML feed uploaded by the browser extension, converted to AS1.
Key is token generated by browser extension.
"""
html = ndb.TextProperty(required=True)
actor_json = ndb.StringProperty()
as1_json = ndb.TextProperty(required=True)
created = ndb.DateTimeProperty(auto_now_add=True,
required=True,
tzinfo=timezone.utc)
updated = ndb.DateTimeProperty(auto_now=True,
required=True,
tzinfo=timezone.utc)
class MediumAuth(BaseAuth):
"""An authenticated Medium user.
Provides methods that return information about this user and make OAuth-signed
requests to the Medium REST API. Stores OAuth credentials in the datastore.
See models.BaseAuth for usage details.
Medium-specific details: implements get() but not urlopen() or api().
The key name is the user id (*not* username).
"""
access_token_str = ndb.StringProperty(required=True)
user_json = ndb.TextProperty()
# used by bridgy in
# https://github.com/snarfed/bridgy/commit/58cce60790e746d300e7e5dac331543c56bd9108
# background: https://github.com/snarfed/bridgy/issues/506
publications_json = ndb.TextProperty()
def site_name(self):
return 'Medium'
def user_display_name(self):
"""Returns the user's full name or username.
"""
if self.user_json:
data = json_loads(self.user_json).get('data')
if data:
return data.get('name') or data.get('username')
return self.key_id()
def access_token(self):
"""Returns the OAuth access token string.
"""
return self.access_token_str
def get(self, *args, **kwargs):
"""Wraps requests.get() and adds the Bearer token header.
"""
headers = kwargs.setdefault('headers', {})
headers['Authorization'] = 'Bearer ' + self.access_token_str
headers.setdefault('User-Agent', USER_AGENT)
resp = util.requests_get(*args, **kwargs)
try:
resp.raise_for_status()
except BaseException as e:
util.interpret_http_exception(e)
raise
return resp
class BloggerV2Auth(models.BaseAuth):
"""An authenticated Blogger user.
Provides methods that return information about this user (or page) and make
OAuth-signed requests to the Blogger API. Stores OAuth credentials in the
datastore. See models.BaseAuth for usage details.
Blogger-specific details: implements api() but not urlopen(). api() returns a
:class:`gdata.blogger.client.BloggerClient`. The datastore entity key name is
the Blogger user id.
"""
name = ndb.StringProperty(required=True)
creds_json = ndb.TextProperty(required=True)
user_atom = ndb.TextProperty(required=True)
blogs_atom = ndb.TextProperty(required=True)
picture_url = ndb.TextProperty(required=True)
# the elements in both of these lists match
blog_ids = ndb.StringProperty(repeated=True)
blog_titles = ndb.StringProperty(repeated=True)
blog_hostnames = ndb.StringProperty(repeated=True)
def site_name(self):
return 'Blogger'
def user_display_name(self):
"""Returns the user's Blogger username.
"""
return self.name
def access_token(self):
"""Returns the OAuth access token string.
"""
return json_loads(self.creds_json)['access_token']
def _api(self):
"""Returns a gdata.blogger.client.BloggerClient.
"""
return BloggerClient(auth_token=self)
def modify_request(self, http_request):
"""Makes this class usable as an auth_token object in a gdata Client.
Background in :class:`gdata.client.GDClient` and
:meth:`gdata.client.GDClient.request`. Other similar classes include
:class:`gdata.gauth.ClientLoginToken` and :class:`gdata.gauth.AuthSubToken`.
"""
http_request.headers['Authorization'] = 'Bearer %s' % self.access_token()
class Follower(StringIdModel):
"""A follower of a Bridgy Fed user.
Key name is 'USER_DOMAIN FOLLOWER_ID', e.g.:
'snarfed.org https://mastodon.social/@swentel'.
"""
STATUSES = ('active', 'inactive')
# most recent AP Follow activity (JSON). must have a composite actor object
# with an inbox, publicInbox, or sharedInbox!
last_follow = ndb.TextProperty()
status = ndb.StringProperty(choices=STATUSES, default='active')
created = ndb.DateTimeProperty(auto_now_add=True)
updated = ndb.DateTimeProperty(auto_now=True)
@classmethod
def _id(cls, user_domain, follower_id):
assert user_domain
assert follower_id
return '%s %s' % (user_domain, follower_id)
@classmethod
def get_or_create(cls, user_domain, follower_id, **kwargs):
logging.info('new Follower for %s %s', user_domain, follower_id)
return cls.get_or_insert(cls._id(user_domain, follower_id), **kwargs)
class DisqusAuth(models.BaseAuth):
"""An authenticated Disqus user.
Provides methods that return information about this user (or page)
and make OAuth-signed requests to Instagram's HTTP-based
APIs. Stores OAuth credentials in the datastore. See models.BaseAuth
for usage details.
Disqus-specific details: implements urlopen() but not api().
The key name is the Disqus user id.
"""
auth_code = ndb.StringProperty(required=True)
access_token_str = ndb.StringProperty(required=True)
user_json = ndb.TextProperty(required=True)
def site_name(self):
return 'Disqus'
def user_display_name(self):
"""Returns the user's name.
"""
return json_loads(self.user_json)['name']
def access_token(self):
"""Returns the OAuth access token string.
"""
return self.access_token_str
def urlopen(self, url, **kwargs):
"""Wraps urlopen() and adds OAuth credentials to the request.
"""
# TODO does work for POST requests? key is always passed as a
# query param, regardless of method.
return models.BaseAuth.urlopen_access_token(url, self.access_token_str,
DISQUS_CLIENT_ID, **kwargs)
class House(ndb.Model):
address = ndb.StringProperty()
json_property = ndb.JsonProperty()
indexed_int = ndb.IntegerProperty()
unindexed_int = ndb.IntegerProperty(indexed=False)
text_property = ndb.TextProperty()
datetime_property = ndb.DateTimeProperty()
class SessionModel(ndb.Model):
created_on = ndb.DateTimeProperty(auto_now_add=True, indexed=False)
updated_on = ndb.DateTimeProperty(auto_now=True, indexed=False)
expires_on = ndb.DateTimeProperty(indexed=True)
data = ndb.TextProperty(indexed=False)
def delete(self):
self.key.delete()
@classmethod
def delete_by_id(cls, sid):
ndb.Key(cls, sid).delete()
def has_expired(self):
return self.expires_on and self.expires_on <= datetime.utcnow()
def should_slide_expiry(self):
if not self.expires_on or not self.updated_on:
return False
# Use a precision of 5 minutes
return (datetime.utcnow() - self.updated_on).total_seconds() > 300
def get_data(self):
return json.loads(want_bytes(self.data))
def set_data(self, data):
self.data = json.dumps(dict(data))
class InstagramAuth(models.BaseAuth):
"""An authenticated Instagram user or page.
Provides methods that return information about this user (or page) and make
OAuth-signed requests to Instagram's HTTP-based APIs. Stores OAuth credentials
in the datastore. See models.BaseAuth for usage details.
Instagram-specific details: implements urlopen() but not api(). The key name
is the Instagram username.
"""
auth_code = ndb.StringProperty(required=True)
access_token_str = ndb.StringProperty(required=True)
user_json = ndb.TextProperty(required=True)
def site_name(self):
return 'Instagram'
def user_display_name(self):
"""Returns the Instagram username.
"""
return self.key_id()
def access_token(self):
"""Returns the OAuth access token string.
"""
return self.access_token_str
def urlopen(self, url, **kwargs):
"""Wraps urlopen() and adds OAuth credentials to the request.
"""
return models.BaseAuth.urlopen_access_token(url, self.access_token_str,
**kwargs)
class Publish(ndb.Model):
"""A comment, like, repost, or RSVP published into a silo.
Child of a :class:`PublishedPage` entity.
"""
STATUSES = ('new', 'complete', 'failed', 'deleted')
# Turn off instance and memcache caching. See Source for details.
_use_cache = False
_use_memcache = False
type = ndb.StringProperty(choices=PUBLISH_TYPES)
status = ndb.StringProperty(choices=STATUSES, default='new')
source = ndb.KeyProperty()
html = ndb.TextProperty() # raw HTML fetched from source
published = ndb.JsonProperty(compressed=True)
created = ndb.DateTimeProperty(auto_now_add=True)
updated = ndb.DateTimeProperty(auto_now=True)
def type_label(self):
"""Returns silo-specific string type, e.g. 'favorite' instead of 'like'."""
for cls in sources.values(): # global
if cls.__name__ == self.source.kind():
return cls.TYPE_LABELS.get(self.type, self.type)
return self.type
class MeetupAuth(BaseAuth):
"""An authenticated Meetup.com user.
Provides methods that return information about this user and make
OAuth-signed requests to Meetup's HTTP-based APIs. Stores OAuth credentials
in the datastore. See models.BaseAuth for usage details.
Implements urlopen() but not http() or api().
"""
access_token_str = ndb.StringProperty(required=True)
user_json = ndb.TextProperty(required=True)
def site_name(self):
return 'Meetup.com'
def user_display_name(self):
"""Returns the Meetup.com user id.
"""
return json_loads(self.user_json)['name']
def access_token(self):
"""Returns the OAuth access token string.
"""
return self.access_token_str
def urlopen(self, url, **kwargs):
return urlopen_bearer_token(url, self.access_token_str, kwargs)
class Note(ndb.Model):
# """NDB model class for a user's note.
# Key is user id from decrypted token.
# """
friendly_id = ndb.StringProperty()
message = ndb.TextProperty()
created = ndb.DateTimeProperty(auto_now_add=True)
class Program(ndb.Model):
"""A single program"""
# Parent is a Folder
# key is the program's name (unique for a folder)
description = ndb.StringProperty()
source = ndb.TextProperty()
screenshot = ndb.BlobProperty()
datetime = ndb.DateTimeProperty() # this is UTC date and time
class Trial(Model): """Trials for specific binaries.""" # App name that this trial is applied to. E.g. "d8" or "chrome". app_name = ndb.StringProperty() # Chance to select this set of arguments. Zero to one. probability = ndb.FloatProperty() # Additional arguments to apply if selected. app_args = ndb.TextProperty()
class Love(ndb.Model):
"""Models an instance of sent love."""
message = ndb.TextProperty()
recipient_key = ndb.KeyProperty(kind=Employee)
secret = ndb.BooleanProperty(default=False)
sender_key = ndb.KeyProperty(kind=Employee)
timestamp = ndb.DateTimeProperty(auto_now_add=True)
@property
def seconds_since_epoch(self):
return int(mktime(self.timestamp.timetuple()))
class GitHubAuth(BaseAuth):
"""An authenticated GitHub user.
Provides methods that return information about this user and make OAuth-signed
requests to the GitHub REST API. Stores OAuth credentials in the datastore.
See models.BaseAuth for usage details.
GitHub-specific details: implements get() but not urlopen(), or api().
The key name is the username.
"""
access_token_str = ndb.StringProperty(required=True)
user_json = ndb.TextProperty()
def site_name(self):
return 'GitHub'
def user_display_name(self):
"""Returns the user's full name or username.
"""
return self.key_id()
def access_token(self):
"""Returns the OAuth access token string.
"""
return self.access_token_str
def get(self, *args, **kwargs):
"""Wraps requests.get() and adds the Bearer token header.
TODO: unify with medium.py.
"""
return self._requests_call(util.requests_get, *args, **kwargs)
def post(self, *args, **kwargs):
"""Wraps requests.post() and adds the Bearer token header.
TODO: unify with medium.py.
"""
return self._requests_call(util.requests_post, *args, **kwargs)
def _requests_call(self, fn, *args, **kwargs):
headers = kwargs.setdefault('headers', {})
headers['Authorization'] = 'Bearer ' + self.access_token_str
resp = fn(*args, **kwargs)
assert 'errors' not in resp, resp
try:
resp.raise_for_status()
except BaseException as e:
util.interpret_http_exception(e)
raise
return resp
class AffectedPackage(ndb.Model):
"""Affected packages."""
# The affected package identifier.
package = ndb.StructuredProperty(Package)
# The list of affected ranges.
ranges = ndb.LocalStructuredProperty(AffectedRange2, repeated=True)
# The list of explicit affected versions.
versions = ndb.TextProperty(repeated=True)
# Database specific metadata.
database_specific = ndb.JsonProperty()
# Ecosystem specific metadata.
ecosystem_specific = ndb.JsonProperty()
class GoogleUser(models.BaseAuth):
"""An authenticated Google user.
Provides methods that return information about this user and make OAuth-signed
requests to Google APIs. Stores OAuth credentials in the datastore. See
models.BaseAuth for usage details.
To make Google API calls: https://google-auth.readthedocs.io/
"""
user_json = ndb.TextProperty()
token_json = ndb.TextProperty()
def site_name(self):
return 'Google'
def user_display_name(self):
"""Returns the user's name."""
return json_loads(self.user_json).get('name') or 'unknown'
def access_token(self):
"""Returns the OAuth access token string."""
return json_loads(self.token_json)['access_token']
class Tune(ndb.Model):
id_tune = ndb.IntegerProperty()
ref_tune = ndb.StringProperty()
titre = ndb.StringProperty() # name of the tune
auteur = ndb.StringProperty()
youtubelink = ndb.StringProperty()
text_ly = ndb.TextProperty()
chords = ndb.TextProperty()
image_file = ndb.StringProperty()
pdf_file = ndb.StringProperty()
id_rythme = ndb.IntegerProperty()
lastChanged = ndb.DateTimeProperty(auto_now=True)
@classmethod
def getAll(cls):
cache_tunes = redis_client.get("tunes")
if cache_tunes:
b_tunes = json.loads(cache_tunes)
else:
b_tunes = cls.query()
redis_client.set("tunes", json.dumps(b_tunes))
return b_tunes
class FixResult(ndb.Model):
"""Fix results."""
# The commit hash.
commit = ndb.StringProperty(default='')
# Vulnerability summary.
summary = ndb.TextProperty()
# Vulnerability details.
details = ndb.TextProperty()
# Error (if any).
error = ndb.StringProperty()
# OSS-Fuzz issue ID.
issue_id = ndb.StringProperty()
# Project for the bug.
project = ndb.StringProperty()
# Package ecosystem for the project.
ecosystem = ndb.StringProperty()
# Repo URL.
repo_url = ndb.StringProperty()
# Severity of the bug.
severity = ndb.StringProperty(validator=_check_valid_severity)
# Reference URLs.
reference_urls = ndb.StringProperty(repeated=True)
# Source timestamp.
timestamp = ndb.DateTimeProperty()
class AccessKey(ndb.Model):
"""Models an instance of an API key."""
access_key = ndb.StringProperty()
description = ndb.TextProperty()
@staticmethod
def generate_uuid():
return uuid4().hex
@classmethod
def create(cls, description):
new_key = cls()
new_key.access_key = cls.generate_uuid()
new_key.description = description
new_key.put()
return new_key
class CachedPage(StringIdModel):
"""Cached HTML for pages that changes rarely. Key id is path.
Stored in the datastore since datastore entities in memcache (mostly
:class:`models.Response`) are requested way more often, so it would get
evicted out of memcache easily.
Keys, useful for deleting from memcache:
/: aglzfmJyaWQtZ3lyEQsSCkNhY2hlZFBhZ2UiAS8M
/users: aglzfmJyaWQtZ3lyFgsSCkNhY2hlZFBhZ2UiBi91c2Vycww
"""
html = ndb.TextProperty()
expires = ndb.DateTimeProperty()
@classmethod
def load(cls, path):
cached = CachedPage.get_by_id(path)
if cached:
if cached.expires and now_fn() > cached.expires:
logging.info('Deleting expired cached page for %s', path)
cached.key.delete()
return None
else:
logging.info('Found cached page for %s', path)
return cached
@classmethod
def store(cls, path, html, expires=None):
"""Stores new page contents.
Args:
path: string
html: string
expires: :class:`datetime.timedelta`
"""
logging.info('Storing new page in cache for %s', path)
if expires is not None:
logging.info(' (expires in %s)', expires)
expires = now_fn() + expires
CachedPage(id=path, html=html, expires=expires).put()
@classmethod
def invalidate(cls, path):
logging.info('Deleting cached page for %s', path)
CachedPage(id=path).key.delete()
