class Result(BaseModel):
competition = ndb.KeyProperty(kind=Competition)
event = ndb.KeyProperty(kind=Event)
round_type = ndb.KeyProperty(kind=RoundType)
person = ndb.KeyProperty(kind=Person)
fmt = ndb.KeyProperty(kind=Format)
person_name = ndb.StringProperty()
person_country = ndb.KeyProperty(kind=Country)
pos = ndb.IntegerProperty()
best = ndb.IntegerProperty()
average = ndb.IntegerProperty()
values = ndb.IntegerProperty(repeated=True)
regional_single_record = ndb.StringProperty()
regional_average_record = ndb.StringProperty()
def ParseFromDict(self, row):
self.competition = ndb.Key(Competition, row['competitionId'])
self.event = ndb.Key(Event, row['eventId'])
self.round_type = ndb.Key(RoundType, row['roundTypeId'])
self.person = ndb.Key(Person, row['personId'])
self.fmt = ndb.Key(Format, row['formatId'])
self.person_name = row['personName']
self.person_country = ndb.Key(Country, row['personCountryId'])
self.pos = int(row['pos'])
self.best = int(row['best'])
self.average = int(row['average'])
self.values = [
v for v in [int(row['value%d' % n]) for n in (1, 2, 3, 4, 5)]
if v != 0
]
self.regional_single_record = row['regionalSingleRecord']
self.regional_average_record = row['regionalAverageRecord']
@staticmethod
def Filter():
# Only include results of championships that are in the datastore.
known_competitions = set([
championship.competition.id()
for championship in Championship.query().iter()
])
def filter_row(row):
return row['competitionId'] in known_competitions
return filter_row
@staticmethod
def GetId(row):
return '%s_%s_%s_%s' % (row['competitionId'], row['eventId'],
row['roundTypeId'], row['personId'])
@staticmethod
def ColumnsUsed():
return [
'competitionId', 'eventId', 'roundTypeId', 'personId', 'formatId',
'personName', 'personCountryId', 'pos', 'best', 'average',
'value1', 'value2', 'value3', 'value4', 'value5',
'regionalSingleRecord', 'regionalAverageRecord'
]
class SomeKind(ndb.Model):
foo = ndb.IntegerProperty()
class OtherKind(ndb.Model):
bar = ndb.IntegerProperty()
class BuildCrashStatsJobHistory(Model):
"""Represents the record of build_crash_stats run."""
# End time in hours from epoch, inclusively.
end_time_in_hours = ndb.IntegerProperty()
class Job(Model):
"""Definition of a job type used by the bots."""
VALID_NAME_REGEX = NAME_CHECK_REGEX
# Job type name.
name = StringProperty()
# Job environment string.
environment_string = TextProperty()
# The platform that this job can run on.
platform = StringProperty()
# Blobstore key of the custom binary for this job.
custom_binary_key = StringProperty()
# Filename for the custom binary.
custom_binary_filename = StringProperty()
# Revision of the custom binary.
custom_binary_revision = ndb.IntegerProperty()
# Description of the job.
description = TextProperty()
# Template to use, if any.
templates = StringProperty(repeated=True)
# Project name.
project = StringProperty()
def get_environment(self):
"""Get the environment as a dict for this job, including any environment
variables in its template."""
if not self.templates:
return environment.parse_environment_definition(
self.environment_string)
job_environment = {}
for template_name in self.templates:
template = JobTemplate.query(
JobTemplate.name == template_name).get()
if not template:
continue
template_environment = environment.parse_environment_definition(
template.environment_string)
job_environment.update(template_environment)
environment_overrides = environment.parse_environment_definition(
self.environment_string)
job_environment.update(environment_overrides)
return job_environment
def get_environment_string(self):
"""Get the environment string for this job, including any environment
variables in its template. Avoid using this if possible."""
environment_string = ''
job_environment = self.get_environment()
for key, value in six.iteritems(job_environment):
environment_string += '%s = %s\n' % (key, value)
return environment_string
def _pre_put_hook(self):
"""Pre-put hook."""
self.project = self.get_environment().get('PROJECT_NAME',
utils.default_project_name())
class Source(StringIdModel, metaclass=SourceMeta):
"""A silo account, e.g. a Facebook or Google+ account.
Each concrete silo class should subclass this class.
"""
# Turn off NDB instance and memcache caching.
# https://developers.google.com/appengine/docs/python/ndb/cache
# https://github.com/snarfed/bridgy/issues/558
# https://github.com/snarfed/bridgy/issues/68
_use_cache = False
STATUSES = ('enabled', 'disabled')
POLL_STATUSES = ('ok', 'error', 'polling')
FEATURES = ('listen', 'publish', 'webmention', 'email')
# short name for this site type. used in URLs, etc.
SHORT_NAME = None
# the corresponding granary class
GR_CLASS = None
# oauth-dropins StartHandler class
OAUTH_START_HANDLER = None
# whether Bridgy supports listen for this silo - this is unlikely, so we default to True
CAN_LISTEN = True
# whether Bridgy supports publish for this silo
CAN_PUBLISH = None
# whether this source should poll automatically, or only when triggered
# (eg Instagram)
AUTO_POLL = True
# how often to poll for responses
FAST_POLL = datetime.timedelta(minutes=30)
# how often to poll sources that have never sent a webmention
SLOW_POLL = datetime.timedelta(days=1)
# how often to poll sources that are currently rate limited by their silo
RATE_LIMITED_POLL = SLOW_POLL
# how long to wait after signup for a successful webmention before dropping to
# the lower frequency poll
FAST_POLL_GRACE_PERIOD = datetime.timedelta(days=7)
# how often refetch author url to look for updated syndication links
FAST_REFETCH = datetime.timedelta(hours=6)
# refetch less often (this often) if it's been >2w since the last synd link
SLOW_REFETCH = datetime.timedelta(days=2)
# rate limiting HTTP status codes returned by this silo. e.g. twitter returns
# 429, instagram 503, google+ 403.
RATE_LIMIT_HTTP_CODES = ('429', )
DISABLE_HTTP_CODES = ('401', )
TRANSIENT_ERROR_HTTP_CODES = ()
# whether granary supports fetching block lists
HAS_BLOCKS = False
# whether to require a u-syndication link for backfeed
BACKFEED_REQUIRES_SYNDICATION_LINK = False
# ignore fragments when comparing syndication links in OPD
IGNORE_SYNDICATION_LINK_FRAGMENTS = False
# Maps Publish.type (e.g. 'like') to source-specific human readable type label
# (e.g. 'favorite'). Subclasses should override this.
TYPE_LABELS = {}
# subclasses should override this
URL_CANONICALIZER = util.UrlCanonicalizer(headers=util.REQUEST_HEADERS)
# Regexps for URL paths that don't accept incoming webmentions. Currently used
# by Blogger.
PATH_BLOCKLIST = ()
created = ndb.DateTimeProperty(auto_now_add=True, required=True)
url = ndb.StringProperty()
status = ndb.StringProperty(choices=STATUSES, default='enabled')
poll_status = ndb.StringProperty(choices=POLL_STATUSES, default='ok')
rate_limited = ndb.BooleanProperty(default=False)
name = ndb.StringProperty() # full human-readable name
picture = ndb.StringProperty()
domains = ndb.StringProperty(repeated=True)
domain_urls = ndb.StringProperty(repeated=True)
features = ndb.StringProperty(repeated=True, choices=FEATURES)
superfeedr_secret = ndb.StringProperty()
webmention_endpoint = ndb.StringProperty()
# points to an oauth-dropins auth entity. The model class should be a subclass
# of oauth_dropins.BaseAuth. the token should be generated with the
# offline_access scope so that it doesn't expire.
auth_entity = ndb.KeyProperty()
#
# listen-only properties
#
last_polled = ndb.DateTimeProperty(default=util.EPOCH)
last_poll_attempt = ndb.DateTimeProperty(default=util.EPOCH)
last_webmention_sent = ndb.DateTimeProperty()
last_public_post = ndb.DateTimeProperty()
recent_private_posts = ndb.IntegerProperty(default=0)
# the last time we re-fetched the author's url looking for updated
# syndication links
last_hfeed_refetch = ndb.DateTimeProperty(default=util.EPOCH)
# the last time we've seen a rel=syndication link for this Source.
# we won't spend the time to re-fetch and look for updates if there's
# never been one
last_syndication_url = ndb.DateTimeProperty()
# the last time we saw a syndication link in an h-feed, as opposed to just on
# permalinks. background: https://github.com/snarfed/bridgy/issues/624
last_feed_syndication_url = ndb.DateTimeProperty()
last_activity_id = ndb.StringProperty()
last_activities_etag = ndb.StringProperty()
last_activities_cache_json = ndb.TextProperty()
seen_responses_cache_json = ndb.TextProperty(compressed=True)
# populated in Poll.poll(), used by handlers
blocked_ids = ndb.JsonProperty(compressed=True)
# maps updated property names to values that put_updates() writes back to the
# datastore transactionally. set this to {} before beginning.
updates = None
# gr_source is *not* set to None by default here, since it needs to be unset
# for __getattr__ to run when it's accessed.
def __init__(self, *args, id=None, **kwargs):
"""Constructor. Escapes the key string id if it starts with `__`."""
if id and id.startswith('__'):
id = '\\' + id
super().__init__(*args, id=id, **kwargs)
def key_id(self):
"""Returns the key's unescaped string id."""
id = self.key.id()
return id[1:] if id[0] == '\\' else id
@classmethod
def new(cls, handler, **kwargs):
"""Factory method. Creates and returns a new instance for the current user.
To be implemented by subclasses.
"""
raise NotImplementedError()
def __getattr__(self, name):
"""Lazily load the auth entity and instantiate :attr:`self.gr_source`.
Once :attr:`self.gr_source` is set, this method will *not* be called;
:attr:`gr_source` will be returned normally.
"""
if name == 'gr_source':
super_attr = getattr(super(Source, self), name, None)
if super_attr:
return super_attr
elif not self.auth_entity:
return None
auth_entity = self.auth_entity.get()
try:
refresh_token = auth_entity.refresh_token
self.gr_source = self.GR_CLASS(refresh_token)
return self.gr_source
except AttributeError:
logging.info('no refresh_token')
args = auth_entity.access_token()
if not isinstance(args, tuple):
args = (args, )
kwargs = {}
if self.key.kind(
) == 'FacebookPage' and auth_entity.type == 'user':
kwargs = {'user_id': self.key_id()}
elif self.key.kind() == 'Instagram':
kwargs = {'scrape': True, 'cookie': INSTAGRAM_SESSIONID_COOKIE}
elif self.key.kind() == 'Mastodon':
args = (auth_entity.instance(), ) + args
inst = auth_entity.app.get().instance_info
kwargs = {
'user_id':
json_loads(auth_entity.user_json).get('id'),
# https://docs-develop.pleroma.social/backend/API/differences_in_mastoapi_responses/#instance
'truncate_text_length':
json_loads(inst).get('max_toot_chars') if inst else None,
}
elif self.key.kind() == 'Twitter':
kwargs = {
'username': self.key_id(),
'scrape_headers': TWITTER_SCRAPE_HEADERS
}
self.gr_source = self.GR_CLASS(*args, **kwargs)
return self.gr_source
return getattr(super(Source, self), name)
@classmethod
def lookup(cls, id):
"""Returns the entity with the given id.
By default, interprets id as just the key id. Subclasses may extend this to
support usernames, etc.
"""
if id and id.startswith('__'):
id = '\\' + id
return ndb.Key(cls, id).get()
def user_tag_id(self):
"""Returns the tag URI for this source, e.g. 'tag:plus.google.com:123456'."""
return self.gr_source.tag_uri(self.key_id())
def bridgy_path(self):
"""Returns the Bridgy page URL path for this source."""
return '/%s/%s' % (self.SHORT_NAME, self.key_id())
def bridgy_url(self, handler):
"""Returns the Bridgy page URL for this source."""
return util.host_url(handler) + self.bridgy_path()
def silo_url(self, handler):
"""Returns the silo account URL, e.g. https://twitter.com/foo."""
raise NotImplementedError()
def label(self):
"""Human-readable label for this source."""
return '%s (%s)' % (self.label_name(), self.GR_CLASS.NAME)
def label_name(self):
"""Human-readable name or username for this source, whichever is preferred."""
return self.name or self.key_id()
@classmethod
@ndb.transactional()
def put_updates(cls, source):
"""Writes source.updates to the datastore transactionally.
Returns:
source: :class:`Source`
Returns:
the updated :class:`Source`
"""
if not source.updates:
return source
logging.info(
'Updating %s %s : %r', source.label(), source.bridgy_path(), {
k: v
for k, v in source.updates.items() if not k.endswith('_json')
})
updates = source.updates
source = source.key.get()
source.updates = updates
for name, val in updates.items():
setattr(source, name, val)
source.put()
return source
def poll_period(self):
"""Returns the poll frequency for this source, as a :class:`datetime.timedelta`.
Defaults to ~15m, depending on silo. If we've never sent a webmention for
this source, or the last one we sent was over a month ago, we drop them down
to ~1d after a week long grace period.
"""
now = datetime.datetime.now()
if self.rate_limited:
return self.RATE_LIMITED_POLL
elif now < self.created + self.FAST_POLL_GRACE_PERIOD:
return self.FAST_POLL
elif not self.last_webmention_sent:
return self.SLOW_POLL
elif self.last_webmention_sent > now - datetime.timedelta(days=7):
return self.FAST_POLL
elif self.last_webmention_sent > now - datetime.timedelta(days=30):
return self.FAST_POLL * 10
else:
return self.SLOW_POLL
def should_refetch(self):
"""Returns True if we should run OPD refetch on this source now."""
now = datetime.datetime.now()
if self.last_hfeed_refetch == REFETCH_HFEED_TRIGGER:
return True
elif not self.last_syndication_url:
return False
period = (self.FAST_REFETCH if self.last_syndication_url > now -
datetime.timedelta(days=14) else self.SLOW_REFETCH)
return self.last_poll_attempt >= self.last_hfeed_refetch + period
@classmethod
def bridgy_webmention_endpoint(cls, domain='brid.gy'):
"""Returns the Bridgy webmention endpoint for this source type."""
return 'https://%s/webmention/%s' % (domain, cls.SHORT_NAME)
def has_bridgy_webmention_endpoint(self):
"""Returns True if this source uses Bridgy's webmention endpoint."""
return self.webmention_endpoint in (self.bridgy_webmention_endpoint(),
self.bridgy_webmention_endpoint(
domain='www.brid.gy'))
def get_author_urls(self):
"""Determine the author urls for a particular source.
In debug mode, replace test domains with localhost.
Return:
a list of string URLs, possibly empty
"""
return [
util.replace_test_domains_with_localhost(u)
for u in self.domain_urls
]
def search_for_links(self):
"""Searches for activities with links to any of this source's web sites.
https://github.com/snarfed/bridgy/issues/456
https://github.com/snarfed/bridgy/issues/565
Returns:
sequence of ActivityStreams activity dicts
"""
return []
def get_activities_response(self, **kwargs):
"""Returns recent posts and embedded comments for this source.
May be overridden by subclasses.
"""
kwargs.setdefault('group_id', gr_source.SELF)
resp = self.gr_source.get_activities_response(**kwargs)
for activity in resp['items']:
self._inject_user_urls(activity)
return resp
def get_activities(self, **kwargs):
return self.get_activities_response(**kwargs)['items']
def get_comment(self, comment_id, **kwargs):
"""Returns a comment from this source.
Passes through to granary by default. May be overridden by subclasses.
Args:
comment_id: string, site-specific comment id
kwargs: passed to :meth:`granary.source.Source.get_comment`
Returns:
dict, decoded ActivityStreams comment object, or None
"""
comment = self.gr_source.get_comment(comment_id, **kwargs)
if comment:
self._inject_user_urls(comment)
return comment
def get_like(self, activity_user_id, activity_id, like_user_id, **kwargs):
"""Returns an ActivityStreams 'like' activity object.
Passes through to granary by default. May be overridden
by subclasses.
Args:
activity_user_id: string id of the user who posted the original activity
activity_id: string activity id
like_user_id: string id of the user who liked the activity
kwargs: passed to granary.Source.get_comment
"""
return self.gr_source.get_like(activity_user_id, activity_id,
like_user_id, **kwargs)
def _inject_user_urls(self, activity):
"""Adds this user's web site URLs to their user mentions (in tags), in place."""
obj = activity.get('object') or activity
user_tag_id = self.user_tag_id()
for tag in obj.get('tags', []):
if tag.get('id') == user_tag_id:
tag.setdefault('urls', []).extend([{
'value': u
} for u in self.domain_urls])
def create_comment(self, post_url, author_name, author_url, content):
"""Creates a new comment in the source silo.
Must be implemented by subclasses.
Args:
post_url: string
author_name: string
author_url: string
content: string
Returns:
response dict with at least 'id' field
"""
raise NotImplementedError()
def feed_url(self):
"""Returns the RSS or Atom (or similar) feed URL for this source.
Must be implemented by subclasses. Currently only implemented by
:mod:`blogger`, :mod:`medium`, :mod:`tumblr`, and :mod:`wordpress_rest`.
Returns:
string URL
"""
raise NotImplementedError()
def edit_template_url(self):
"""Returns the URL for editing this blog's template HTML.
Must be implemented by subclasses. Currently only implemented by
:mod:`blogger`, :mod:`medium`, :mod:`tumblr`, and :mod:`wordpress_rest`.
Returns:
string URL
"""
raise NotImplementedError()
@classmethod
def button_html(cls, feature, **kwargs):
"""Returns an HTML string with a login form and button for this site.
Mostly just passes through to
:meth:`oauth_dropins.handlers.StartHandler.button_html`.
Returns: string, HTML
"""
assert set(feature.split(',')) <= set(cls.FEATURES)
form_extra = (
kwargs.pop('form_extra', '') +
'<input name="feature" type="hidden" value="%s" />' % feature)
source = kwargs.pop('source', None)
if source:
form_extra += ('\n<input name="id" type="hidden" value="%s" />' %
source.key_id())
if cls.OAUTH_START_HANDLER:
return cls.OAUTH_START_HANDLER.button_html(
'/%s/start' % cls.SHORT_NAME,
form_extra=form_extra,
image_prefix='/oauth_dropins/static/',
**kwargs)
return ''
@classmethod
def create_new(cls, handler, user_url=None, **kwargs):
"""Creates and saves a new :class:`Source` and adds a poll task for it.
Args:
handler: the current :class:`webapp2.RequestHandler`
user_url: a string, optional. if provided, supersedes other urls when
determining the author_url
**kwargs: passed to :meth:`new()`
Returns: newly created :class:`Source`
"""
source = cls.new(handler, **kwargs)
if source is None:
return None
if not source.domain_urls: # defer to the source if it already set this
auth_entity = kwargs.get('auth_entity')
if auth_entity and hasattr(auth_entity, 'user_json'):
source.domain_urls, source.domains = source._urls_and_domains(
auth_entity, user_url)
logging.debug('URLs/domains: %s %s', source.domain_urls,
source.domains)
# check if this source already exists
existing = source.key.get()
if existing:
# merge some fields
source.features = set(source.features + existing.features)
source.populate(**existing.to_dict(
include=('created', 'last_hfeed_refetch', 'last_poll_attempt',
'last_polled', 'last_syndication_url',
'last_webmention_sent', 'superfeedr_secret',
'webmention_endpoint')))
verb = 'Updated'
else:
verb = 'Added'
author_urls = source.get_author_urls()
link = ('http://indiewebify.me/send-webmentions/?url=' + author_urls[0]
if author_urls else 'http://indiewebify.me/#send-webmentions')
feature = source.features[0] if source.features else 'listen'
blurb = '%s %s. %s' % (
verb, source.label(), 'Try previewing a post from your web site!'
if feature == 'publish' else '<a href="%s">Try a webmention!</a>' %
link if feature == 'webmention' else
"Refresh in a minute to see what we've found!")
logging.info('%s %s', blurb, source.bridgy_url(handler))
# uncomment to send email notification for each new user
# if not existing:
# util.email_me(subject=blurb, body=source.bridgy_url(handler))
source.verify()
if source.verified():
handler.messages = {blurb}
# TODO: ugh, *all* of this should be transactional
source.put()
if 'webmention' in source.features:
superfeedr.subscribe(source, handler)
if 'listen' in source.features and source.AUTO_POLL:
util.add_poll_task(source, now=True)
util.add_poll_task(source)
return source
def verified(self):
"""Returns True if this source is ready to be used, false otherwise.
See :meth:`verify()` for details. May be overridden by subclasses, e.g.
:class:`tumblr.Tumblr`.
"""
if not self.domains or not self.domain_urls:
return False
if 'webmention' in self.features and not self.webmention_endpoint:
return False
if ('listen' in self.features and
not (self.webmention_endpoint or self.last_webmention_sent)):
return False
return True
def verify(self, force=False):
"""Checks that this source is ready to be used.
For blog and listen sources, this fetches their front page HTML and
discovers their webmention endpoint. For publish sources, this checks that
they have a domain.
May be overridden by subclasses, e.g. :class:`tumblr.Tumblr`.
Args:
force: if True, fully verifies (e.g. re-fetches the blog's HTML and
performs webmention discovery) even we already think this source is
verified.
"""
author_urls = [
u for u, d in zip(self.get_author_urls(), self.domains)
if not util.in_webmention_blocklist(d)
]
if ((self.verified() and not force) or self.status == 'disabled'
or not self.features or not author_urls):
return
author_url = author_urls[0]
logging.info('Attempting to discover webmention endpoint on %s',
author_url)
mention = send.WebmentionSend('https://brid.gy/', author_url)
mention.requests_kwargs = {
'timeout': util.HTTP_TIMEOUT,
'headers': util.REQUEST_HEADERS
}
try:
mention._discoverEndpoint()
except BaseException as e:
logging.info('Error discovering webmention endpoint', exc_info=e)
mention.error = {'code': 'EXCEPTION'}
self._fetched_html = getattr(mention, 'html', None)
error = getattr(mention, 'error', None)
endpoint = getattr(mention, 'receiver_endpoint', None)
if error or not endpoint:
logging.info("No webmention endpoint found: %s %r", error,
endpoint)
self.webmention_endpoint = None
else:
logging.info("Discovered webmention endpoint %s", endpoint)
self.webmention_endpoint = endpoint
self.put()
def _urls_and_domains(self, auth_entity, user_url, actor=None):
"""Returns this user's valid (not webmention-blocklisted) URLs and domains.
Converts the auth entity's user_json to an ActivityStreams actor and uses
its 'urls' and 'url' fields. May be overridden by subclasses.
Args:
auth_entity: :class:`oauth_dropins.models.BaseAuth`
user_url: string, optional URL passed in when authorizing
actor: dict, optional AS actor for the user. If provided, overrides
auth_entity
Returns:
([string url, ...], [string domain, ...])
"""
if not actor:
actor = self.gr_source.user_to_actor(
json_loads(auth_entity.user_json))
logging.debug('Extracting URLs and domains from actor: %s',
json_dumps(actor, indent=2))
candidates = util.trim_nulls(
util.uniquify([user_url] + microformats2.object_urls(actor)))
if len(candidates) > MAX_AUTHOR_URLS:
logging.info(
'Too many profile links! Only resolving the first %s: %s',
MAX_AUTHOR_URLS, candidates)
urls = []
for i, url in enumerate(candidates):
resolved = self.resolve_profile_url(url,
resolve=i < MAX_AUTHOR_URLS)
if resolved:
urls.append(resolved)
final_urls = []
domains = []
for url in util.dedupe_urls(urls): # normalizes domains to lower case
# skip links on this source's domain itself. only currently needed for
# Mastodon; the other silo domains are in the webmention blocklist.
domain = util.domain_from_link(url)
if domain != self.gr_source.DOMAIN:
final_urls.append(url)
domains.append(domain)
return final_urls, domains
@staticmethod
def resolve_profile_url(url, resolve=True):
"""Resolves a profile URL to be added to a source.
Args:
url: string
resolve: boolean, whether to make HTTP requests to follow redirects, etc.
Returns: string, resolved URL, or None
"""
final, _, ok = util.get_webmention_target(url, resolve=resolve)
if not ok:
return None
final = final.lower()
if util.schemeless(final).startswith(util.schemeless(url.lower())):
# redirected to a deeper path. use the original higher level URL. #652
final = url
# If final has a path segment check if root has a matching rel=me.
match = re.match(r'^(https?://[^/]+)/.+', final)
if match and resolve:
root = match.group(1)
try:
mf2 = util.fetch_mf2(root)
me_urls = mf2['rels'].get('me', [])
if final in me_urls:
final = root
except requests.RequestException:
logging.warning("Couldn't fetch %s, preserving path in %s",
root,
final,
stack_info=True)
return final
def canonicalize_url(self, url, activity=None, **kwargs):
"""Canonicalizes a post or object URL.
Wraps :class:`oauth_dropins.webutil.util.UrlCanonicalizer`.
"""
return self.URL_CANONICALIZER(
url, **kwargs) if self.URL_CANONICALIZER else url
def infer_profile_url(self, url):
"""Given an arbitrary URL representing a person, try to find their
profile URL for *this* service.
Queries Bridgy's registered accounts for users with a particular
domain in their silo profile.
Args:
url: string, a person's URL
Return:
a string URL for their profile on this service (or None)
"""
domain = util.domain_from_link(url)
if domain == self.gr_source.DOMAIN:
return url
user = self.__class__.query(self.__class__.domains == domain).get()
if user:
return self.gr_source.user_url(user.key_id())
def preprocess_for_publish(self, obj):
"""Preprocess an object before trying to publish it.
By default this tries to massage person tags so that the tag's
"url" points to the person's profile on this service (as opposed
to a person's homepage).
The object is modified in place.
Args:
obj: ActivityStreams activity or object dict
"""
for tag in obj.get('tags', []):
if tag.get('objectType') == 'person':
silo_url = None
for url in microformats2.object_urls(tag):
silo_url = url and self.infer_profile_url(url)
if silo_url:
break
if silo_url:
tag['url'] = silo_url
# recurse on contained object(s)
for obj in util.get_list(obj, 'object'):
self.preprocess_for_publish(obj)
def on_new_syndicated_post(self, syndpost):
"""Called when a new :class:`SyndicatedPost` is stored for this source.
Args:
syndpost: :class:`SyndicatedPost`
"""
pass
def is_private(self):
"""Returns True if this source is private aka protected.
...ie their posts are not public.
"""
return False
def is_activity_public(self, activity):
"""Returns True if the given activity is public, False otherwise.
Just wraps :meth:`granary.source.Source.is_public`. Subclasses may override.
"""
return gr_source.Source.is_public(activity)
def is_beta_user(self):
"""Returns True if this is a "beta" user opted into new features.
Beta users come from beta_users.txt.
"""
return self.bridgy_path() in util.BETA_USER_PATHS
def load_blocklist(self):
"""Fetches this user's blocklist, if supported, and stores it in the entity."""
if not self.HAS_BLOCKS:
return
try:
ids = self.gr_source.get_blocklist_ids()
except gr_source.RateLimited as e:
ids = e.partial or []
self.blocked_ids = ids[:BLOCKLIST_MAX_IDS]
self.put()
def is_blocked(self, obj):
"""Returns True if an object's author is being blocked.
...ie they're in this user's block list.
Note that this method is tested in test_twitter.py, not test_models.py, for
historical reasons.
"""
if not self.blocked_ids:
return False
for o in [obj] + util.get_list(obj, 'object'):
for field in 'author', 'actor':
if o.get(field, {}).get('numeric_id') in self.blocked_ids:
return True
class FuzzTargetsCount(Model):
"""Fuzz targets count for every job. Key IDs are the job name."""
count = ndb.IntegerProperty(indexed=False)
class Bug(ndb.Model):
"""Bug entity."""
OSV_ID_PREFIX = 'OSV-'
# Status of the bug.
status = ndb.IntegerProperty()
# Timestamp when Bug was allocated.
timestamp = ndb.DateTimeProperty()
# When the entry was last edited.
last_modified = ndb.DateTimeProperty()
# The source identifier.
# For OSS-Fuzz, this oss-fuzz:<ClusterFuzz testcase ID>.
# For others this is <source>:<path/to/source>.
source_id = ndb.StringProperty()
# Main repo url.
repo_url = ndb.StringProperty()
# The main fixed commit.
fixed = ndb.StringProperty()
# The main regressing commit.
regressed = ndb.StringProperty()
# Additional affected commit ranges derived from the main fixed and regressed
# commits.
additional_commit_ranges = ndb.StructuredProperty(CommitRange, repeated=True)
# List of affected versions.
affected = ndb.StringProperty(repeated=True)
# List of normalized versions for fuzzy matching.
affected_fuzzy = ndb.StringProperty(repeated=True)
# OSS-Fuzz issue ID.
issue_id = ndb.StringProperty()
# Project/package name for the bug.
project = ndb.StringProperty()
# Package ecosystem for the project.
ecosystem = ndb.StringProperty()
# Summary for the bug.
summary = ndb.StringProperty()
# Vulnerability details.
details = ndb.StringProperty()
# Severity of the bug.
severity = ndb.StringProperty(validator=_check_valid_severity)
# Whether or not the bug is public (OSS-Fuzz only).
public = ndb.BooleanProperty()
# Reference URLs.
reference_urls = ndb.StringProperty(repeated=True)
# Search indices (auto-populated)
search_indices = ndb.StringProperty(repeated=True)
# Whether or not the bug has any affected versions (auto-populated).
has_affected = ndb.BooleanProperty()
# Sort key.
sort_key = ndb.StringProperty()
# Source of truth for this Bug.
source_of_truth = ndb.IntegerProperty(default=SourceOfTruth.INTERNAL)
def id(self):
"""Get the bug ID."""
if re.match(r'^\d+', self.key.id()):
return self.OSV_ID_PREFIX + self.key.id()
return self.key.id()
@classmethod
def get_by_id(cls, vuln_id, *args, **kwargs):
"""Overridden get_by_id to handle OSV allocated IDs."""
# OSV allocated bug IDs are stored without the prefix.
if vuln_id.startswith(cls.OSV_ID_PREFIX):
vuln_id = vuln_id[len(cls.OSV_ID_PREFIX):]
return super().get_by_id(vuln_id, *args, **kwargs)
def _pre_put_hook(self):
"""Pre-put hook for populating search indices."""
self.search_indices = []
if self.project:
self.search_indices.append(self.project)
key_parts = self.key.id().split('-')
self.search_indices.append(self.key.id())
self.search_indices.extend(key_parts)
self.has_affected = bool(self.affected)
self.affected_fuzzy = bug.normalize_tags(self.affected)
self.sort_key = key_parts[0] + '-' + key_parts[1].zfill(7)
if not self.last_modified:
self.last_modified = utcnow()
def update_from_vulnerability(self, vulnerability):
"""Set fields from vulnerability."""
self.summary = vulnerability.summary
self.details = vulnerability.details
self.severity = (
vulnerability_pb2.Vulnerability.Severity.Name(vulnerability.severity))
self.reference_urls = list(vulnerability.references)
self.last_modified = vulnerability.modified.ToDatetime()
self.project = vulnerability.package.name
self.ecosystem = vulnerability.package.ecosystem
self.affected = list(vulnerability.affects.versions)
found_first = False
for affected_range in vulnerability.affects.ranges:
if affected_range.type != vulnerability_pb2.AffectedRange.Type.GIT:
continue
if found_first:
self.additional_commit_ranges.append(
CommitRange(
introduced_in=affected_range.introduced,
fixed_in=affected_range.fixed))
else:
self.regressed = affected_range.introduced
self.fixed = affected_range.fixed
self.repo_url = affected_range.repo
found_first = True
def to_vulnerability(self):
"""Convert to Vulnerability proto."""
package = vulnerability_pb2.Package(
name=self.project, ecosystem=self.ecosystem)
affects = vulnerability_pb2.Affects(versions=self.affected)
affects.ranges.add(
type=vulnerability_pb2.AffectedRange.Type.GIT,
repo=self.repo_url,
introduced=self.regressed,
fixed=self.fixed)
for additional_range in self.additional_commit_ranges:
affects.ranges.add(
type=vulnerability_pb2.AffectedRange.Type.GIT,
repo=self.repo_url,
introduced=additional_range.introduced_in,
fixed=additional_range.fixed_in)
if self.severity:
severity = vulnerability_pb2.Vulnerability.Severity.Value(self.severity)
else:
severity = vulnerability_pb2.Vulnerability.Severity.NONE
details = self.details
if self.status == bug.BugStatus.INVALID:
affects = None
details = 'INVALID'
severity = vulnerability_pb2.Vulnerability.Severity.NONE
if self.last_modified:
modified = timestamp_pb2.Timestamp()
modified.FromDatetime(self.last_modified)
else:
modified = None
created = timestamp_pb2.Timestamp()
created.FromDatetime(self.timestamp)
result = vulnerability_pb2.Vulnerability(
id=self.id(),
created=created,
modified=modified,
summary=self.summary,
details=details,
package=package,
severity=severity,
affects=affects,
references=self.reference_urls)
return result
class IDCounter(ndb.Model): """Counter for ID allocations.""" # Next ID to allocate. next_id = ndb.IntegerProperty()
class SomeKind(ndb.Expando):
foo = ndb.IntegerProperty()
class Bug(ndb.Model):
"""Bug entity."""
OSV_ID_PREFIX = 'OSV-'
# Very large fake version to use when there is no fix available.
_NOT_FIXED_SEMVER = '999999.999999.999999'
# Display ID as used by the source database. The full qualified database that
# OSV tracks this as may be different.
db_id = ndb.StringProperty()
# Other IDs this bug is known as.
aliases = ndb.StringProperty(repeated=True)
# Related IDs.
related = ndb.StringProperty(repeated=True)
# Status of the bug.
status = ndb.IntegerProperty()
# Timestamp when Bug was allocated.
timestamp = ndb.DateTimeProperty()
# When the entry was last edited.
last_modified = ndb.DateTimeProperty()
# When the entry was withdrawn.
withdrawn = ndb.DateTimeProperty()
# The source identifier.
# For OSS-Fuzz, this oss-fuzz:<ClusterFuzz testcase ID>.
# For others this is <source>:<path/to/source>.
source_id = ndb.StringProperty()
# The main fixed commit (from bisection).
fixed = ndb.StringProperty(default='')
# The main regressing commit (from bisection).
regressed = ndb.StringProperty(default='')
# All affected ranges. TODO(ochang): To be removed.
affected_ranges = ndb.StructuredProperty(AffectedRange, repeated=True)
# List of affected versions.
affected = ndb.TextProperty(repeated=True)
# List of normalized versions indexed for fuzzy matching.
affected_fuzzy = ndb.StringProperty(repeated=True)
# OSS-Fuzz issue ID.
issue_id = ndb.StringProperty()
# Package URL for this package.
purl = ndb.StringProperty(repeated=True)
# Project/package name for the bug.
project = ndb.StringProperty(repeated=True)
# Package ecosystem for the project.
ecosystem = ndb.StringProperty(repeated=True)
# Summary for the bug.
summary = ndb.TextProperty()
# Vulnerability details.
details = ndb.TextProperty()
# Severity of the bug.
severity = ndb.StringProperty(validator=_check_valid_severity)
# Whether or not the bug is public (OSS-Fuzz only).
public = ndb.BooleanProperty()
# Reference URL types (dict of url -> type).
reference_url_types = ndb.JsonProperty()
# Search indices (auto-populated)
search_indices = ndb.StringProperty(repeated=True)
# Whether or not the bug has any affected versions (auto-populated).
has_affected = ndb.BooleanProperty()
# Source of truth for this Bug.
source_of_truth = ndb.IntegerProperty(default=SourceOfTruth.INTERNAL)
# Whether the bug is fixed (indexed for querying).
is_fixed = ndb.BooleanProperty()
# Database specific.
database_specific = ndb.JsonProperty()
# Ecosystem specific.
ecosystem_specific = ndb.JsonProperty()
# Normalized SEMVER fixed indexes for querying.
semver_fixed_indexes = ndb.StringProperty(repeated=True)
# Affected packages and versions.
affected_packages = ndb.LocalStructuredProperty(AffectedPackage,
repeated=True)
# The source of this Bug.
source = ndb.StringProperty()
def id(self):
"""Get the bug ID."""
if self.db_id:
return self.db_id
# TODO(ochang): Remove once all existing bugs have IDs migrated.
if re.match(r'^\d+', self.key.id()):
return self.OSV_ID_PREFIX + self.key.id()
return self.key.id()
@property
def repo_url(self):
"""Repo URL."""
for affected_package in self.affected_packages:
for affected_range in affected_package.ranges:
if affected_range.repo_url:
return affected_range.repo_url
return None
@classmethod
def get_by_id(cls, vuln_id, *args, **kwargs):
"""Overridden get_by_id to handle OSV allocated IDs."""
result = cls.query(cls.db_id == vuln_id).get()
if result:
return result
# TODO(ochang): Remove once all exsting bugs have IDs migrated.
if vuln_id.startswith(cls.OSV_ID_PREFIX):
vuln_id = vuln_id[len(cls.OSV_ID_PREFIX):]
return super().get_by_id(vuln_id, *args, **kwargs)
def _tokenize(self, value):
"""Tokenize value for indexing."""
if not value:
return []
value_lower = value.lower()
return re.split(r'\W+', value_lower) + [value_lower]
def _pre_put_hook(self):
"""Pre-put hook for populating search indices."""
search_indices = set()
search_indices.update(self._tokenize(self.id()))
if self.affected_packages:
self.project = [
pkg.package.name for pkg in self.affected_packages
if pkg.package.name
]
self.ecosystem = [
pkg.package.ecosystem for pkg in self.affected_packages
if pkg.package.ecosystem
]
self.purl = [
pkg.package.purl for pkg in self.affected_packages
if pkg.package.purl
]
for project in self.project:
search_indices.update(self._tokenize(project))
for ecosystem in self.ecosystem:
search_indices.update(self._tokenize(ecosystem))
self.search_indices = sorted(list(search_indices))
self.affected_fuzzy = []
self.semver_fixed_indexes = []
self.has_affected = False
self.is_fixed = False
for affected_package in self.affected_packages:
# Indexes used for querying by exact version.
self.affected_fuzzy.extend(
bug.normalize_tags(affected_package.versions))
self.has_affected |= bool(affected_package.versions)
for affected_range in affected_package.ranges:
fixed_version = None
for event in affected_range.events:
# Index used to query by fixed/unfixed.
if event.type == 'fixed':
self.is_fixed = True
fixed_version = event.value
if affected_range.type == 'SEMVER':
# Indexes used for querying by semver.
fixed = fixed_version or self._NOT_FIXED_SEMVER
self.semver_fixed_indexes.append(
semver_index.normalize(fixed))
self.has_affected |= (affected_range.type
in ('SEMVER', 'ECOSYSTEM'))
if not self.last_modified:
self.last_modified = utcnow()
if self.source_id:
self.source, _ = sources.parse_source_id(self.source_id)
if not self.source:
raise ValueError('Source not specified for Bug.')
if not self.db_id:
raise ValueError('DB ID not specified for Bug.')
if not self.key: # pylint: disable=access-member-before-definition
source_repo = get_source_repository(self.source)
if not source_repo:
raise ValueError(f'Invalid source {self.source}')
if source_repo.db_prefix and self.db_id.startswith(
source_repo.db_prefix):
key_id = self.db_id
else:
key_id = f'{self.source}:{self.db_id}'
self.key = ndb.Key(Bug, key_id)
def _update_from_pre_0_8(self, vulnerability):
"""Update from pre 0.8 import."""
if self.affected_packages:
affected_package = self.affected_packages[0]
else:
affected_package = AffectedPackage()
self.affected_packages.append(affected_package)
affected_package.package = Package(
name=vulnerability.package.name,
ecosystem=vulnerability.package.ecosystem,
purl=vulnerability.package.purl)
vuln_dict = sources.vulnerability_to_dict(vulnerability)
if vulnerability.database_specific:
affected_package.database_specific = vuln_dict['database_specific']
if vulnerability.ecosystem_specific:
affected_package.ecosystem_specific = vuln_dict[
'ecosystem_specific']
affected_package.versions = list(vulnerability.affects.versions)
affected_package.ranges = []
events_by_type = {}
for affected_range in vulnerability.affects.ranges:
events = events_by_type.setdefault(
(vulnerability_pb2.AffectedRange.Type.Name(
affected_range.type), affected_range.repo), [])
# An empty introduced in 0.7 now needs to be represented as '0' in 0.8.
introduced = AffectedEvent(type='introduced',
value=affected_range.introduced or '0')
if introduced not in events:
events.append(introduced)
if affected_range.fixed:
fixed = AffectedEvent(type='fixed', value=affected_range.fixed)
if fixed not in events:
events.append(fixed)
for (range_type, repo_url), events in events_by_type.items():
affected_range = AffectedRange2(type=range_type, events=events)
if range_type == 'GIT' and repo_url:
affected_range.repo_url = repo_url
affected_package.ranges.append(affected_range)
def update_from_vulnerability(self, vulnerability):
"""Set fields from vulnerability. Does not set the ID."""
self.summary = vulnerability.summary
self.details = vulnerability.details
self.reference_url_types = {
ref.url: vulnerability_pb2.Reference.Type.Name(ref.type)
for ref in vulnerability.references
}
if vulnerability.HasField('modified'):
self.last_modified = vulnerability.modified.ToDatetime()
if vulnerability.HasField('published'):
self.timestamp = vulnerability.published.ToDatetime()
if vulnerability.HasField('withdrawn'):
self.withdrawn = vulnerability.withdrawn.ToDatetime()
self.aliases = list(vulnerability.aliases)
self.related = list(vulnerability.related)
if not vulnerability.affected:
self._update_from_pre_0_8(vulnerability)
return
self.affected_packages = []
for affected_package in vulnerability.affected:
current = AffectedPackage()
current.package = Package(
name=affected_package.package.name,
ecosystem=affected_package.package.ecosystem,
purl=affected_package.package.purl)
current.ranges = []
for affected_range in affected_package.ranges:
current_range = AffectedRange2(
type=vulnerability_pb2.Range.Type.Name(
affected_range.type),
repo_url=affected_range.repo,
events=[])
for evt in affected_range.events:
if evt.introduced:
current_range.events.append(
AffectedEvent(type='introduced',
value=evt.introduced))
continue
if evt.fixed:
current_range.events.append(
AffectedEvent(type='fixed', value=evt.fixed))
continue
if evt.limit:
current_range.events.append(
AffectedEvent(type='limit', value=evt.limit))
continue
current.ranges.append(current_range)
current.versions = list(affected_package.versions)
if affected_package.database_specific:
current.database_specific = json_format.MessageToDict(
affected_package.database_specific,
preserving_proto_field_name=True)
if affected_package.ecosystem_specific:
current.ecosystem_specific = json_format.MessageToDict(
affected_package.ecosystem_specific,
preserving_proto_field_name=True)
self.affected_packages.append(current)
def _get_pre_0_8_affects(self):
"""Get pre 0.8 schema affects field."""
affected_package = self.affected_packages[0]
affects = vulnerability_pb2.Affects(versions=affected_package.versions)
for affected_range in affected_package.ranges:
# Convert flattened events to range pairs (pre-0.8 schema).
# TODO(ochang): Remove this once all consumers are migrated.
# pylint: disable=cell-var-from-loop
new_range = lambda x, y: vulnerability_pb2.AffectedRange(
type=vulnerability_pb2.AffectedRange.Type.Value(affected_range.
type),
repo=affected_range.repo_url,
introduced=x,
fixed=y)
last_introduced = None
# Sort the flattened events, then find corresponding [introduced,
# fixed) pairs.
for event in sorted_events(affected_package.package.ecosystem,
affected_range.type,
affected_range.events):
if event.type == 'introduced':
if last_introduced is not None and affected_range.type == 'GIT':
# If this is GIT, then we need to store all "introduced", even if
# they overlap.
affects.ranges.append(new_range(last_introduced, ''))
last_introduced = None
if last_introduced is None:
# If not GIT, ignore overlapping introduced versions since they're
# redundant.
last_introduced = event.value
if last_introduced == '0':
last_introduced = ''
if event.type == 'fixed':
if affected_range.type != 'GIT' and last_introduced is None:
# No prior introduced, so ignore this invalid entry.
continue
# Found a complete pair.
affects.ranges.append(
new_range(last_introduced, event.value))
last_introduced = None
if last_introduced is not None:
affects.ranges.append(new_range(last_introduced, ''))
return affects
def to_vulnerability(self, include_source=False, v0_7=True, v0_8=False):
"""Convert to Vulnerability proto."""
package = None
ecosystem_specific = None
database_specific = None
affected = []
affects = None
source_link = None
if self.source and include_source:
source_repo = get_source_repository(self.source)
if source_repo and source_repo.link:
source_link = source_repo.link + sources.source_path(
source_repo, self)
if self.affected_packages:
if v0_7:
# The pre-0.8 schema only supports a single package, so we take the
# first.
affected_package = self.affected_packages[0]
package = vulnerability_pb2.Package(
name=affected_package.package.name,
ecosystem=affected_package.package.ecosystem,
purl=affected_package.package.purl)
affects = self._get_pre_0_8_affects()
if affected_package.ecosystem_specific:
ecosystem_specific = affected_package.ecosystem_specific
if affected_package.database_specific:
database_specific = affected_package.database_specific
if v0_8:
for affected_package in self.affected_packages:
ranges = []
for affected_range in affected_package.ranges:
events = []
for event in affected_range.events:
kwargs = {event.type: event.value}
events.append(vulnerability_pb2.Event(**kwargs))
current_range = vulnerability_pb2.Range(
type=vulnerability_pb2.Range.Type.Value(
affected_range.type),
repo=affected_range.repo_url,
events=events)
ranges.append(current_range)
current = vulnerability_pb2.Affected(
package=vulnerability_pb2.Package(
name=affected_package.package.name,
ecosystem=affected_package.package.ecosystem,
purl=affected_package.package.purl),
ranges=ranges,
versions=affected_package.versions)
if affected_package.database_specific:
current.database_specific.update(
affected_package.database_specific)
if source_link:
current.database_specific.update(
{'source': source_link})
if affected_package.ecosystem_specific:
current.ecosystem_specific.update(
affected_package.ecosystem_specific)
affected.append(current)
details = self.details
if self.status == bug.BugStatus.INVALID:
affects = None
affected = None
details = 'INVALID'
if self.last_modified:
modified = timestamp_pb2.Timestamp()
modified.FromDatetime(self.last_modified)
else:
modified = None
if self.withdrawn:
withdrawn = timestamp_pb2.Timestamp()
withdrawn.FromDatetime(self.withdrawn)
else:
withdrawn = None
published = timestamp_pb2.Timestamp()
published.FromDatetime(self.timestamp)
references = []
if self.reference_url_types:
for url, url_type in self.reference_url_types.items():
references.append(
vulnerability_pb2.Reference(
url=url,
type=vulnerability_pb2.Reference.Type.Value(url_type)))
result = vulnerability_pb2.Vulnerability(id=self.id(),
published=published,
modified=modified,
aliases=self.aliases,
related=self.related,
withdrawn=withdrawn,
summary=self.summary,
details=details,
package=package,
affects=affects,
affected=affected,
references=references)
if ecosystem_specific:
result.ecosystem_specific.update(ecosystem_specific)
if database_specific:
result.database_specific.update(database_specific)
if source_link:
result.database_specific.update({'source': source_link})
return result
class Xml(ndb.Model):
# A row in the database.
xml_hash = ndb.IntegerProperty()
xml_content = ndb.TextProperty()
class A(ndb.Model):
some_prop = ndb.IntegerProperty()
source = ndb.StringProperty()
class Xml(ndb.Model):
# A row in the database.
xml_hash = ndb.IntegerProperty()
xml_content = ndb.TextProperty()
last_accessed = ndb.DateTimeProperty(auto_now=True)
class EmailListUsers(ndb.Model):
uid: str = ndb.StringProperty(required=True)
email: str = ndb.StringProperty(required=True)
name: str = ndb.StringProperty(required=True)
time_created: int = ndb.IntegerProperty(default=timestamp())
class AnyKind(ndb.Model):
foo = ndb.IntegerProperty()
bar = ndb.StringProperty()
baz = ndb.IntegerProperty()
qux = ndb.StringProperty()
class Person(ndb.Model):
chat_id = ndb.IntegerProperty()
name = ndb.StringProperty()
last_name = ndb.StringProperty()
username = ndb.StringProperty()
state = ndb.StringProperty(indexed=True)
last_state = ndb.StringProperty()
last_mod = ndb.DateTimeProperty(auto_now=True)
enabled = ndb.BooleanProperty(default=True)
variables = ndb.JsonProperty(indexed=False)
language_interface = ndb.StringProperty(
default=params.default_language_interface)
language_exercise = ndb.StringProperty(
default=params.default_language_exercise)
ux_lang = None # see method ux()
def user_telegram_id(self):
return 'telegram_{}'.format(self.chat_id)
def ux(self):
if self.ux_lang is None:
self.ux_lang = bot_ux.UX_LANG(self.language_interface)
return self.ux_lang
def get_name(self):
return self.name
def get_last_name(self):
return self.last_name if self.last_name else ''
def get_username(self):
return self.username
def get_name_last_name(self):
return self.get_name() + ' ' + self.get_last_name()
def get_name_last_name_username(self):
result = self.get_name_last_name()
if self.username:
result += ' @' + self.get_username()
return result
def set_enabled(self, enabled, put=False):
self.enabled = enabled
if put:
self.put()
def switch_notifications(self, put=True):
self.enabled = not self.enabled
if put:
self.put()
def update_user(self, name, last_name, username, put=False):
import params
changed = False
if self.name != name:
self.name = name
changed = True
if self.last_name != last_name:
self.last_name = last_name
changed = True
if self.username != username:
self.username = username
changed = True
if self.language_exercise not in params.LANGUAGES:
self.language_exercise = params.default_language_exercise
if self.language_interface not in params.LANGUAGES:
self.language_interface = params.default_language_interface
if changed and put:
self.put()
def set_language_interface(self, lang):
self.language_interface = lang
self.ux_lang = bot_ux.UX_LANG(lang)
def set_language_exercise(self, lang):
self.language_exercise = lang
def set_state(self, newstate, put=True):
self.last_state = self.state
self.state = newstate
if put:
self.put()
def is_administrator(self):
result = self.chat_id in key.ADMIN_IDS
#logging.debug("Amministratore: " + str(result))
return result
def set_last_exercise_id_and_options(self, ex_id, options):
self.set_variable('Exercise_ID', ex_id)
self.set_variable('Exercise_Options', options)
self.put()
def get_last_exercise_id_and_options(self):
options = [x for x in self.get_variable('Exercise_Options')]
return self.get_variable('Exercise_ID'), options
def set_keyboard(self, kb, put=True):
self.set_variable("keyboard", kb, put=put)
def get_keyboard(self):
return self.get_variable("keyboard", [])
def set_variable(self, var_key, var_value, put=True):
self.variables[var_key] = var_value
if put:
self.put()
def get_variable(self, var_key, default_value=None):
return self.variables.get(var_key, default_value)
def is_admin(self):
import key
return self.chat_id in key.ADMIN_IDS
def is_manager(self):
import key
return self.chat_id in key.MANAGER_IDS
class AnyKind(ndb.Model):
foo = ndb.IntegerProperty()
class SomeKind(ndb.Model):
bar = ndb.IntegerProperty(name="foo")
class Animal(ndb.PolyModel):
foo = ndb.IntegerProperty()
class Fuzzer(Model):
"""Represents a fuzzer."""
VALID_NAME_REGEX = NAME_CHECK_REGEX
# Last update time.
timestamp = ndb.DateTimeProperty()
# Fuzzer Name.
name = StringProperty()
# The name of the archive that the user uploaded.
filename = StringProperty()
# Blobstore key for this fuzzer.
blobstore_key = StringProperty()
# String representation of the file size.
file_size = StringProperty()
# Fuzzer's main executable path, relative to root.
executable_path = StringProperty()
# Revision number of the fuzzer.
revision = ndb.IntegerProperty()
# Fuzzer's source (for accountability).
source = StringProperty()
# Testcase timeout.
timeout = ndb.IntegerProperty()
# Supported platforms.
supported_platforms = StringProperty()
# Custom script that should be used to launch chrome for this fuzzer.
launcher_script = StringProperty()
# Result from the last fuzzer run showing the number of testcases generated.
result = StringProperty()
# Last result update timestamp.
result_timestamp = ndb.DateTimeProperty()
# Console output from last fuzzer run.
console_output = TextProperty()
# Return code from last fuzzer run.
return_code = ndb.IntegerProperty()
# Blobstore key for the sample testcase generated by the fuzzer.
sample_testcase = StringProperty()
# Job types for this fuzzer.
jobs = StringProperty(repeated=True)
# Is the fuzzer coming from an external contributor ? Useful for adding
# reward flags.
external_contribution = ndb.BooleanProperty(default=False)
# Max testcases to generate for this fuzzer.
max_testcases = ndb.IntegerProperty()
# Does it run un-trusted content ? Examples including running live sites.
untrusted_content = ndb.BooleanProperty(default=False)
# Data bundle name.
data_bundle_name = StringProperty(default='')
# Additional environment variables that need to be set for this fuzzer.
additional_environment_string = TextProperty()
# Column specification for stats.
stats_columns = TextProperty()
# Helpful descriptions for the stats_columns. In a yaml format.
stats_column_descriptions = TextProperty(indexed=False)
# Whether this is a builtin fuzzer.
builtin = ndb.BooleanProperty(indexed=False, default=False)
# Whether this is a differential fuzzer.
differential = ndb.BooleanProperty(default=False)
class SomeKind(ndb.Model):
foo = ndb.IntegerProperty()
bar = ndb.StructuredProperty(OtherKind, repeated=True)
class Testcase(Model):
"""Represents a single testcase."""
# Crash on an invalid read/write.
crash_type = StringProperty()
# Crashing address.
crash_address = TextProperty()
# First x stack frames.
crash_state = StringProperty()
# Complete stacktrace.
crash_stacktrace = TextProperty(indexed=False)
# Last tested crash stacktrace using the latest revision.
last_tested_crash_stacktrace = TextProperty(indexed=False)
# Blobstore keys for various things like original testcase, minimized
# testcase, etc.
fuzzed_keys = TextProperty()
minimized_keys = TextProperty()
minidump_keys = TextProperty()
# Tracking issue tracker bug. One bug number per line (future extension).
bug_information = StringProperty()
# Regression range.
regression = StringProperty(default='')
# Revisions where this issue has been fixed.
fixed = StringProperty(default='')
# Is it a security bug ?
security_flag = ndb.BooleanProperty(default=False)
# Security severity of the bug.
security_severity = ndb.IntegerProperty(indexed=False)
# Did the bug only reproduced once ?
one_time_crasher_flag = ndb.BooleanProperty(default=False)
# Any additional comments.
comments = TextProperty(default='', indexed=False)
# Revision that we discovered the crash in.
crash_revision = ndb.IntegerProperty()
# The file on the bot that generated the testcase.
original_absolute_path = TextProperty(default='')
absolute_path = TextProperty()
# Minimized argument list.
minimized_arguments = TextProperty(default='', indexed=False)
# Window argument (usually width, height, top, left, etc).
window_argument = TextProperty(default='', indexed=False)
# Type of job associated with this testcase.
job_type = StringProperty()
# Original job queue used for tasks created for this testcase.
queue = TextProperty()
# State representing whether the fuzzed or minimized testcases are archived.
archive_state = ndb.IntegerProperty(default=0, indexed=False)
# File name of the original uploaded archive.
archive_filename = TextProperty()
# Is this a binary file?
binary_flag = ndb.BooleanProperty(default=False, indexed=False)
# Timestamp.
timestamp = ndb.DateTimeProperty()
# Does the testcase crash stack vary b/w crashes ?
flaky_stack = ndb.BooleanProperty(default=False, indexed=False)
# Do we need to test this testcase using an HTTP/HTTPS server?
http_flag = ndb.BooleanProperty(default=False, indexed=False)
# Name of the fuzzer used to generate this testcase.
fuzzer_name = StringProperty()
# Status of this testcase (pending, processed, unreproducible, etc).
status = StringProperty(default='Processed')
# Id of the testcase that this is marked as a duplicate of.
duplicate_of = ndb.IntegerProperty(indexed=False)
# Flag indicating whether or not the testcase has been symbolized.
symbolized = ndb.BooleanProperty(default=False, indexed=False)
# Id for this testcase's associated group.
group_id = ndb.IntegerProperty(default=0)
# Tracking issue tracker bug for this testcase group.
group_bug_information = ndb.IntegerProperty(default=0)
# Fake user interaction sequences like key clicks, mouse movements, etc.
gestures = TextProperty(repeated=True)
# ASAN redzone size in bytes.
redzone = ndb.IntegerProperty(default=128, indexed=False)
# Flag indicating if UBSan detection should be disabled. This is needed for
# cases when ASan and UBSan are bundled in the same build configuration
# and we need to disable UBSan in some runs to find the potentially more
# interesting ASan bugs.
disable_ubsan = ndb.BooleanProperty(default=False)
# Whether testcase is open.
open = ndb.BooleanProperty(default=True)
# Adjusts timeout based on multiplier value.
timeout_multiplier = ndb.FloatProperty(default=1.0, indexed=False)
# Additional metadata stored as a JSON object. This should be used for
# properties that are not commonly accessed and do not need to be indexed.
additional_metadata = TextProperty(indexed=False)
# Boolean attribute indicating if cleanup triage needs to be done.
triaged = ndb.BooleanProperty(default=False)
# Project name associated with this test case.
project_name = StringProperty()
# keywords is used for searching.
keywords = StringProperty(repeated=True)
# Whether testcase has a bug (either bug_information or
# group_bug_information).
has_bug_flag = ndb.BooleanProperty()
# Indices for bug_information and group_bug_information.
bug_indices = StringProperty(repeated=True)
# Overridden fuzzer name because actual fuzzer name can be different in many
# scenarios (libfuzzer, afl, etc).
overridden_fuzzer_name = StringProperty()
# Platform (e.g. windows, linux, android).
platform = StringProperty()
# Platform id (e.g. windows, linux, android:hammerhead:l).
# For Android, includes device type and underlying OS version.
platform_id = StringProperty()
# Impact indices for searching.
impact_indices = StringProperty(repeated=True)
# Whether or not a testcase is a duplicate of other testcase.
is_a_duplicate_flag = ndb.BooleanProperty()
# Whether or not a testcase is the leader of its group.
# If the testcase is not in a group, it's the leader of a group of 1.
# The default is false because we prefer not to show crashes until we are
# sure. And group_task will correctly set the value within 30 minutes.
is_leader = ndb.BooleanProperty(default=False)
# Fuzzer name indices
fuzzer_name_indices = StringProperty(repeated=True)
# The impacted version indices (including both beta and stable).
impact_version_indices = StringProperty(repeated=True)
# The impacted stable version.
impact_stable_version = StringProperty()
# The impacted stable version indices.
impact_stable_version_indices = StringProperty(repeated=True)
# The impacted stable version is merely probable (not definite). Because
# for a non-asan build, we don't have a stable/beta build. Therefore, we
# make an intelligent guess on the version.
impact_stable_version_likely = ndb.BooleanProperty()
# The impacted beta version.
impact_beta_version = StringProperty()
# The impacted beta version indices.
impact_beta_version_indices = StringProperty(repeated=True)
# The impacted beta version is merely probable (not definite). See the
# comment on impact_stable_version_likely.
impact_beta_version_likely = ndb.BooleanProperty()
# Whether or not impact task has been run on this testcase.
is_impact_set_flag = ndb.BooleanProperty()
# Uploader email address.
uploader_email = StringProperty()
def has_blame(self):
return self.project_name == 'chromium'
def has_impacts(self):
return self.project_name == 'chromium' and not self.one_time_crasher_flag
def impacts_production(self):
return bool(self.impact_stable_version) or bool(
self.impact_beta_version)
def is_status_unreproducible(self):
return self.status and self.status.startswith('Unreproducible')
def is_crash(self):
return bool(self.crash_state)
def populate_indices(self):
"""Populate keywords for fast test case list searching."""
self.keywords = list(
search_tokenizer.tokenize(self.crash_state)
| search_tokenizer.tokenize(self.crash_type)
| search_tokenizer.tokenize(self.fuzzer_name)
| search_tokenizer.tokenize(self.overridden_fuzzer_name)
| search_tokenizer.tokenize(self.job_type)
| search_tokenizer.tokenize(self.platform_id))
self.bug_indices = search_tokenizer.tokenize_bug_information(self)
self.has_bug_flag = bool(self.bug_indices)
self.is_a_duplicate_flag = bool(self.duplicate_of)
fuzzer_name_indices = list(
set([self.fuzzer_name, self.overridden_fuzzer_name]))
self.fuzzer_name_indices = [f for f in fuzzer_name_indices if f]
# If the impact task hasn't been run (aka is_impact_set_flag=False) OR
# if impact isn't applicable (aka has_impacts() is False), we wipe all
# the impact fields' indices.
if self.has_impacts() and self.is_impact_set_flag:
self.impact_stable_version_indices = (
search_tokenizer.tokenize_impact_version(
self.impact_stable_version))
self.impact_beta_version_indices = (
search_tokenizer.tokenize_impact_version(
self.impact_beta_version))
self.impact_version_indices = list(
set(self.impact_stable_version_indices +
self.impact_beta_version_indices))
if self.impact_beta_version:
self.impact_version_indices.append('beta')
if self.impact_stable_version:
self.impact_version_indices.append('stable')
if not self.impacts_production():
self.impact_version_indices.append('head')
else:
self.impact_version_indices = []
self.impact_stable_version_indices = []
self.impact_beta_version_indices = []
def _pre_put_hook(self):
self.populate_indices()
def _post_put_hook(self, _):
if not self.key:
# Failed put. An exception will be thrown automatically afterwards.
return
logs.log('Updated testcase %d (bug %s).' %
(self.key.id(), self.bug_information or '-'))
def set_impacts_as_na(self):
self.impact_stable_version = self.impact_beta_version = None
self.impact_stable_version_likely = self.impact_beta_version_likely = False
self.is_impact_set_flag = False
def _ensure_metadata_is_cached(self):
"""Ensure that the metadata for this has been cached."""
if hasattr(self, 'metadata_cache'):
return
try:
cache = json_utils.loads(self.additional_metadata)
except (TypeError, ValueError):
cache = {}
setattr(self, 'metadata_cache', cache)
def get_metadata(self, key=None, default=None):
"""Get metadata for a test case. Slow on first access."""
self._ensure_metadata_is_cached()
# If no key is specified, return all metadata.
if not key:
return self.metadata_cache
try:
return self.metadata_cache[key]
except KeyError:
return default
def set_metadata(self, key, value, update_testcase=True):
"""Set metadata for a test case."""
self._ensure_metadata_is_cached()
self.metadata_cache[key] = value
self.additional_metadata = json_utils.dumps(self.metadata_cache)
if update_testcase:
self.put()
def delete_metadata(self, key, update_testcase=True):
"""Remove metadata key for a test case."""
self._ensure_metadata_is_cached()
# Make sure that the key exists in cache. If not, no work to do here.
if key not in self.metadata_cache:
return
del self.metadata_cache[key]
self.additional_metadata = json_utils.dumps(self.metadata_cache)
if update_testcase:
self.put()
def actual_fuzzer_name(self):
"""Actual fuzzer name, uses one from overridden attribute if available."""
return self.overridden_fuzzer_name or self.fuzzer_name
def get_fuzz_target(self):
"""Get the associated FuzzTarget entity for this test case."""
name = self.actual_fuzzer_name()
if not name:
return None
target = ndb.Key(FuzzTarget, name).get()
if environment.get_value('ORIGINAL_JOB_NAME'):
# Overridden engine (e.g. for minimization).
target.engine = environment.get_engine_for_job()
return target
class User(ndb.Model):
# Currently unused
firstName = ndb.StringProperty()
# User state in the state machine
status = ndb.StringProperty(default=UserState.INIT_DEFAULT)
# From temptaking website
groupId = ndb.StringProperty()
groupName = ndb.StringProperty()
groupMembers = ndb.TextProperty()
memberName = ndb.StringProperty()
memberId = ndb.StringProperty()
pin = ndb.StringProperty()
# Set when user confirms their name but have not set a PIN on the website
PIN_NOTSET = "no pin"
# Last recorded temperature
temp = ndb.StringProperty()
# Reset at the start of a new session
TEMP_NONE = "none"
# Temperature wasn't set because of an error
TEMP_ERROR = "error"
# Time for reminders
# Only store the hour since the reminders are sent at the start of the configured hour
remindAM = ndb.IntegerProperty(default=-1)
remindPM = ndb.IntegerProperty(default=-1)
VALID_AM_TIMES = [f"{x:02}:01" for x in range(12)]
VALID_PM_TIMES = [f"{x:02}:01" for x in range(12, 24)]
blocked = ndb.BooleanProperty(default=False)
# To maintain back-compatability with the Cloud Datastore
@classmethod
def _get_kind(cls):
return "Client"
def reset(self):
self.status = UserState.INIT_START
self.groupId = None
self.groupName = None
self.groupMembers = None
self.memberName = None
self.memberId = None
self.pin = None
self.temp = "init"
self.remindAM = -1
self.remindPM = -1
self.blocked = False
# User can only issue commands after initialization
def canIssueCommand(self) -> bool:
return self.status in [
UserState.TEMP_DEFAULT,
UserState.TEMP_REPORT,
UserState.REMIND_SET_AM,
UserState.REMIND_SET_PM,
"offline,endgame 1",
"offline,endgame 2",
"offline,remind wizard 1",
"offline,remind wizard 2",
]
class SomeKind(ndb.Model):
foo = ndb.IntegerProperty()
bar = ndb.StringProperty()
class ThreadMessage(ndb.Model):
"""Models the individual messages in a thread."""
number = ndb.IntegerProperty(indexed=True)
created_by = ndb.StringProperty(default=VERSION)
text = ndb.TextProperty()
created = ndb.DateTimeProperty(auto_now=True, indexed=True)
class PickleOtherKind(ndb.Model):
foo = ndb.IntegerProperty()
@classmethod
def _get_kind(cls):
return "OtherKind"
class StoreCache(ndb.Model):
cache_key = ndb.StringProperty()
response = ndb.PickleProperty()
last_accessed = ndb.IntegerProperty()
class SomeKind(ndb.Model):
foos = ndb.KeyProperty(repeated=True)
bar = ndb.IntegerProperty(default=42)
class DummyEntity(ndb.Model):
string_property = ndb.StringProperty()
datetime_property = ndb.DateTimeProperty()
integer_property = ndb.IntegerProperty()
