class PolicyModel(ndb.Model):
"""Class that represents a tags and their associated schedule."""
Name = ndb.StringProperty(indexed=True, required=True)
DisplayName = ndb.JsonProperty(indexed=True)
Tags = ndb.JsonProperty(repeated=True)
Projects = ndb.JsonProperty(repeated=True)
Schedule = ndb.StringProperty(indexed=True, required=True, repeated=False)
class AffectedPackage(ndb.Model):
"""Affected packages."""
# The affected package identifier.
package = ndb.StructuredProperty(Package)
# The list of affected ranges.
ranges = ndb.LocalStructuredProperty(AffectedRange2, repeated=True)
# The list of explicit affected versions.
versions = ndb.TextProperty(repeated=True)
# Database specific metadata.
database_specific = ndb.JsonProperty()
# Ecosystem specific metadata.
ecosystem_specific = ndb.JsonProperty()
class Publish(ndb.Model):
"""A comment, like, repost, or RSVP published into a silo.
Child of a :class:`PublishedPage` entity.
"""
STATUSES = ('new', 'complete', 'failed', 'deleted')
# Turn off instance and memcache caching. See Source for details.
_use_cache = False
_use_memcache = False
type = ndb.StringProperty(choices=PUBLISH_TYPES)
status = ndb.StringProperty(choices=STATUSES, default='new')
source = ndb.KeyProperty()
html = ndb.TextProperty() # raw HTML fetched from source
published = ndb.JsonProperty(compressed=True)
created = ndb.DateTimeProperty(auto_now_add=True)
updated = ndb.DateTimeProperty(auto_now=True)
def type_label(self):
"""Returns silo-specific string type, e.g. 'favorite' instead of 'like'."""
for cls in sources.values(): # global
if cls.__name__ == self.source.kind():
return cls.TYPE_LABELS.get(self.type, self.type)
return self.type
class House(ndb.Model):
address = ndb.StringProperty()
json_property = ndb.JsonProperty()
indexed_int = ndb.IntegerProperty()
unindexed_int = ndb.IntegerProperty(indexed=False)
text_property = ndb.TextProperty()
datetime_property = ndb.DateTimeProperty()
class SchedulesModel(ndb.Model):
"""Stores scheduling data."""
Name = ndb.StringProperty(indexed=True, required=True)
DisplayName = ndb.StringProperty(indexed=True)
Timezone = ndb.StringProperty(default='UTC',
choices=tz.get_all_timezones(),
required=True)
Schedule = ndb.JsonProperty(required=True)
class BlogPost(Webmentions):
"""A blog post to be processed for links to send webmentions to.
The key name is the URL.
"""
feed_item = ndb.JsonProperty(compressed=True) # from Superfeedr
def label(self):
url = self.feed_item.get('permalinkUrl') if self.feed_item else None
return ' '.join((self.key.kind(), self.key.id(), url or '[no url]'))
def add_task(self):
util.add_propagate_blogpost_task(self)
class SomeKind(ndb.Model):
foo = ndb.JsonProperty(compressed=True)
class SomeKind(ndb.Model):
foo = ndb.JsonProperty()
class Bug(ndb.Model):
"""Bug entity."""
OSV_ID_PREFIX = 'OSV-'
# Status of the bug.
status = ndb.IntegerProperty()
# Timestamp when Bug was allocated.
timestamp = ndb.DateTimeProperty()
# When the entry was last edited.
last_modified = ndb.DateTimeProperty()
# The source identifier.
# For OSS-Fuzz, this oss-fuzz:<ClusterFuzz testcase ID>.
# For others this is <source>:<path/to/source>.
source_id = ndb.StringProperty()
# The main fixed commit (from bisection).
fixed = ndb.StringProperty(default='')
# The main regressing commit (from bisection).
regressed = ndb.StringProperty(default='')
# All affected ranges.
affected_ranges = ndb.StructuredProperty(AffectedRange, repeated=True)
# List of affected versions.
affected = ndb.StringProperty(repeated=True)
# List of normalized versions for fuzzy matching.
affected_fuzzy = ndb.StringProperty(repeated=True)
# OSS-Fuzz issue ID.
issue_id = ndb.StringProperty()
# Project/package name for the bug.
project = ndb.StringProperty()
# Package ecosystem for the project.
ecosystem = ndb.StringProperty()
# Summary for the bug.
summary = ndb.StringProperty()
# Vulnerability details.
details = ndb.StringProperty()
# Severity of the bug.
severity = ndb.StringProperty(validator=_check_valid_severity)
# Whether or not the bug is public (OSS-Fuzz only).
public = ndb.BooleanProperty()
# Reference URL types (dict of url -> type).
reference_url_types = ndb.JsonProperty()
# Search indices (auto-populated)
search_indices = ndb.StringProperty(repeated=True)
# Whether or not the bug has any affected versions (auto-populated).
has_affected = ndb.BooleanProperty()
# Sort key.
sort_key = ndb.StringProperty()
# Source of truth for this Bug.
source_of_truth = ndb.IntegerProperty(default=SourceOfTruth.INTERNAL)
# Whether the bug is fixed (indexed for querying).
is_fixed = ndb.BooleanProperty()
def id(self):
"""Get the bug ID."""
if re.match(r'^\d+', self.key.id()):
return self.OSV_ID_PREFIX + self.key.id()
return self.key.id()
@property
def repo_url(self):
"""Repo URL."""
for affected_range in self.affected_ranges:
if affected_range.repo_url:
return affected_range.repo_url
return None
@classmethod
def get_by_id(cls, vuln_id, *args, **kwargs):
"""Overridden get_by_id to handle OSV allocated IDs."""
# OSV allocated bug IDs are stored without the prefix.
if vuln_id.startswith(cls.OSV_ID_PREFIX):
vuln_id = vuln_id[len(cls.OSV_ID_PREFIX):]
return super().get_by_id(vuln_id, *args, **kwargs)
def _pre_put_hook(self):
"""Pre-put hook for populating search indices."""
self.search_indices = []
if self.project:
self.search_indices.append(self.project)
key_parts = self.key.id().split('-')
self.search_indices.append(self.key.id())
self.search_indices.extend(key_parts)
self.has_affected = bool(self.affected)
self.affected_fuzzy = bug.normalize_tags(self.affected)
self.sort_key = key_parts[0] + '-' + key_parts[1].zfill(7)
if not self.last_modified:
self.last_modified = utcnow()
self.is_fixed = any(
affected_range.fixed for affected_range in self.affected_ranges)
def update_from_vulnerability(self, vulnerability):
"""Set fields from vulnerability."""
self.summary = vulnerability.summary
self.details = vulnerability.details
self.severity = vulnerability_pb2.Severity.Name(vulnerability.severity)
self.reference_url_types = {
ref.url: vulnerability_pb2.Reference.Type.Name(ref.type)
for ref in vulnerability.references
}
if vulnerability.HasField('modified'):
self.last_modified = vulnerability.modified.ToDatetime()
if vulnerability.HasField('published'):
self.timestamp = vulnerability.published.ToDatetime()
self.project = vulnerability.package.name
self.ecosystem = vulnerability.package.ecosystem
self.affected = list(vulnerability.affects.versions)
self.affected_ranges = []
for affected_range in vulnerability.affects.ranges:
self.affected_ranges.append(
AffectedRange(
type=vulnerability_pb2.AffectedRange.Type.Name(
affected_range.type),
repo_url=affected_range.repo,
introduced=affected_range.introduced or '',
fixed=affected_range.fixed or ''))
def to_vulnerability(self):
"""Convert to Vulnerability proto."""
package = vulnerability_pb2.Package(
name=self.project, ecosystem=self.ecosystem)
affects = vulnerability_pb2.Affects(versions=self.affected)
for affected_range in self.affected_ranges:
affects.ranges.add(
type=vulnerability_pb2.AffectedRange.Type.Value(affected_range.type),
repo=affected_range.repo_url,
introduced=affected_range.introduced,
fixed=affected_range.fixed)
if self.severity:
severity = vulnerability_pb2.Severity.Value(self.severity)
else:
severity = vulnerability_pb2.Severity.NONE
details = self.details
if self.status == bug.BugStatus.INVALID:
affects = None
details = 'INVALID'
severity = vulnerability_pb2.Severity.NONE
if self.last_modified:
modified = timestamp_pb2.Timestamp()
modified.FromDatetime(self.last_modified)
else:
modified = None
published = timestamp_pb2.Timestamp()
published.FromDatetime(self.timestamp)
references = []
if self.reference_url_types:
for url, url_type in self.reference_url_types.items():
references.append(
vulnerability_pb2.Reference(
url=url, type=vulnerability_pb2.Reference.Type.Value(url_type)))
result = vulnerability_pb2.Vulnerability(
id=self.id(),
published=published,
modified=modified,
summary=self.summary,
details=details,
package=package,
severity=severity,
affects=affects,
references=references)
return result
class NDB_User(ndb.Model):
chat_id = ndb.StringProperty()
name = ndb.StringProperty()
last_name = ndb.StringProperty()
application = ndb.StringProperty() # 'telegram', 'messenger'
username = ndb.StringProperty()
last_mod = ndb.DateTimeProperty(auto_now=True)
state = ndb.StringProperty()
enabled = ndb.BooleanProperty(default=True)
tmp_variables = ndb.JsonProperty(indexed=False)
def update_info(self, name, last_name, username):
modified, was_disabled = False, False
if self.get_first_name() != name:
self.name = name
modified = True
if self.get_last_name() != last_name:
self.last_name = last_name
modified = True
if self.username != username:
self.username = username
modified = True
if not self.enabled:
self.enabled = True
modified = True
was_disabled = True
if modified:
self.put()
return modified, was_disabled
def get_id(self):
return self.key.id()
def is_admin(self):
import key
return self.get_id() in key.TELEGRAM_BOT_MASTER_ID
def get_first_name(self, escape_markdown=True):
return utility.escape_markdown(
self.name) if escape_markdown else self.name
def get_last_name(self, escape_markdown=True):
if self.last_name is None:
return None
return utility.escape_markdown(
self.last_name) if escape_markdown else self.name
def get_username(self, escape_markdown=True):
if self.username is None:
return None
return utility.escape_markdown(
self.username) if escape_markdown else self.name
def get_first_last_name(self, escape_markdown=True):
if self.last_name is None:
return self.get_first_name(escape_markdown)
return self.get_first_name(escape_markdown) + ' ' + self.get_last_name(
escape_markdown)
def get_first_last_username(self, escape_markdown=True):
result = self.get_first_last_name(escape_markdown)
if self.username:
result += ' @' + self.get_username(escape_markdown)
return result
def get_state(self):
return self.state
def set_enabled(self, enabled, put=False):
self.enabled = enabled
if put:
self.put()
def set_state(self, newstate, put=True):
self.state = newstate
if put:
self.put()
def set_keyboard(self, kb, put=True):
self.set_tmp_variable("keyboard", value=kb, put=put)
def get_keyboard(self):
return self.get_tmp_variable("keyboard", [])
def reset_tmp_variables(self):
self.tmp_variables = {}
def set_tmp_variable(self, var_name, value, put=False):
self.tmp_variables[var_name] = value
if put: self.put()
def get_tmp_variable(self, var_name, initValue=None):
if var_name in self.tmp_variables:
return self.tmp_variables[var_name]
self.tmp_variables[var_name] = initValue
return initValue
def switch_notifications(self):
self.enabled = not self.enabled
self.put()
class WordPress(models.Source):
"""A WordPress blog.
The key name is the blog hostname.
"""
GR_CLASS = collections.namedtuple('FakeGrClass',
('NAME', ))(NAME='WordPress.com')
OAUTH_START_HANDLER = oauth_wordpress.StartHandler
SHORT_NAME = 'wordpress'
site_info = ndb.JsonProperty(compressed=True) # from /sites/$site API call
def feed_url(self):
# http://en.support.wordpress.com/feeds/
return urllib.parse.urljoin(self.silo_url(), 'feed/')
def silo_url(self):
return self.domain_urls[0]
def edit_template_url(self):
return urllib.parse.urljoin(self.silo_url(), 'wp-admin/widgets.php')
@staticmethod
def new(handler, auth_entity=None, **kwargs):
"""Creates and returns a WordPress for the logged in user.
Args:
handler: the current :class:`webapp2.RequestHandler`
auth_entity: :class:`oauth_dropins.wordpress_rest.WordPressAuth`
"""
site_info = WordPress.get_site_info(handler, auth_entity)
if site_info is None:
return
urls = util.dedupe_urls(
util.trim_nulls([site_info.get('URL'), auth_entity.blog_url]))
domains = [util.domain_from_link(u) for u in urls]
avatar = (json_loads(auth_entity.user_json).get('avatar_URL')
if auth_entity.user_json else None)
return WordPress(id=domains[0],
auth_entity=auth_entity.key,
name=auth_entity.user_display_name(),
picture=avatar,
superfeedr_secret=util.generate_secret(),
url=urls[0],
domain_urls=urls,
domains=domains,
site_info=site_info,
**kwargs)
def _urls_and_domains(self, auth_entity):
"""Returns this blog's URL and domain.
Args:
auth_entity: unused
Returns:
([string url], [string domain])
"""
return [self.url], [self.key_id()]
def create_comment(self, post_url, author_name, author_url, content):
"""Creates a new comment in the source silo.
If the last part of the post URL is numeric, e.g. http://site/post/123999,
it's used as the post id. Otherwise, we extract the last part of
the path as the slug, e.g. http: / / site / post / the-slug,
and look up the post id via the API.
Args:
post_url: string
author_name: string
author_url: string
content: string
Returns:
JSON response dict with 'id' and other fields
"""
auth_entity = self.auth_entity.get()
logging.info('Determining WordPress.com post id for %s', post_url)
# extract the post's slug and look up its post id
path = urllib.parse.urlparse(post_url).path
if path.endswith('/'):
path = path[:-1]
slug = path.split('/')[-1]
try:
post_id = int(slug)
except ValueError:
logging.info('Looking up post id for slug %s', slug)
url = API_POST_SLUG_URL % (auth_entity.blog_id, slug)
post_id = self.urlopen(auth_entity, url).get('ID')
if not post_id:
return self.error('Could not find post id', report=False)
logging.info('Post id is %d', post_id)
# create the comment
url = API_CREATE_COMMENT_URL % (auth_entity.blog_id, post_id)
content = '<a href="%s">%s</a>: %s' % (author_url, author_name,
content)
data = {'content': content.encode()}
try:
resp = self.urlopen(auth_entity,
url,
data=urllib.parse.urlencode(data))
except urllib.error.HTTPError as e:
code, body = util.interpret_http_exception(e)
try:
parsed = json_loads(body) if body else {}
if ((code == '400' and parsed.get('error') == 'invalid_input')
or (code == '403' and parsed.get('message')
== 'Comments on this post are closed')):
return parsed # known error: https://github.com/snarfed/bridgy/issues/161
except ValueError:
pass # fall through
raise e
resp['id'] = resp.pop('ID', None)
return resp
@classmethod
def get_site_info(cls, handler, auth_entity):
"""Fetches the site info from the API.
Args:
handler: the current :class:`webapp2.RequestHandler`
auth_entity: :class:`oauth_dropins.wordpress_rest.WordPressAuth`
Returns:
site info dict, or None if API calls are disabled for this blog
"""
try:
return cls.urlopen(auth_entity, API_SITE_URL % auth_entity.blog_id)
except urllib.error.HTTPError as e:
code, body = util.interpret_http_exception(e)
if (code == '403' and
'"API calls to this blog have been disabled."' in body):
handler.messages.add(
'You need to <a href="http://jetpack.me/support/json-api/">enable '
'the Jetpack JSON API</a> in %s\'s WordPress admin console.'
% util.pretty_link(auth_entity.blog_url))
handler.redirect('/')
return None
raise
@staticmethod
def urlopen(auth_entity, url, **kwargs):
resp = auth_entity.urlopen(url, **kwargs).read()
logging.debug(resp)
return json_loads(resp)
class OtherKind(ndb.Model):
data = ndb.JsonProperty(compressed=True)
class Bug(ndb.Model):
"""Bug entity."""
OSV_ID_PREFIX = 'OSV-'
# Very large fake version to use when there is no fix available.
_NOT_FIXED_SEMVER = '999999.999999.999999'
# Display ID as used by the source database. The full qualified database that
# OSV tracks this as may be different.
db_id = ndb.StringProperty()
# Other IDs this bug is known as.
aliases = ndb.StringProperty(repeated=True)
# Related IDs.
related = ndb.StringProperty(repeated=True)
# Status of the bug.
status = ndb.IntegerProperty()
# Timestamp when Bug was allocated.
timestamp = ndb.DateTimeProperty()
# When the entry was last edited.
last_modified = ndb.DateTimeProperty()
# When the entry was withdrawn.
withdrawn = ndb.DateTimeProperty()
# The source identifier.
# For OSS-Fuzz, this oss-fuzz:<ClusterFuzz testcase ID>.
# For others this is <source>:<path/to/source>.
source_id = ndb.StringProperty()
# The main fixed commit (from bisection).
fixed = ndb.StringProperty(default='')
# The main regressing commit (from bisection).
regressed = ndb.StringProperty(default='')
# All affected ranges.
affected_ranges = ndb.StructuredProperty(AffectedRange, repeated=True)
# List of affected versions.
affected = ndb.TextProperty(repeated=True)
# List of normalized versions indexed for fuzzy matching.
affected_fuzzy = ndb.StringProperty(repeated=True)
# OSS-Fuzz issue ID.
issue_id = ndb.StringProperty()
# Package URL for this package.
purl = ndb.StringProperty()
# Project/package name for the bug.
project = ndb.StringProperty()
# Package ecosystem for the project.
ecosystem = ndb.StringProperty()
# Summary for the bug.
summary = ndb.TextProperty()
# Vulnerability details.
details = ndb.TextProperty()
# Severity of the bug.
severity = ndb.StringProperty(validator=_check_valid_severity)
# Whether or not the bug is public (OSS-Fuzz only).
public = ndb.BooleanProperty()
# Reference URL types (dict of url -> type).
reference_url_types = ndb.JsonProperty()
# Search indices (auto-populated)
search_indices = ndb.StringProperty(repeated=True)
# Whether or not the bug has any affected versions (auto-populated).
has_affected = ndb.BooleanProperty()
# Source of truth for this Bug.
source_of_truth = ndb.IntegerProperty(default=SourceOfTruth.INTERNAL)
# Whether the bug is fixed (indexed for querying).
is_fixed = ndb.BooleanProperty()
# Database specific.
database_specific = ndb.JsonProperty()
# Ecosystem specific.
ecosystem_specific = ndb.JsonProperty()
# Normalized SEMVER fixed indexes for querying.
semver_fixed_indexes = ndb.StringProperty(repeated=True)
# The source of this Bug.
source = ndb.StringProperty()
def id(self):
"""Get the bug ID."""
if self.db_id:
return self.db_id
# TODO(ochang): Remove once all existing bugs have IDs migrated.
if re.match(r'^\d+', self.key.id()):
return self.OSV_ID_PREFIX + self.key.id()
return self.key.id()
@property
def repo_url(self):
"""Repo URL."""
for affected_range in self.affected_ranges:
if affected_range.repo_url:
return affected_range.repo_url
return None
@classmethod
def get_by_id(cls, vuln_id, *args, **kwargs):
"""Overridden get_by_id to handle OSV allocated IDs."""
result = cls.query(cls.db_id == vuln_id).get()
if result:
return result
# TODO(ochang): Remove once all exsting bugs have IDs migrated.
if vuln_id.startswith(cls.OSV_ID_PREFIX):
vuln_id = vuln_id[len(cls.OSV_ID_PREFIX):]
return super().get_by_id(vuln_id, *args, **kwargs)
def _tokenize(self, value):
"""Tokenize value for indexing."""
if not value:
return []
value_lower = value.lower()
return re.split(r'\W+', value_lower) + [value_lower]
def _pre_put_hook(self):
"""Pre-put hook for populating search indices."""
search_indices = set()
search_indices.update(self._tokenize(self.id()))
if self.project:
search_indices.update(self._tokenize(self.project))
if self.ecosystem:
search_indices.update(self._tokenize(self.ecosystem))
self.search_indices = sorted(list(search_indices))
self.has_affected = bool(self.affected) or any(
r.type in ('SEMVER', 'ECOSYSTEM') for r in self.affected_ranges)
self.affected_fuzzy = bug.normalize_tags(self.affected)
if not self.last_modified:
self.last_modified = utcnow()
self.is_fixed = any(affected_range.fixed
for affected_range in self.affected_ranges)
self.semver_fixed_indexes = []
for affected_range in self.affected_ranges:
if affected_range.type == 'SEMVER':
fixed = affected_range.fixed or self._NOT_FIXED_SEMVER
self.semver_fixed_indexes.append(semver_index.normalize(fixed))
if self.source_id:
self.source, _ = sources.parse_source_id(self.source_id)
if not self.source:
raise ValueError('Source not specified for Bug.')
if not self.db_id:
raise ValueError('DB ID not specified for Bug.')
if not self.key:
source_repo = get_source_repository(self.source)
if not source_repo:
raise ValueError(f'Invalid source {self.source}')
if source_repo.db_prefix and self.db_id.startswith(
source_repo.db_prefix):
key_id = self.db_id
else:
key_id = f'{self.source}:{self.db_id}'
self.key = ndb.Key(Bug, key_id)
def update_from_vulnerability(self, vulnerability):
"""Set fields from vulnerability. Does not set the ID."""
self.summary = vulnerability.summary
self.details = vulnerability.details
self.reference_url_types = {
ref.url: vulnerability_pb2.Reference.Type.Name(ref.type)
for ref in vulnerability.references
}
if vulnerability.HasField('modified'):
self.last_modified = vulnerability.modified.ToDatetime()
if vulnerability.HasField('published'):
self.timestamp = vulnerability.published.ToDatetime()
if vulnerability.HasField('withdrawn'):
self.withdrawn = vulnerability.withdrawn.ToDatetime()
self.project = vulnerability.package.name
self.ecosystem = vulnerability.package.ecosystem
if vulnerability.package.purl:
self.purl = vulnerability.package.purl
self.affected = list(vulnerability.affects.versions)
self.aliases = list(vulnerability.aliases)
self.related = list(vulnerability.related)
vuln_dict = sources.vulnerability_to_dict(vulnerability)
if vulnerability.database_specific:
self.database_specific = vuln_dict['database_specific']
if vulnerability.ecosystem_specific:
self.ecosystem_specific = vuln_dict['ecosystem_specific']
self.affected_ranges = []
for affected_range in vulnerability.affects.ranges:
self.affected_ranges.append(
AffectedRange(type=vulnerability_pb2.AffectedRange.Type.Name(
affected_range.type),
repo_url=affected_range.repo,
introduced=affected_range.introduced or '',
fixed=affected_range.fixed or ''))
def to_vulnerability(self, include_source=False):
"""Convert to Vulnerability proto."""
package = vulnerability_pb2.Package(name=self.project,
ecosystem=self.ecosystem,
purl=self.purl)
affects = vulnerability_pb2.Affects(versions=self.affected)
for affected_range in self.affected_ranges:
affects.ranges.add(type=vulnerability_pb2.AffectedRange.Type.Value(
affected_range.type),
repo=affected_range.repo_url,
introduced=affected_range.introduced,
fixed=affected_range.fixed)
details = self.details
if self.status == bug.BugStatus.INVALID:
affects = None
details = 'INVALID'
if self.last_modified:
modified = timestamp_pb2.Timestamp()
modified.FromDatetime(self.last_modified)
else:
modified = None
if self.withdrawn:
withdrawn = timestamp_pb2.Timestamp()
withdrawn.FromDatetime(self.withdrawn)
else:
withdrawn = None
published = timestamp_pb2.Timestamp()
published.FromDatetime(self.timestamp)
references = []
if self.reference_url_types:
for url, url_type in self.reference_url_types.items():
references.append(
vulnerability_pb2.Reference(
url=url,
type=vulnerability_pb2.Reference.Type.Value(url_type)))
result = vulnerability_pb2.Vulnerability(id=self.id(),
published=published,
modified=modified,
aliases=self.aliases,
related=self.related,
withdrawn=withdrawn,
summary=self.summary,
details=details,
package=package,
affects=affects,
references=references)
if self.ecosystem_specific:
result.ecosystem_specific.update(self.ecosystem_specific)
if self.database_specific:
result.database_specific.update(self.database_specific)
if self.source and include_source:
source_repo = get_source_repository(self.source)
if not source_repo or not source_repo.link:
return result
result.database_specific.update({
'source':
source_repo.link + sources.source_path(source_repo, self),
})
return result
class Bug(ndb.Model):
"""Bug entity."""
OSV_ID_PREFIX = 'OSV-'
# Very large fake version to use when there is no fix available.
_NOT_FIXED_SEMVER = '999999.999999.999999'
# Display ID as used by the source database. The full qualified database that
# OSV tracks this as may be different.
db_id = ndb.StringProperty()
# Other IDs this bug is known as.
aliases = ndb.StringProperty(repeated=True)
# Related IDs.
related = ndb.StringProperty(repeated=True)
# Status of the bug.
status = ndb.IntegerProperty()
# Timestamp when Bug was allocated.
timestamp = ndb.DateTimeProperty()
# When the entry was last edited.
last_modified = ndb.DateTimeProperty()
# When the entry was withdrawn.
withdrawn = ndb.DateTimeProperty()
# The source identifier.
# For OSS-Fuzz, this oss-fuzz:<ClusterFuzz testcase ID>.
# For others this is <source>:<path/to/source>.
source_id = ndb.StringProperty()
# The main fixed commit (from bisection).
fixed = ndb.StringProperty(default='')
# The main regressing commit (from bisection).
regressed = ndb.StringProperty(default='')
# All affected ranges. TODO(ochang): To be removed.
affected_ranges = ndb.StructuredProperty(AffectedRange, repeated=True)
# List of affected versions.
affected = ndb.TextProperty(repeated=True)
# List of normalized versions indexed for fuzzy matching.
affected_fuzzy = ndb.StringProperty(repeated=True)
# OSS-Fuzz issue ID.
issue_id = ndb.StringProperty()
# Package URL for this package.
purl = ndb.StringProperty(repeated=True)
# Project/package name for the bug.
project = ndb.StringProperty(repeated=True)
# Package ecosystem for the project.
ecosystem = ndb.StringProperty(repeated=True)
# Summary for the bug.
summary = ndb.TextProperty()
# Vulnerability details.
details = ndb.TextProperty()
# Severity of the bug.
severity = ndb.StringProperty(validator=_check_valid_severity)
# Whether or not the bug is public (OSS-Fuzz only).
public = ndb.BooleanProperty()
# Reference URL types (dict of url -> type).
reference_url_types = ndb.JsonProperty()
# Search indices (auto-populated)
search_indices = ndb.StringProperty(repeated=True)
# Whether or not the bug has any affected versions (auto-populated).
has_affected = ndb.BooleanProperty()
# Source of truth for this Bug.
source_of_truth = ndb.IntegerProperty(default=SourceOfTruth.INTERNAL)
# Whether the bug is fixed (indexed for querying).
is_fixed = ndb.BooleanProperty()
# Database specific.
database_specific = ndb.JsonProperty()
# Ecosystem specific.
ecosystem_specific = ndb.JsonProperty()
# Normalized SEMVER fixed indexes for querying.
semver_fixed_indexes = ndb.StringProperty(repeated=True)
# Affected packages and versions.
affected_packages = ndb.LocalStructuredProperty(AffectedPackage,
repeated=True)
# The source of this Bug.
source = ndb.StringProperty()
def id(self):
"""Get the bug ID."""
if self.db_id:
return self.db_id
# TODO(ochang): Remove once all existing bugs have IDs migrated.
if re.match(r'^\d+', self.key.id()):
return self.OSV_ID_PREFIX + self.key.id()
return self.key.id()
@property
def repo_url(self):
"""Repo URL."""
for affected_package in self.affected_packages:
for affected_range in affected_package.ranges:
if affected_range.repo_url:
return affected_range.repo_url
return None
@classmethod
def get_by_id(cls, vuln_id, *args, **kwargs):
"""Overridden get_by_id to handle OSV allocated IDs."""
result = cls.query(cls.db_id == vuln_id).get()
if result:
return result
# TODO(ochang): Remove once all exsting bugs have IDs migrated.
if vuln_id.startswith(cls.OSV_ID_PREFIX):
vuln_id = vuln_id[len(cls.OSV_ID_PREFIX):]
return super().get_by_id(vuln_id, *args, **kwargs)
def _tokenize(self, value):
"""Tokenize value for indexing."""
if not value:
return []
value_lower = value.lower()
return re.split(r'\W+', value_lower) + [value_lower]
def _pre_put_hook(self):
"""Pre-put hook for populating search indices."""
search_indices = set()
search_indices.update(self._tokenize(self.id()))
if self.affected_packages:
self.project = [
pkg.package.name for pkg in self.affected_packages
if pkg.package.name
]
self.ecosystem = [
pkg.package.ecosystem for pkg in self.affected_packages
if pkg.package.ecosystem
]
self.purl = [
pkg.package.purl for pkg in self.affected_packages
if pkg.package.purl
]
for project in self.project:
search_indices.update(self._tokenize(project))
for ecosystem in self.ecosystem:
search_indices.update(self._tokenize(ecosystem))
self.search_indices = sorted(list(search_indices))
self.affected_fuzzy = []
self.semver_fixed_indexes = []
self.has_affected = False
self.is_fixed = False
for affected_package in self.affected_packages:
# Indexes used for querying by exact version.
self.affected_fuzzy.extend(
bug.normalize_tags(affected_package.versions))
self.has_affected |= bool(affected_package.versions)
for affected_range in affected_package.ranges:
fixed_version = None
for event in affected_range.events:
# Index used to query by fixed/unfixed.
if event.type == 'fixed':
self.is_fixed = True
fixed_version = event.value
if affected_range.type == 'SEMVER':
# Indexes used for querying by semver.
fixed = fixed_version or self._NOT_FIXED_SEMVER
self.semver_fixed_indexes.append(
semver_index.normalize(fixed))
self.has_affected |= (affected_range.type
in ('SEMVER', 'ECOSYSTEM'))
if not self.last_modified:
self.last_modified = utcnow()
if self.source_id:
self.source, _ = sources.parse_source_id(self.source_id)
if not self.source:
raise ValueError('Source not specified for Bug.')
if not self.db_id:
raise ValueError('DB ID not specified for Bug.')
if not self.key: # pylint: disable=access-member-before-definition
source_repo = get_source_repository(self.source)
if not source_repo:
raise ValueError(f'Invalid source {self.source}')
if source_repo.db_prefix and self.db_id.startswith(
source_repo.db_prefix):
key_id = self.db_id
else:
key_id = f'{self.source}:{self.db_id}'
self.key = ndb.Key(Bug, key_id)
def _update_from_pre_0_8(self, vulnerability):
"""Update from pre 0.8 import."""
if self.affected_packages:
affected_package = self.affected_packages[0]
else:
affected_package = AffectedPackage()
self.affected_packages.append(affected_package)
affected_package.package = Package(
name=vulnerability.package.name,
ecosystem=vulnerability.package.ecosystem,
purl=vulnerability.package.purl)
vuln_dict = sources.vulnerability_to_dict(vulnerability)
if vulnerability.database_specific:
affected_package.database_specific = vuln_dict['database_specific']
if vulnerability.ecosystem_specific:
affected_package.ecosystem_specific = vuln_dict[
'ecosystem_specific']
affected_package.versions = list(vulnerability.affects.versions)
affected_package.ranges = []
events_by_type = {}
for affected_range in vulnerability.affects.ranges:
events = events_by_type.setdefault(
(vulnerability_pb2.AffectedRange.Type.Name(
affected_range.type), affected_range.repo), [])
# An empty introduced in 0.7 now needs to be represented as '0' in 0.8.
introduced = AffectedEvent(type='introduced',
value=affected_range.introduced or '0')
if introduced not in events:
events.append(introduced)
if affected_range.fixed:
fixed = AffectedEvent(type='fixed', value=affected_range.fixed)
if fixed not in events:
events.append(fixed)
for (range_type, repo_url), events in events_by_type.items():
affected_range = AffectedRange2(type=range_type, events=events)
if range_type == 'GIT' and repo_url:
affected_range.repo_url = repo_url
affected_package.ranges.append(affected_range)
def update_from_vulnerability(self, vulnerability):
"""Set fields from vulnerability. Does not set the ID."""
self.summary = vulnerability.summary
self.details = vulnerability.details
self.reference_url_types = {
ref.url: vulnerability_pb2.Reference.Type.Name(ref.type)
for ref in vulnerability.references
}
if vulnerability.HasField('modified'):
self.last_modified = vulnerability.modified.ToDatetime()
if vulnerability.HasField('published'):
self.timestamp = vulnerability.published.ToDatetime()
if vulnerability.HasField('withdrawn'):
self.withdrawn = vulnerability.withdrawn.ToDatetime()
self.aliases = list(vulnerability.aliases)
self.related = list(vulnerability.related)
if not vulnerability.affected:
self._update_from_pre_0_8(vulnerability)
return
self.affected_packages = []
for affected_package in vulnerability.affected:
current = AffectedPackage()
current.package = Package(
name=affected_package.package.name,
ecosystem=affected_package.package.ecosystem,
purl=affected_package.package.purl)
current.ranges = []
for affected_range in affected_package.ranges:
current_range = AffectedRange2(
type=vulnerability_pb2.Range.Type.Name(
affected_range.type),
repo_url=affected_range.repo,
events=[])
for evt in affected_range.events:
if evt.introduced:
current_range.events.append(
AffectedEvent(type='introduced',
value=evt.introduced))
continue
if evt.fixed:
current_range.events.append(
AffectedEvent(type='fixed', value=evt.fixed))
continue
if evt.limit:
current_range.events.append(
AffectedEvent(type='limit', value=evt.limit))
continue
current.ranges.append(current_range)
current.versions = list(affected_package.versions)
if affected_package.database_specific:
current.database_specific = json_format.MessageToDict(
affected_package.database_specific,
preserving_proto_field_name=True)
if affected_package.ecosystem_specific:
current.ecosystem_specific = json_format.MessageToDict(
affected_package.ecosystem_specific,
preserving_proto_field_name=True)
self.affected_packages.append(current)
def _get_pre_0_8_affects(self):
"""Get pre 0.8 schema affects field."""
affected_package = self.affected_packages[0]
affects = vulnerability_pb2.Affects(versions=affected_package.versions)
for affected_range in affected_package.ranges:
# Convert flattened events to range pairs (pre-0.8 schema).
# TODO(ochang): Remove this once all consumers are migrated.
# pylint: disable=cell-var-from-loop
new_range = lambda x, y: vulnerability_pb2.AffectedRange(
type=vulnerability_pb2.AffectedRange.Type.Value(affected_range.
type),
repo=affected_range.repo_url,
introduced=x,
fixed=y)
last_introduced = None
# Sort the flattened events, then find corresponding [introduced,
# fixed) pairs.
for event in sorted_events(affected_package.package.ecosystem,
affected_range.type,
affected_range.events):
if event.type == 'introduced':
if last_introduced is not None and affected_range.type == 'GIT':
# If this is GIT, then we need to store all "introduced", even if
# they overlap.
affects.ranges.append(new_range(last_introduced, ''))
last_introduced = None
if last_introduced is None:
# If not GIT, ignore overlapping introduced versions since they're
# redundant.
last_introduced = event.value
if last_introduced == '0':
last_introduced = ''
if event.type == 'fixed':
if affected_range.type != 'GIT' and last_introduced is None:
# No prior introduced, so ignore this invalid entry.
continue
# Found a complete pair.
affects.ranges.append(
new_range(last_introduced, event.value))
last_introduced = None
if last_introduced is not None:
affects.ranges.append(new_range(last_introduced, ''))
return affects
def to_vulnerability(self, include_source=False, v0_7=True, v0_8=False):
"""Convert to Vulnerability proto."""
package = None
ecosystem_specific = None
database_specific = None
affected = []
affects = None
source_link = None
if self.source and include_source:
source_repo = get_source_repository(self.source)
if source_repo and source_repo.link:
source_link = source_repo.link + sources.source_path(
source_repo, self)
if self.affected_packages:
if v0_7:
# The pre-0.8 schema only supports a single package, so we take the
# first.
affected_package = self.affected_packages[0]
package = vulnerability_pb2.Package(
name=affected_package.package.name,
ecosystem=affected_package.package.ecosystem,
purl=affected_package.package.purl)
affects = self._get_pre_0_8_affects()
if affected_package.ecosystem_specific:
ecosystem_specific = affected_package.ecosystem_specific
if affected_package.database_specific:
database_specific = affected_package.database_specific
if v0_8:
for affected_package in self.affected_packages:
ranges = []
for affected_range in affected_package.ranges:
events = []
for event in affected_range.events:
kwargs = {event.type: event.value}
events.append(vulnerability_pb2.Event(**kwargs))
current_range = vulnerability_pb2.Range(
type=vulnerability_pb2.Range.Type.Value(
affected_range.type),
repo=affected_range.repo_url,
events=events)
ranges.append(current_range)
current = vulnerability_pb2.Affected(
package=vulnerability_pb2.Package(
name=affected_package.package.name,
ecosystem=affected_package.package.ecosystem,
purl=affected_package.package.purl),
ranges=ranges,
versions=affected_package.versions)
if affected_package.database_specific:
current.database_specific.update(
affected_package.database_specific)
if source_link:
current.database_specific.update(
{'source': source_link})
if affected_package.ecosystem_specific:
current.ecosystem_specific.update(
affected_package.ecosystem_specific)
affected.append(current)
details = self.details
if self.status == bug.BugStatus.INVALID:
affects = None
affected = None
details = 'INVALID'
if self.last_modified:
modified = timestamp_pb2.Timestamp()
modified.FromDatetime(self.last_modified)
else:
modified = None
if self.withdrawn:
withdrawn = timestamp_pb2.Timestamp()
withdrawn.FromDatetime(self.withdrawn)
else:
withdrawn = None
published = timestamp_pb2.Timestamp()
published.FromDatetime(self.timestamp)
references = []
if self.reference_url_types:
for url, url_type in self.reference_url_types.items():
references.append(
vulnerability_pb2.Reference(
url=url,
type=vulnerability_pb2.Reference.Type.Value(url_type)))
result = vulnerability_pb2.Vulnerability(id=self.id(),
published=published,
modified=modified,
aliases=self.aliases,
related=self.related,
withdrawn=withdrawn,
summary=self.summary,
details=details,
package=package,
affects=affects,
affected=affected,
references=references)
if ecosystem_specific:
result.ecosystem_specific.update(ecosystem_specific)
if database_specific:
result.database_specific.update(database_specific)
if source_link:
result.database_specific.update({'source': source_link})
return result
class Source(StringIdModel, metaclass=SourceMeta):
"""A silo account, e.g. a Facebook or Google+ account.
Each concrete silo class should subclass this class.
"""
# Turn off NDB instance and memcache caching.
# https://developers.google.com/appengine/docs/python/ndb/cache
# https://github.com/snarfed/bridgy/issues/558
# https://github.com/snarfed/bridgy/issues/68
_use_cache = False
STATUSES = ('enabled', 'disabled', 'error') # 'error' is deprecated
POLL_STATUSES = ('ok', 'error', 'polling')
FEATURES = ('listen', 'publish', 'webmention', 'email')
# short name for this site type. used in URLs, etc.
SHORT_NAME = None
# the corresponding granary class
GR_CLASS = None
# oauth-dropins StartHandler class
OAUTH_START_HANDLER = None
# whether Bridgy supports listen for this silo - this is unlikely, so we default to True
CAN_LISTEN = True
# whether Bridgy supports publish for this silo
CAN_PUBLISH = None
# how often to poll for responses
FAST_POLL = datetime.timedelta(minutes=30)
# how often to poll sources that have never sent a webmention
SLOW_POLL = datetime.timedelta(days=1)
# how often to poll sources that are currently rate limited by their silo
RATE_LIMITED_POLL = SLOW_POLL
# how long to wait after signup for a successful webmention before dropping to
# the lower frequency poll
FAST_POLL_GRACE_PERIOD = datetime.timedelta(days=7)
# how often refetch author url to look for updated syndication links
FAST_REFETCH = datetime.timedelta(hours=6)
# refetch less often (this often) if it's been >2w since the last synd link
SLOW_REFETCH = datetime.timedelta(days=2)
# rate limiting HTTP status codes returned by this silo. e.g. twitter returns
# 429, instagram 503, google+ 403.
RATE_LIMIT_HTTP_CODES = ('429',)
DISABLE_HTTP_CODES = ('401',)
TRANSIENT_ERROR_HTTP_CODES = ()
# whether granary supports fetching block lists
HAS_BLOCKS = False
# whether to require a u-syndication link for backfeed
BACKFEED_REQUIRES_SYNDICATION_LINK = False
# Maps Publish.type (e.g. 'like') to source-specific human readable type label
# (e.g. 'favorite'). Subclasses should override this.
TYPE_LABELS = {}
# subclasses should override this
URL_CANONICALIZER = util.UrlCanonicalizer(headers=util.REQUEST_HEADERS)
# Regexps for URL paths that don't accept incoming webmentions. Currently used
# by Blogger.
PATH_BLACKLIST = ()
created = ndb.DateTimeProperty(auto_now_add=True, required=True)
url = ndb.StringProperty()
status = ndb.StringProperty(choices=STATUSES, default='enabled')
poll_status = ndb.StringProperty(choices=POLL_STATUSES, default='ok')
rate_limited = ndb.BooleanProperty(default=False)
name = ndb.StringProperty() # full human-readable name
picture = ndb.StringProperty()
domains = ndb.StringProperty(repeated=True)
domain_urls = ndb.StringProperty(repeated=True)
features = ndb.StringProperty(repeated=True, choices=FEATURES)
superfeedr_secret = ndb.StringProperty()
webmention_endpoint = ndb.StringProperty()
# points to an oauth-dropins auth entity. The model class should be a subclass
# of oauth_dropins.BaseAuth. the token should be generated with the
# offline_access scope so that it doesn't expire.
auth_entity = ndb.KeyProperty()
#
# listen-only properties
#
last_polled = ndb.DateTimeProperty(default=util.EPOCH)
last_poll_attempt = ndb.DateTimeProperty(default=util.EPOCH)
last_webmention_sent = ndb.DateTimeProperty()
last_public_post = ndb.DateTimeProperty()
recent_private_posts = ndb.IntegerProperty(default=0)
# the last time we re-fetched the author's url looking for updated
# syndication links
last_hfeed_refetch = ndb.DateTimeProperty(default=util.EPOCH)
# the last time we've seen a rel=syndication link for this Source.
# we won't spend the time to re-fetch and look for updates if there's
# never been one
last_syndication_url = ndb.DateTimeProperty()
# the last time we saw a syndication link in an h-feed, as opposed to just on
# permalinks. background: https://github.com/snarfed/bridgy/issues/624
last_feed_syndication_url = ndb.DateTimeProperty()
last_activity_id = ndb.StringProperty()
last_activities_etag = ndb.StringProperty()
last_activities_cache_json = ndb.TextProperty()
seen_responses_cache_json = ndb.TextProperty(compressed=True)
# populated in Poll.poll(), used by handlers
blocked_ids = ndb.JsonProperty(compressed=True)
# maps updated property names to values that put_updates() writes back to the
# datastore transactionally. set this to {} before beginning.
updates = None
# gr_source is *not* set to None by default here, since it needs to be unset
# for __getattr__ to run when it's accessed.
def __init__(self, *args, id=None, **kwargs):
"""Constructor. Escapes the key string id if it starts with `__`."""
if id and id.startswith('__'):
id = '\\' + id
super().__init__(*args, id=id, **kwargs)
def key_id(self):
"""Returns the key's unescaped string id."""
id = self.key.id()
return id[1:] if id[0] == '\\' else id
@classmethod
def new(cls, handler, **kwargs):
"""Factory method. Creates and returns a new instance for the current user.
To be implemented by subclasses.
"""
raise NotImplementedError()
def __getattr__(self, name):
"""Lazily load the auth entity and instantiate :attr:`self.gr_source`.
Once :attr:`self.gr_source` is set, this method will *not* be called;
:attr:`gr_source` will be returned normally.
"""
if name == 'gr_source' and self.auth_entity:
auth_entity = self.auth_entity.get()
args = auth_entity.access_token()
if not isinstance(args, tuple):
args = (args,)
kwargs = {}
if self.key.kind() == 'FacebookPage' and auth_entity.type == 'user':
kwargs = {'user_id': self.key_id()}
elif self.key.kind() == 'Instagram':
kwargs = {'scrape': True, 'cookie': INSTAGRAM_SESSIONID_COOKIE}
elif self.key.kind() == 'Mastodon':
args = (auth_entity.instance(),) + args
kwargs = {'user_id': json_loads(auth_entity.user_json).get('id')}
elif self.key.kind() == 'Twitter':
kwargs = {'username': self.key_id()}
self.gr_source = self.GR_CLASS(*args, **kwargs)
return self.gr_source
return getattr(super(Source, self), name)
@classmethod
def lookup(cls, id):
"""Returns the entity with the given id.
By default, interprets id as just the key id. Subclasses may extend this to
support usernames, etc.
"""
if id and id.startswith('__'):
id = '\\' + id
return ndb.Key(cls, id).get()
def user_tag_id(self):
"""Returns the tag URI for this source, e.g. 'tag:plus.google.com:123456'."""
return self.gr_source.tag_uri(self.key_id())
def bridgy_path(self):
"""Returns the Bridgy page URL path for this source."""
return '/%s/%s' % (self.SHORT_NAME, self.key_id())
def bridgy_url(self, handler):
"""Returns the Bridgy page URL for this source."""
return util.host_url(handler) + self.bridgy_path()
def silo_url(self, handler):
"""Returns the silo account URL, e.g. https://twitter.com/foo."""
raise NotImplementedError()
def label(self):
"""Human-readable label for this source."""
return '%s (%s)' % (self.label_name(), self.GR_CLASS.NAME)
def label_name(self):
"""Human-readable name or username for this source, whichever is preferred."""
return self.name or self.key_id()
@classmethod
@ndb.transactional()
def put_updates(cls, source):
"""Writes source.updates to the datastore transactionally.
Returns:
source: :class:`Source`
Returns:
the updated :class:`Source`
"""
if not source.updates:
return source
logging.info('Updating %s %s : %r', source.label(), source.bridgy_path(),
{k: v for k, v in source.updates.items() if not k.endswith('_json')})
updates = source.updates
source = source.key.get()
source.updates = updates
for name, val in updates.items():
setattr(source, name, val)
if source.status == 'error': # deprecated
logging.warning('Resetting status from error to enabled')
source.status = 'enabled'
source.put()
return source
def poll_period(self):
"""Returns the poll frequency for this source, as a :class:`datetime.timedelta`.
Defaults to ~15m, depending on silo. If we've never sent a webmention for
this source, or the last one we sent was over a month ago, we drop them down
to ~1d after a week long grace period.
"""
now = datetime.datetime.now()
if self.rate_limited:
return self.RATE_LIMITED_POLL
elif now < self.created + self.FAST_POLL_GRACE_PERIOD:
return self.FAST_POLL
elif not self.last_webmention_sent:
return self.SLOW_POLL
elif self.last_webmention_sent > now - datetime.timedelta(days=7):
return self.FAST_POLL
elif self.last_webmention_sent > now - datetime.timedelta(days=30):
return self.FAST_POLL * 10
else:
return self.SLOW_POLL
def should_refetch(self):
"""Returns True if we should run OPD refetch on this source now."""
now = datetime.datetime.now()
if self.last_hfeed_refetch == REFETCH_HFEED_TRIGGER:
return True
elif not self.last_syndication_url:
return False
period = (self.FAST_REFETCH
if self.last_syndication_url > now - datetime.timedelta(days=14)
else self.SLOW_REFETCH)
return self.last_poll_attempt >= self.last_hfeed_refetch + period
@classmethod
def bridgy_webmention_endpoint(cls, domain='brid.gy'):
"""Returns the Bridgy webmention endpoint for this source type."""
return 'https://%s/webmention/%s' % (domain, cls.SHORT_NAME)
def has_bridgy_webmention_endpoint(self):
"""Returns True if this source uses Bridgy's webmention endpoint."""
return self.webmention_endpoint in (
self.bridgy_webmention_endpoint(),
self.bridgy_webmention_endpoint(domain='www.brid.gy'))
def get_author_urls(self):
"""Determine the author urls for a particular source.
In debug mode, replace test domains with localhost.
Return:
a list of string URLs, possibly empty
"""
return [util.replace_test_domains_with_localhost(u) for u in self.domain_urls]
def search_for_links(self):
"""Searches for activities with links to any of this source's web sites.
https://github.com/snarfed/bridgy/issues/456
https://github.com/snarfed/bridgy/issues/565
Returns:
sequence of ActivityStreams activity dicts
"""
return []
def get_activities_response(self, **kwargs):
"""Returns recent posts and embedded comments for this source.
May be overridden by subclasses.
"""
kwargs.setdefault('group_id', gr_source.SELF)
resp = self.gr_source.get_activities_response(**kwargs)
for activity in resp['items']:
self._inject_user_urls(activity)
return resp
def get_activities(self, **kwargs):
return self.get_activities_response(**kwargs)['items']
def get_comment(self, comment_id, **kwargs):
"""Returns a comment from this source.
Passes through to granary by default. May be overridden by subclasses.
Args:
comment_id: string, site-specific comment id
kwargs: passed to :meth:`granary.source.Source.get_comment`
Returns:
dict, decoded ActivityStreams comment object, or None
"""
comment = self.gr_source.get_comment(comment_id, **kwargs)
if comment:
self._inject_user_urls(comment)
return comment
def get_like(self, activity_user_id, activity_id, like_user_id, **kwargs):
"""Returns an ActivityStreams 'like' activity object.
Passes through to granary by default. May be overridden
by subclasses.
Args:
activity_user_id: string id of the user who posted the original activity
activity_id: string activity id
like_user_id: string id of the user who liked the activity
kwargs: passed to granary.Source.get_comment
"""
return self.gr_source.get_like(activity_user_id, activity_id, like_user_id,
**kwargs)
def _inject_user_urls(self, activity):
"""Adds this user's web site URLs to their user mentions (in tags), in place."""
obj = activity.get('object') or activity
user_tag_id = self.user_tag_id()
for tag in obj.get('tags', []):
if tag.get('id') == user_tag_id:
tag.setdefault('urls', []).extend([{'value': u} for u in self.domain_urls])
def create_comment(self, post_url, author_name, author_url, content):
"""Creates a new comment in the source silo.
Must be implemented by subclasses.
Args:
post_url: string
author_name: string
author_url: string
content: string
Returns:
response dict with at least 'id' field
"""
raise NotImplementedError()
def feed_url(self):
"""Returns the RSS or Atom (or similar) feed URL for this source.
Must be implemented by subclasses. Currently only implemented by
:mod:`blogger`, :mod:`medium`, :mod:`tumblr`, and :mod:`wordpress_rest`.
Returns:
string URL
"""
raise NotImplementedError()
def edit_template_url(self):
"""Returns the URL for editing this blog's template HTML.
Must be implemented by subclasses. Currently only implemented by
:mod:`blogger`, :mod:`medium`, :mod:`tumblr`, and :mod:`wordpress_rest`.
Returns:
string URL
"""
raise NotImplementedError()
@classmethod
def button_html(cls, feature, **kwargs):
"""Returns an HTML string with a login form and button for this site.
Mostly just passes through to
:meth:`oauth_dropins.handlers.StartHandler.button_html`.
Returns: string, HTML
"""
assert feature in cls.FEATURES
form_extra = (kwargs.pop('form_extra', '') +
'<input name="feature" type="hidden" value="%s" />' % feature)
source = kwargs.pop('source', None)
if source:
form_extra += ('\n<input name="id" type="hidden" value="%s" />' %
source.key_id())
return cls.OAUTH_START_HANDLER.button_html(
'/%s/start' % cls.SHORT_NAME,
form_extra=form_extra,
image_prefix='/oauth_dropins/static/',
**kwargs)
@classmethod
def create_new(cls, handler, user_url=None, **kwargs):
"""Creates and saves a new :class:`Source` and adds a poll task for it.
Args:
handler: the current :class:`webapp2.RequestHandler`
user_url: a string, optional. if provided, supersedes other urls when
determining the author_url
**kwargs: passed to :meth:`new()`
Returns: newly created :class:`Source`
"""
source = cls.new(handler, **kwargs)
if source is None:
return None
if not source.domain_urls: # defer to the source if it already set this
auth_entity = kwargs.get('auth_entity')
if auth_entity and hasattr(auth_entity, 'user_json'):
source.domain_urls, source.domains = source._urls_and_domains(
auth_entity, user_url)
logging.debug('URLs/domains: %s %s', source.domain_urls, source.domains)
# check if this source already exists
existing = source.key.get()
if existing:
# merge some fields
source.features = set(source.features + existing.features)
source.populate(**existing.to_dict(include=(
'created', 'last_hfeed_refetch', 'last_poll_attempt', 'last_polled',
'last_syndication_url', 'last_webmention_sent', 'superfeedr_secret',
'webmention_endpoint')))
verb = 'Updated'
else:
verb = 'Added'
author_urls = source.get_author_urls()
link = ('http://indiewebify.me/send-webmentions/?url=' + author_urls[0]
if author_urls else 'http://indiewebify.me/#send-webmentions')
feature = source.features[0] if source.features else 'listen'
blurb = '%s %s. %s' % (
verb, source.label(),
'Try previewing a post from your web site!' if feature == 'publish'
else '<a href="%s">Try a webmention!</a>' % link if feature == 'webmention'
else "Refresh in a minute to see what we've found!")
logging.info('%s %s', blurb, source.bridgy_url(handler))
# uncomment to send email notification for each new user
# if not existing:
# util.email_me(subject=blurb, body=source.bridgy_url(handler))
source.verify()
if source.verified():
handler.messages = {blurb}
# TODO: ugh, *all* of this should be transactional
source.put()
if 'webmention' in source.features:
superfeedr.subscribe(source, handler)
if 'listen' in source.features:
util.add_poll_task(source, now=True)
util.add_poll_task(source)
return source
def verified(self):
"""Returns True if this source is ready to be used, false otherwise.
See :meth:`verify()` for details. May be overridden by subclasses, e.g.
:class:`tumblr.Tumblr`.
"""
if not self.domains or not self.domain_urls:
return False
if 'webmention' in self.features and not self.webmention_endpoint:
return False
if ('listen' in self.features and
not (self.webmention_endpoint or self.last_webmention_sent)):
return False
return True
def verify(self, force=False):
"""Checks that this source is ready to be used.
For blog and listen sources, this fetches their front page HTML and
discovers their webmention endpoint. For publish sources, this checks that
they have a domain.
May be overridden by subclasses, e.g. :class:`tumblr.Tumblr`.
Args:
force: if True, fully verifies (e.g. re-fetches the blog's HTML and
performs webmention discovery) even we already think this source is
verified.
"""
author_urls = [u for u, d in zip(self.get_author_urls(), self.domains)
if not util.in_webmention_blocklist(d)]
if ((self.verified() and not force) or self.status == 'disabled' or
not self.features or not author_urls):
return
author_url = author_urls[0]
logging.info('Attempting to discover webmention endpoint on %s', author_url)
mention = send.WebmentionSend('https://brid.gy/', author_url)
mention.requests_kwargs = {'timeout': util.HTTP_TIMEOUT,
'headers': util.REQUEST_HEADERS}
try:
mention._discoverEndpoint()
except BaseException:
logging.info('Error discovering webmention endpoint', stack_info=True)
mention.error = {'code': 'EXCEPTION'}
self._fetched_html = getattr(mention, 'html', None)
error = getattr(mention, 'error', None)
endpoint = getattr(mention, 'receiver_endpoint', None)
if error or not endpoint:
logging.info("No webmention endpoint found: %s %r", error, endpoint)
self.webmention_endpoint = None
else:
logging.info("Discovered webmention endpoint %s", endpoint)
self.webmention_endpoint = endpoint
self.put()
def _urls_and_domains(self, auth_entity, user_url):
"""Returns this user's valid (not webmention-blocklisted) URLs and domains.
Converts the auth entity's user_json to an ActivityStreams actor and uses
its 'urls' and 'url' fields. May be overridden by subclasses.
Args:
auth_entity: :class:`oauth_dropins.models.BaseAuth`
user_url: string, optional URL passed in when authorizing
Returns:
([string url, ...], [string domain, ...])
"""
user = json_loads(auth_entity.user_json)
actor = (user.get('actor') # for Instagram; its user_json is IndieAuth
or self.gr_source.user_to_actor(user))
logging.debug('Extracting URLs and domains from actor: %s',
json_dumps(actor, indent=2))
candidates = util.trim_nulls(util.uniquify(
[user_url] + microformats2.object_urls(actor)))
if len(candidates) > MAX_AUTHOR_URLS:
logging.info('Too many profile links! Only resolving the first %s: %s',
MAX_AUTHOR_URLS, candidates)
urls = []
for i, url in enumerate(candidates):
resolved = self.resolve_profile_url(url, resolve=i < MAX_AUTHOR_URLS)
if resolved:
urls.append(resolved)
final_urls = []
domains = []
for url in util.dedupe_urls(urls): # normalizes domains to lower case
# skip links on this source's domain itself. only currently needed for
# Mastodon; the other silo domains are in the webmention blocklist.
domain = util.domain_from_link(url)
if domain != self.gr_source.DOMAIN:
final_urls.append(url)
domains.append(domain)
return final_urls, domains
@staticmethod
def resolve_profile_url(url, resolve=True):
"""Resolves a profile URL to be added to a source.
Args:
url: string
resolve: boolean, whether to make HTTP requests to follow redirects, etc.
Returns: string, resolved URL, or None
"""
final, _, ok = util.get_webmention_target(url, resolve=resolve)
if not ok:
return None
final = final.lower()
if util.schemeless(final).startswith(util.schemeless(url.lower())):
# redirected to a deeper path. use the original higher level URL. #652
final = url
# If final has a path segment check if root has a matching rel=me.
match = re.match(r'^(https?://[^/]+)/.+', final)
if match and resolve:
root = match.group(1)
try:
mf2 = util.fetch_mf2(root)
me_urls = mf2['rels'].get('me', [])
if final in me_urls:
final = root
except requests.RequestException:
logging.warning("Couldn't fetch %s, preserving path in %s",
root, final, stack_info=True)
return final
def canonicalize_url(self, url, activity=None, **kwargs):
"""Canonicalizes a post or object URL.
Wraps :class:`oauth_dropins.webutil.util.UrlCanonicalizer`.
"""
return self.URL_CANONICALIZER(url, **kwargs) if self.URL_CANONICALIZER else url
def infer_profile_url(self, url):
"""Given an arbitrary URL representing a person, try to find their
profile URL for *this* service.
Queries Bridgy's registered accounts for users with a particular
domain in their silo profile.
Args:
url: string, a person's URL
Return:
a string URL for their profile on this service (or None)
"""
domain = util.domain_from_link(url)
if domain == self.gr_source.DOMAIN:
return url
user = self.__class__.query(self.__class__.domains == domain).get()
if user:
return self.gr_source.user_url(user.key_id())
def preprocess_for_publish(self, obj):
"""Preprocess an object before trying to publish it.
By default this tries to massage person tags so that the tag's
"url" points to the person's profile on this service (as opposed
to a person's homepage).
The object is modified in place.
Args:
obj: ActivityStreams activity or object dict
"""
for tag in obj.get('tags', []):
if tag.get('objectType') == 'person':
silo_url = None
for url in microformats2.object_urls(tag):
silo_url = url and self.infer_profile_url(url)
if silo_url:
break
if silo_url:
tag['url'] = silo_url
# recurse on contained object(s)
for obj in util.get_list(obj, 'object'):
self.preprocess_for_publish(obj)
def on_new_syndicated_post(self, syndpost):
"""Called when a new :class:`SyndicatedPost` is stored for this source.
Args:
syndpost: :class:`SyndicatedPost`
"""
pass
def is_private(self):
"""Returns True if this source is private aka protected.
...ie their posts are not public.
"""
return False
def is_activity_public(self, activity):
"""Returns True if the given activity is public, False otherwise.
Just wraps :meth:`granary.source.Source.is_public`. Subclasses may override.
"""
return gr_source.Source.is_public(activity)
def is_beta_user(self):
"""Returns True if this is a "beta" user opted into new features.
Beta users come from beta_users.txt.
"""
return self.bridgy_path() in util.BETA_USER_PATHS
def load_blocklist(self):
"""Fetches this user's blocklist, if supported, and stores it in the entity."""
if not self.HAS_BLOCKS:
return
try:
ids = self.gr_source.get_blocklist_ids()
except gr_source.RateLimited as e:
ids = e.partial or []
self.blocked_ids = ids[:BLOCKLIST_MAX_IDS]
self.put()
def is_blocked(self, obj):
"""Returns True if an object's author is being blocked.
...ie they're in this user's block list.
Note that this method is tested in test_twitter.py, not test_models.py, for
historical reasons.
"""
if not self.blocked_ids:
return False
for o in [obj] + util.get_list(obj, 'object'):
for field in 'author', 'actor':
if o.get(field, {}).get('numeric_id') in self.blocked_ids:
return True
class Person(ndb.Model):
chat_id = ndb.IntegerProperty()
name = ndb.StringProperty()
last_name = ndb.StringProperty()
username = ndb.StringProperty()
state = ndb.StringProperty(indexed=True)
last_state = ndb.StringProperty()
last_mod = ndb.DateTimeProperty(auto_now=True)
enabled = ndb.BooleanProperty(default=True)
variables = ndb.JsonProperty(indexed=False)
language_interface = ndb.StringProperty(
default=params.default_language_interface)
language_exercise = ndb.StringProperty(
default=params.default_language_exercise)
ux_lang = None # see method ux()
def user_telegram_id(self):
return 'telegram_{}'.format(self.chat_id)
def ux(self):
if self.ux_lang is None:
self.ux_lang = bot_ux.UX_LANG(self.language_interface)
return self.ux_lang
def get_name(self):
return self.name
def get_last_name(self):
return self.last_name if self.last_name else ''
def get_username(self):
return self.username
def get_name_last_name(self):
return self.get_name() + ' ' + self.get_last_name()
def get_name_last_name_username(self):
result = self.get_name_last_name()
if self.username:
result += ' @' + self.get_username()
return result
def set_enabled(self, enabled, put=False):
self.enabled = enabled
if put:
self.put()
def switch_notifications(self, put=True):
self.enabled = not self.enabled
if put:
self.put()
def update_user(self, name, last_name, username, put=False):
import params
changed = False
if self.name != name:
self.name = name
changed = True
if self.last_name != last_name:
self.last_name = last_name
changed = True
if self.username != username:
self.username = username
changed = True
if self.language_exercise not in params.LANGUAGES:
self.language_exercise = params.default_language_exercise
if self.language_interface not in params.LANGUAGES:
self.language_interface = params.default_language_interface
if changed and put:
self.put()
def set_language_interface(self, lang):
self.language_interface = lang
self.ux_lang = bot_ux.UX_LANG(lang)
def set_language_exercise(self, lang):
self.language_exercise = lang
def set_state(self, newstate, put=True):
self.last_state = self.state
self.state = newstate
if put:
self.put()
def is_administrator(self):
result = self.chat_id in key.ADMIN_IDS
#logging.debug("Amministratore: " + str(result))
return result
def set_last_exercise_id_and_options(self, ex_id, options):
self.set_variable('Exercise_ID', ex_id)
self.set_variable('Exercise_Options', options)
self.put()
def get_last_exercise_id_and_options(self):
options = [x for x in self.get_variable('Exercise_Options')]
return self.get_variable('Exercise_ID'), options
def set_keyboard(self, kb, put=True):
self.set_variable("keyboard", kb, put=put)
def get_keyboard(self):
return self.get_variable("keyboard", [])
def set_variable(self, var_key, var_value, put=True):
self.variables[var_key] = var_value
if put:
self.put()
def get_variable(self, var_key, default_value=None):
return self.variables.get(var_key, default_value)
def is_admin(self):
import key
return self.chat_id in key.ADMIN_IDS
def is_manager(self):
import key
return self.chat_id in key.MANAGER_IDS
class SomeKind(ndb.Model):
foo = ndb.JsonProperty(indexed=True)