def _get_binary_bug_stats(self, binary_name):
bug_stats, implemented = vendor.call(
'get_binary_package_bug_stats', binary_name)
if not implemented:
# The vendor does not provide a custom list of bugs, so the default
# is to display all bug info known for the package.
stats = get_or_none(
BinaryPackageBugStats, package__name=binary_name)
if stats is not None:
bug_stats = stats.stats
if bug_stats is None:
return
# Try to get the URL to the bug tracker for the given categories
for category in bug_stats:
url, implemented = vendor.call(
'get_bug_tracker_url',
binary_name,
'binary',
category['category_name'])
if not implemented:
continue
category['url'] = url
# Include the total bug count and corresponding tracker URL
all_bugs_url, implemented = vendor.call(
'get_bug_tracker_url', binary_name, 'binary', 'all')
return {
'total_count': sum(
category['bug_count'] for category in bug_stats),
'all_bugs_url': all_bugs_url,
'categories': bug_stats,
}
def add_developer_extras(general, url_only=False):
"""
Receives a general dict with package data and add to it more data
regarding that package's developers
"""
if 'maintainer' in general:
maintainer_email = general['maintainer']['email']
url = get_developer_information_url(maintainer_email)
if url:
general['maintainer']['developer_info_url'] = url
if not url_only:
extra, implemented = vendor.call(
'get_maintainer_extra', maintainer_email, general['name'])
general['maintainer']['extra'] = extra
uploaders = general.get('uploaders', None)
if uploaders:
for uploader in uploaders:
url = get_developer_information_url(uploader['email'])
if url:
uploader['developer_info_url'] = url
if url_only:
continue
# Vendor specific extras.
extra, implemented = vendor.call(
'get_uploader_extra', uploader['email'], general['name'])
if implemented and extra:
uploader['extra'] = extra
return general
def get_keyword(local_part, msg):
"""
Extracts the keywoword from the given message.
The function first tries using a vendor-provided function
:func:`get_keyword <distro_tracker.vendor.skeleton.rules.get_keyword>`.
If the vendor did not implement this function or does not return a keyword
for the given message, the function tries extracting the keyword from the
``local_part`` of the address (using :func:`get_keyword_from_address`).
If this also does not yield a keyword, ``default`` is returned.
:param local_part: The local part of the email address to which the message
was sent.
:type local_part: string
:param msg: The received package message
:type msg: :py:class:`email.message.Message` or an equivalent interface
object
:returns: The name of the keyword.
:rtype: string
"""
# Use a vendor-provided function to try and classify the message.
keyword, _ = vendor.call('get_keyword', local_part, msg)
if keyword:
return keyword
# Otherwise try getting the keyword from the address
keyword = get_keyword_from_address(local_part)
if keyword:
return keyword
# If we still do not have the keyword
return 'default'
def classify_message(msg, package=None, keyword=None):
"""
Analyzes a message to identify what package it is about and
what keyword is appropriate.
:param msg: The received message
:type msg: :py:class:`email.message.Message`
:param str package: The suggested package name.
:param str keyword: The suggested keyword under which the message can be
forwarded.
"""
if package is None:
package = msg.get('X-Distro-Tracker-Package')
if keyword is None:
keyword = msg.get('X-Distro-Tracker-Keyword')
result, implemented = vendor.call('classify_message', msg,
package=package, keyword=keyword)
if implemented:
package, keyword = result
if package and keyword is None:
keyword = 'default'
return (package, keyword)
def context(self):
try:
info = PackageExtractedInfo.objects.get(
package=self.package, key='binaries')
except PackageExtractedInfo.DoesNotExist:
return
binaries = info.value
for binary in binaries:
# For each binary try to include known bug stats
bug_stats = self._get_binary_bug_stats(binary['name'])
if bug_stats is not None:
binary['bug_stats'] = bug_stats
# For each binary try to include a link to an external package-info
# site.
if 'repository' in binary:
url, implemented = vendor.call(
'get_package_information_site_url', **{
'package_name': binary['name'],
'repository': binary['repository'],
'source_package': False,
}
)
if implemented and url:
binary['url'] = url
return binaries
def add_new_headers(received_message, package_name, keyword):
"""
The function adds new distro-tracker specific headers to the received
message. This is used before forwarding the message to subscribers.
The headers added by this function are used regardless whether the
message is forwarded due to direct package subscriptions or a team
subscription.
:param received_message: The received package message
:type received_message: :py:class:`email.message.Message` or an equivalent
interface object
:param package_name: The name of the package for which this message was
intended.
:type package_name: string
:param keyword: The keyword with which the message should be tagged
:type keyword: string
"""
new_headers = [
('X-Loop', '{package}@{distro_tracker_fqdn}'.format(
package=package_name,
distro_tracker_fqdn=DISTRO_TRACKER_FQDN)),
('X-Distro-Tracker-Package', package_name),
('X-Distro-Tracker-Keyword', keyword),
]
extra_vendor_headers, implemented = vendor.call(
'add_new_headers', received_message, package_name, keyword)
if implemented:
new_headers.extend(extra_vendor_headers)
add_headers(received_message, new_headers)
def context(self):
result, implemented = vendor.call(
'get_bug_panel_stats', self.package.name)
# implemented = False
if not implemented:
# If the vendor does not provide custom categories to be displayed
# in the panel, the default is to make each stored category a
# separate entry.
try:
stats = self.package.bug_stats.stats
except ObjectDoesNotExist:
return
# Also adds a total of all those bugs
total = sum(category['bug_count'] for category in stats)
stats.insert(0, {
'category_name': 'all',
'bug_count': total,
})
result = stats
# Either the vendor decided not to provide any info for this package
# or there is no known info.
if not result:
return []
return result
def context(self):
try:
info = PackageExtractedInfo.objects.get(
package=self.package, key='general')
except PackageExtractedInfo.DoesNotExist:
# There is no general info for the package
return
general = info.value
# Add source package URL
url, implemented = vendor.call('get_package_information_site_url', **{
'package_name': general['name'],
'source_package': True,
})
if implemented and url:
general['url'] = url
# Map the VCS type to its name.
if 'vcs' in general and 'type' in general['vcs']:
shorthand = general['vcs']['type']
general['vcs']['full_name'] = get_vcs_name(shorthand)
# Add mailing list archive URLs
self._add_archive_urls(general)
# Add developer information links and any other vendor-specific extras
self._add_developer_extras(general)
return general
def update_pseudo_package_list():
"""
Retrieves the list of all allowed pseudo packages and updates the stored
list if necessary.
Uses a vendor-provided function
:func:`get_pseudo_package_list
<distro_tracker.vendor.skeleton.rules.get_pseudo_package_list>`
to get the list of currently available pseudo packages.
"""
try:
pseudo_packages, implemented = vendor.call('get_pseudo_package_list')
except:
# Error accessing pseudo package resource: do not update the list
return
if not implemented or pseudo_packages is None:
return
# Faster lookups than if this were a list
pseudo_packages = set(pseudo_packages)
for existing_package in PseudoPackageName.objects.all():
if existing_package.name not in pseudo_packages:
# Existing packages which are no longer considered pseudo packages
# are demoted -- losing their pseudo package flag.
existing_package.pseudo = False
existing_package.save()
else:
# If an existing package remained a pseudo package there will be no
# action required so it is removed from the set.
pseudo_packages.remove(existing_package.name)
# The left over packages in the set are the ones that do not exist.
for package_name in pseudo_packages:
PseudoPackageName.objects.create(name=package_name)
def get_developer_information_url(email):
"""
Returns developer's information url based on his/her email
through vendor-specific function
"""
info_url, implemented = vendor.call(
'get_developer_information_url', **{'developer_email': email, })
if implemented and info_url:
return info_url
def context(self):
try:
info = PackageExtractedInfo.objects.get(
package=self.package, key='versions')
except PackageExtractedInfo.DoesNotExist:
info = None
context = {}
if info:
version_info = info.value
package_name = info.package.name
for item in version_info.get('version_list', ()):
url, implemented = vendor.call(
'get_package_information_site_url',
**
{'package_name': package_name,
'repository': item.get(
'repository'),
'source_package': True,
'version':
item.get('version'), })
if implemented and url:
item['url'] = url
context['version_info'] = version_info
# Add in any external version resource links
external_resources, implemented = (
vendor.call('get_external_version_information_urls',
self.package.name)
)
if implemented and external_resources:
context['external_resources'] = external_resources
# Add any vendor-provided versions
vendor_versions, implemented = vendor.call(
'get_extra_versions', self.package)
if implemented and vendor_versions:
context['vendor_versions'] = vendor_versions
return context
def table_fields(self):
"""
Returns the tuple of :class:`BaseTableField` that will compose the
table
"""
fields, implemented = vendor.call('get_table_fields', **{
'table': self,
})
if implemented and fields:
return tuple(fields)
else:
return tuple(self.default_fields)
def _add_developer_extras(self, general):
maintainer_email = general['maintainer']['email']
url = self._get_developer_information_url(maintainer_email)
if url:
general['maintainer']['developer_info_url'] = url
extra, implemented = vendor.call(
'get_maintainer_extra', maintainer_email, general['name'])
general['maintainer']['extra'] = extra
uploaders = general.get('uploaders', None)
if not uploaders:
return
for uploader in uploaders:
# Vendor specific extras.
extra, implemented = vendor.call(
'get_uploader_extra', uploader['email'], general['name'])
if implemented and extra:
uploader['extra'] = extra
url = self._get_developer_information_url(uploader['email'])
if url:
uploader['developer_info_url'] = url
def _create_tables(self):
result, implemented = vendor.call(
'get_tables_for_team_page', self.object, self.table_limit)
if implemented:
return result
return [
create_table(
slug='general', scope=self.object, limit=self.table_limit),
create_table(
slug='general', scope=self.object,
limit=self.table_limit, tag='tag:bugs'
),
]
def approved_default(msg):
"""
The function checks whether a message tagged with the default keyword should
be approved, meaning that it gets forwarded to subscribers.
:param msg: The received package message
:type msg: :py:class:`email.message.Message` or an equivalent interface
object
"""
if 'X-Distro-Tracker-Approved' in msg:
return True
approved, implemented = vendor.call('approve_default_message', msg)
if implemented:
return approved
else:
return False
def process(message):
"""
Process an incoming message which is potentially a news item.
The function first tries to call the vendor-provided function
:func:`create_news_from_email_message
<distro_tracker.vendor.skeleton.rules.create_news_from_email_message>`.
If this function does not exist a news item is created only if there is a
``X-Distro-Tracker-Package`` header set giving the name of an existing
source or pseudo package.
If the ``X-Distro-Tracker-Url`` is also set then the content of the message
will not be the email content, rather the URL given in this header.
:param message: The received message
:type message: :class:`bytes`
"""
assert isinstance(message, bytes), 'Message must be given as bytes'
msg = message_from_bytes(message)
# Try asking the vendor function first.
created, implemented = vendor.call('create_news_from_email_message', msg)
if implemented and created:
return
# If the message has an X-Distro-Tracker-Package header, it is
# automatically made into a news item.
if 'X-Distro-Tracker-Package' in msg:
package_name = msg['X-Distro-Tracker-Package']
package = get_or_none(PackageName, name=package_name)
if not package:
return
if 'X-Distro-Tracker-Url' not in msg:
create_news(msg, package)
else:
distro_tracker_url = msg['X-Distro-Tracker-Url']
News.objects.create(
title=distro_tracker_url,
content="<a href={url}>{url}</a>".format(
url=escape(distro_tracker_url)),
package=package,
content_type='text/html')
def add_new_headers(received_message, package_name=None, keyword=None,
team=None):
"""
The function adds new distro-tracker specific headers to the received
message. This is used before forwarding the message to subscribers.
The headers added by this function are used regardless whether the
message is forwarded due to direct package subscriptions or a team
subscription.
:param received_message: The received package message
:type received_message: :py:class:`email.message.Message` or an equivalent
interface object
:param package_name: The name of the package for which this message was
intended.
:type package_name: string
:param keyword: The keyword with which the message should be tagged
:type keyword: string
"""
new_headers = [
('X-Loop', 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)),
]
if keyword:
new_headers.append(('X-Distro-Tracker-Keyword', keyword))
if package_name:
new_headers.extend([
('X-Distro-Tracker-Package', package_name),
('List-Id', '<{}.{}>'.format(package_name, DISTRO_TRACKER_FQDN)),
])
if team:
new_headers.append(('X-Distro-Tracker-Team', team))
extra_vendor_headers, implemented = vendor.call(
'add_new_headers', received_message, package_name, keyword, team)
if implemented:
new_headers.extend(extra_vendor_headers)
for header_name, header_value in new_headers:
received_message[header_name] = header_value
def context(self, package):
try:
info = package.general_data[0]
except IndexError:
# There is no general info for the package
return
general = {}
if 'vcs' in info.value:
general['vcs'] = info.value['vcs']
# Map the VCS type to its name.
shorthand = general['vcs'].get('type', 'Unknown')
general['vcs']['full_name'] = get_vcs_name(shorthand)
result, implemented = vendor.call(
'get_vcs_data', package)
if implemented:
general.update(result)
return general
def _update_sources_file(self, repository, component, sources_file):
for stanza in deb822.Sources.iter_paragraphs(sources_file):
allow, implemented = vendor.call('allow_package', stanza)
if allow is not None and implemented and not allow:
# The vendor-provided function indicates that the package
# should not be included
continue
src_pkg_name, _ = SourcePackageName.objects.get_or_create(
name=stanza['package']
)
src_pkg, created_new_version = SourcePackage.objects.get_or_create(
source_package_name=src_pkg_name,
version=stanza['version']
)
if created_new_version or self.force_update:
# Extract package data from Sources
entry = self._extract_information_from_sources_entry(
src_pkg, stanza)
# Update the source package information based on the newly
# extracted data.
src_pkg.update(**entry)
src_pkg.save()
if not repository.has_source_package(src_pkg):
# Add it to the repository
entry = repository.add_source_package(
src_pkg, component=component)
else:
# We get the entry to mark that the package version is still in
# the repository.
entry = SourcePackageRepositoryEntry.objects.get(
repository=repository,
source_package=src_pkg
)
self._add_processed_repository_entry(entry)
def context(self):
try:
info = PackageData.objects.get(package=self.package, key='general')
except PackageData.DoesNotExist:
# There is no general info for the package
return
general = info.value
# Add source package URL
url, implemented = vendor.call('get_package_information_site_url', **{
'package_name': general['name'],
'source_package': True,
})
if implemented and url:
general['url'] = url
# Map the VCS type to its name.
if 'vcs' in general:
shorthand = general['vcs'].get('type', 'unknown')
general['vcs']['full_name'] = get_vcs_name(shorthand)
# Add vcs extra links (including Vcs-Browser)
try:
vcs_extra_links = PackageData.objects.get(
package=self.package, key='vcs_extra_links').value
except PackageData.DoesNotExist:
vcs_extra_links = {}
if 'browser' in general['vcs']:
vcs_extra_links['Browse'] = general['vcs']['browser']
vcs_extra_links.pop('checksum', None)
general['vcs']['extra_links'] = [
(key, vcs_extra_links[key])
for key in sorted(vcs_extra_links.keys())
]
# Add mailing list archive URLs
self._add_archive_urls(general)
# Add developer information links and any other vendor-specific extras
general = add_developer_extras(general)
return general
def packages_with_prefetch_related(self):
"""
Returns the list of packages with prefetched relationships defined by
table fields
"""
attributes_name = set()
package_query_set = self.packages
for field in self.table_fields:
for l in field.prefetch_related_lookups:
if isinstance(l, Prefetch):
if l.to_attr in attributes_name:
continue
else:
attributes_name.add(l.to_attr)
package_query_set = package_query_set.prefetch_related(l)
additional_data, implemented = vendor.call(
'additional_prefetch_related_lookups'
)
if implemented and additional_data:
for l in additional_data:
package_query_set = package_query_set.prefetch_related(l)
return package_query_set
def _update_sources_file(self, repository, sources_file):
for stanza in deb822.Sources.iter_paragraphs(sources_file):
allow, implemented = vendor.call('allow_package', stanza)
if allow is not None and implemented and not allow:
# The vendor-provided function indicates that the package
# should not be included
continue
src_pkg_name, created = SourcePackageName.objects.get_or_create(
name=stanza['package']
)
if created:
self.raise_event('new-source-package', {
'name': src_pkg_name.name
})
src_pkg, created_new_version = SourcePackage.objects.get_or_create(
source_package_name=src_pkg_name,
version=stanza['version']
)
if created_new_version:
self.raise_event('new-source-package-version', {
'name': src_pkg.name,
'version': src_pkg.version,
'pk': src_pkg.pk,
})
# Since it's a new version, extract package data from Sources
entry = self._extract_information_from_sources_entry(
src_pkg, stanza)
# Update the source package information based on the newly
# extracted data.
src_pkg.update(**entry)
src_pkg.save()
if not repository.has_source_package(src_pkg):
# Does it have any version of the package?
if not repository.has_source_package_name(src_pkg.name):
self.raise_event('new-source-package-in-repository', {
'name': src_pkg.name,
'repository': repository.name,
})
# Add it to the repository
kwargs = {
'priority': stanza.get('priority', ''),
'section': stanza.get('section', ''),
}
entry = repository.add_source_package(src_pkg, **kwargs)
self.raise_event('new-source-package-version-in-repository', {
'name': src_pkg.name,
'version': src_pkg.version,
'repository': repository.name,
})
else:
# We get the entry to mark that the package version is still in
# the repository.
entry = SourcePackageRepositoryEntry.objects.get(
repository=repository,
source_package=src_pkg
)
self._add_processed_repository_entry(entry)
def _get_developer_information_url(self, email):
info_url, implemented = vendor.call(
'get_developer_information_url', **{'developer_email': email, })
if implemented and info_url:
return info_url
