def _get_binary_bug_stats(self, binary_name): bug_stats, implemented = vendor.call( 'get_binary_package_bug_stats', binary_name) if not implemented: # The vendor does not provide a custom list of bugs, so the default # is to display all bug info known for the package. stats = get_or_none( BinaryPackageBugStats, package__name=binary_name) if stats is not None: bug_stats = stats.stats if bug_stats is None: return # Try to get the URL to the bug tracker for the given categories for category in bug_stats: url, implemented = vendor.call( 'get_bug_tracker_url', binary_name, 'binary', category['category_name']) if not implemented: continue category['url'] = url # Include the total bug count and corresponding tracker URL all_bugs_url, implemented = vendor.call( 'get_bug_tracker_url', binary_name, 'binary', 'all') return { 'total_count': sum( category['bug_count'] for category in bug_stats), 'all_bugs_url': all_bugs_url, 'categories': bug_stats, }
def add_developer_extras(general, url_only=False): """ Receives a general dict with package data and add to it more data regarding that package's developers """ if 'maintainer' in general: maintainer_email = general['maintainer']['email'] url = get_developer_information_url(maintainer_email) if url: general['maintainer']['developer_info_url'] = url if not url_only: extra, implemented = vendor.call( 'get_maintainer_extra', maintainer_email, general['name']) general['maintainer']['extra'] = extra uploaders = general.get('uploaders', None) if uploaders: for uploader in uploaders: url = get_developer_information_url(uploader['email']) if url: uploader['developer_info_url'] = url if url_only: continue # Vendor specific extras. extra, implemented = vendor.call( 'get_uploader_extra', uploader['email'], general['name']) if implemented and extra: uploader['extra'] = extra return general
def get_keyword(local_part, msg): """ Extracts the keywoword from the given message. The function first tries using a vendor-provided function :func:`get_keyword <distro_tracker.vendor.skeleton.rules.get_keyword>`. If the vendor did not implement this function or does not return a keyword for the given message, the function tries extracting the keyword from the ``local_part`` of the address (using :func:`get_keyword_from_address`). If this also does not yield a keyword, ``default`` is returned. :param local_part: The local part of the email address to which the message was sent. :type local_part: string :param msg: The received package message :type msg: :py:class:`email.message.Message` or an equivalent interface object :returns: The name of the keyword. :rtype: string """ # Use a vendor-provided function to try and classify the message. keyword, _ = vendor.call('get_keyword', local_part, msg) if keyword: return keyword # Otherwise try getting the keyword from the address keyword = get_keyword_from_address(local_part) if keyword: return keyword # If we still do not have the keyword return 'default'
def classify_message(msg, package=None, keyword=None): """ Analyzes a message to identify what package it is about and what keyword is appropriate. :param msg: The received message :type msg: :py:class:`email.message.Message` :param str package: The suggested package name. :param str keyword: The suggested keyword under which the message can be forwarded. """ if package is None: package = msg.get('X-Distro-Tracker-Package') if keyword is None: keyword = msg.get('X-Distro-Tracker-Keyword') result, implemented = vendor.call('classify_message', msg, package=package, keyword=keyword) if implemented: package, keyword = result if package and keyword is None: keyword = 'default' return (package, keyword)
def context(self): try: info = PackageExtractedInfo.objects.get( package=self.package, key='binaries') except PackageExtractedInfo.DoesNotExist: return binaries = info.value for binary in binaries: # For each binary try to include known bug stats bug_stats = self._get_binary_bug_stats(binary['name']) if bug_stats is not None: binary['bug_stats'] = bug_stats # For each binary try to include a link to an external package-info # site. if 'repository' in binary: url, implemented = vendor.call( 'get_package_information_site_url', **{ 'package_name': binary['name'], 'repository': binary['repository'], 'source_package': False, } ) if implemented and url: binary['url'] = url return binaries
def add_new_headers(received_message, package_name, keyword): """ The function adds new distro-tracker specific headers to the received message. This is used before forwarding the message to subscribers. The headers added by this function are used regardless whether the message is forwarded due to direct package subscriptions or a team subscription. :param received_message: The received package message :type received_message: :py:class:`email.message.Message` or an equivalent interface object :param package_name: The name of the package for which this message was intended. :type package_name: string :param keyword: The keyword with which the message should be tagged :type keyword: string """ new_headers = [ ('X-Loop', '{package}@{distro_tracker_fqdn}'.format( package=package_name, distro_tracker_fqdn=DISTRO_TRACKER_FQDN)), ('X-Distro-Tracker-Package', package_name), ('X-Distro-Tracker-Keyword', keyword), ] extra_vendor_headers, implemented = vendor.call( 'add_new_headers', received_message, package_name, keyword) if implemented: new_headers.extend(extra_vendor_headers) add_headers(received_message, new_headers)
def context(self): result, implemented = vendor.call( 'get_bug_panel_stats', self.package.name) # implemented = False if not implemented: # If the vendor does not provide custom categories to be displayed # in the panel, the default is to make each stored category a # separate entry. try: stats = self.package.bug_stats.stats except ObjectDoesNotExist: return # Also adds a total of all those bugs total = sum(category['bug_count'] for category in stats) stats.insert(0, { 'category_name': 'all', 'bug_count': total, }) result = stats # Either the vendor decided not to provide any info for this package # or there is no known info. if not result: return [] return result
def context(self): try: info = PackageExtractedInfo.objects.get( package=self.package, key='general') except PackageExtractedInfo.DoesNotExist: # There is no general info for the package return general = info.value # Add source package URL url, implemented = vendor.call('get_package_information_site_url', **{ 'package_name': general['name'], 'source_package': True, }) if implemented and url: general['url'] = url # Map the VCS type to its name. if 'vcs' in general and 'type' in general['vcs']: shorthand = general['vcs']['type'] general['vcs']['full_name'] = get_vcs_name(shorthand) # Add mailing list archive URLs self._add_archive_urls(general) # Add developer information links and any other vendor-specific extras self._add_developer_extras(general) return general
def update_pseudo_package_list(): """ Retrieves the list of all allowed pseudo packages and updates the stored list if necessary. Uses a vendor-provided function :func:`get_pseudo_package_list <distro_tracker.vendor.skeleton.rules.get_pseudo_package_list>` to get the list of currently available pseudo packages. """ try: pseudo_packages, implemented = vendor.call('get_pseudo_package_list') except: # Error accessing pseudo package resource: do not update the list return if not implemented or pseudo_packages is None: return # Faster lookups than if this were a list pseudo_packages = set(pseudo_packages) for existing_package in PseudoPackageName.objects.all(): if existing_package.name not in pseudo_packages: # Existing packages which are no longer considered pseudo packages # are demoted -- losing their pseudo package flag. existing_package.pseudo = False existing_package.save() else: # If an existing package remained a pseudo package there will be no # action required so it is removed from the set. pseudo_packages.remove(existing_package.name) # The left over packages in the set are the ones that do not exist. for package_name in pseudo_packages: PseudoPackageName.objects.create(name=package_name)
def get_developer_information_url(email): """ Returns developer's information url based on his/her email through vendor-specific function """ info_url, implemented = vendor.call( 'get_developer_information_url', **{'developer_email': email, }) if implemented and info_url: return info_url
def context(self): try: info = PackageExtractedInfo.objects.get( package=self.package, key='versions') except PackageExtractedInfo.DoesNotExist: info = None context = {} if info: version_info = info.value package_name = info.package.name for item in version_info.get('version_list', ()): url, implemented = vendor.call( 'get_package_information_site_url', ** {'package_name': package_name, 'repository': item.get( 'repository'), 'source_package': True, 'version': item.get('version'), }) if implemented and url: item['url'] = url context['version_info'] = version_info # Add in any external version resource links external_resources, implemented = ( vendor.call('get_external_version_information_urls', self.package.name) ) if implemented and external_resources: context['external_resources'] = external_resources # Add any vendor-provided versions vendor_versions, implemented = vendor.call( 'get_extra_versions', self.package) if implemented and vendor_versions: context['vendor_versions'] = vendor_versions return context
def table_fields(self): """ Returns the tuple of :class:`BaseTableField` that will compose the table """ fields, implemented = vendor.call('get_table_fields', **{ 'table': self, }) if implemented and fields: return tuple(fields) else: return tuple(self.default_fields)
def _add_developer_extras(self, general): maintainer_email = general['maintainer']['email'] url = self._get_developer_information_url(maintainer_email) if url: general['maintainer']['developer_info_url'] = url extra, implemented = vendor.call( 'get_maintainer_extra', maintainer_email, general['name']) general['maintainer']['extra'] = extra uploaders = general.get('uploaders', None) if not uploaders: return for uploader in uploaders: # Vendor specific extras. extra, implemented = vendor.call( 'get_uploader_extra', uploader['email'], general['name']) if implemented and extra: uploader['extra'] = extra url = self._get_developer_information_url(uploader['email']) if url: uploader['developer_info_url'] = url
def _create_tables(self): result, implemented = vendor.call( 'get_tables_for_team_page', self.object, self.table_limit) if implemented: return result return [ create_table( slug='general', scope=self.object, limit=self.table_limit), create_table( slug='general', scope=self.object, limit=self.table_limit, tag='tag:bugs' ), ]
def approved_default(msg): """ The function checks whether a message tagged with the default keyword should be approved, meaning that it gets forwarded to subscribers. :param msg: The received package message :type msg: :py:class:`email.message.Message` or an equivalent interface object """ if 'X-Distro-Tracker-Approved' in msg: return True approved, implemented = vendor.call('approve_default_message', msg) if implemented: return approved else: return False
def process(message): """ Process an incoming message which is potentially a news item. The function first tries to call the vendor-provided function :func:`create_news_from_email_message <distro_tracker.vendor.skeleton.rules.create_news_from_email_message>`. If this function does not exist a news item is created only if there is a ``X-Distro-Tracker-Package`` header set giving the name of an existing source or pseudo package. If the ``X-Distro-Tracker-Url`` is also set then the content of the message will not be the email content, rather the URL given in this header. :param message: The received message :type message: :class:`bytes` """ assert isinstance(message, bytes), 'Message must be given as bytes' msg = message_from_bytes(message) # Try asking the vendor function first. created, implemented = vendor.call('create_news_from_email_message', msg) if implemented and created: return # If the message has an X-Distro-Tracker-Package header, it is # automatically made into a news item. if 'X-Distro-Tracker-Package' in msg: package_name = msg['X-Distro-Tracker-Package'] package = get_or_none(PackageName, name=package_name) if not package: return if 'X-Distro-Tracker-Url' not in msg: create_news(msg, package) else: distro_tracker_url = msg['X-Distro-Tracker-Url'] News.objects.create( title=distro_tracker_url, content="<a href={url}>{url}</a>".format( url=escape(distro_tracker_url)), package=package, content_type='text/html')
def add_new_headers(received_message, package_name=None, keyword=None, team=None): """ The function adds new distro-tracker specific headers to the received message. This is used before forwarding the message to subscribers. The headers added by this function are used regardless whether the message is forwarded due to direct package subscriptions or a team subscription. :param received_message: The received package message :type received_message: :py:class:`email.message.Message` or an equivalent interface object :param package_name: The name of the package for which this message was intended. :type package_name: string :param keyword: The keyword with which the message should be tagged :type keyword: string """ new_headers = [ ('X-Loop', 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)), ] if keyword: new_headers.append(('X-Distro-Tracker-Keyword', keyword)) if package_name: new_headers.extend([ ('X-Distro-Tracker-Package', package_name), ('List-Id', '<{}.{}>'.format(package_name, DISTRO_TRACKER_FQDN)), ]) if team: new_headers.append(('X-Distro-Tracker-Team', team)) extra_vendor_headers, implemented = vendor.call( 'add_new_headers', received_message, package_name, keyword, team) if implemented: new_headers.extend(extra_vendor_headers) for header_name, header_value in new_headers: received_message[header_name] = header_value
def context(self, package): try: info = package.general_data[0] except IndexError: # There is no general info for the package return general = {} if 'vcs' in info.value: general['vcs'] = info.value['vcs'] # Map the VCS type to its name. shorthand = general['vcs'].get('type', 'Unknown') general['vcs']['full_name'] = get_vcs_name(shorthand) result, implemented = vendor.call( 'get_vcs_data', package) if implemented: general.update(result) return general
def _update_sources_file(self, repository, component, sources_file): for stanza in deb822.Sources.iter_paragraphs(sources_file): allow, implemented = vendor.call('allow_package', stanza) if allow is not None and implemented and not allow: # The vendor-provided function indicates that the package # should not be included continue src_pkg_name, _ = SourcePackageName.objects.get_or_create( name=stanza['package'] ) src_pkg, created_new_version = SourcePackage.objects.get_or_create( source_package_name=src_pkg_name, version=stanza['version'] ) if created_new_version or self.force_update: # Extract package data from Sources entry = self._extract_information_from_sources_entry( src_pkg, stanza) # Update the source package information based on the newly # extracted data. src_pkg.update(**entry) src_pkg.save() if not repository.has_source_package(src_pkg): # Add it to the repository entry = repository.add_source_package( src_pkg, component=component) else: # We get the entry to mark that the package version is still in # the repository. entry = SourcePackageRepositoryEntry.objects.get( repository=repository, source_package=src_pkg ) self._add_processed_repository_entry(entry)
def context(self): try: info = PackageData.objects.get(package=self.package, key='general') except PackageData.DoesNotExist: # There is no general info for the package return general = info.value # Add source package URL url, implemented = vendor.call('get_package_information_site_url', **{ 'package_name': general['name'], 'source_package': True, }) if implemented and url: general['url'] = url # Map the VCS type to its name. if 'vcs' in general: shorthand = general['vcs'].get('type', 'unknown') general['vcs']['full_name'] = get_vcs_name(shorthand) # Add vcs extra links (including Vcs-Browser) try: vcs_extra_links = PackageData.objects.get( package=self.package, key='vcs_extra_links').value except PackageData.DoesNotExist: vcs_extra_links = {} if 'browser' in general['vcs']: vcs_extra_links['Browse'] = general['vcs']['browser'] vcs_extra_links.pop('checksum', None) general['vcs']['extra_links'] = [ (key, vcs_extra_links[key]) for key in sorted(vcs_extra_links.keys()) ] # Add mailing list archive URLs self._add_archive_urls(general) # Add developer information links and any other vendor-specific extras general = add_developer_extras(general) return general
def packages_with_prefetch_related(self): """ Returns the list of packages with prefetched relationships defined by table fields """ attributes_name = set() package_query_set = self.packages for field in self.table_fields: for l in field.prefetch_related_lookups: if isinstance(l, Prefetch): if l.to_attr in attributes_name: continue else: attributes_name.add(l.to_attr) package_query_set = package_query_set.prefetch_related(l) additional_data, implemented = vendor.call( 'additional_prefetch_related_lookups' ) if implemented and additional_data: for l in additional_data: package_query_set = package_query_set.prefetch_related(l) return package_query_set
def _update_sources_file(self, repository, sources_file): for stanza in deb822.Sources.iter_paragraphs(sources_file): allow, implemented = vendor.call('allow_package', stanza) if allow is not None and implemented and not allow: # The vendor-provided function indicates that the package # should not be included continue src_pkg_name, created = SourcePackageName.objects.get_or_create( name=stanza['package'] ) if created: self.raise_event('new-source-package', { 'name': src_pkg_name.name }) src_pkg, created_new_version = SourcePackage.objects.get_or_create( source_package_name=src_pkg_name, version=stanza['version'] ) if created_new_version: self.raise_event('new-source-package-version', { 'name': src_pkg.name, 'version': src_pkg.version, 'pk': src_pkg.pk, }) # Since it's a new version, extract package data from Sources entry = self._extract_information_from_sources_entry( src_pkg, stanza) # Update the source package information based on the newly # extracted data. src_pkg.update(**entry) src_pkg.save() if not repository.has_source_package(src_pkg): # Does it have any version of the package? if not repository.has_source_package_name(src_pkg.name): self.raise_event('new-source-package-in-repository', { 'name': src_pkg.name, 'repository': repository.name, }) # Add it to the repository kwargs = { 'priority': stanza.get('priority', ''), 'section': stanza.get('section', ''), } entry = repository.add_source_package(src_pkg, **kwargs) self.raise_event('new-source-package-version-in-repository', { 'name': src_pkg.name, 'version': src_pkg.version, 'repository': repository.name, }) else: # We get the entry to mark that the package version is still in # the repository. entry = SourcePackageRepositoryEntry.objects.get( repository=repository, source_package=src_pkg ) self._add_processed_repository_entry(entry)
def _get_developer_information_url(self, email): info_url, implemented = vendor.call( 'get_developer_information_url', **{'developer_email': email, }) if implemented and info_url: return info_url