def _process_url(self, url, project_name=None, follow_links=True): """Process an url and search for distributions packages. For each URL found, if it's a download, creates a PyPIdistribution object. If it's a homepage and we can follow links, process it too. :param url: the url to process :param project_name: the project name we are searching for. :param follow_links: Do not want to follow links more than from one level. This parameter tells if we want to follow the links we find (eg. run recursively this method on it) """ with self._open_url(url) as f: base_url = f.url if url not in self._processed_urls: self._processed_urls.append(url) link_matcher = self._get_link_matcher(url) for link, is_download in link_matcher(f.read().decode(), base_url): if link not in self._processed_urls: if self._is_distribution(link) or is_download: self._processed_urls.append(link) # it's a distribution, so create a dist object try: infos = get_infos_from_url(link, project_name, is_external=self.index_url not in url) except CantParseArchiveName as e: logger.warning( "version has not been parsed: %s", e) else: self._register_release(release_info=infos) else: if self._is_browsable(link) and follow_links: self._process_url(link, project_name, follow_links=False)