def find_new_urls(vcs_type, vcs_url, package, maintainer_email, net_access=False): if net_access and ( vcs_url.startswith('https://') or vcs_url.startswith('http://')): headers = {'User-Agent': USER_AGENT} try: response = urlopen( Request(vcs_url, headers=headers), timeout=DEFAULT_URLLIB_TIMEOUT) except (urllib.error.HTTPError, urllib.error.URLError): pass except socket.timeout: pass else: redirected_url = response.geturl() if not is_on_obsolete_host(redirected_url): vcs_url = redirected_url vcs_browser = determine_browser_url(vcs_type, vcs_url) print("Update Vcs-* headers from URL redirect.") return (vcs_type, vcs_url, vcs_browser) # If possible, we use vcswatch to find the VCS repository URL if net_access: loop = asyncio.get_event_loop() try: (vcs_type, vcs_url, vcs_browser) = loop.run_until_complete( retrieve_vcswatch_urls(package)) except VcsWatchError as e: warn('vcswatch URL unusable: %s' % e.args[0]) except KeyError: pass else: if not is_on_obsolete_host(vcs_url): print("Update Vcs-* headers from vcswatch.") if is_on_obsolete_host(vcs_browser): vcs_browser = ( determine_browser_url(vcs_type, vcs_url) or vcs_browser) return (vcs_type, vcs_url, vcs_browser) warn('vcswatch URL %s is still on old infrastructure.' % vcs_url) # Otherwise, attempt to guess based on maintainer email. guessed_url = guess_repository_url(package, maintainer_email) if guessed_url is not None: vcs_type = "Git" vcs_url = guessed_url else: vcs_url = salsa_url_from_alioth_url(vcs_type, vcs_url) if vcs_url is None: raise NewRepositoryURLUnknown(vcs_type, vcs_url) vcs_type = "Git" # Verify that there is actually a repository there if net_access: if verify_salsa_repository(vcs_url) is False: raise NewRepositoryURLUnknown(vcs_type, vcs_url) print("Update Vcs-* headers to use salsa repository.") vcs_browser = determine_salsa_browser_url(vcs_url) return (vcs_type, vcs_url, vcs_browser)
def get_upstream_branch_location(tree, subpath, config, trust_package=False): from lintian_brush.vcs import sanitize_url as sanitize_vcs_url if config.upstream_branch is not None: note("Using upstream branch %s (from configuration)", config.upstream_branch) # TODO(jelmer): Make brz-debian sanitize the URL? upstream_branch_location = sanitize_vcs_url(config.upstream_branch) upstream_branch_browse = getattr(config, "upstream_branch_browse", None) else: from upstream_ontologist.guess import guess_upstream_metadata guessed_upstream_metadata = guess_upstream_metadata( tree.abspath(subpath), trust_package=trust_package, net_access=True, consult_external_directory=False, ) upstream_branch_location = guessed_upstream_metadata.get("Repository") upstream_branch_browse = guessed_upstream_metadata.get( "Repository-Browse") if upstream_branch_location: note("Using upstream branch %s (guessed)", upstream_branch_location) if upstream_branch_browse is None and upstream_branch_location is not None: try: from lintian_brush.vcs import determine_browser_url except ImportError: pass else: upstream_branch_browse = determine_browser_url( None, upstream_branch_location) return (upstream_branch_location, upstream_branch_browse)
#!/usr/bin/python3 from lintian_brush.fixer import control, report_result, fixed_lintian_tag from lintian_brush.vcs import determine_browser_url with control as updater: if "Vcs-Browser" not in updater.source: try: vcs_git = updater.source["Vcs-Git"] except KeyError: pass else: browser_url = determine_browser_url('git', vcs_git) if browser_url is not None: updater.source["Vcs-Browser"] = browser_url fixed_lintian_tag(updater.source, 'missing-vcs-browser-field', info='Vcs-Git %s' % vcs_git) report_result("debian/control: Add Vcs-Browser field")
def test_github(self): self.assertEqual( "https://github.com/jelmer/dulwich", determine_browser_url("git", "https://github.com/jelmer/dulwich.git"), ) self.assertEqual( "https://github.com/jelmer/dulwich/tree/master", determine_browser_url( "git", "https://github.com/jelmer/dulwich.git -b master"), ) self.assertEqual( "https://github.com/jelmer/dulwich/tree/master", determine_browser_url("git", "git://github.com/jelmer/dulwich -b master"), ) self.assertEqual( "https://github.com/jelmer/dulwich/tree/master/blah", determine_browser_url( "git", "git://github.com/jelmer/dulwich -b master [blah]"), ) self.assertEqual( "https://github.com/jelmer/dulwich/tree/HEAD/blah", determine_browser_url("git", "git://github.com/jelmer/dulwich [blah]"), ) self.assertEqual( "https://git.sv.gnu.org/cgit/rcs.git", determine_browser_url("git", "https://git.sv.gnu.org/git/rcs.git"), ) self.assertEqual( "https://git.savannah.gnu.org/cgit/rcs.git", determine_browser_url("git", "git://git.savannah.gnu.org/rcs.git"), ) self.assertEqual( "https://sourceforge.net/p/shorewall/debian", determine_browser_url("git", "git://git.code.sf.net/p/shorewall/debian"), ) self.assertEqual( "https://sourceforge.net/p/shorewall/debian/ci/foo/tree", determine_browser_url( "git", "git://git.code.sf.net/p/shorewall/debian -b foo"), ) self.assertEqual( "https://sourceforge.net/p/shorewall/debian/ci/HEAD/tree/sp", determine_browser_url( "git", "git://git.code.sf.net/p/shorewall/debian [sp]"), ) self.assertEqual( "https://sourceforge.net/p/shorewall/debian/ci/foo/tree/sp", determine_browser_url( "git", "git://git.code.sf.net/p/shorewall/debian -b foo [sp]"), )
def test_salsa(self): self.assertEqual( "https://salsa.debian.org/jelmer/dulwich", determine_browser_url( "git", "https://salsa.debian.org/jelmer/dulwich.git"), )
async def update_package_metadata( conn, distribution: str, provided_packages, package_overrides, origin ): logging.info("Updating package metadata.") packages = [] for package in provided_packages: try: override = package_overrides[package.name] except KeyError: vcs_url = package.vcs_url else: vcs_url = override.branch_url or package.vcs_url or None vcs_last_revision = None if package.vcs_type and package.vcs_type.capitalize() == "Git": new_vcs_url = fixup_broken_git_url(vcs_url) if new_vcs_url != vcs_url: logging.info("Fixing up VCS URL: %s -> %s", vcs_url, new_vcs_url) vcs_url = new_vcs_url if package.commit_id: vcs_last_revision = default_mapping.revision_id_foreign_to_bzr( package.commit_id.encode("ascii") ) if package.vcs_type: # Drop the subpath, we're storing it separately. (url, branch, subpath) = split_vcs_url(vcs_url) url = unsplit_vcs_url(url, branch) url = canonicalize_vcs_url(package.vcs_type, url) try: branch_url = convert_debian_vcs_url(package.vcs_type.capitalize(), url) except ValueError as e: logging.info("%s: %s", package.name, e) branch_url = None else: subpath = None branch_url = None if vcs_url: vcs_browser = determine_browser_url(package.vcs_type, vcs_url) else: vcs_browser = None if vcs_browser is None and package.vcs_browser: vcs_browser = package.vcs_browser packages.append( ( package.name, distribution, branch_url if branch_url else None, subpath if subpath else None, package.maintainer_email if package.maintainer_email else None, package.uploader_email if package.uploader_email else [], package.archive_version if package.archive_version else None, package.vcs_type.lower() if package.vcs_type else None, vcs_url, vcs_browser, vcs_last_revision.decode("utf-8") if vcs_last_revision else None, package.vcswatch_status.lower() if package.vcswatch_status else None, package.vcswatch_version if package.vcswatch_version else None, package.insts, package.removed, package.in_base, origin ) ) await conn.executemany( "INSERT INTO package " "(name, distribution, branch_url, subpath, maintainer_email, " "uploader_emails, archive_version, vcs_type, vcs_url, vcs_browse, " "vcs_last_revision, vcswatch_status, vcswatch_version, popcon_inst, " "removed, in_base, origin) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, " "$13, $14, $15, $16, $17) ON CONFLICT (name, distribution) DO UPDATE SET " "branch_url = EXCLUDED.branch_url, " "subpath = EXCLUDED.subpath, " "maintainer_email = EXCLUDED.maintainer_email, " "uploader_emails = EXCLUDED.uploader_emails, " "archive_version = EXCLUDED.archive_version, " "vcs_type = EXCLUDED.vcs_type, " "vcs_url = EXCLUDED.vcs_url, " "vcs_last_revision = EXCLUDED.vcs_last_revision, " "vcs_browse = EXCLUDED.vcs_browse, " "vcswatch_status = EXCLUDED.vcswatch_status, " "vcswatch_version = EXCLUDED.vcswatch_version, " "popcon_inst = EXCLUDED.popcon_inst, " "removed = EXCLUDED.removed, " "in_base = EXCLUDED.in_base", packages, )