def find_new_urls(vcs_type, vcs_url, package, maintainer_email,
net_access=False):
if net_access and (
vcs_url.startswith('https://') or vcs_url.startswith('http://')):
headers = {'User-Agent': USER_AGENT}
try:
response = urlopen(
Request(vcs_url, headers=headers),
timeout=DEFAULT_URLLIB_TIMEOUT)
except (urllib.error.HTTPError, urllib.error.URLError):
pass
except socket.timeout:
pass
else:
redirected_url = response.geturl()
if not is_on_obsolete_host(redirected_url):
vcs_url = redirected_url
vcs_browser = determine_browser_url(vcs_type, vcs_url)
print("Update Vcs-* headers from URL redirect.")
return (vcs_type, vcs_url, vcs_browser)
# If possible, we use vcswatch to find the VCS repository URL
if net_access:
loop = asyncio.get_event_loop()
try:
(vcs_type, vcs_url, vcs_browser) = loop.run_until_complete(
retrieve_vcswatch_urls(package))
except VcsWatchError as e:
warn('vcswatch URL unusable: %s' % e.args[0])
except KeyError:
pass
else:
if not is_on_obsolete_host(vcs_url):
print("Update Vcs-* headers from vcswatch.")
if is_on_obsolete_host(vcs_browser):
vcs_browser = (
determine_browser_url(vcs_type, vcs_url) or
vcs_browser)
return (vcs_type, vcs_url, vcs_browser)
warn('vcswatch URL %s is still on old infrastructure.' % vcs_url)
# Otherwise, attempt to guess based on maintainer email.
guessed_url = guess_repository_url(package, maintainer_email)
if guessed_url is not None:
vcs_type = "Git"
vcs_url = guessed_url
else:
vcs_url = salsa_url_from_alioth_url(vcs_type, vcs_url)
if vcs_url is None:
raise NewRepositoryURLUnknown(vcs_type, vcs_url)
vcs_type = "Git"
# Verify that there is actually a repository there
if net_access:
if verify_salsa_repository(vcs_url) is False:
raise NewRepositoryURLUnknown(vcs_type, vcs_url)
print("Update Vcs-* headers to use salsa repository.")
vcs_browser = determine_salsa_browser_url(vcs_url)
return (vcs_type, vcs_url, vcs_browser)
def get_upstream_branch_location(tree, subpath, config, trust_package=False):
from lintian_brush.vcs import sanitize_url as sanitize_vcs_url
if config.upstream_branch is not None:
note("Using upstream branch %s (from configuration)",
config.upstream_branch)
# TODO(jelmer): Make brz-debian sanitize the URL?
upstream_branch_location = sanitize_vcs_url(config.upstream_branch)
upstream_branch_browse = getattr(config, "upstream_branch_browse",
None)
else:
from upstream_ontologist.guess import guess_upstream_metadata
guessed_upstream_metadata = guess_upstream_metadata(
tree.abspath(subpath),
trust_package=trust_package,
net_access=True,
consult_external_directory=False,
)
upstream_branch_location = guessed_upstream_metadata.get("Repository")
upstream_branch_browse = guessed_upstream_metadata.get(
"Repository-Browse")
if upstream_branch_location:
note("Using upstream branch %s (guessed)",
upstream_branch_location)
if upstream_branch_browse is None and upstream_branch_location is not None:
try:
from lintian_brush.vcs import determine_browser_url
except ImportError:
pass
else:
upstream_branch_browse = determine_browser_url(
None, upstream_branch_location)
return (upstream_branch_location, upstream_branch_browse)
#!/usr/bin/python3
from lintian_brush.fixer import control, report_result, fixed_lintian_tag
from lintian_brush.vcs import determine_browser_url
with control as updater:
if "Vcs-Browser" not in updater.source:
try:
vcs_git = updater.source["Vcs-Git"]
except KeyError:
pass
else:
browser_url = determine_browser_url('git', vcs_git)
if browser_url is not None:
updater.source["Vcs-Browser"] = browser_url
fixed_lintian_tag(updater.source,
'missing-vcs-browser-field',
info='Vcs-Git %s' % vcs_git)
report_result("debian/control: Add Vcs-Browser field")
def test_github(self):
self.assertEqual(
"https://github.com/jelmer/dulwich",
determine_browser_url("git",
"https://github.com/jelmer/dulwich.git"),
)
self.assertEqual(
"https://github.com/jelmer/dulwich/tree/master",
determine_browser_url(
"git", "https://github.com/jelmer/dulwich.git -b master"),
)
self.assertEqual(
"https://github.com/jelmer/dulwich/tree/master",
determine_browser_url("git",
"git://github.com/jelmer/dulwich -b master"),
)
self.assertEqual(
"https://github.com/jelmer/dulwich/tree/master/blah",
determine_browser_url(
"git", "git://github.com/jelmer/dulwich -b master [blah]"),
)
self.assertEqual(
"https://github.com/jelmer/dulwich/tree/HEAD/blah",
determine_browser_url("git",
"git://github.com/jelmer/dulwich [blah]"),
)
self.assertEqual(
"https://git.sv.gnu.org/cgit/rcs.git",
determine_browser_url("git", "https://git.sv.gnu.org/git/rcs.git"),
)
self.assertEqual(
"https://git.savannah.gnu.org/cgit/rcs.git",
determine_browser_url("git", "git://git.savannah.gnu.org/rcs.git"),
)
self.assertEqual(
"https://sourceforge.net/p/shorewall/debian",
determine_browser_url("git",
"git://git.code.sf.net/p/shorewall/debian"),
)
self.assertEqual(
"https://sourceforge.net/p/shorewall/debian/ci/foo/tree",
determine_browser_url(
"git", "git://git.code.sf.net/p/shorewall/debian -b foo"),
)
self.assertEqual(
"https://sourceforge.net/p/shorewall/debian/ci/HEAD/tree/sp",
determine_browser_url(
"git", "git://git.code.sf.net/p/shorewall/debian [sp]"),
)
self.assertEqual(
"https://sourceforge.net/p/shorewall/debian/ci/foo/tree/sp",
determine_browser_url(
"git", "git://git.code.sf.net/p/shorewall/debian -b foo [sp]"),
)
def test_salsa(self):
self.assertEqual(
"https://salsa.debian.org/jelmer/dulwich",
determine_browser_url(
"git", "https://salsa.debian.org/jelmer/dulwich.git"),
)
async def update_package_metadata(
conn, distribution: str, provided_packages, package_overrides, origin
):
logging.info("Updating package metadata.")
packages = []
for package in provided_packages:
try:
override = package_overrides[package.name]
except KeyError:
vcs_url = package.vcs_url
else:
vcs_url = override.branch_url or package.vcs_url or None
vcs_last_revision = None
if package.vcs_type and package.vcs_type.capitalize() == "Git":
new_vcs_url = fixup_broken_git_url(vcs_url)
if new_vcs_url != vcs_url:
logging.info("Fixing up VCS URL: %s -> %s", vcs_url, new_vcs_url)
vcs_url = new_vcs_url
if package.commit_id:
vcs_last_revision = default_mapping.revision_id_foreign_to_bzr(
package.commit_id.encode("ascii")
)
if package.vcs_type:
# Drop the subpath, we're storing it separately.
(url, branch, subpath) = split_vcs_url(vcs_url)
url = unsplit_vcs_url(url, branch)
url = canonicalize_vcs_url(package.vcs_type, url)
try:
branch_url = convert_debian_vcs_url(package.vcs_type.capitalize(), url)
except ValueError as e:
logging.info("%s: %s", package.name, e)
branch_url = None
else:
subpath = None
branch_url = None
if vcs_url:
vcs_browser = determine_browser_url(package.vcs_type, vcs_url)
else:
vcs_browser = None
if vcs_browser is None and package.vcs_browser:
vcs_browser = package.vcs_browser
packages.append(
(
package.name,
distribution,
branch_url if branch_url else None,
subpath if subpath else None,
package.maintainer_email if package.maintainer_email else None,
package.uploader_email if package.uploader_email else [],
package.archive_version if package.archive_version else None,
package.vcs_type.lower() if package.vcs_type else None,
vcs_url,
vcs_browser,
vcs_last_revision.decode("utf-8") if vcs_last_revision else None,
package.vcswatch_status.lower() if package.vcswatch_status else None,
package.vcswatch_version if package.vcswatch_version else None,
package.insts,
package.removed,
package.in_base,
origin
)
)
await conn.executemany(
"INSERT INTO package "
"(name, distribution, branch_url, subpath, maintainer_email, "
"uploader_emails, archive_version, vcs_type, vcs_url, vcs_browse, "
"vcs_last_revision, vcswatch_status, vcswatch_version, popcon_inst, "
"removed, in_base, origin) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, "
"$13, $14, $15, $16, $17) ON CONFLICT (name, distribution) DO UPDATE SET "
"branch_url = EXCLUDED.branch_url, "
"subpath = EXCLUDED.subpath, "
"maintainer_email = EXCLUDED.maintainer_email, "
"uploader_emails = EXCLUDED.uploader_emails, "
"archive_version = EXCLUDED.archive_version, "
"vcs_type = EXCLUDED.vcs_type, "
"vcs_url = EXCLUDED.vcs_url, "
"vcs_last_revision = EXCLUDED.vcs_last_revision, "
"vcs_browse = EXCLUDED.vcs_browse, "
"vcswatch_status = EXCLUDED.vcswatch_status, "
"vcswatch_version = EXCLUDED.vcswatch_version, "
"popcon_inst = EXCLUDED.popcon_inst, "
"removed = EXCLUDED.removed, "
"in_base = EXCLUDED.in_base",
packages,
)