def open_packaging_branch(location, possible_transports=None, vcs_type=None):
"""Open a packaging branch from a location string.
location can either be a package name or a full URL
"""
if "/" not in location and ":" not in location:
pkg_source = apt_get_source_package(location)
try:
(vcs_type, vcs_url) = source_package_vcs(pkg_source)
except KeyError:
raise NoVcsInformation(location)
(url, branch_name, subpath) = split_vcs_url(vcs_url)
else:
url, params = urlutils.split_segment_parameters(location)
try:
branch_name = urlutils.unquote(params["branch"])
except KeyError:
branch_name = None
subpath = ""
probers = select_probers(vcs_type)
branch = open_branch(url,
possible_transports=possible_transports,
probers=probers,
name=branch_name)
return branch, subpath or ""
def find_secure_vcs_url(url: str, net_access: bool = True) -> Optional[str]:
(repo_url, branch, subpath) = split_vcs_url(url)
repo_url = find_secure_repo_url(repo_url,
branch=branch,
net_access=net_access)
if repo_url is None:
return None
return unsplit_vcs_url(repo_url, branch, subpath)
def fixup_broken_git_url(url: str) -> str:
"""Attempt to fix up broken Git URLs.
A common misspelling is to add an extra ":" after the hostname
"""
repo_url, branch, subpath = split_vcs_url(url)
newrepo_url, newbranch, newsubpath = fixup_broken_git_details(
repo_url, branch, subpath)
if newrepo_url != repo_url or newbranch != branch or newsubpath != subpath:
return unsplit_vcs_url(newrepo_url, newbranch, newsubpath)
return url
def determine_browser_url(vcs_type, vcs_url: str) -> Optional[str]:
repo_url, branch, subpath = split_vcs_url(vcs_url)
parsed = urlparse(repo_url.rstrip("/"))
if is_gitlab_site(parsed.netloc):
return determine_gitlab_browser_url(vcs_url)
if parsed.netloc == "github.com":
path = parsed.path
if path.endswith(".git"):
path = path[:-4]
if subpath and not branch:
branch = "HEAD"
if branch:
path = posixpath.join(path, "tree", branch)
if subpath:
path = posixpath.join(path, subpath)
return urlunparse(("https", parsed.netloc, path, parsed.query,
parsed.params, parsed.fragment))
if (parsed.netloc in ("code.launchpad.net", "launchpad.net") and not branch
and not subpath):
return urlunparse((
"https",
"code.launchpad.net",
parsed.path,
parsed.query,
parsed.params,
parsed.fragment,
))
if parsed.hostname in ("git.savannah.gnu.org", "git.sv.gnu.org"):
path_elements = parsed.path.strip("/").split("/")
if parsed.scheme == "https" and path_elements[0] == "git":
path_elements.pop(0)
# Why cgit and not gitweb?
path_elements.insert(0, "cgit")
return urlunparse(("https", parsed.netloc, "/".join(path_elements),
None, None, None))
if parsed.hostname in ("git.code.sf.net", "git.code.sourceforge.net"):
path_elements = parsed.path.strip("/").split("/")
if path_elements[0] != "p":
return None
project = path_elements[1]
repository = path_elements[2]
path_elements = ["p", project, repository]
if branch is not None:
path_elements.extend(["ci", branch, "tree"])
elif subpath is not None:
path_elements.extend(["ci", "HEAD", "tree"])
if subpath is not None:
path_elements.append(subpath)
return urlunparse(("https", "sourceforge.net", "/".join(path_elements),
None, None, None))
return None
def determine_gitlab_browser_url(url: str) -> str:
(url, branch, subpath) = split_vcs_url(url)
parsed_url = urlparse(url.rstrip("/"))
# TODO(jelmer): Add support for branches
path = parsed_url.path
if path.endswith(".git"):
path = path[:-len(".git")]
if subpath and not branch:
branch = "HEAD"
if branch:
path = path + "/tree/%s" % branch
if subpath:
path = path + "/" + subpath
return "https://%s%s" % (parsed_url.hostname, path)
def fixup_rcp_style_git_url(url: str) -> str:
(repo_url, branch, subpath) = split_vcs_url(url)
repo_url = fixup_rcp_style_git_repo_url(repo_url)
return unsplit_vcs_url(repo_url, branch, subpath)
def find_public_vcs_url(url: str) -> Optional[str]:
(repo_url, branch, subpath) = split_vcs_url(url)
revised_url = find_public_repo_url(repo_url)
if revised_url is not None:
return unsplit_vcs_url(revised_url, branch, subpath)
return None
def canonical_vcs_git_url(url: str) -> str:
(repo_url, branch, subpath) = split_vcs_url(url)
repo_url = canonical_git_repo_url(repo_url)
return unsplit_vcs_url(repo_url, branch, subpath)
async def update_package_metadata(
conn, distribution: str, provided_packages, package_overrides, origin
):
logging.info("Updating package metadata.")
packages = []
for package in provided_packages:
try:
override = package_overrides[package.name]
except KeyError:
vcs_url = package.vcs_url
else:
vcs_url = override.branch_url or package.vcs_url or None
vcs_last_revision = None
if package.vcs_type and package.vcs_type.capitalize() == "Git":
new_vcs_url = fixup_broken_git_url(vcs_url)
if new_vcs_url != vcs_url:
logging.info("Fixing up VCS URL: %s -> %s", vcs_url, new_vcs_url)
vcs_url = new_vcs_url
if package.commit_id:
vcs_last_revision = default_mapping.revision_id_foreign_to_bzr(
package.commit_id.encode("ascii")
)
if package.vcs_type:
# Drop the subpath, we're storing it separately.
(url, branch, subpath) = split_vcs_url(vcs_url)
url = unsplit_vcs_url(url, branch)
url = canonicalize_vcs_url(package.vcs_type, url)
try:
branch_url = convert_debian_vcs_url(package.vcs_type.capitalize(), url)
except ValueError as e:
logging.info("%s: %s", package.name, e)
branch_url = None
else:
subpath = None
branch_url = None
if vcs_url:
vcs_browser = determine_browser_url(package.vcs_type, vcs_url)
else:
vcs_browser = None
if vcs_browser is None and package.vcs_browser:
vcs_browser = package.vcs_browser
packages.append(
(
package.name,
distribution,
branch_url if branch_url else None,
subpath if subpath else None,
package.maintainer_email if package.maintainer_email else None,
package.uploader_email if package.uploader_email else [],
package.archive_version if package.archive_version else None,
package.vcs_type.lower() if package.vcs_type else None,
vcs_url,
vcs_browser,
vcs_last_revision.decode("utf-8") if vcs_last_revision else None,
package.vcswatch_status.lower() if package.vcswatch_status else None,
package.vcswatch_version if package.vcswatch_version else None,
package.insts,
package.removed,
package.in_base,
origin
)
)
await conn.executemany(
"INSERT INTO package "
"(name, distribution, branch_url, subpath, maintainer_email, "
"uploader_emails, archive_version, vcs_type, vcs_url, vcs_browse, "
"vcs_last_revision, vcswatch_status, vcswatch_version, popcon_inst, "
"removed, in_base, origin) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, "
"$13, $14, $15, $16, $17) ON CONFLICT (name, distribution) DO UPDATE SET "
"branch_url = EXCLUDED.branch_url, "
"subpath = EXCLUDED.subpath, "
"maintainer_email = EXCLUDED.maintainer_email, "
"uploader_emails = EXCLUDED.uploader_emails, "
"archive_version = EXCLUDED.archive_version, "
"vcs_type = EXCLUDED.vcs_type, "
"vcs_url = EXCLUDED.vcs_url, "
"vcs_last_revision = EXCLUDED.vcs_last_revision, "
"vcs_browse = EXCLUDED.vcs_browse, "
"vcswatch_status = EXCLUDED.vcswatch_status, "
"vcswatch_version = EXCLUDED.vcswatch_version, "
"popcon_inst = EXCLUDED.popcon_inst, "
"removed = EXCLUDED.removed, "
"in_base = EXCLUDED.in_base",
packages,
)
