def open_packaging_branch(location, possible_transports=None, vcs_type=None): """Open a packaging branch from a location string. location can either be a package name or a full URL """ if "/" not in location and ":" not in location: pkg_source = apt_get_source_package(location) try: (vcs_type, vcs_url) = source_package_vcs(pkg_source) except KeyError: raise NoVcsInformation(location) (url, branch_name, subpath) = split_vcs_url(vcs_url) else: url, params = urlutils.split_segment_parameters(location) try: branch_name = urlutils.unquote(params["branch"]) except KeyError: branch_name = None subpath = "" probers = select_probers(vcs_type) branch = open_branch(url, possible_transports=possible_transports, probers=probers, name=branch_name) return branch, subpath or ""
def find_secure_vcs_url(url: str, net_access: bool = True) -> Optional[str]: (repo_url, branch, subpath) = split_vcs_url(url) repo_url = find_secure_repo_url(repo_url, branch=branch, net_access=net_access) if repo_url is None: return None return unsplit_vcs_url(repo_url, branch, subpath)
def fixup_broken_git_url(url: str) -> str: """Attempt to fix up broken Git URLs. A common misspelling is to add an extra ":" after the hostname """ repo_url, branch, subpath = split_vcs_url(url) newrepo_url, newbranch, newsubpath = fixup_broken_git_details( repo_url, branch, subpath) if newrepo_url != repo_url or newbranch != branch or newsubpath != subpath: return unsplit_vcs_url(newrepo_url, newbranch, newsubpath) return url
def determine_browser_url(vcs_type, vcs_url: str) -> Optional[str]: repo_url, branch, subpath = split_vcs_url(vcs_url) parsed = urlparse(repo_url.rstrip("/")) if is_gitlab_site(parsed.netloc): return determine_gitlab_browser_url(vcs_url) if parsed.netloc == "github.com": path = parsed.path if path.endswith(".git"): path = path[:-4] if subpath and not branch: branch = "HEAD" if branch: path = posixpath.join(path, "tree", branch) if subpath: path = posixpath.join(path, subpath) return urlunparse(("https", parsed.netloc, path, parsed.query, parsed.params, parsed.fragment)) if (parsed.netloc in ("code.launchpad.net", "launchpad.net") and not branch and not subpath): return urlunparse(( "https", "code.launchpad.net", parsed.path, parsed.query, parsed.params, parsed.fragment, )) if parsed.hostname in ("git.savannah.gnu.org", "git.sv.gnu.org"): path_elements = parsed.path.strip("/").split("/") if parsed.scheme == "https" and path_elements[0] == "git": path_elements.pop(0) # Why cgit and not gitweb? path_elements.insert(0, "cgit") return urlunparse(("https", parsed.netloc, "/".join(path_elements), None, None, None)) if parsed.hostname in ("git.code.sf.net", "git.code.sourceforge.net"): path_elements = parsed.path.strip("/").split("/") if path_elements[0] != "p": return None project = path_elements[1] repository = path_elements[2] path_elements = ["p", project, repository] if branch is not None: path_elements.extend(["ci", branch, "tree"]) elif subpath is not None: path_elements.extend(["ci", "HEAD", "tree"]) if subpath is not None: path_elements.append(subpath) return urlunparse(("https", "sourceforge.net", "/".join(path_elements), None, None, None)) return None
def determine_gitlab_browser_url(url: str) -> str: (url, branch, subpath) = split_vcs_url(url) parsed_url = urlparse(url.rstrip("/")) # TODO(jelmer): Add support for branches path = parsed_url.path if path.endswith(".git"): path = path[:-len(".git")] if subpath and not branch: branch = "HEAD" if branch: path = path + "/tree/%s" % branch if subpath: path = path + "/" + subpath return "https://%s%s" % (parsed_url.hostname, path)
def fixup_rcp_style_git_url(url: str) -> str: (repo_url, branch, subpath) = split_vcs_url(url) repo_url = fixup_rcp_style_git_repo_url(repo_url) return unsplit_vcs_url(repo_url, branch, subpath)
def find_public_vcs_url(url: str) -> Optional[str]: (repo_url, branch, subpath) = split_vcs_url(url) revised_url = find_public_repo_url(repo_url) if revised_url is not None: return unsplit_vcs_url(revised_url, branch, subpath) return None
def canonical_vcs_git_url(url: str) -> str: (repo_url, branch, subpath) = split_vcs_url(url) repo_url = canonical_git_repo_url(repo_url) return unsplit_vcs_url(repo_url, branch, subpath)
async def update_package_metadata( conn, distribution: str, provided_packages, package_overrides, origin ): logging.info("Updating package metadata.") packages = [] for package in provided_packages: try: override = package_overrides[package.name] except KeyError: vcs_url = package.vcs_url else: vcs_url = override.branch_url or package.vcs_url or None vcs_last_revision = None if package.vcs_type and package.vcs_type.capitalize() == "Git": new_vcs_url = fixup_broken_git_url(vcs_url) if new_vcs_url != vcs_url: logging.info("Fixing up VCS URL: %s -> %s", vcs_url, new_vcs_url) vcs_url = new_vcs_url if package.commit_id: vcs_last_revision = default_mapping.revision_id_foreign_to_bzr( package.commit_id.encode("ascii") ) if package.vcs_type: # Drop the subpath, we're storing it separately. (url, branch, subpath) = split_vcs_url(vcs_url) url = unsplit_vcs_url(url, branch) url = canonicalize_vcs_url(package.vcs_type, url) try: branch_url = convert_debian_vcs_url(package.vcs_type.capitalize(), url) except ValueError as e: logging.info("%s: %s", package.name, e) branch_url = None else: subpath = None branch_url = None if vcs_url: vcs_browser = determine_browser_url(package.vcs_type, vcs_url) else: vcs_browser = None if vcs_browser is None and package.vcs_browser: vcs_browser = package.vcs_browser packages.append( ( package.name, distribution, branch_url if branch_url else None, subpath if subpath else None, package.maintainer_email if package.maintainer_email else None, package.uploader_email if package.uploader_email else [], package.archive_version if package.archive_version else None, package.vcs_type.lower() if package.vcs_type else None, vcs_url, vcs_browser, vcs_last_revision.decode("utf-8") if vcs_last_revision else None, package.vcswatch_status.lower() if package.vcswatch_status else None, package.vcswatch_version if package.vcswatch_version else None, package.insts, package.removed, package.in_base, origin ) ) await conn.executemany( "INSERT INTO package " "(name, distribution, branch_url, subpath, maintainer_email, " "uploader_emails, archive_version, vcs_type, vcs_url, vcs_browse, " "vcs_last_revision, vcswatch_status, vcswatch_version, popcon_inst, " "removed, in_base, origin) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, " "$13, $14, $15, $16, $17) ON CONFLICT (name, distribution) DO UPDATE SET " "branch_url = EXCLUDED.branch_url, " "subpath = EXCLUDED.subpath, " "maintainer_email = EXCLUDED.maintainer_email, " "uploader_emails = EXCLUDED.uploader_emails, " "archive_version = EXCLUDED.archive_version, " "vcs_type = EXCLUDED.vcs_type, " "vcs_url = EXCLUDED.vcs_url, " "vcs_last_revision = EXCLUDED.vcs_last_revision, " "vcs_browse = EXCLUDED.vcs_browse, " "vcswatch_status = EXCLUDED.vcswatch_status, " "vcswatch_version = EXCLUDED.vcswatch_version, " "popcon_inst = EXCLUDED.popcon_inst, " "removed = EXCLUDED.removed, " "in_base = EXCLUDED.in_base", packages, )