Beispiel #1
0
def setup_transport_and_pubs(repo_uri, remote=True):

        if repo_uri.startswith("null:"):
                return None, None

        xport, xport_cfg = transport.setup_transport()
        targ_pub = transport.setup_publisher(repo_uri, "default",
            xport, xport_cfg, remote_prefix=remote)

        return xport, targ_pub
Beispiel #2
0
def setup_transport_and_pubs(repo_uri, remote=True, ssl_key=None,
    ssl_cert=None):

        if repo_uri.startswith("null:"):
                return None, None

        xport, xport_cfg = transport.setup_transport()
        targ_pub = transport.setup_publisher(repo_uri, "default",
            xport, xport_cfg, remote_prefix=remote, ssl_key=ssl_key,
            ssl_cert=ssl_cert)

        return xport, targ_pub
Beispiel #3
0
def main_func():
        global dry_run, tmpdir, xport, dest_xport, target_pub

        dest_repo     = None
        source_list   = []
        variant_list  = []
        pub_list      = []
        use_pub_list  = False

        try:
                opts, pargs = getopt.getopt(sys.argv[1:], "d:np:s:?",
                    ["help"])
                for opt, arg in opts:
                        if opt == "-d":
                                dest_repo = misc.parse_uri(arg)
                        elif opt == "-n":
                                dry_run = True
                        elif opt == "-s":
                                s = arg.split(",")
                                if len(s) < 2:
                                        usage("-s option must specify "
                                            "variant=value,repo_uri")

                                # All but last part should be variant.
                                src_vars = {}
                                for v in s[:-1]:
                                        try:
                                                vname, vval = v.split("=")
                                        except ValueError:
                                                usage("-s option must specify "
                                                    "variant=value,repo_uri")

                                        if not vname.startswith("variant."):
                                                vname = "variant.{0}".format(vname)
                                        src_vars[vname] = vval

                                variant_list.append(src_vars)
                                source_list.append(publisher.RepositoryURI(
                                    misc.parse_uri(s[-1])))
                        elif opt == "-p":
                                use_pub_list = True
                                pub_list.append(arg)

                        if opt in ("--help", "-?"):
                                usage(exitcode=0)
        except getopt.GetoptError as e:
                usage(_("illegal option -- {0}").format(e.opt))

        if not source_list:
                usage(_("At least one variant name, value, and package source "
                   "must be provided using -s."))

        if not dest_repo:
                usage(_("A destination package repository must be provided "
                    "using -d."))

        # Determine the unique set of variants across all sources.
        variants = set()
        vcombos = collections.defaultdict(set)
        for src_vars in variant_list:
                for v, vval in src_vars.iteritems():
                        variants.add(v)
                        vcombos[v].add((v, vval))

        # merge_fmris() expects this to be a list.
        variants = list(variants)

        # Require that the user specified the same variants for all sources.
        for i, src_vars in enumerate(variant_list):
                missing = set(v for v in variants if v not in variant_list[i])
                if missing:
                        missing = ", ".join(missing)
                        source = source_list[i]
                        usage(_("Source {source} missing values for "
                            "variants: {missing}").format(**locals()))

        # Require that each unique variant combination has a source.
        for combo in itertools.product(*vcombos.values()):
                found = False
                for i, src in enumerate(source_list):
                        for vname, vval in combo:
                                if variant_list[i].get(vname,
                                    None) != vval:
                                        found = False
                                        break
                        else:
                                found = True
                                break

                if not found:
                        combo = " ".join(
                            "{0}={1}".format(vname, vval)
                            for vname, vval in combo
                        )
                        usage(_("No source was specified for variant "
                            "combination {combo}.").format(**locals()))

        # initialize transport
        # we use a single endpoint for now, since the transport code
        # uses publisher as a unique key... so we just flop the repo
        # list as needed to access the different catalogs/manifests/files.
        temp_root = misc.config_temp_root()

        tmpdir = tempfile.mkdtemp(dir=temp_root, prefix="pkgmerge")
        xport, xport_cfg = transport.setup_transport()
        xport_cfg.incoming_root = tmpdir

        # we don't use the publisher returned by setup_publisher, as that only
        # returns one of the publishers in source_list.  Instead we use
        # xport_cfg to access all publishers.
        transport.setup_publisher(source_list,
            "pkgmerge", xport, xport_cfg, remote_prefix=True)
        cat_dir = tempfile.mkdtemp(dir=tmpdir)

        # we must have at least one matching publisher if -p was used.
        known_pubs = set([pub.prefix for pub in xport_cfg.gen_publishers()])
        if pub_list and len(set(pub_list).intersection(known_pubs)) == 0:
                error(_("no publishers from source repositories match "
                    "the given -p options."))

        errors = set()
        tracker = get_tracker()

        # iterate over all publishers in our source repositories.  If errors
        # are encountered for a given publisher, we accumulate those, and
        # skip to the next publisher.
        for pub in xport_cfg.gen_publishers():

                if use_pub_list:
                        if pub.prefix not in pub_list:
                                continue
                        else:
                                # remove publishers from pub_list as we go, so
                                # that when we're finished, any remaining
                                # publishers in pub_list suggest superfluous
                                # -p options, which will cause us to exit with
                                # an error.
                                pub_list.remove(pub.prefix)

                pub.meta_root = cat_dir
                pub.transport = xport

                # Use separate transport for destination repository in case
                # source and destination have identical publisher configuration.
                dest_xport, dest_xport_cfg = transport.setup_transport()
                dest_xport_cfg.incoming_root = tmpdir

                # retrieve catalogs for all specified repositories
                for s in source_list:
                        load_catalog(s, pub)

                # determine the list of packages we'll be processing
                if not pargs:
                        # use the latest versions and merge everything
                        fmri_arguments = list(set(
                            name
                            for s in source_list
                            for name in get_all_pkg_names(s)
                        ))
                        exclude_args = []
                else:
                        fmri_arguments = [
                            f
                            for f in pargs
                            if not f.startswith("!")
                        ]

                        exclude_args = [
                            f[1:]
                            for f in pargs
                            if f.startswith("!")
                        ]

                # build fmris to be merged
                masterlist = [
                    build_merge_list(fmri_arguments, exclude_args,
                        catalog_dict[s.uri])
                    for s in source_list
                ]

                # check for unmatched patterns
                in_none = reduce(lambda x, y: x & y,
                    (set(u) for d, u in masterlist))
                if in_none:
                        errors.add(
                            _("The following pattern(s) did not match any "
                            "packages in any of the specified repositories for "
                            "publisher {pub_name}:"
                            "\n{patterns}").format(patterns="\n".join(in_none),
                            pub_name=pub.prefi))
                        continue

                # generate set of all package names to be processed, and dict
                # of lists indexed by order in source_list; if that repo has no
                # fmri for this pkg then use None.
                allpkgs = set(name for d, u in masterlist for name in d)

                processdict = {}
                for p in allpkgs:
                        for d, u in masterlist:
                                processdict.setdefault(p, []).append(
                                    d.setdefault(p, None))

                # check to make sure all fmris are at same version modulo
                # timestamp
                for entry in processdict:
                        if len(set([
                                str(a).rsplit(":")[0]
                                for a in processdict[entry]
                                if a is not None
                            ])) > 1:
                                errors.add(
                                    _("fmris matching the following patterns do"
                                    " not have matching versions across all "
                                    "repositories for publisher {pubs}: "
                                    "{patterns}").format(pub=pub.prefix,
                                    patterns=processdict[entry]))
                                continue

                # we're ready to merge
                if not dry_run:
                        target_pub = transport.setup_publisher(dest_repo,
                            pub.prefix, dest_xport, dest_xport_cfg,
                            remote_prefix=True)
                else:
                        target_pub = None

                tracker.republish_set_goal(len(processdict), 0, 0)
                # republish packages for this publisher. If we encounter any
                # publication errors, we move on to the next publisher.
                try:
                        pkg_tmpdir = tempfile.mkdtemp(dir=tmpdir)
                        republish_packages(pub, target_pub,
                            processdict, source_list, variant_list, variants,
                            tracker, xport, dest_repo, dest_xport, pkg_tmpdir,
                            dry_run=dry_run)
                except (trans.TransactionError, PkgmergeException) as e:
                        errors.add(str(e))
                        tracker.reset()
                        continue
                finally:
                        # if we're handling an exception, this still gets called
                        # in spite of the 'continue' that the handler ends with.
                        if os.path.exists(pkg_tmpdir):
                                shutil.rmtree(pkg_tmpdir)

                tracker.republish_done(dryrun=dry_run)
                tracker.reset()

        # If -p options were supplied, we should have processed all of them
        # by now. Remaining entries suggest -p options that were not merged.
        if use_pub_list and pub_list:
                errors.add(_("the following publishers were not found in "
                    "source repositories: {0}").format(" ".join(pub_list)))

        # If we have encountered errors for some publishers, print them now
        # and exit.
        tracker.flush()
        for message in errors:
                error(message, exitcode=None)
        if errors:
                exit(EXIT_OOPS)

        return EXIT_OK
Beispiel #4
0
def main_func():

    global temp_root, repo_modified, repo_finished, repo_uri, tracker
    global dry_run

    global_settings.client_name = PKG_CLIENT_NAME

    try:
        opts, pargs = getopt.getopt(sys.argv[1:], "?c:i:np:r:s:u", ["help"])
    except getopt.GetoptError as e:
        usage(_("illegal option -- {0}").format(e.opt))

    dry_run = False
    ref_repo_uri = None
    repo_uri = os.getenv("PKG_REPO", None)
    changes = set()
    ignores = set()
    publishers = set()
    cmp_policy = CMP_ALL

    processed_pubs = 0

    for opt, arg in opts:
        if opt == "-c":
            changes.add(arg)
        elif opt == "-i":
            ignores.add(arg)
        elif opt == "-n":
            dry_run = True
        elif opt == "-p":
            publishers.add(arg)
        elif opt == "-r":
            ref_repo_uri = misc.parse_uri(arg)
        elif opt == "-s":
            repo_uri = misc.parse_uri(arg)
        elif opt == "-u":
            cmp_policy = CMP_UNSIGNED
        elif opt == "-?" or opt == "--help":
            usage(retcode=pkgdefs.EXIT_OK)

    if pargs:
        usage(_("Unexpected argument(s): {0}").format(" ".join(pargs)))

    if not repo_uri:
        usage(_("A target repository must be provided."))

    if not ref_repo_uri:
        usage(_("A reference repository must be provided."))

    target = publisher.RepositoryURI(misc.parse_uri(repo_uri))
    if target.scheme != "file":
        abort(err=_("Target repository must be filesystem-based."))
    try:
        target_repo = sr.Repository(read_only=dry_run,
                                    root=target.get_pathname())
    except sr.RepositoryError as e:
        abort(str(e))

    # Use the tmp directory in target repo for efficient file rename since
    # files are in the same file system.
    temp_root = target_repo.temp_root
    if not os.path.exists(temp_root):
        os.makedirs(temp_root)

    ref_incoming_dir = tempfile.mkdtemp(dir=temp_root)
    ref_pkg_root = tempfile.mkdtemp(dir=temp_root)

    ref_xport, ref_xport_cfg = transport.setup_transport()
    ref_xport_cfg.incoming_root = ref_incoming_dir
    ref_xport_cfg.pkg_root = ref_pkg_root
    transport.setup_publisher(ref_repo_uri,
                              "ref",
                              ref_xport,
                              ref_xport_cfg,
                              remote_prefix=True)

    ref_repo = None
    ref = publisher.RepositoryURI(misc.parse_uri(ref_repo_uri))
    if ref.scheme == "file":
        try:
            ref_repo = sr.Repository(read_only=dry_run,
                                     root=ref.get_pathname())
        except sr.RepositoryError as e:
            abort(str(e))

    tracker = get_tracker()

    for pub in target_repo.publishers:
        if publishers and pub not in publishers \
            and '*' not in publishers:
            continue

        msg(_("Processing packages for publisher {0} ...").format(pub))
        # Find the matching pub in the ref repo.
        for ref_pub in ref_xport_cfg.gen_publishers():
            if ref_pub.prefix == pub:
                found = True
                break
        else:
            txt = _("Publisher {0} not found in reference "
                    "repository.").format(pub)
            if publishers:
                abort(err=txt)
            else:
                txt += _(" Skipping.")
                msg(txt)
            continue

        processed_pubs += 1

        rev = do_reversion(pub, ref_pub, target_repo, ref_xport, changes,
                           ignores, cmp_policy, ref_repo, ref, ref_xport_cfg)

        # Only rebuild catalog if anything got actually reversioned.
        if rev and not dry_run:
            msg(_("Rebuilding repository catalog."))
            target_repo.rebuild(pub=pub)
        repo_finished = True

    ret = pkgdefs.EXIT_OK
    if processed_pubs == 0:
        msg(_("No matching publishers could be found."))
        ret = pkgdefs.EXIT_OOPS
    cleanup()
    return ret
Beispiel #5
0
def main_func():
        global_settings.client_name = "pkgsign"

        try:
                opts, pargs = getopt.getopt(sys.argv[1:], "a:c:i:k:ns:D:",
                    ["help", "no-index", "no-catalog"])
        except getopt.GetoptError as e:
                usage(_("illegal global option -- {0}").format(e.opt))

        show_usage = False
        sig_alg = "rsa-sha256"
        cert_path = None
        key_path = None
        chain_certs = []
        add_to_catalog = True
        set_alg = False
        dry_run = False

        repo_uri = os.getenv("PKG_REPO", None)
        for opt, arg in opts:
                if opt == "-a":
                        sig_alg = arg
                        set_alg = True
                elif opt == "-c":
                        cert_path = os.path.abspath(arg)
                        if not os.path.isfile(cert_path):
                                usage(_("{0} was expected to be a certificate "
                                    "but isn't a file.").format(cert_path))
                elif opt == "-i":
                        p = os.path.abspath(arg)
                        if not os.path.isfile(p):
                                usage(_("{0} was expected to be a certificate "
                                    "but isn't a file.").format(p))
                        chain_certs.append(p)
                elif opt == "-k":
                        key_path = os.path.abspath(arg)
                        if not os.path.isfile(key_path):
                                usage(_("{0} was expected to be a key file "
                                    "but isn't a file.").format(key_path))
                elif opt == "-n":
                        dry_run = True
                elif opt == "-s":
                        repo_uri = misc.parse_uri(arg)
                elif opt == "--help":
                        show_usage = True
                elif opt == "--no-catalog":
                        add_to_catalog = False
                elif opt == "-D":
                        try:
                                key, value = arg.split("=", 1)
                                DebugValues.set_value(key, value)
                        except (AttributeError, ValueError):
                                error(_("{opt} takes argument of form "
                                    "name=value, not {arg}").format(
                                    opt=opt, arg=arg))
        if show_usage:
                usage(retcode=EXIT_OK)

        if not repo_uri:
                usage(_("a repository must be provided"))

        if key_path and not cert_path:
                usage(_("If a key is given to sign with, its associated "
                    "certificate must be given."))

        if cert_path and not key_path:
                usage(_("If a certificate is given, its associated key must be "
                    "given."))

        if chain_certs and not cert_path:
                usage(_("Intermediate certificates are only valid if a key "
                    "and certificate are also provided."))

        if not pargs:
                usage(_("At least one fmri or pattern must be provided to "
                    "sign."))

        if not set_alg and not key_path:
                sig_alg = "sha256"

        s, h = actions.signature.SignatureAction.decompose_sig_alg(sig_alg)
        if h is None:
                usage(_("{0} is not a recognized signature algorithm.").format(
                    sig_alg))
        if s and not key_path:
                usage(_("Using {0} as the signature algorithm requires that a "
                    "key and certificate pair be presented using the -k and -c "
                    "options.").format(sig_alg))
        if not s and key_path:
                usage(_("The {0} hash algorithm does not use a key or "
                    "certificate.  Do not use the -k or -c options with this "
                    "algorithm.").format(sig_alg))

        if DebugValues:
                reload(digest)

        errors = []

        t = misc.config_temp_root()
        temp_root = tempfile.mkdtemp(dir=t)
        del t

        cache_dir = tempfile.mkdtemp(dir=temp_root)
        incoming_dir = tempfile.mkdtemp(dir=temp_root)
        chash_dir = tempfile.mkdtemp(dir=temp_root)
        cert_dir = tempfile.mkdtemp(dir=temp_root)

        try:
                chain_certs = [
                    __make_tmp_cert(cert_dir, c) for c in chain_certs
                ]
                if cert_path is not None:
                        cert_path = __make_tmp_cert(cert_dir, cert_path)

                xport, xport_cfg = transport.setup_transport()
                xport_cfg.add_cache(cache_dir, readonly=False)
                xport_cfg.incoming_root = incoming_dir

                # Configure publisher(s)
                transport.setup_publisher(repo_uri, "source", xport,
                    xport_cfg, remote_prefix=True)
                pats = pargs
                successful_publish = False

                concrete_fmris = []
                unmatched_pats = set(pats)
                all_pats = frozenset(pats)
                get_all_pubs = False
                pub_prefs = set()
                # Gather the publishers whose catalogs will be needed.
                for pat in pats:
                        try:
                                p_obj = fmri.MatchingPkgFmri(pat)
                        except fmri.IllegalMatchingFmri as e:
                                errors.append(e)
                                continue
                        pub_prefix = p_obj.get_publisher()
                        if pub_prefix:
                                pub_prefs.add(pub_prefix)
                        else:
                                get_all_pubs = True
                # Check each publisher for matches to our patterns.
                for p in xport_cfg.gen_publishers():
                        if not get_all_pubs and p.prefix not in pub_prefs:
                                continue
                        cat = fetch_catalog(p, xport, temp_root)
                        ms, tmp1, u = cat.get_matching_fmris(pats)
                        # Find which patterns matched.
                        matched_pats = all_pats - u
                        # Remove those patterns from the unmatched set.
                        unmatched_pats -= matched_pats
                        for v_list in ms.values():
                                concrete_fmris.extend([(v, p) for v in v_list])
                if unmatched_pats:
                        raise api_errors.PackageMatchErrors(
                            unmatched_fmris=unmatched_pats)

                for pfmri, src_pub in sorted(set(concrete_fmris)):
                        try:
                                # Get the existing manifest for the package to
                                # be signed.
                                m_str = xport.get_manifest(pfmri,
                                    content_only=True, pub=src_pub)
                                m = manifest.Manifest()
                                m.set_content(content=m_str)

                                # Construct the base signature action.
                                attrs = { "algorithm": sig_alg }
                                a = actions.signature.SignatureAction(cert_path,
                                    **attrs)
                                a.hash = cert_path

                                # Add the action to the manifest to be signed
                                # since the action signs itself.
                                m.add_action(a, misc.EmptyI)

                                # Set the signature value and certificate
                                # information for the signature action.
                                a.set_signature(m.gen_actions(),
                                    key_path=key_path, chain_paths=chain_certs,
                                    chash_dir=chash_dir)

                                # The hash of 'a' is currently a path, we need
                                # to find the hash of that file to allow
                                # comparison to existing signatures.
                                hsh = None
                                if cert_path:
                                        # Action identity still uses the 'hash'
                                        # member of the action, so we need to
                                        # stay with the sha1 hash.
                                        hsh, _dummy = \
                                            misc.get_data_digest(cert_path,
                                            hash_func=hashlib.sha1)

                                # Check whether the signature about to be added
                                # is identical, or almost identical, to existing
                                # signatures on the package.  Because 'a' has
                                # already been added to the manifest, it is
                                # generated by gen_actions_by_type, so the cnt
                                # must be 2 or higher to be an issue.
                                cnt = 0
                                almost_identical = False
                                for a2 in m.gen_actions_by_type("signature"):
                                        try:
                                                if a.identical(a2, hsh):
                                                        cnt += 1
                                        except api_errors.AlmostIdentical as e:
                                                e.pkg = pfmri
                                                errors.append(e)
                                                almost_identical = True
                                if almost_identical:
                                        continue
                                if cnt == 2:
                                        continue
                                elif cnt > 2:
                                        raise api_errors.DuplicateSignaturesAlreadyExist(pfmri)
                                assert cnt == 1, "Cnt was:{0}".format(cnt)

                                if not dry_run:
                                        # Append the finished signature action
                                        # to the published manifest.
                                        t = trans.Transaction(repo_uri,
                                            pkg_name=str(pfmri), xport=xport,
                                            pub=src_pub)
                                        try:
                                                t.append()
                                                t.add(a)
                                                for c in chain_certs:
                                                        t.add_file(c)
                                                t.close(add_to_catalog=
                                                    add_to_catalog)
                                        except:
                                                if t.trans_id:
                                                        t.close(abandon=True)
                                                raise
                                msg(_("Signed {0}").format(pfmri.get_fmri(
                                    include_build=False)))
                                successful_publish = True
                        except (api_errors.ApiException, fmri.FmriError,
                            trans.TransactionError) as e:
                                errors.append(e)
                if errors:
                        error("\n".join([str(e) for e in errors]))
                        if successful_publish:
                                return EXIT_PARTIAL
                        else:
                                return EXIT_OOPS
                return EXIT_OK
        except api_errors.ApiException as e:
                error(e)
                return EXIT_OOPS
        finally:
                shutil.rmtree(temp_root)
Beispiel #6
0
    if len(varlist) < 2:
        usage(_("at least two -v arguments needed to merge"))

    if not basedir:
        basedir = os.getcwd()

    incomingdir = os.path.normpath(
        os.path.join(basedir, "incoming-%d" % os.getpid()))
    os.makedirs(incomingdir)
    tmpdirs.append(incomingdir)

    server_list = [
        publisher.RepositoryURI(v.split(",", 1)[1]) for v in varlist
    ]

    xport, xport_cfg = transport.setup_transport()
    xport_cfg.incoming_root = incomingdir
    pub = transport.setup_publisher(server_list,
                                    "merge",
                                    xport,
                                    xport_cfg,
                                    remote_prefix=True)

    if len(pargs) == 1:
        recursive = False
        overall_set = set()
        for s in server_list:
            for name in get_all_pkg_names(s):
                overall_set.add(name)
        fmri_arguments = list(overall_set)
Beispiel #7
0
        else:
                src_uri = misc.parse_uri(src_uri)

        if not cache_dir:
                cache_dir = tempfile.mkdtemp(dir=temp_root,
                    prefix=global_settings.client_name + "-")
                # Only clean-up cache dir if implicitly created by pkgrecv.
                # User's cache-dirs should be preserved
                tmpdirs.append(cache_dir)

        incoming_dir = tempfile.mkdtemp(dir=temp_root,
            prefix=global_settings.client_name + "-")
        tmpdirs.append(incoming_dir)

        # Create transport and transport config
        xport, xport_cfg = transport.setup_transport()
        xport_cfg.add_cache(cache_dir, readonly=False)
        xport_cfg.incoming_root = incoming_dir

        # Since publication destinations may only have one repository configured
        # per publisher, create destination as separate transport in case source
        # and destination have identical publisher configuration but different
        # repository endpoints.
        dest_xport, dest_xport_cfg = transport.setup_transport()
        dest_xport_cfg.add_cache(cache_dir, readonly=False)
        dest_xport_cfg.incoming_root = incoming_dir

        # Configure src publisher(s).
        transport.setup_publisher(src_uri, "source", xport, xport_cfg,
            remote_prefix=True, ssl_key=key, ssl_cert=cert)
Beispiel #8
0
        temp_root = tempfile.mkdtemp(dir=t)
        del t
        
        cache_dir = tempfile.mkdtemp(dir=temp_root)
        incoming_dir = tempfile.mkdtemp(dir=temp_root)
        chash_dir = tempfile.mkdtemp(dir=temp_root)
        cert_dir = tempfile.mkdtemp(dir=temp_root)

        try:
                chain_certs = [
                    __make_tmp_cert(cert_dir, c) for c in chain_certs
                ]
                if cert_path is not None:
                        cert_path = __make_tmp_cert(cert_dir, cert_path)

                xport, xport_cfg = transport.setup_transport()
                xport_cfg.add_cache(cache_dir, readonly=False)
                xport_cfg.incoming_root = incoming_dir

                # Configure publisher(s)
                transport.setup_publisher(repo_uri, "source", xport,
                    xport_cfg, remote_prefix=True)
                pats = pargs
                successful_publish = False

                concrete_fmris = []
                unmatched_pats = set(pats)
                all_pats = frozenset(pats)
                get_all_pubs = False
                pub_prefs = set()
                # Gather the publishers whose catalogs will be needed.