Exemplo n.º 1
0
    def test_createrepo_generated_commands_comps_xml(self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        for path in [path_epel_5, path_fedora]:
            for add_comps in [True, False]:
                os.makedirs(path)

                comps_path = os.path.join(path, "comps.xml")
                if add_comps:
                    with open(comps_path, "w") as handle:
                        handle.write("1")

                repo_path = os.path.join(path, "repodata")
                os.makedirs(repo_path)
                with open(os.path.join(repo_path, "repomd.xml"),
                          "w") as handle:
                    handle.write("1")

                createrepo_unsafe(path)
                if add_comps:
                    assert "--groupfile" in mc_run_cmd_unsafe.call_args[0][0]
                else:
                    assert "--groupfile" not in mc_run_cmd_unsafe.call_args[0][
                        0]
                mc_run_cmd_unsafe.mock_reset()
                shutil.rmtree(path, ignore_errors=True)
Exemplo n.º 2
0
    def test_createrepo_unsafe_lock_usage(self, mc_popen):
        mocked_lock = MagicMock()
        self.shared_state = dict(in_lock=False, lock_status=None)

        def enter_lock(*args, **kwargs):
            self.shared_state["in_lock"] = True

        def exit_lock(*args, **kwargs):
            self.shared_state["in_lock"] = False

        def popen_side_effect(*args, **kwargs):
            self.shared_state["lock_status"] = copy.copy(self.shared_state["in_lock"])
            return mock.DEFAULT

        mocked_lock.__enter__.side_effect = enter_lock
        mocked_lock.__exit__.side_effect = exit_lock

        mc_popen.side_effect = popen_side_effect
        mc_popen.return_value.communicate.return_value = ("", "")

        createrepo_unsafe(self.tmp_dir_name, lock=mocked_lock)
        assert self.shared_state["lock_status"]

        self.shared_state["lock_status"] = None
        createrepo_unsafe(self.tmp_dir_name, lock=None)
        assert not self.shared_state["lock_status"]
Exemplo n.º 3
0
    def test_createrepo_unsafe_lock_usage(self, mc_popen):
        mocked_lock = MagicMock()
        self.shared_state = dict(in_lock=False, lock_status=None)

        def enter_lock(*args, **kwargs):
            self.shared_state["in_lock"] = True

        def exit_lock(*args, **kwargs):
            self.shared_state["in_lock"] = False

        def popen_side_effect(*args, **kwargs):
            self.shared_state["lock_status"] = copy.copy(
                self.shared_state["in_lock"])
            return mock.DEFAULT

        mocked_lock.__enter__.side_effect = enter_lock
        mocked_lock.__exit__.side_effect = exit_lock

        mc_popen.side_effect = popen_side_effect
        mc_popen.return_value.communicate.return_value = ("", "")

        createrepo_unsafe(self.tmp_dir_name, lock=mocked_lock)
        assert self.shared_state["lock_status"]

        self.shared_state["lock_status"] = None
        createrepo_unsafe(self.tmp_dir_name, lock=None)
        assert not self.shared_state["lock_status"]
Exemplo n.º 4
0
    def test_createrepo_devel_generated_commands(self, mc_popen):

        mc_popen.return_value.communicate.return_value = ("", "")
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = [
            '/usr/bin/createrepo_c', '--database', '--ignore-lock', '-s',
            'sha', '--checksum', 'md5', '--outputdir',
            os.path.join(path_epel_5,
                         "devel"), '--baseurl', self.base_url, path_epel_5
        ]
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = [
            '/usr/bin/createrepo_c', '--database', '--ignore-lock',
            '--outputdir',
            os.path.join(path_fedora,
                         "devel"), '--baseurl', self.base_url, path_fedora
        ]
        for path, expected in [(path_epel_5, expected_epel_5),
                               (path_fedora, expected_fedora)]:
            os.makedirs(path)

            createrepo_unsafe(path,
                              lock=None,
                              base_url=self.base_url,
                              dest_dir="devel")
            assert os.path.exists(os.path.join(path, "devel"))
            assert mc_popen.call_args == mock.call(expected,
                                                   stderr=-1,
                                                   stdout=-1)
Exemplo n.º 5
0
    def test_createrepo_generated_commands_existing_repodata(self, mc_popen):
        mc_popen.return_value.communicate.return_value = ("", "")
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = [
            '/usr/bin/createrepo_c', '--database', '--ignore-lock', '--update',
            '-s', 'sha', '--checksum', 'md5', path_epel_5
        ]
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = [
            '/usr/bin/createrepo_c', '--database', '--ignore-lock', '--update',
            path_fedora
        ]
        for path, expected in [(path_epel_5, expected_epel_5),
                               (path_fedora, expected_fedora)]:
            os.makedirs(path)

            repo_path = os.path.join(path, "repodata")
            os.makedirs(repo_path)
            with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
                handle.write("1")

            createrepo_unsafe(path, None)
            assert mc_popen.call_args == mock.call(expected,
                                                   stderr=-1,
                                                   stdout=-1)
Exemplo n.º 6
0
    def test_createrepo_devel_creates_folder(self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")

        for path in [path_epel_5, path_fedora]:
            os.makedirs(path)

            createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")
            assert os.path.exists(os.path.join(path, "devel"))
Exemplo n.º 7
0
    def test_createrepo_devel_creates_folder(self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")

        for path in [path_epel_5, path_fedora]:
            os.makedirs(path)

            createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")
            assert os.path.exists(os.path.join(path, "devel"))
Exemplo n.º 8
0
    def test_createrepo_devel_generated_commands(self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock "
                           "-s sha --checksum md5 "
                           "--outputdir " + os.path.join(path_epel_5, "devel") + " "
                           "--baseurl " + self.base_url + " " + path_epel_5)
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock "
                           "--outputdir " + os.path.join(path_fedora, "devel") + " "
                           "--baseurl " + self.base_url + " " + path_fedora)
        for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:
            os.makedirs(path)

            createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")
            assert os.path.exists(os.path.join(path, "devel"))
            assert mc_run_cmd_unsafe.call_args[0][0] == expected
Exemplo n.º 9
0
    def test_createrepo_devel_generated_commands(self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 "
                           "-s sha --checksum md5 "
                           "--outputdir " + os.path.join(path_epel_5, "devel") + " "
                           "--baseurl " + self.base_url + " " + path_epel_5)
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 "
                           "--outputdir " + os.path.join(path_fedora, "devel") + " "
                           "--baseurl " + self.base_url + " " + path_fedora)
        for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:
            os.makedirs(path)

            createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")
            assert os.path.exists(os.path.join(path, "devel"))
            assert mc_run_cmd_unsafe.call_args[0][0] == expected
Exemplo n.º 10
0
    def test_createrepo_generated_commands_existing_repodata(self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = ('/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 '
                           '--update -s sha --checksum md5 ' + path_epel_5)
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = ('/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 '
                           '--update ' + path_fedora)
        for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:
            os.makedirs(path)

            repo_path = os.path.join(path, "repodata")
            os.makedirs(repo_path)
            with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
                handle.write("1")

            createrepo_unsafe(path)
            assert mc_run_cmd_unsafe.call_args[0][0] == expected
Exemplo n.º 11
0
    def test_createrepo_generated_commands_existing_repodata(self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = ('/usr/bin/createrepo_c --database '
                           '--ignore-lock --update -s sha --checksum md5 ' + path_epel_5)
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = ('/usr/bin/createrepo_c --database '
                           '--ignore-lock --update ' + path_fedora)
        for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:
            os.makedirs(path)

            repo_path = os.path.join(path, "repodata")
            os.makedirs(repo_path)
            with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
                handle.write("1")

            createrepo_unsafe(path)
            assert mc_run_cmd_unsafe.call_args[0][0] == expected
Exemplo n.º 12
0
def fix_copr(opts, copr_full_name):
    log.info('Going to fix {}:'.format(copr_full_name))

    owner, coprname = tuple(copr_full_name.split('/'))
    copr_path = os.path.abspath(os.path.join(opts.destdir, owner, coprname))

    if not os.path.isdir(copr_path):
        log.info('Ignoring {}. Directory does not exist.'.format(copr_path))
        return

    log.info(
        '> Generate key-pair on copr-keygen (if not generated) for email {}.'.
        format(create_gpg_email(owner, coprname)))
    create_user_keys(owner, coprname, opts)

    log.info('> Regenerate pubkey.gpg in copr {}.'.format(copr_path))
    get_pubkey(owner, coprname, os.path.join(copr_path, 'pubkey.gpg'))

    log.info('> Re-sign rpms and call createrepo in copr\'s chroots:')
    for dir_name in os.listdir(copr_path):
        dir_path = os.path.join(copr_path, dir_name)
        if not os.path.isdir(dir_path):
            log.info('> > Ignoring {}'.format(dir_path))
            continue

        for builddir_name in os.listdir(dir_path):
            builddir_path = os.path.join(dir_path, builddir_name)
            if not os.path.isdir(builddir_path):
                continue
            log.info(
                '> > Processing rpms in builddir {}:'.format(builddir_path))
            try:
                unsign_rpms_in_dir(
                    builddir_path, opts, log
                )  # first we need to unsign by using rpm-sign before we sign with obs-sign
                sign_rpms_in_dir(owner, coprname, builddir_path, opts, log)
            except Exception as e:
                log.exception(str(e))
                continue

        log.info("> > Running createrepo_unsafe for {}".format(dir_path))
        createrepo_unsafe(dir_path)

        log.info("> > Running add_appdata for {}".format(dir_path))
        add_appdata(dir_path, owner, coprname)
Exemplo n.º 13
0
    def test_createrepo_devel_generated_commands(self, mc_popen):

        mc_popen.return_value.communicate.return_value = ("", "")
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = ['/usr/bin/createrepo_c', '--database', '--ignore-lock',
                           '-s', 'sha', '--checksum', 'md5',
                           '--outputdir', os.path.join(path_epel_5, "devel"),
                           '--baseurl', self.base_url, path_epel_5]
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = ['/usr/bin/createrepo_c', '--database', '--ignore-lock',
                           '--outputdir', os.path.join(path_fedora, "devel"),
                           '--baseurl', self.base_url, path_fedora]
        for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:
            os.makedirs(path)

            createrepo_unsafe(path, lock=None, base_url=self.base_url, dest_dir="devel")
            assert os.path.exists(os.path.join(path, "devel"))
            assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1)
Exemplo n.º 14
0
    def test_createrepo_generated_commands_existing_repodata(self, mc_popen):
        mc_popen.return_value.communicate.return_value = ("", "")
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = ['/usr/bin/createrepo_c', '--database',
                           '--ignore-lock', '--update', '-s', 'sha', '--checksum', 'md5', path_epel_5]
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = ['/usr/bin/createrepo_c', '--database',
                           '--ignore-lock', '--update', path_fedora]
        for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:
            os.makedirs(path)

            repo_path = os.path.join(path, "repodata")
            os.makedirs(repo_path)
            with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
                handle.write("1")

            createrepo_unsafe(path, None)
            assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1)
Exemplo n.º 15
0
    def test_createrepo_devel_generated_commands_existing_repodata(self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock "
                           "-s sha --checksum md5 "
                           "--outputdir " + os.path.join(path_epel_5, "devel") + " "
                           "--baseurl " + self.base_url + " " + path_epel_5)
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock "
                           "--outputdir " + os.path.join(path_fedora, "devel") + " "
                           "--baseurl " + self.base_url + " " + path_fedora)
        for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:
            os.makedirs(path)

            repo_path = os.path.join(path, "devel", "repodata")
            os.makedirs(repo_path)
            with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
                handle.write("1")

            createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")
            assert mc_run_cmd_unsafe.call_args[0][0] == expected
Exemplo n.º 16
0
def fix_copr(opts, copr_full_name):
    log.info('Going to fix {}:'.format(copr_full_name))

    owner, coprname = tuple(copr_full_name.split('/'))
    copr_path = os.path.abspath(os.path.join(opts.destdir, owner, coprname))

    if not os.path.isdir(copr_path):
        log.info('Ignoring {}. Directory does not exist.'.format(copr_path))
        return

    log.info('> Generate key-pair on copr-keygen (if not generated) for email {}.'.format(create_gpg_email(owner, coprname)))
    create_user_keys(owner, coprname, opts)

    log.info('> Regenerate pubkey.gpg in copr {}.'.format(copr_path))
    get_pubkey(owner, coprname, os.path.join(copr_path, 'pubkey.gpg'))

    log.info('> Re-sign rpms and call createrepo in copr\'s chroots:')
    for dir_name in os.listdir(copr_path):
        dir_path = os.path.join(copr_path, dir_name)
        if not os.path.isdir(dir_path):
            log.info('> > Ignoring {}'.format(dir_path))
            continue

        for builddir_name in os.listdir(dir_path):
            builddir_path = os.path.join(dir_path, builddir_name)
            if not os.path.isdir(builddir_path):
                continue
            log.info('> > Processing rpms in builddir {}:'.format(builddir_path))
            try:
                unsign_rpms_in_dir(builddir_path, opts, log) # first we need to unsign by using rpm-sign before we sign with obs-sign
                sign_rpms_in_dir(owner, coprname, builddir_path, opts, log)
            except Exception as e:
                log.exception(str(e))
                continue

        log.info("> > Running createrepo_unsafe for {}".format(dir_path))
        createrepo_unsafe(dir_path)

        log.info("> > Running add_appdata for {}".format(dir_path))
        add_appdata(dir_path, owner, coprname)
Exemplo n.º 17
0
    def prune_project(self, project_path, username, projectname):
        log.info("Going to prune {}/{}".format(username, projectname))
        # get ACR
        try:
            if not get_auto_createrepo_status(self.opts.frontend_base_url, username, projectname):
                log.debug("Skipped {}/{} since auto createrepo option is disabled"
                          .format(username, projectname))
                return
        except (CoprException, CoprRequestException) as exception:
            log.debug("Failed to get project details for {}/{} with error: {}".format(
                username, projectname, exception))
            return

        # run prune project sh
        for sub_dir_name in os.listdir(project_path):
            chroot_path = os.path.join(project_path, sub_dir_name)
            if not os.path.isdir(chroot_path):
                continue

            try:
                self.prune_failed_builds(chroot_path)
                self.prune_obsolete_success_builds(chroot_path)
            except Exception as err:
                log.exception(err)
                log.error("Error during prune copr {}/{}:{}".format(username, projectname, sub_dir_name))

            log.debug("Prune done for {}/{}:{}".format(username, projectname, sub_dir_name))

            try:
                createrepo_unsafe(chroot_path)
                log.info("Createrepo done for copr {}/{}:{}"
                         .format(username, projectname, sub_dir_name))
            except CreateRepoError as exception:
                log.exception("Createrepo for copr {}/{}:{} failed with error: {}"
                              .format(username, projectname, sub_dir_name, exception))

        log.info("Prune finished for copr {}/{}".format(username, projectname))
Exemplo n.º 18
0
    def prune_project(self, project_path, username, projectname):
        log.info("Going to prune {}/{}".format(username, projectname))
        # get ACR
        try:
            if not get_auto_createrepo_status(self.opts.frontend_base_url, username, projectname):
                log.debug("Skipped {}/{} since auto createrepo option is disabled"
                          .format(username, projectname))
                return
        except (CoprException, CoprRequestException) as exception:
            log.debug("Failed to get project details for {}/{} with error: {}".format(
                username, projectname, exception))
            return

        # run prune project sh
        for sub_dir_name in os.listdir(project_path):
            chroot_path = os.path.join(project_path, sub_dir_name)
            if not os.path.isdir(chroot_path):
                continue

            try:
                self.prune_failed_builds(chroot_path)
                self.prune_obsolete_success_builds(chroot_path)
            except Exception as err:
                log.exception(err)
                log.error("Error during prune copr {}/{}:{}".format(username, projectname, sub_dir_name))

            log.debug("Prune done for {}/{}:{}".format(username, projectname, sub_dir_name))

            try:
                createrepo_unsafe(chroot_path)
                log.info("Createrepo done for copr {}/{}:{}"
                         .format(username, projectname, sub_dir_name))
            except CreateRepoError as exception:
                log.exception("Createrepo for copr {}/{}:{} failed with error: {}"
                              .format(username, projectname, sub_dir_name, exception))

        log.info("Prune finished for copr {}/{}".format(username, projectname))
Exemplo n.º 19
0
    def test_createrepo_devel_generated_commands_existing_repodata(
            self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock "
                           "-s sha --checksum md5 "
                           "--outputdir " +
                           os.path.join(path_epel_5, "devel") + " "
                           "--baseurl " + self.base_url + " " + path_epel_5)
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock "
                           "--outputdir " +
                           os.path.join(path_fedora, "devel") + " "
                           "--baseurl " + self.base_url + " " + path_fedora)
        for path, expected in [(path_epel_5, expected_epel_5),
                               (path_fedora, expected_fedora)]:
            os.makedirs(path)

            repo_path = os.path.join(path, "devel", "repodata")
            os.makedirs(repo_path)
            with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
                handle.write("1")

            createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")
            assert mc_run_cmd_unsafe.call_args[0][0] == expected
Exemplo n.º 20
0
    def test_createrepo_generated_commands_comps_xml(self, mc_run_cmd_unsafe):
        path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
        path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
        for path in [path_epel_5, path_fedora]:
            for add_comps in [True, False]:
                os.makedirs(path)

                comps_path = os.path.join(path, "comps.xml")
                if add_comps:
                    with open(comps_path, "w") as handle:
                        handle.write("1")

                repo_path = os.path.join(path, "repodata")
                os.makedirs(repo_path)
                with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
                    handle.write("1")

                createrepo_unsafe(path)
                if add_comps:
                    assert "--groupfile" in mc_run_cmd_unsafe.call_args[0][0]
                else:
                    assert "--groupfile" not in mc_run_cmd_unsafe.call_args[0][0]
                mc_run_cmd_unsafe.mock_reset()
                shutil.rmtree(path, ignore_errors=True)
Exemplo n.º 21
0
def prune_project(opts, path, username, projectname):
    log.debug("Going to prune {}/{}".format(username, projectname))
    # get ACR
    try:
        if not get_auto_createrepo_status(opts.frontend_base_url, username, projectname):
            log.debug("Skipped {}/{} since auto createrepo option is disabled"
                      .format(username, projectname))
            return
    except (CoprException, CoprRequestException) as exception:
        log.debug("Failed to get project details for {}/{} with error: {}".format(
            username, projectname, exception))
        return

    # run prune project sh
    days = getattr(opts, "prune_days", DEF_DAYS)

    cmd = map(str, [DEF_PRUNE_SCRIPT, path, days])

    handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
    stdout, stderr = handle.communicate()

    if handle.returncode != 0:
        print("Failed to prune old builds for copr {}/{}".format(username, projectname))
        print("STDOUT: \n{}".format(stdout.decode()))
        print("STDERR: \n{}".format(stderr.decode()))
        return

    # run createrepo
    log.debug("Prune done for {}/{}".format(username, projectname))
    try:
        retcode, stdout, stderr = createrepo_unsafe(path)
        if retcode != 0:
            print("Createrepo for {}/{} failed".format(username, projectname))
            print("STDOUT: \n{}".format(stdout.decode()))
            print("STDERR: \n{}".format(stderr.decode()))
    except Exception as exception:
        print("Createrepo for {}/{} failed with error: {}"
              .format(username, projectname, exception))