def test_createrepo_generated_commands_comps_xml(self, mc_run_cmd_unsafe): path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") for path in [path_epel_5, path_fedora]: for add_comps in [True, False]: os.makedirs(path) comps_path = os.path.join(path, "comps.xml") if add_comps: with open(comps_path, "w") as handle: handle.write("1") repo_path = os.path.join(path, "repodata") os.makedirs(repo_path) with open(os.path.join(repo_path, "repomd.xml"), "w") as handle: handle.write("1") createrepo_unsafe(path) if add_comps: assert "--groupfile" in mc_run_cmd_unsafe.call_args[0][0] else: assert "--groupfile" not in mc_run_cmd_unsafe.call_args[0][ 0] mc_run_cmd_unsafe.mock_reset() shutil.rmtree(path, ignore_errors=True)
def test_createrepo_unsafe_lock_usage(self, mc_popen): mocked_lock = MagicMock() self.shared_state = dict(in_lock=False, lock_status=None) def enter_lock(*args, **kwargs): self.shared_state["in_lock"] = True def exit_lock(*args, **kwargs): self.shared_state["in_lock"] = False def popen_side_effect(*args, **kwargs): self.shared_state["lock_status"] = copy.copy(self.shared_state["in_lock"]) return mock.DEFAULT mocked_lock.__enter__.side_effect = enter_lock mocked_lock.__exit__.side_effect = exit_lock mc_popen.side_effect = popen_side_effect mc_popen.return_value.communicate.return_value = ("", "") createrepo_unsafe(self.tmp_dir_name, lock=mocked_lock) assert self.shared_state["lock_status"] self.shared_state["lock_status"] = None createrepo_unsafe(self.tmp_dir_name, lock=None) assert not self.shared_state["lock_status"]
def test_createrepo_unsafe_lock_usage(self, mc_popen): mocked_lock = MagicMock() self.shared_state = dict(in_lock=False, lock_status=None) def enter_lock(*args, **kwargs): self.shared_state["in_lock"] = True def exit_lock(*args, **kwargs): self.shared_state["in_lock"] = False def popen_side_effect(*args, **kwargs): self.shared_state["lock_status"] = copy.copy( self.shared_state["in_lock"]) return mock.DEFAULT mocked_lock.__enter__.side_effect = enter_lock mocked_lock.__exit__.side_effect = exit_lock mc_popen.side_effect = popen_side_effect mc_popen.return_value.communicate.return_value = ("", "") createrepo_unsafe(self.tmp_dir_name, lock=mocked_lock) assert self.shared_state["lock_status"] self.shared_state["lock_status"] = None createrepo_unsafe(self.tmp_dir_name, lock=None) assert not self.shared_state["lock_status"]
def test_createrepo_devel_generated_commands(self, mc_popen): mc_popen.return_value.communicate.return_value = ("", "") path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = [ '/usr/bin/createrepo_c', '--database', '--ignore-lock', '-s', 'sha', '--checksum', 'md5', '--outputdir', os.path.join(path_epel_5, "devel"), '--baseurl', self.base_url, path_epel_5 ] path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = [ '/usr/bin/createrepo_c', '--database', '--ignore-lock', '--outputdir', os.path.join(path_fedora, "devel"), '--baseurl', self.base_url, path_fedora ] for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) createrepo_unsafe(path, lock=None, base_url=self.base_url, dest_dir="devel") assert os.path.exists(os.path.join(path, "devel")) assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1)
def test_createrepo_generated_commands_existing_repodata(self, mc_popen): mc_popen.return_value.communicate.return_value = ("", "") path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = [ '/usr/bin/createrepo_c', '--database', '--ignore-lock', '--update', '-s', 'sha', '--checksum', 'md5', path_epel_5 ] path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = [ '/usr/bin/createrepo_c', '--database', '--ignore-lock', '--update', path_fedora ] for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) repo_path = os.path.join(path, "repodata") os.makedirs(repo_path) with open(os.path.join(repo_path, "repomd.xml"), "w") as handle: handle.write("1") createrepo_unsafe(path, None) assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1)
def test_createrepo_devel_creates_folder(self, mc_run_cmd_unsafe): path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") for path in [path_epel_5, path_fedora]: os.makedirs(path) createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel") assert os.path.exists(os.path.join(path, "devel"))
def test_createrepo_devel_generated_commands(self, mc_run_cmd_unsafe): path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock " "-s sha --checksum md5 " "--outputdir " + os.path.join(path_epel_5, "devel") + " " "--baseurl " + self.base_url + " " + path_epel_5) path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock " "--outputdir " + os.path.join(path_fedora, "devel") + " " "--baseurl " + self.base_url + " " + path_fedora) for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel") assert os.path.exists(os.path.join(path, "devel")) assert mc_run_cmd_unsafe.call_args[0][0] == expected
def test_createrepo_devel_generated_commands(self, mc_run_cmd_unsafe): path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 " "-s sha --checksum md5 " "--outputdir " + os.path.join(path_epel_5, "devel") + " " "--baseurl " + self.base_url + " " + path_epel_5) path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 " "--outputdir " + os.path.join(path_fedora, "devel") + " " "--baseurl " + self.base_url + " " + path_fedora) for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel") assert os.path.exists(os.path.join(path, "devel")) assert mc_run_cmd_unsafe.call_args[0][0] == expected
def test_createrepo_generated_commands_existing_repodata(self, mc_run_cmd_unsafe): path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = ('/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 ' '--update -s sha --checksum md5 ' + path_epel_5) path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = ('/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 ' '--update ' + path_fedora) for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) repo_path = os.path.join(path, "repodata") os.makedirs(repo_path) with open(os.path.join(repo_path, "repomd.xml"), "w") as handle: handle.write("1") createrepo_unsafe(path) assert mc_run_cmd_unsafe.call_args[0][0] == expected
def test_createrepo_generated_commands_existing_repodata(self, mc_run_cmd_unsafe): path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = ('/usr/bin/createrepo_c --database ' '--ignore-lock --update -s sha --checksum md5 ' + path_epel_5) path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = ('/usr/bin/createrepo_c --database ' '--ignore-lock --update ' + path_fedora) for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) repo_path = os.path.join(path, "repodata") os.makedirs(repo_path) with open(os.path.join(repo_path, "repomd.xml"), "w") as handle: handle.write("1") createrepo_unsafe(path) assert mc_run_cmd_unsafe.call_args[0][0] == expected
def fix_copr(opts, copr_full_name): log.info('Going to fix {}:'.format(copr_full_name)) owner, coprname = tuple(copr_full_name.split('/')) copr_path = os.path.abspath(os.path.join(opts.destdir, owner, coprname)) if not os.path.isdir(copr_path): log.info('Ignoring {}. Directory does not exist.'.format(copr_path)) return log.info( '> Generate key-pair on copr-keygen (if not generated) for email {}.'. format(create_gpg_email(owner, coprname))) create_user_keys(owner, coprname, opts) log.info('> Regenerate pubkey.gpg in copr {}.'.format(copr_path)) get_pubkey(owner, coprname, os.path.join(copr_path, 'pubkey.gpg')) log.info('> Re-sign rpms and call createrepo in copr\'s chroots:') for dir_name in os.listdir(copr_path): dir_path = os.path.join(copr_path, dir_name) if not os.path.isdir(dir_path): log.info('> > Ignoring {}'.format(dir_path)) continue for builddir_name in os.listdir(dir_path): builddir_path = os.path.join(dir_path, builddir_name) if not os.path.isdir(builddir_path): continue log.info( '> > Processing rpms in builddir {}:'.format(builddir_path)) try: unsign_rpms_in_dir( builddir_path, opts, log ) # first we need to unsign by using rpm-sign before we sign with obs-sign sign_rpms_in_dir(owner, coprname, builddir_path, opts, log) except Exception as e: log.exception(str(e)) continue log.info("> > Running createrepo_unsafe for {}".format(dir_path)) createrepo_unsafe(dir_path) log.info("> > Running add_appdata for {}".format(dir_path)) add_appdata(dir_path, owner, coprname)
def test_createrepo_devel_generated_commands(self, mc_popen): mc_popen.return_value.communicate.return_value = ("", "") path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = ['/usr/bin/createrepo_c', '--database', '--ignore-lock', '-s', 'sha', '--checksum', 'md5', '--outputdir', os.path.join(path_epel_5, "devel"), '--baseurl', self.base_url, path_epel_5] path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = ['/usr/bin/createrepo_c', '--database', '--ignore-lock', '--outputdir', os.path.join(path_fedora, "devel"), '--baseurl', self.base_url, path_fedora] for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) createrepo_unsafe(path, lock=None, base_url=self.base_url, dest_dir="devel") assert os.path.exists(os.path.join(path, "devel")) assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1)
def test_createrepo_generated_commands_existing_repodata(self, mc_popen): mc_popen.return_value.communicate.return_value = ("", "") path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = ['/usr/bin/createrepo_c', '--database', '--ignore-lock', '--update', '-s', 'sha', '--checksum', 'md5', path_epel_5] path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = ['/usr/bin/createrepo_c', '--database', '--ignore-lock', '--update', path_fedora] for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) repo_path = os.path.join(path, "repodata") os.makedirs(repo_path) with open(os.path.join(repo_path, "repomd.xml"), "w") as handle: handle.write("1") createrepo_unsafe(path, None) assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1)
def test_createrepo_devel_generated_commands_existing_repodata(self, mc_run_cmd_unsafe): path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock " "-s sha --checksum md5 " "--outputdir " + os.path.join(path_epel_5, "devel") + " " "--baseurl " + self.base_url + " " + path_epel_5) path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock " "--outputdir " + os.path.join(path_fedora, "devel") + " " "--baseurl " + self.base_url + " " + path_fedora) for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) repo_path = os.path.join(path, "devel", "repodata") os.makedirs(repo_path) with open(os.path.join(repo_path, "repomd.xml"), "w") as handle: handle.write("1") createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel") assert mc_run_cmd_unsafe.call_args[0][0] == expected
def fix_copr(opts, copr_full_name): log.info('Going to fix {}:'.format(copr_full_name)) owner, coprname = tuple(copr_full_name.split('/')) copr_path = os.path.abspath(os.path.join(opts.destdir, owner, coprname)) if not os.path.isdir(copr_path): log.info('Ignoring {}. Directory does not exist.'.format(copr_path)) return log.info('> Generate key-pair on copr-keygen (if not generated) for email {}.'.format(create_gpg_email(owner, coprname))) create_user_keys(owner, coprname, opts) log.info('> Regenerate pubkey.gpg in copr {}.'.format(copr_path)) get_pubkey(owner, coprname, os.path.join(copr_path, 'pubkey.gpg')) log.info('> Re-sign rpms and call createrepo in copr\'s chroots:') for dir_name in os.listdir(copr_path): dir_path = os.path.join(copr_path, dir_name) if not os.path.isdir(dir_path): log.info('> > Ignoring {}'.format(dir_path)) continue for builddir_name in os.listdir(dir_path): builddir_path = os.path.join(dir_path, builddir_name) if not os.path.isdir(builddir_path): continue log.info('> > Processing rpms in builddir {}:'.format(builddir_path)) try: unsign_rpms_in_dir(builddir_path, opts, log) # first we need to unsign by using rpm-sign before we sign with obs-sign sign_rpms_in_dir(owner, coprname, builddir_path, opts, log) except Exception as e: log.exception(str(e)) continue log.info("> > Running createrepo_unsafe for {}".format(dir_path)) createrepo_unsafe(dir_path) log.info("> > Running add_appdata for {}".format(dir_path)) add_appdata(dir_path, owner, coprname)
def prune_project(self, project_path, username, projectname): log.info("Going to prune {}/{}".format(username, projectname)) # get ACR try: if not get_auto_createrepo_status(self.opts.frontend_base_url, username, projectname): log.debug("Skipped {}/{} since auto createrepo option is disabled" .format(username, projectname)) return except (CoprException, CoprRequestException) as exception: log.debug("Failed to get project details for {}/{} with error: {}".format( username, projectname, exception)) return # run prune project sh for sub_dir_name in os.listdir(project_path): chroot_path = os.path.join(project_path, sub_dir_name) if not os.path.isdir(chroot_path): continue try: self.prune_failed_builds(chroot_path) self.prune_obsolete_success_builds(chroot_path) except Exception as err: log.exception(err) log.error("Error during prune copr {}/{}:{}".format(username, projectname, sub_dir_name)) log.debug("Prune done for {}/{}:{}".format(username, projectname, sub_dir_name)) try: createrepo_unsafe(chroot_path) log.info("Createrepo done for copr {}/{}:{}" .format(username, projectname, sub_dir_name)) except CreateRepoError as exception: log.exception("Createrepo for copr {}/{}:{} failed with error: {}" .format(username, projectname, sub_dir_name, exception)) log.info("Prune finished for copr {}/{}".format(username, projectname))
def test_createrepo_devel_generated_commands_existing_repodata( self, mc_run_cmd_unsafe): path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock " "-s sha --checksum md5 " "--outputdir " + os.path.join(path_epel_5, "devel") + " " "--baseurl " + self.base_url + " " + path_epel_5) path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock " "--outputdir " + os.path.join(path_fedora, "devel") + " " "--baseurl " + self.base_url + " " + path_fedora) for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: os.makedirs(path) repo_path = os.path.join(path, "devel", "repodata") os.makedirs(repo_path) with open(os.path.join(repo_path, "repomd.xml"), "w") as handle: handle.write("1") createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel") assert mc_run_cmd_unsafe.call_args[0][0] == expected
def test_createrepo_generated_commands_comps_xml(self, mc_run_cmd_unsafe): path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") for path in [path_epel_5, path_fedora]: for add_comps in [True, False]: os.makedirs(path) comps_path = os.path.join(path, "comps.xml") if add_comps: with open(comps_path, "w") as handle: handle.write("1") repo_path = os.path.join(path, "repodata") os.makedirs(repo_path) with open(os.path.join(repo_path, "repomd.xml"), "w") as handle: handle.write("1") createrepo_unsafe(path) if add_comps: assert "--groupfile" in mc_run_cmd_unsafe.call_args[0][0] else: assert "--groupfile" not in mc_run_cmd_unsafe.call_args[0][0] mc_run_cmd_unsafe.mock_reset() shutil.rmtree(path, ignore_errors=True)
def prune_project(opts, path, username, projectname): log.debug("Going to prune {}/{}".format(username, projectname)) # get ACR try: if not get_auto_createrepo_status(opts.frontend_base_url, username, projectname): log.debug("Skipped {}/{} since auto createrepo option is disabled" .format(username, projectname)) return except (CoprException, CoprRequestException) as exception: log.debug("Failed to get project details for {}/{} with error: {}".format( username, projectname, exception)) return # run prune project sh days = getattr(opts, "prune_days", DEF_DAYS) cmd = map(str, [DEF_PRUNE_SCRIPT, path, days]) handle = Popen(cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = handle.communicate() if handle.returncode != 0: print("Failed to prune old builds for copr {}/{}".format(username, projectname)) print("STDOUT: \n{}".format(stdout.decode())) print("STDERR: \n{}".format(stderr.decode())) return # run createrepo log.debug("Prune done for {}/{}".format(username, projectname)) try: retcode, stdout, stderr = createrepo_unsafe(path) if retcode != 0: print("Createrepo for {}/{} failed".format(username, projectname)) print("STDOUT: \n{}".format(stdout.decode())) print("STDERR: \n{}".format(stderr.decode())) except Exception as exception: print("Createrepo for {}/{} failed with error: {}" .format(username, projectname, exception))