def main():
config_file = os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf")
pruner = Pruner(BackendConfigReader(config_file).read())
try:
pruner.run()
except Exception as e:
logexception(e)
def main():
config_file = os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf")
config = BackendConfigReader(config_file).read()
client = Client({"copr_url": config.frontend_base_url})
for ownername in os.listdir(config.destdir):
ownerpath = os.path.join(config.destdir, ownername)
for projectname in os.listdir(ownerpath):
projectpath = os.path.join(ownerpath, projectname)
# It may be a good idea, to not DoS attack the frontend
# Set whatever number of seconds is necessary
time.sleep(0)
# If a project doesn't exist in frontend, it should be removed
try:
client.project_proxy.get(ownername=ownername,
projectname=projectname)
except CoprNoResultException:
print(projectpath)
continue
# If a chroot is not enabled in the project, it should be removed
for chroot in os.listdir(projectpath):
if chroot in ["srpm-builds", "modules"]:
continue
if not is_outdated_to_be_deleted(
get_chroot_safe(client, ownername, projectname,
chroot)):
continue
print(os.path.join(projectpath, chroot))
def get_managed_vms_names():
result = []
if VmManager:
opts = BackendConfigReader().read()
vmm = VmManager(opts, log)
result.extend(vmd.vm_name.lower() for vmd in vmm.get_all_vm())
return result
def main():
opts = BackendConfigReader().read()
with open(args.coprs_file_path) as coprs_file:
for copr_full_name in coprs_file:
try:
fix_copr(opts, copr_full_name.strip())
except Exception as e:
log.exception(str(e))
def main():
# shutil.rmtree("/tmp/users_failed.txt", ignore_errors=True)
# shutil.rmtree("/tmp/users_done.txt", ignore_errors=True)
users_done_old = set()
try:
with open("/tmp/users_done.txt") as handle:
for line in handle:
users_done_old.add(line.strip())
except Exception as err:
log.exception(err)
log.debug("error during read old users done")
opts = BackendConfigReader().read()
log.info("Starting pubkey fill, destdir: {}".format(opts.destdir))
log.debug("list dir: {}".format(os.listdir(opts.destdir)))
for user_name in os.listdir(opts.destdir):
if user_name in users_done_old:
log.info("skipping user: {}".format(user_name))
continue
failed = False
log.info("Started processing user dir: {}".format(user_name))
user_dir = os.path.join(opts.destdir, user_name)
for project_name in os.listdir(user_dir):
log.info("Checking project dir: {}".format(project_name))
try:
get_pubkey(user_name, project_name)
log.info("Key-pair exists for {}/{}".format(
user_name, project_name))
except CoprSignNoKeyError:
create_user_keys(user_name, project_name, opts)
log.info("Created new key-pair for {}/{}".format(
user_name, project_name))
except Exception as err:
log.error(
"Failed to get pubkey for {}/{}, mark as failed, skipping")
log.exception(err)
failed = True
continue
project_dir = os.path.join(user_dir, project_name)
pubkey_path = os.path.join(project_dir, "pubkey.gpg")
if not check_signed_rpms(project_dir, user_name, project_name,
opts):
failed = False
if not check_pubkey(pubkey_path, user_name, project_name, opts):
failed = False
if failed:
with open("/tmp/users_failed.txt", "a") as handle:
handle.write("{}\n".format(user_name))
else:
with open("/tmp/users_done.txt", "a") as handle:
handle.write("{}\n".format(user_name))
def get_managed_vms():
result = {}
if VmManager:
opts = BackendConfigReader().read()
vmm = VmManager(opts, log)
for vmd in vmm.get_all_vm():
result[vmd.vm_name.lower()] = {
'unused': vmd.state == VmStates.READY,
}
return result
def main():
opts = BackendConfigReader().read()
conn = get_redis_connection(opts)
key = CONSECUTIVE_FAILURE_REDIS_KEY
value = int(conn.get(key) or 0)
if value > opts.consecutive_failure_threshold:
print("Critical")
sys.exit(2)
elif value > int(0.5 * opts.consecutive_failure_threshold):
print("Warning")
sys.exit(1)
else:
print("OK")
sys.exit(0)
def main():
opts = BackendConfigReader().read()
conn = StrictRedis() # connecting to default local redis instance
key = CONSECUTIVE_FAILURE_REDIS_KEY
value = int(conn.get(key) or 0)
if value > opts.consecutive_failure_threshold:
print("Critical")
sys.exit(2)
elif value > int(0.5 * opts.consecutive_failure_threshold):
print("Warning")
sys.exit(1)
else:
print("OK")
sys.exit(0)
def main(args):
parser = SortedOptParser("copr_create_repo")
parser.add_option("-u",
"--user",
dest="user",
help="copr project owner username")
parser.add_option("-p",
"--project",
dest="project",
help="copr project name")
parser.add_option("-f",
"--front_url",
dest="front_url",
help="copr frontend url")
cli_opts, args = parser.parse_args(args)
if not cli_opts.user:
print("No user was specified, exiting", file=sys.stderr)
sys.exit(1)
if not cli_opts.project:
print("No project was specified, exiting", file=sys.stderr)
sys.exit(1)
opts = BackendConfigReader().read()
front_url = cli_opts.front_url or opts.frontend_base_url
project_path = os.path.join(opts.destdir, cli_opts.user, cli_opts.project)
log.info("start processing {}/{}".format(cli_opts.user, cli_opts.project))
for subdir in os.listdir(project_path):
if os.path.isdir(subdir):
path = os.path.join(project_path, subdir)
log.info("entering dir: {}".format(subdir))
createrepo(path=path,
front_url=front_url,
username=cli_opts.user,
projectname=cli_opts.project)
log.info("done dir: {}".format(subdir))
log.info("finished processing {}/{}".format(cli_opts.user,
cli_opts.project))
def main():
config_file = os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf")
opts = BackendConfigReader(config_file).read()
results_dir = opts.destdir
log.info("Pruning results dir: {} ".format(results_dir))
user_dir_names, user_dirs = list_subdir(results_dir)
print("Going to process total number: {} of user's directories".format(len(user_dir_names)))
log.info("Going to process user's directories: {}".format(user_dir_names))
counter = 0
for username, subpath in zip(user_dir_names, user_dirs):
log.debug("For user `{}` exploring path: {}".format(username, subpath))
for projectname, project_path in zip(*list_subdir(subpath)):
log.debug("Exploring project `{}` with path: {}".format(projectname, project_path))
prune_project(opts, project_path, username, projectname)
counter += 1
print("Pruned {}. projects".format(counter))
print("Pruning finished")
def main():
config_file = os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf")
config = BackendConfigReader(config_file).read()
client = Client({"copr_url": config.frontend_base_url})
for ownername in os.listdir(config.destdir):
ownerpath = os.path.join(config.destdir, ownername)
try:
for projectname in os.listdir(ownerpath):
projectpath = os.path.join(ownerpath, projectname)
# I don't know how to determine whether a PR dir can be deleted or not
# just ommit the logic for the time being.
if ":pr:" in projectname:
continue
# It may be a good idea, to not DoS attack the frontend
# Set whatever number of seconds is necessary
time.sleep(0)
# If a project doesn't exist in frontend, it should be removed
try:
client.project_proxy.get(ownername=ownername, projectname=projectname)
except CoprNoResultException:
print(projectpath)
continue
# If a chroot is not enabled in the project, it should be removed
for chroot in os.listdir(projectpath):
if chroot in ["srpm-builds", "modules", "repodata", "pubkey.gpg"]:
continue
if not is_outdated_to_be_deleted(get_chroot_safe(client, ownername, projectname, chroot)):
continue
print(os.path.join(projectpath, chroot))
except NotADirectoryError as ex:
print(str(ex))
def main():
opts = BackendConfigReader().read()
vmm = VmManager(opts, None)
print(vmm.info())
#!/usr/bin/python
# coding: utf-8
NUM_QUEUES = 2
import sys
sys.path.append("/usr/share/copr/")
from retask.task import Task
from retask.queue import Queue
from backend.helpers import BackendConfigReader
opts = BackendConfigReader().read()
redis_config = {
'host': opts['redis_host'],
'port': opts['redis_port'],
'db': opts['redis_db'],
}
for i in range(0, NUM_QUEUES):
print("## Queue {}".format(i))
q = Queue("copr-be-{}".format(i), config=redis_config)
q.connect()
save_q = []
while q.length != 0:
task = q.dequeue()
print task.data
save_q.append(task)
for t in save_q:
q.enqueue(t)
def main():
config_file = os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf")
pruner = Pruner(BackendConfigReader(config_file).read())
pruner.run()
