# MAIN PART # if args.dryrun: print("Running in simulation mode\n") gitlab_lib.debug("Setting up work queue") if args.project: queue.put(gitlab_lib.get_project(args.project)) delete_old_jobs(queue) else: for project in gitlab_lib.get_projects(): queue.put(project) gitlab_lib.debug("Processing work queue") for _ in range(args.number): processes.append(gitlab_lib.create_process(delete_old_jobs, (queue, ))) # wait for processes to finish the work while queue.qsize() > 0: time.sleep(5) # check if a process crashed and must be restarted if len(processes) < int( args.number) and queue.qsize() > len(processes): gitlab_lib.debug("Starting new process") processes.append( gitlab_lib.create_process(delete_old_jobs, (queue, )))
gitlab_lib.log(u"Backing up %s from project %s [ID %s]" % (component, project['name'], project['id'])) dump(backup_dir, component + ".json", gitlab_lib.fetch(api_url % (gitlab_lib.API_URL, project['id']))) else: gitlab_lib.log("Component %s disabled for project %s [ID %s]" % (component, project['name'], project['id'])) # # MAIN PART # OUTPUT_BASEDIR = args.output or "." queue = Queue() if not os.path.exists(OUTPUT_BASEDIR): os.mkdir(OUTPUT_BASEDIR) # Backup metadata of a single user if args.user: backup_user_metadata(args.user) # Backup all projects or only the projects of a single user for project in gitlab_lib.get_projects(args.user, personal=True): queue.put(project) # Start processes and let em backup every project for process in range(int(args.number)): gitlab_lib.create_process(backup, (args.repository, queue))
# Lookup metadata of destination project project_data = gitlab_lib.get_project_metadata(args.project) if not project_data or len(project_data) == 0: gitlab_lib.log("Cannot find project " + args.project) sys.exit(1) if len(project_data) > 1: gitlab_lib.log("Found more then one project for " + args.project) sys.exit(1) # Restore only one component? if args.component: fill_restore_queue(project_data[0], args.component) # Restore all else: for component in gitlab_lib.PROJECT_COMPONENTS.keys(): fill_restore_queue(project_data[0], component) # spawn some processes to do the actual restore nr_of_processes = args.number if queue.qsize() < args.number: nr_of_processes = queue.qsize() map(lambda _: gitlab_lib.create_process(restore_entry, (project_data[0], queue)), range(int(nr_of_processes)))
if os.path.exists(backup_archive): gitlab_lib.log("Restoring repository " + backup_archive) gitlab_lib.restore_repository(backup_archive, args.repository, args.project, ".wiki.git") # Restore only one component? if args.component: fill_restore_queue(project_data, args.component) # Restore all (but issues at the end, they link to lots of other components) else: for component in filter(lambda x: x != "issues", gitlab_lib.PROJECT_COMPONENTS.keys()): fill_restore_queue(project_data, component) fill_restore_queue(project_data, "issues") # spawn some processes to do the actual restore nr_of_processes = int(args.number) if work_queue.qsize() < nr_of_processes: nr_of_processes = work_queue.qsize() for process in range(nr_of_processes): processes.append( gitlab_lib.create_process(gitlab_lib.restore, (args.backup_dir, project_data, work_queue))) sys.exit(0)
if not gitlab_lib.core.QUIET: sys.stdout.write(".") work_queue.put(project) if not gitlab_lib.core.QUIET: sys.stdout.write("\n") if work_queue.qsize() == 0: gitlab_lib.error("Cannot find any projects to backup!") else: nr_of_jobs = work_queue.qsize() if nr_of_processes > nr_of_jobs: nr_of_processes = nr_of_jobs # Start processes and let em backup every project for process in range(nr_of_processes): processes.append( gitlab_lib.create_process(gitlab_lib.backup, (work_queue, result_queue, args.output, args.archive)) ) # Check if a process died and must be restarted while result_queue.qsize() < nr_of_jobs -1: gitlab_lib.debug("Work queue size: %d Result queue size: %d Nr of jobs: %d " % (work_queue.qsize(), result_queue.qsize(), nr_of_jobs)) for (i, process) in enumerate(processes): if not process.is_alive(): gitlab_lib.debug("Found dead process") del processes[i] if len(processes) < int(args.number) and work_queue.qsize() > len(processes): gitlab_lib.debug("Starting new process") processes.append( gitlab_lib.create_process(gitlab_lib.backup, (work_queue, result_queue, args.output, args.archive)) ) time.sleep(10)