Exemplo n.º 1
0
# MAIN PART
#

if args.dryrun:
    print("Running in simulation mode\n")

gitlab_lib.debug("Setting up work queue")

if args.project:
    queue.put(gitlab_lib.get_project(args.project))
    delete_old_jobs(queue)
else:
    for project in gitlab_lib.get_projects():
        queue.put(project)

    gitlab_lib.debug("Processing work queue")

    for _ in range(args.number):
        processes.append(gitlab_lib.create_process(delete_old_jobs, (queue, )))

    # wait for processes to finish the work
    while queue.qsize() > 0:
        time.sleep(5)

        # check if a process crashed and must be restarted
        if len(processes) < int(
                args.number) and queue.qsize() > len(processes):
            gitlab_lib.debug("Starting new process")
            processes.append(
                gitlab_lib.create_process(delete_old_jobs, (queue, )))
Exemplo n.º 2
0
                gitlab_lib.log(u"Backing up %s from project %s [ID %s]" % (component, project['name'], project['id']))
                dump(backup_dir,
                     component + ".json",
                     gitlab_lib.fetch(api_url % (gitlab_lib.API_URL, project['id'])))

            else:
                gitlab_lib.log("Component %s disabled for project %s [ID %s]" % (component, project['name'], project['id']))


#
# MAIN PART
#

OUTPUT_BASEDIR = args.output or "."
queue = Queue()

if not os.path.exists(OUTPUT_BASEDIR):
    os.mkdir(OUTPUT_BASEDIR)

# Backup metadata of a single user
if args.user:
    backup_user_metadata(args.user)

# Backup all projects or only the projects of a single user
for project in gitlab_lib.get_projects(args.user, personal=True):
    queue.put(project)

# Start processes and let em backup every project
for process in range(int(args.number)):
    gitlab_lib.create_process(backup, (args.repository, queue))
Exemplo n.º 3
0
# Lookup metadata of destination project
project_data = gitlab_lib.get_project_metadata(args.project)

if not project_data or len(project_data) == 0:
    gitlab_lib.log("Cannot find project " + args.project)
    sys.exit(1)

if len(project_data) > 1:
    gitlab_lib.log("Found more then one project for " + args.project)
    sys.exit(1)

# Restore only one component?
if args.component:
    fill_restore_queue(project_data[0], args.component)

# Restore all
else:
    for component in gitlab_lib.PROJECT_COMPONENTS.keys():
        fill_restore_queue(project_data[0], component)

# spawn some processes to do the actual restore
nr_of_processes = args.number

if queue.qsize() < args.number:
    nr_of_processes = queue.qsize()

map(lambda _: gitlab_lib.create_process(restore_entry,
                                        (project_data[0], queue)),
    range(int(nr_of_processes)))
Exemplo n.º 4
0
    if os.path.exists(backup_archive):
        gitlab_lib.log("Restoring repository " + backup_archive)
        gitlab_lib.restore_repository(backup_archive, args.repository,
                                      args.project, ".wiki.git")

# Restore only one component?
if args.component:
    fill_restore_queue(project_data, args.component)

# Restore all (but issues at the end, they link to lots of other components)
else:
    for component in filter(lambda x: x != "issues",
                            gitlab_lib.PROJECT_COMPONENTS.keys()):
        fill_restore_queue(project_data, component)

    fill_restore_queue(project_data, "issues")

# spawn some processes to do the actual restore
nr_of_processes = int(args.number)

if work_queue.qsize() < nr_of_processes:
    nr_of_processes = work_queue.qsize()

for process in range(nr_of_processes):
    processes.append(
        gitlab_lib.create_process(gitlab_lib.restore,
                                  (args.backup_dir, project_data, work_queue)))

sys.exit(0)
Exemplo n.º 5
0
        if not gitlab_lib.core.QUIET: sys.stdout.write(".")
        work_queue.put(project)

if not gitlab_lib.core.QUIET: sys.stdout.write("\n")

if work_queue.qsize() == 0:
    gitlab_lib.error("Cannot find any projects to backup!")
else:
    nr_of_jobs = work_queue.qsize()

    if nr_of_processes > nr_of_jobs:
        nr_of_processes = nr_of_jobs

    # Start processes and let em backup every project
    for process in range(nr_of_processes):
        processes.append( gitlab_lib.create_process(gitlab_lib.backup, (work_queue, result_queue, args.output, args.archive)) )

    # Check if a process died and must be restarted
    while result_queue.qsize() < nr_of_jobs -1:
        gitlab_lib.debug("Work queue size: %d Result queue size: %d Nr of jobs: %d " % (work_queue.qsize(), result_queue.qsize(), nr_of_jobs))

        for (i, process) in enumerate(processes):
            if not process.is_alive():
                gitlab_lib.debug("Found dead process")
                del processes[i]

        if len(processes) < int(args.number) and work_queue.qsize() > len(processes):
            gitlab_lib.debug("Starting new process")
            processes.append( gitlab_lib.create_process(gitlab_lib.backup, (work_queue, result_queue, args.output, args.archive)) )

        time.sleep(10)