def backup_user_metadata(username):
"""
Backup all metadata including email addresses and SSH keys of a single user
"""
user = gitlab_lib.get_user_metadata(username)
backup_dir = os.path.join(OUTPUT_BASEDIR, "user_%s_%s" % (user['id'], user['username']))
if not os.path.exists(backup_dir): os.mkdir(backup_dir)
gitlab_lib.log(u"Backing up metadata of user %s [ID %s]" % (user["username"], user["id"]))
dump(backup_dir, "user.json", user)
dump(backup_dir, "projects.json", gitlab_lib.get_projects(username))
dump(backup_dir, "ssh.json", gitlab_lib.fetch(gitlab_lib.USER_SSHKEYS % (gitlab_lib.API_URL, user["id"])))
dump(backup_dir, "email.json", gitlab_lib.fetch(gitlab_lib.USER_EMAILS % (gitlab_lib.API_URL, user["id"])))
def backup_issues(api_url, project, token, backup_dir):
"""
Backup all issues of a project
issue notes must be backuped by additional api call
"""
gitlab_lib.log(u"Backing up issues from project %s [ID %s]" % (project['name'], project['id']))
issues = gitlab_lib.fetch(api_url % (gitlab_lib.API_URL, project['id']))
dump(backup_dir, "issues.json", issues)
for issue in issues:
notes = gitlab_lib.fetch(gitlab_lib.NOTES_FOR_ISSUE % (gitlab_lib.API_URL, project['id'], issue['id']))
if notes:
dump(backup_dir, "issue_%d_notes.dump" % (issue['id'],), notes)
def backup_snippets(api_url, project, backup_dir):
"""
Backup snippets and their contents
snippet contents must be backuped by additional api call
"""
gitlab_lib.log(u"Backing up snippets from project %s [ID %s]" % (project['name'], project['id']))
snippets = gitlab_lib.fetch(api_url % (gitlab_lib.API_URL, project['id']))
dump(backup_dir, "snippets.json", snippets)
for snippet in snippets:
dump(backup_dir,
"snippet_%d_content.dump" % (snippet['id'],),
gitlab_lib.rest_api_call(gitlab_lib.GET_SNIPPET_CONTENT % (gitlab_lib.API_URL, project['id'], snippet['id']), method="GET").text)
notes = gitlab_lib.fetch(gitlab_lib.NOTES_FOR_SNIPPET % (gitlab_lib.API_URL, project['id'], snippet['id']))
if notes:
dump(backup_dir, "snippet_%d_notes.dump" % (snippet['id'],), notes)
def backup(repository_dir, queue):
"""
Backup everything for the given project
For every project create a dictionary with id_name as pattern
Dump project metadata and each component as separate JSON files
"""
while not queue.empty():
project = queue.get()
backup_dir = os.path.join(OUTPUT_BASEDIR, "%s_%s_%s" % (project['id'], project['namespace']['name'], project['name']))
if not os.path.exists(backup_dir): os.mkdir(backup_dir)
dump(backup_dir, "project.json", project)
# shall we backup local data like repository and wiki?
if repository_dir:
backup_local_data(repository_dir, args.upload, backup_dir, project)
# backup metadata of each component
for (component, api_url) in gitlab_lib.PROJECT_COMPONENTS.items():
# issues
if component == "issues" and \
project.get(component + "_enabled") == True:
backup_issues(api_url, project, args.token, backup_dir)
# snippets
elif component == "snippets" and \
project.get(component + "_enabled") == True:
backup_snippets(api_url, project, backup_dir)
# milestones are enabled if either issues or merge_requests are enabled
# labels cannot be disabled therefore no labels_enabled field exists
# otherwise check if current component is enabled in project
elif component == "milestones" and \
(project.get("issues_enabled") == True or project.get("merge_requests_enabled") == True) or \
component == "labels" or \
project.get(component + "_enabled") == True:
gitlab_lib.log(u"Backing up %s from project %s [ID %s]" % (component, project['name'], project['id']))
dump(backup_dir,
component + ".json",
gitlab_lib.fetch(api_url % (gitlab_lib.API_URL, project['id'])))
else:
gitlab_lib.log("Component %s disabled for project %s [ID %s]" % (component, project['name'], project['id']))
def get_objects(obj, obj_id):
"""
Returns a list of all gitlab objects
Obj can be projects or users or groups etc
"""
objects = []
chunk_size = 100
page = 1
url = ""
suffix = ""
if obj == "projects":
suffix = "/all"
if obj_id:
try:
url = "%s/%s/%d" % (gitlab_lib.API_URL, obj, int(obj_id))
except ValueError:
if obj == 'users':
url = "%s/%s?username=%s" % (gitlab_lib.API_URL, obj, obj_id)
else:
url = "%s/%s?search=%s" % (gitlab_lib.API_URL, obj, obj_id)
else:
url = "%s/%s%s?per_page=%d&page=%d" % (gitlab_lib.API_URL, obj, suffix, chunk_size, page)
while 1:
buff = gitlab_lib.fetch(url)
if buff and type(buff) == list:
objects.extend(buff)
page += 1
elif buff:
objects.append(buff)
if obj_id or not buff:
break
return objects
def get_objects(obj, obj_id):
"""
Returns a list of all gitlab objects
Obj can be projects or users or groups etc
"""
objects = []
chunk_size = 100
page = 1
url = ""
suffix = ""
while 1:
if obj_id:
try:
url = "%s/%s/%d" % (gitlab_lib.API_BASE_URL, obj, int(obj_id))
except ValueError:
if obj == 'users':
url = "%s/%s?username=%s" % (gitlab_lib.API_BASE_URL, obj, obj_id)
else:
url = "%s/%s?search=%s" % (gitlab_lib.API_BASE_URL, obj, obj_id)
else:
url = "%s/%s%s?per_page=%d&page=%d" % (gitlab_lib.API_BASE_URL, obj, suffix, chunk_size, page)
buff = gitlab_lib.fetch(url)
page += 1
if buff and type(buff) == list:
objects.extend(buff)
elif buff:
objects.append(buff)
if obj_id or not buff:
break
return objects
def get_property(obj, obj_id, obj_property):
metadata = gitlab_lib.fetch("%s/%s/%d" % (gitlab_lib.API_URL, obj, obj_id))
return metadata.get(obj_property)
parser.add_argument("-s", "--server", help="Gitlab server name", default=backup_config.SERVER)
parser.add_argument("-t", "--token", help="Private token", default=backup_config.TOKEN)
args = parser.parse_args()
if not args.server or not args.token or not args.project:
print "You must at least specify --server, --token and --project"
sys.exit(1)
#
# MAIN PART
#
projects = []
try:
projects.append(gitlab_lib.fetch(gitlab_lib.PROJECT_METADATA % (int(args.project),)))
except ValueError:
projects = [x for x in gitlab_lib.get_projects() \
if args.project in (x['name_with_namespace'] or x['description'])]
if len(projects) == 0:
print "Found nothing"
else:
for project in projects:
if args.details:
keys = project.keys()
keys.sort()
for key in keys:
print u"%s: %s" % (key, project[key])
print u"-" * 79
