def delete_push_mirror(push_mirror_id: int) -> None:
mirror = PushMirror.query.filter_by(id=push_mirror_id,
is_deleted=True).first()
if mirror:
# Check if project clone exists
project_path = get_repository_path(
get_namespace_path(mirror, flask.current_app.config['USER']),
mirror)
if os.path.isdir(project_path):
shutil.rmtree(project_path)
db.session.delete(mirror)
db.session.commit()
def sync_push_mirror(push_mirror_id: int) -> None:
mirror = PushMirror.query.filter_by(id=push_mirror_id).first()
if not mirror.source:
raise Exception('Mirror {} has no source'.format(mirror.id))
if not mirror.target:
raise Exception('Mirror {} has no target'.format(mirror.id))
namespace_path = get_namespace_path(mirror, flask.current_app.config['USER'])
git_remote_source = GitRemote(mirror.source, mirror.is_force_update, mirror.is_prune_mirrors)
git_remote_target = GitRemote(mirror.target, mirror.is_force_update, mirror.is_prune_mirrors)
Git.sync_mirror(namespace_path, str(mirror.id), git_remote_source, git_remote_target)
# 5. Set last_sync date to mirror
mirror.last_sync = datetime.datetime.now()
db.session.add(mirror)
db.session.commit()
def save_pull_mirror(mirror_id: int) -> None:
mirror = PullMirror.query.filter_by(id=mirror_id).first()
gl = None # !FIXME this is here cos hotfix on the end of script
gitlab_project = None # !FIXME this is here cos hotfix on the end of script
if not mirror.is_no_create and not mirror.is_no_remote:
gl = gitlab.Gitlab(
flask.current_app.config['GITLAB_URL'],
oauth_token=mirror.user.access_token,
api_version=flask.current_app.config['GITLAB_API_VERSION'])
gl.auth()
# 0. Check if group/s exists
try:
gl.groups.get(mirror.group.gitlab_id)
except gitlab.exceptions.GitlabError as e:
if e.response_code == 404:
raise Exception('Selected group ({}) not found'.format(
mirror.group.gitlab_id))
else:
raise
# 1. check if project mirror exists in group/s if not create
# If we have project_id check if exists and use it if does, or create new project if not
if mirror.project_id:
try:
gitlab_project = gl.projects.get(mirror.project.gitlab_id)
except gitlab.exceptions.GitlabError as e:
if e.response_code == 404:
gitlab_project = None
else:
raise
# @TODO Project exists, lets check if it is in correct group ? This may not be needed if project.namespace_id bellow works
else:
gitlab_project = None
if gitlab_project:
# Update project
gitlab_project.name = mirror.project_name
gitlab_project.description = 'Mirror of {}.'.format(
mirror.project_mirror)
gitlab_project.issues_enabled = mirror.is_issues_enabled
gitlab_project.jobs_enabled = mirror.is_jobs_enabled
gitlab_project.wall_enabled = mirror.is_wall_enabled
gitlab_project.merge_requests_enabled = mirror.is_merge_requests_enabled
gitlab_project.wiki_enabled = mirror.is_wiki_enabled
gitlab_project.snippets_enabled = mirror.is_snippets_enabled
gitlab_project.visibility = mirror.visibility
gitlab_project.namespace_id = mirror.group.gitlab_id # !FIXME is this enough to move it to different group ?
gitlab_project.save()
else:
gitlab_project = gl.projects.create({
'name':
mirror.project_name,
'description':
'Mirror of {}.'.format(mirror.project_mirror),
'issues_enabled':
mirror.is_issues_enabled,
'wall_enabled':
mirror.is_wall_enabled,
'merge_requests_enabled':
mirror.is_merge_requests_enabled,
'wiki_enabled':
mirror.is_wiki_enabled,
'snippets_enabled':
mirror.is_snippets_enabled,
'visibility':
mirror.visibility,
'namespace_id':
mirror.group.gitlab_id,
'jobs_enabled':
mirror.is_jobs_enabled
})
# !FIXME BUG Trigger housekeeping right after creation to prevent ugly 404/500 project detail bug
# !FIXME BUG See https://gitlab.com/gitlab-org/gitlab-ce/issues/43825
gl.http_post('/projects/{project_id}/housekeeping'.format(
project_id=gitlab_project.id))
found_project = Project.query.filter_by(
gitlab_id=gitlab_project.id).first()
if not found_project:
found_project = Project()
found_project.gitlab_id = gitlab_project.id
found_project.name = gitlab_project.name
found_project.name_with_namespace = gitlab_project.name_with_namespace
found_project.web_url = gitlab_project.web_url
db.session.add(found_project)
db.session.commit()
mirror.project_id = found_project.id
db.session.add(mirror)
db.session.commit()
# Check deploy key exists in gitlab
key = None
if mirror.user.gitlab_deploy_key_id:
try:
key = gl.deploykeys.get(mirror.user.gitlab_deploy_key_id)
except gitlab.exceptions.GitlabError as e:
if e.response_code == 404:
key = None
else:
raise
if key:
# We got here, so key exists! lets check if its enabled for project
try:
gitlab_project.keys.get(mirror.user.gitlab_deploy_key_id)
except gitlab.exceptions.GitlabError as e:
if e.response_code == 404:
# Enable if not enabled
gitlab_project.keys.enable(
mirror.user.gitlab_deploy_key_id)
else:
raise
enabled_key = gitlab_project.keys.get(
mirror.user.gitlab_deploy_key_id)
gl.http_put(
'/projects/{project_id}/deploy_keys/{key_id}'.format(
project_id=gitlab_project.id, key_id=enabled_key.id),
post_data={'can_push': True})
# Make sure that key has can_push=True
""" !FIXME Enable when implemented
enabled_key = project.keys.get(mirror.user.gitlab_deploy_key_id)
enabled_key.can_push = True
enabled_key.save()
"""
if not key:
# No deploy key ID found, that means we need to add that key
key = gitlab_project.keys.create({
'title':
'Gitlab tools deploy key for user {}'.format(mirror.user.name),
'key':
open(
get_user_public_key_path(
mirror.user, flask.current_app.config['USER'])).read(),
'can_push':
True # We need write access
})
mirror.user.gitlab_deploy_key_id = key.id
db.session.add(mirror)
db.session.commit()
git_remote_target_original = GitRemote(gitlab_project.ssh_url_to_repo,
mirror.is_force_update,
mirror.is_prune_mirrors)
git_remote_target = GitRemote(
convert_url_for_user(gitlab_project.ssh_url_to_repo, mirror.user),
mirror.is_force_update, mirror.is_prune_mirrors)
add_ssh_config(mirror.user, flask.current_app.config['USER'],
git_remote_target.hostname,
git_remote_target_original.hostname)
else:
git_remote_target = None
namespace_path = get_namespace_path(mirror,
flask.current_app.config['USER'])
# Check if repository storage group directory exists:
if not os.path.isdir(namespace_path):
mkdir_p(namespace_path)
git_remote_source = GitRemote(mirror.source, mirror.is_force_update,
mirror.is_prune_mirrors)
Git.create_mirror(namespace_path, str(mirror.id), git_remote_source,
git_remote_target)
if gl and gitlab_project:
# !FIXME BUG Trigger housekeeping right after mirror sync to reload homepage of project
# !FIXME BUG Somehow i'm unable to reproduce this in simple script :/ to report this bug
gl.http_post('/projects/{project_id}/housekeeping'.format(
project_id=gitlab_project.id))
# 5. Set last_sync date to mirror
mirror.target = git_remote_target.url
mirror.last_sync = datetime.datetime.now()
db.session.add(mirror)
db.session.commit()
def save_push_mirror(push_mirror_id) -> None:
mirror = PushMirror.query.filter_by(id=push_mirror_id).first()
gl = gitlab.Gitlab(
flask.current_app.config['GITLAB_URL'],
oauth_token=mirror.user.access_token,
api_version=flask.current_app.config['GITLAB_API_VERSION'])
gl.auth()
# 0. Check if project exists
gitlab_project = gl.projects.get(mirror.project.gitlab_id)
if not gitlab_project:
raise Exception('Selected group ({}) not found'.format(
mirror.project.gitlab_id))
found_project = Project.query.filter_by(
gitlab_id=gitlab_project.id).first()
if not found_project:
found_project = Project()
found_project.gitlab_id = gitlab_project.id
found_project.name = gitlab_project.name
found_project.name_with_namespace = gitlab_project.name_with_namespace
found_project.web_url = gitlab_project.web_url
db.session.add(found_project)
db.session.commit()
# Check deploy key exists in gitlab
key = None
if mirror.user.gitlab_deploy_key_id:
try:
key = gl.deploykeys.get(mirror.user.gitlab_deploy_key_id)
except gitlab.exceptions.GitlabError as e:
if e.response_code == 404:
key = None
else:
raise
if key:
# We got here, so key exists! lets check if its enabled for project
try:
gitlab_project.keys.get(mirror.user.gitlab_deploy_key_id)
except gitlab.exceptions.GitlabError as e:
if e.response_code == 404:
# Enable if not enabled
gitlab_project.keys.enable(
mirror.user.gitlab_deploy_key_id)
else:
raise
if not key:
# No deploy key ID found, that means we need to add that key
key = gitlab_project.keys.create({
'title':
'Gitlab tools deploy key for user {}'.format(mirror.user.name),
'key':
open(
get_user_public_key_path(
mirror.user, flask.current_app.config['USER'])).read()
})
mirror.user.gitlab_deploy_key_id = key.id
db.session.add(mirror)
db.session.commit()
# Create hook
gitlab_project.hooks.create({
'url':
flask.url_for('api.index.schedule_sync_push_mirror',
mirror_id=mirror.id,
token=mirror.hook_token,
_external=True),
'push_events':
True,
'tag_push_events':
True
})
git_remote_source_original = GitRemote(gitlab_project.ssh_url_to_repo,
mirror.is_force_update,
mirror.is_prune_mirrors)
git_remote_source = GitRemote(
convert_url_for_user(gitlab_project.ssh_url_to_repo, mirror.user),
mirror.is_force_update, mirror.is_prune_mirrors)
add_ssh_config(mirror.user, flask.current_app.config['USER'],
git_remote_source.hostname,
git_remote_source_original.hostname)
namespace_path = get_namespace_path(mirror,
flask.current_app.config['USER'])
# Check if repository storage group directory exists:
if not os.path.isdir(namespace_path):
mkdir_p(namespace_path)
git_remote_target = GitRemote(mirror.target, mirror.is_force_update,
mirror.is_prune_mirrors)
Git.create_mirror(namespace_path, str(mirror.id), git_remote_source,
git_remote_target)
# 5. Set last_sync date to mirror
mirror.source = git_remote_source_original.url
mirror.last_sync = datetime.datetime.now()
db.session.add(mirror)
db.session.commit()
