def check_hostname_fingerprint(): json_data = flask.request.get_json() if not json_data: return flask.jsonify({'message': 'No data'}), 400 url = json_data.get('url') if not url: return flask.jsonify({'message': 'Required parameter URL is missing'}), 400 detected_protocol = GitRemote.detect_vcs_protocol(url) if detected_protocol == ProtocolEnum.SSH: parsed_url = urllib.parse.urlparse(url) hostname = parsed_url.hostname if not hostname: parsed_url = GitRemote.parse_scp_like_url(url) if parsed_url: hostname = parsed_url['hostname'] else: hostname = url hostname_string = str(hostname) elif detected_protocol in [ProtocolEnum.HTTP, ProtocolEnum.HTTPS]: parsed_url = urllib.parse.urlparse(url) if parsed_url.hostname: hostname_string = str(parsed_url.hostname) else: hostname_string = url else: return flask.jsonify({ 'message': 'Unknown protocol', }), 400 return check_fingerprint_hostname(hostname_string)
def edit_mirror(mirror_id: int): mirror_detail = PushMirror.query.filter_by( id=mirror_id, user=current_user).first_or_404() form = EditForm(flask.request.form, id=mirror_detail.id, project_mirror=mirror_detail.project_mirror, note=mirror_detail.note, is_force_update=mirror_detail.is_force_update, is_prune_mirrors=mirror_detail.is_prune_mirrors, project=mirror_detail.project.gitlab_id) if flask.request.method == 'POST' and form.validate(): project_mirror_str = form.project_mirror.data.strip() project_mirror = GitRemote(project_mirror_str) target = GitRemote(project_mirror_str) if target.vcs_protocol == ProtocolEnum.SSH: # If protocol is SSH we need to convert URL to use USER RSA pair target = GitRemote( convert_url_for_user(project_mirror_str, current_user)) # PullMirror mirror_detail.project_mirror = project_mirror_str mirror_detail.project = process_project(form.project.data) # Mirror mirror_detail.is_force_update = form.is_force_update.data mirror_detail.is_prune_mirrors = form.is_prune_mirrors.data mirror_detail.is_deleted = False mirror_detail.user = current_user mirror_detail.foreign_vcs_type = target.vcs_type mirror_detail.note = form.note.data mirror_detail.target = target mirror_detail.source = None # We are getting source wia gitlab API db.session.add(mirror_detail) db.session.commit() if target.vcs_protocol == ProtocolEnum.SSH: # If source is SSH, create SSH Config for it also task_result = chain( create_ssh_config.si(current_user.id, target.hostname, project_mirror.hostname), save_push_mirror.si(mirror_detail.id)).apply_async() parent = log_task_pending(task_result.parent, mirror_detail, create_ssh_config, InvokedByEnum.MANUAL) log_task_pending(task_result, mirror_detail, save_push_mirror, InvokedByEnum.MANUAL, parent) else: task = save_push_mirror.delay(mirror_detail.id) log_task_pending(task, mirror_detail, save_push_mirror, InvokedByEnum.MANUAL) flask.flash('Push mirror was saved successfully.', 'success') return flask.redirect(flask.url_for('push_mirror.index.get_mirror')) return flask.render_template('push_mirror.index.edit.html', form=form, mirror_detail=mirror_detail)
def new_mirror(): form = NewForm(flask.request.form, is_force_update=False, is_prune_mirrors=False) if flask.request.method == 'POST' and form.validate(): project_mirror_str = form.project_mirror.data.strip() project_mirror = GitRemote(project_mirror_str) target = GitRemote(project_mirror_str) if target.vcs_protocol == ProtocolEnum.SSH: # If protocol is SSH we need to convert URL to use USER RSA pair target = GitRemote( convert_url_for_user(project_mirror_str, current_user)) mirror_new = PushMirror() # PushMirror mirror_new.project_mirror = project_mirror_str mirror_new.project = process_project(form.project.data) # Mirror mirror_new.is_force_update = form.is_force_update.data mirror_new.is_prune_mirrors = form.is_prune_mirrors.data mirror_new.is_deleted = False mirror_new.user = current_user mirror_new.foreign_vcs_type = target.vcs_type mirror_new.note = form.note.data mirror_new.target = target.url mirror_new.source = None # We are getting source wia gitlab API mirror_new.last_sync = None mirror_new.hook_token = random_password() db.session.add(mirror_new) db.session.commit() if target.vcs_protocol == ProtocolEnum.SSH: # If target is SSH, create SSH Config for it also task_result = chain( create_ssh_config.si(current_user.id, target.hostname, project_mirror.hostname), save_push_mirror.si(mirror_new.id)).apply_async() parent = log_task_pending(task_result.parent, mirror_new, create_ssh_config, InvokedByEnum.MANUAL) log_task_pending(task_result, mirror_new, save_push_mirror, InvokedByEnum.MANUAL, parent) else: task = save_push_mirror.delay(mirror_new.id) log_task_pending(task, mirror_new, save_push_mirror, InvokedByEnum.MANUAL) flask.flash('New push mirror item was added successfully.', 'success') return flask.redirect(flask.url_for('push_mirror.index.get_mirror')) return flask.render_template('push_mirror.index.new.html', form=form)
def validate(self) -> bool: rv = Form.validate(self) if not rv: return False project_name_exists = PullMirror.query.filter_by( project_name=self.project_name.data).first() if project_name_exists: self.project_name.errors.append( gettext('Project name %(project_name)s already exists.', project_name=self.project_name.data)) return False project_mirror_exists = PullMirror.query.filter_by( project_mirror=self.project_mirror.data).first() if project_mirror_exists: self.project_mirror.errors.append( gettext('Project mirror %(project_mirror)s already exists.', project_mirror=self.project_mirror.data)) return False if not GitRemote.detect_vcs_type(self.project_mirror.data): self.project_mirror.errors.append( gettext('Unknown VCS type or detection failed.')) return False if not self.group.data: self.group.errors.append( gettext( 'You have to select GitLab group where mirrored project will be created.' )) return False try: check_project_visibility_in_group(self.visibility.data, self.group.data) except VisibilityError as e: self.visibility.errors.append(gettext(str(e))) return False if check_project_exists(self.project_name.data, self.group.data): if not self.is_force_create.data: self.project_name.errors.append( gettext( 'Project with name %(project_name)s already exists in selected group and "Create project in GitLab even if it already exists." is not checked in "Advanced options"', project_name=self.project_name.data)) return False if self.periodic_sync.data: try: ExpressionDescriptor(self.periodic_sync.data, throw_exception_on_parse_error=True) except (MissingFieldException, FormatException): self.periodic_sync.errors.append( gettext('Wrong cron expression.')) return True
def sync_push_mirror(push_mirror_id: int) -> None: mirror = PushMirror.query.filter_by(id=push_mirror_id).first() if not mirror.source: raise Exception('Mirror {} has no source'.format(mirror.id)) if not mirror.target: raise Exception('Mirror {} has no target'.format(mirror.id)) namespace_path = get_namespace_path(mirror, flask.current_app.config['USER']) git_remote_source = GitRemote(mirror.source, mirror.is_force_update, mirror.is_prune_mirrors) git_remote_target = GitRemote(mirror.target, mirror.is_force_update, mirror.is_prune_mirrors) Git.sync_mirror(namespace_path, str(mirror.id), git_remote_source, git_remote_target) # 5. Set last_sync date to mirror mirror.last_sync = datetime.datetime.now() db.session.add(mirror) db.session.commit()
def convert_url_for_user(url: str, user: User) -> str: """ Converts url hostname of url to user identified type for SSH config :param url: Url to :param user: User to use :return: Returns modified URL """ hostname = GitRemote.get_url_hostname(url) return url.replace(hostname, '{}_{}'.format(hostname, user.id), 1)
def validate(self) -> bool: rv = Form.validate(self) if not rv: return False project_name_exists = PullMirror.query.filter( PullMirror.project_name == self.project_name.data, PullMirror.id != self.id.data).first() if project_name_exists: self.project_name.errors.append( gettext('Project name %(project_name)s already exists.', project_name=self.project_name.data)) return False project_mirror_exists = PullMirror.query.filter( PullMirror.project_mirror == self.project_mirror.data, PullMirror.id != self.id.data).first() if project_mirror_exists: self.project_mirror.errors.append( gettext('Project mirror %(project_mirror)s already exists.', project_mirror=self.project_mirror.data)) return False if not GitRemote.detect_vcs_type(self.project_mirror.data): self.project_mirror.errors.append( gettext('Unknown VCS type or detection failed.')) return False try: check_project_visibility_in_group(self.visibility.data, self.group.data) except VisibilityError as e: self.visibility.errors.append(gettext(str(e))) return False if self.periodic_sync.data: try: ExpressionDescriptor(self.periodic_sync.data, throw_exception_on_parse_error=True) except (MissingFieldException, FormatException): self.periodic_sync.errors.append( gettext('Wrong cron expression.')) return True
def validate(self) -> bool: rv = Form.validate(self) if not rv: return False project_mirror_exists = PushMirror.query.filter_by( project_mirror=self.project_mirror.data).first() if project_mirror_exists: self.project_mirror.errors.append( gettext('Project mirror %(project_mirror)s already exists.', project_mirror=self.project_mirror.data)) return False if not GitRemote.detect_vcs_type(self.project_mirror.data): self.project_mirror.errors.append( gettext('Unknown VCS type or detection failed.')) return False return True
def edit_mirror(mirror_id: int): mirror_detail = PullMirror.query.filter_by(id=mirror_id, user=current_user).first_or_404() form = EditForm( flask.request.form, id=mirror_detail.id, project_name=mirror_detail.project_name, project_mirror=mirror_detail.project_mirror, note=mirror_detail.note, is_no_create=mirror_detail.is_no_create, is_force_create=mirror_detail.is_force_create, is_no_remote=mirror_detail.is_no_remote, is_issues_enabled=mirror_detail.is_issues_enabled, is_wall_enabled=mirror_detail.is_wall_enabled, is_wiki_enabled=mirror_detail.is_wiki_enabled, is_snippets_enabled=mirror_detail.is_snippets_enabled, is_merge_requests_enabled=mirror_detail.is_merge_requests_enabled, visibility=mirror_detail.visibility, is_force_update=mirror_detail.is_force_update, is_prune_mirrors=mirror_detail.is_prune_mirrors, group=mirror_detail.group.gitlab_id, periodic_sync=mirror_detail.periodic_sync, is_jobs_enabled=mirror_detail.is_jobs_enabled ) if flask.request.method == 'POST' and form.validate(): project_mirror_str = form.project_mirror.data.strip() project_mirror = GitRemote(project_mirror_str) source = GitRemote(project_mirror_str) if source.vcs_protocol == ProtocolEnum.SSH: # If protocol is SSH we need to convert URL to use USER RSA pair source = GitRemote(convert_url_for_user(project_mirror_str, current_user)) # PullMirror mirror_detail.project_name = form.project_name.data mirror_detail.project_mirror = project_mirror_str mirror_detail.is_no_create = form.is_no_create.data mirror_detail.is_force_create = form.is_force_create.data mirror_detail.is_no_remote = form.is_no_remote.data mirror_detail.is_issues_enabled = form.is_issues_enabled.data mirror_detail.is_wall_enabled = form.is_wall_enabled.data mirror_detail.is_wiki_enabled = form.is_wiki_enabled.data mirror_detail.is_snippets_enabled = form.is_snippets_enabled.data mirror_detail.is_jobs_enabled = form.is_jobs_enabled.data mirror_detail.is_merge_requests_enabled = form.is_merge_requests_enabled.data mirror_detail.visibility = form.visibility.data mirror_detail.group = process_group(form.group.data) mirror_detail.periodic_sync = form.periodic_sync.data # Mirror mirror_detail.is_force_update = form.is_force_update.data mirror_detail.is_prune_mirrors = form.is_prune_mirrors.data mirror_detail.is_deleted = False mirror_detail.user = current_user mirror_detail.foreign_vcs_type = source.vcs_type mirror_detail.note = form.note.data mirror_detail.target = None # We are getting target wia gitlab API mirror_detail.source = source.url if process_cron_expression(mirror_detail): PeriodicTasks.changed() db.session.add(mirror_detail) db.session.flush() db.session.commit() if source.vcs_protocol == ProtocolEnum.SSH: # If source is SSH, create SSH Config for it also task_result = chain( create_ssh_config.si( current_user.id, source.hostname, project_mirror.hostname ), save_pull_mirror.si( mirror_detail.id ) ).apply_async() parent = log_task_pending(task_result.parent, mirror_detail, create_ssh_config, InvokedByEnum.MANUAL) log_task_pending(task_result, mirror_detail, save_pull_mirror, InvokedByEnum.MANUAL, parent) else: task = save_pull_mirror.delay(mirror_detail.id) log_task_pending(task, mirror_detail, save_pull_mirror, InvokedByEnum.MANUAL) flask.flash('Pull mirror was saved successfully.', 'success') return flask.redirect(flask.url_for('pull_mirror.index.get_mirror')) return flask.render_template('pull_mirror.index.edit.html', form=form, mirror_detail=mirror_detail)
def new_mirror(): form = NewForm( flask.request.form, is_no_create=False, is_force_create=False, is_no_remote=False, is_issues_enabled=False, is_wall_enabled=False, is_wiki_enabled=False, is_snippets_enabled=False, is_merge_requests_enabled=False, visibility=PullMirror.VISIBILITY_PRIVATE, is_force_update=False, is_prune_mirrors=False, is_jobs_enabled=True ) if flask.request.method == 'POST' and form.validate(): project_mirror_str = form.project_mirror.data.strip() project_mirror = GitRemote(project_mirror_str) source = GitRemote(project_mirror_str) if source.vcs_protocol == ProtocolEnum.SSH: # If protocol is SSH we need to convert URL to use USER RSA pair source = GitRemote(convert_url_for_user(project_mirror_str, current_user)) mirror_new = PullMirror() # PullMirror mirror_new.project_name = form.project_name.data mirror_new.project_mirror = project_mirror_str mirror_new.is_no_create = form.is_no_create.data mirror_new.is_force_create = form.is_force_create.data mirror_new.is_no_remote = form.is_no_remote.data mirror_new.is_issues_enabled = form.is_issues_enabled.data mirror_new.is_wall_enabled = form.is_wall_enabled.data mirror_new.is_wiki_enabled = form.is_wiki_enabled.data mirror_new.is_snippets_enabled = form.is_snippets_enabled.data mirror_new.is_jobs_enabled = form.is_jobs_enabled.data mirror_new.is_merge_requests_enabled = form.is_merge_requests_enabled.data mirror_new.visibility = form.visibility.data mirror_new.group = process_group(form.group.data) mirror_new.periodic_sync = form.periodic_sync.data # Mirror mirror_new.is_force_update = form.is_force_update.data mirror_new.is_prune_mirrors = form.is_prune_mirrors.data mirror_new.is_deleted = False mirror_new.user = current_user mirror_new.foreign_vcs_type = source.vcs_type mirror_new.note = form.note.data mirror_new.target = None # We are getting target wia gitlab API mirror_new.source = source.url mirror_new.last_sync = None mirror_new.hook_token = random_password() if process_cron_expression(mirror_new): PeriodicTasks.changed() db.session.add(mirror_new) db.session.commit() if source.vcs_protocol == ProtocolEnum.SSH: # If source is SSH, create SSH Config for it also task_result = chain( create_ssh_config.si( current_user.id, source.hostname, project_mirror.hostname ), save_pull_mirror.si( mirror_new.id ) ).apply_async() parent = log_task_pending(task_result.parent, mirror_new, create_ssh_config, InvokedByEnum.MANUAL) log_task_pending(task_result, mirror_new, save_pull_mirror, InvokedByEnum.MANUAL, parent) else: task = save_pull_mirror.delay(mirror_new.id) log_task_pending(task, mirror_new, save_pull_mirror, InvokedByEnum.MANUAL) flask.flash('New pull mirror item was added successfully.', 'success') return flask.redirect(flask.url_for('pull_mirror.index.get_mirror')) return flask.render_template('pull_mirror.index.new.html', form=form)
def save_pull_mirror(mirror_id: int) -> None: mirror = PullMirror.query.filter_by(id=mirror_id).first() gl = None # !FIXME this is here cos hotfix on the end of script gitlab_project = None # !FIXME this is here cos hotfix on the end of script if not mirror.is_no_create and not mirror.is_no_remote: gl = gitlab.Gitlab( flask.current_app.config['GITLAB_URL'], oauth_token=mirror.user.access_token, api_version=flask.current_app.config['GITLAB_API_VERSION']) gl.auth() # 0. Check if group/s exists try: gl.groups.get(mirror.group.gitlab_id) except gitlab.exceptions.GitlabError as e: if e.response_code == 404: raise Exception('Selected group ({}) not found'.format( mirror.group.gitlab_id)) else: raise # 1. check if project mirror exists in group/s if not create # If we have project_id check if exists and use it if does, or create new project if not if mirror.project_id: try: gitlab_project = gl.projects.get(mirror.project.gitlab_id) except gitlab.exceptions.GitlabError as e: if e.response_code == 404: gitlab_project = None else: raise # @TODO Project exists, lets check if it is in correct group ? This may not be needed if project.namespace_id bellow works else: gitlab_project = None if gitlab_project: # Update project gitlab_project.name = mirror.project_name gitlab_project.description = 'Mirror of {}.'.format( mirror.project_mirror) gitlab_project.issues_enabled = mirror.is_issues_enabled gitlab_project.jobs_enabled = mirror.is_jobs_enabled gitlab_project.wall_enabled = mirror.is_wall_enabled gitlab_project.merge_requests_enabled = mirror.is_merge_requests_enabled gitlab_project.wiki_enabled = mirror.is_wiki_enabled gitlab_project.snippets_enabled = mirror.is_snippets_enabled gitlab_project.visibility = mirror.visibility gitlab_project.namespace_id = mirror.group.gitlab_id # !FIXME is this enough to move it to different group ? gitlab_project.save() else: gitlab_project = gl.projects.create({ 'name': mirror.project_name, 'description': 'Mirror of {}.'.format(mirror.project_mirror), 'issues_enabled': mirror.is_issues_enabled, 'wall_enabled': mirror.is_wall_enabled, 'merge_requests_enabled': mirror.is_merge_requests_enabled, 'wiki_enabled': mirror.is_wiki_enabled, 'snippets_enabled': mirror.is_snippets_enabled, 'visibility': mirror.visibility, 'namespace_id': mirror.group.gitlab_id, 'jobs_enabled': mirror.is_jobs_enabled }) # !FIXME BUG Trigger housekeeping right after creation to prevent ugly 404/500 project detail bug # !FIXME BUG See https://gitlab.com/gitlab-org/gitlab-ce/issues/43825 gl.http_post('/projects/{project_id}/housekeeping'.format( project_id=gitlab_project.id)) found_project = Project.query.filter_by( gitlab_id=gitlab_project.id).first() if not found_project: found_project = Project() found_project.gitlab_id = gitlab_project.id found_project.name = gitlab_project.name found_project.name_with_namespace = gitlab_project.name_with_namespace found_project.web_url = gitlab_project.web_url db.session.add(found_project) db.session.commit() mirror.project_id = found_project.id db.session.add(mirror) db.session.commit() # Check deploy key exists in gitlab key = None if mirror.user.gitlab_deploy_key_id: try: key = gl.deploykeys.get(mirror.user.gitlab_deploy_key_id) except gitlab.exceptions.GitlabError as e: if e.response_code == 404: key = None else: raise if key: # We got here, so key exists! lets check if its enabled for project try: gitlab_project.keys.get(mirror.user.gitlab_deploy_key_id) except gitlab.exceptions.GitlabError as e: if e.response_code == 404: # Enable if not enabled gitlab_project.keys.enable( mirror.user.gitlab_deploy_key_id) else: raise enabled_key = gitlab_project.keys.get( mirror.user.gitlab_deploy_key_id) gl.http_put( '/projects/{project_id}/deploy_keys/{key_id}'.format( project_id=gitlab_project.id, key_id=enabled_key.id), post_data={'can_push': True}) # Make sure that key has can_push=True """ !FIXME Enable when implemented enabled_key = project.keys.get(mirror.user.gitlab_deploy_key_id) enabled_key.can_push = True enabled_key.save() """ if not key: # No deploy key ID found, that means we need to add that key key = gitlab_project.keys.create({ 'title': 'Gitlab tools deploy key for user {}'.format(mirror.user.name), 'key': open( get_user_public_key_path( mirror.user, flask.current_app.config['USER'])).read(), 'can_push': True # We need write access }) mirror.user.gitlab_deploy_key_id = key.id db.session.add(mirror) db.session.commit() git_remote_target_original = GitRemote(gitlab_project.ssh_url_to_repo, mirror.is_force_update, mirror.is_prune_mirrors) git_remote_target = GitRemote( convert_url_for_user(gitlab_project.ssh_url_to_repo, mirror.user), mirror.is_force_update, mirror.is_prune_mirrors) add_ssh_config(mirror.user, flask.current_app.config['USER'], git_remote_target.hostname, git_remote_target_original.hostname) else: git_remote_target = None namespace_path = get_namespace_path(mirror, flask.current_app.config['USER']) # Check if repository storage group directory exists: if not os.path.isdir(namespace_path): mkdir_p(namespace_path) git_remote_source = GitRemote(mirror.source, mirror.is_force_update, mirror.is_prune_mirrors) Git.create_mirror(namespace_path, str(mirror.id), git_remote_source, git_remote_target) if gl and gitlab_project: # !FIXME BUG Trigger housekeeping right after mirror sync to reload homepage of project # !FIXME BUG Somehow i'm unable to reproduce this in simple script :/ to report this bug gl.http_post('/projects/{project_id}/housekeeping'.format( project_id=gitlab_project.id)) # 5. Set last_sync date to mirror mirror.target = git_remote_target.url mirror.last_sync = datetime.datetime.now() db.session.add(mirror) db.session.commit()
def save_push_mirror(push_mirror_id) -> None: mirror = PushMirror.query.filter_by(id=push_mirror_id).first() gl = gitlab.Gitlab( flask.current_app.config['GITLAB_URL'], oauth_token=mirror.user.access_token, api_version=flask.current_app.config['GITLAB_API_VERSION']) gl.auth() # 0. Check if project exists gitlab_project = gl.projects.get(mirror.project.gitlab_id) if not gitlab_project: raise Exception('Selected group ({}) not found'.format( mirror.project.gitlab_id)) found_project = Project.query.filter_by( gitlab_id=gitlab_project.id).first() if not found_project: found_project = Project() found_project.gitlab_id = gitlab_project.id found_project.name = gitlab_project.name found_project.name_with_namespace = gitlab_project.name_with_namespace found_project.web_url = gitlab_project.web_url db.session.add(found_project) db.session.commit() # Check deploy key exists in gitlab key = None if mirror.user.gitlab_deploy_key_id: try: key = gl.deploykeys.get(mirror.user.gitlab_deploy_key_id) except gitlab.exceptions.GitlabError as e: if e.response_code == 404: key = None else: raise if key: # We got here, so key exists! lets check if its enabled for project try: gitlab_project.keys.get(mirror.user.gitlab_deploy_key_id) except gitlab.exceptions.GitlabError as e: if e.response_code == 404: # Enable if not enabled gitlab_project.keys.enable( mirror.user.gitlab_deploy_key_id) else: raise if not key: # No deploy key ID found, that means we need to add that key key = gitlab_project.keys.create({ 'title': 'Gitlab tools deploy key for user {}'.format(mirror.user.name), 'key': open( get_user_public_key_path( mirror.user, flask.current_app.config['USER'])).read() }) mirror.user.gitlab_deploy_key_id = key.id db.session.add(mirror) db.session.commit() # Create hook gitlab_project.hooks.create({ 'url': flask.url_for('api.index.schedule_sync_push_mirror', mirror_id=mirror.id, token=mirror.hook_token, _external=True), 'push_events': True, 'tag_push_events': True }) git_remote_source_original = GitRemote(gitlab_project.ssh_url_to_repo, mirror.is_force_update, mirror.is_prune_mirrors) git_remote_source = GitRemote( convert_url_for_user(gitlab_project.ssh_url_to_repo, mirror.user), mirror.is_force_update, mirror.is_prune_mirrors) add_ssh_config(mirror.user, flask.current_app.config['USER'], git_remote_source.hostname, git_remote_source_original.hostname) namespace_path = get_namespace_path(mirror, flask.current_app.config['USER']) # Check if repository storage group directory exists: if not os.path.isdir(namespace_path): mkdir_p(namespace_path) git_remote_target = GitRemote(mirror.target, mirror.is_force_update, mirror.is_prune_mirrors) Git.create_mirror(namespace_path, str(mirror.id), git_remote_source, git_remote_target) # 5. Set last_sync date to mirror mirror.source = git_remote_source_original.url mirror.last_sync = datetime.datetime.now() db.session.add(mirror) db.session.commit()