def perform_migrate_roadmap(args): redmine = RedmineClient(args.redmine_key) gitlab = GitlabClient(args.gitlab_key) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) checks = [ (check_no_milestone, 'Gitlab project has no pre-existing milestone'), (check_origin_milestone, 'Redmine project contains versions'), ] for i in checks: check( *i, redmine_project=redmine_project, gitlab_project=gitlab_project) versions = redmine_project.get_versions() versions_data = (convert_version(i) for i in versions) for data, meta in versions_data: if args.check: log.info("Would create version {}".format(data)) else: created = gitlab_project.create_milestone(data, meta) log.info("Version {}".format(created['title']))
def perform_migrate_roadmap(args): redmine = RedmineClient(args.redmine_key, args.no_verify) gitlab = GitlabClient(args.gitlab_key, args.no_verify) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) checks = [ #(check_no_milestone, 'Gitlab project has no pre-existing milestone'), (check_origin_milestone, 'Redmine project contains versions'), ] for i in checks: check( *i, redmine_project=redmine_project, gitlab_project=gitlab_project) versions = redmine_project.get_versions() if args.filter: name_filter = re.compile(args.filter) versions = [version for version in versions if name_filter.match(version['name'])] versions_data = (convert_version(i) for i in versions) for data, meta in versions_data: if args.check: log.info("Would create version {}".format(data)) else: created = gitlab_project.create_milestone(data, meta) log.info("Version {}".format(created['title']))
def setUp(self): self.client = FakeGitlabClient() self.project_1 = GitlabProject( 'http://localhost:3000/diaspora/diaspora-project-site', self.client) self.project_2 = GitlabProject( 'http://localhost:3000/brightbox/puppet', self.client)
def perform_migrate_iid(args): """ Should occur after the issues migration """ # access gitlab database with # gitlab-rails dbconsole gitlab = GitlabClient(args.gitlab_key, args.no_verify) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_project_id = gitlab_project.get_id() regex_saved_iid = r'-RM-([0-9]+)-MR-(.*)' sql_cmd = sql.COUNT_UNMIGRATED_ISSUES.format( regex=regex_saved_iid, project_id=gitlab_project_id) output = sql.run_query(sql_cmd) try: m = re.match(r'\s*(\d+)\s*', output, re.DOTALL | re.MULTILINE) issues_count = int(m.group(1)) except (AttributeError, ValueError): raise ValueError( 'Invalid output from postgres command: "{}"'.format(output)) if issues_count > 0: log.info('Ready to recover iid for {} issues.'.format( issues_count)) else: log.error( "No issue to migrate iid, possible causes: " "you already migrated iid or you haven't migrated issues yet.") exit(1) if not args.check: # first we change the iid to large values to prevent a # duplicate key value violates unique_constraint "index_issues_on_project_id_and_iid" # KEY (project_id, iid)=(37, 83) already exists sql_cmd1 = sql.UPDATE_IID_ISSUES.format( regex=regex_saved_iid, project_id=gitlab_project_id) out1 = sql.run_query(sql_cmd1) sql_cmd2 = sql.MIGRATE_IID_ISSUES.format( regex=regex_saved_iid, project_id=gitlab_project_id) out2 = sql.run_query(sql_cmd2) try: m = re.match( r'\s*(\d+)\s*', output, re.DOTALL | re.MULTILINE) migrated_count = int(m.group(1)) log.info('Migrated successfully iid for {} issues'.format( migrated_count)) except (IndexError, AttributeError): raise ValueError( 'Invalid output from postgres command: "{}"'.format(output))
def perform_delete_issues(args): """ Delete all issues in the gitlab repo """ gitlab = GitlabClient(args.gitlab_key, args.no_verify) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_issues = gitlab_project.get_issues() log.debug('Got {} issue(s) from gitlab.'.format(len(gitlab_issues))) for issue in gitlab_issues: log.debug('delete issue {}'.format(issue['id'])) gitlab_project.delete_issue(issue['iid'])
def perform_migrate_iid(args): """ Shoud occur after the issues migration """ gitlab = GitlabClient(args.gitlab_key) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_project_id = gitlab_project.get_id() regex_saved_iid = r'-RM-([0-9]+)-MR-(.*)' sql_cmd = sql.COUNT_UNMIGRATED_ISSUES.format( regex=regex_saved_iid, project_id=gitlab_project_id) output = sql.run_query(sql_cmd) try: m = re.match(r'\s*(\d+)\s*', output, re.DOTALL | re.MULTILINE) issues_count = int(m.group(1)) except (AttributeError, ValueError): raise ValueError( 'Invalid output from postgres command: "{}"'.format(output)) if issues_count > 0: log.info('Ready to recover iid for {} issues.'.format( issues_count)) else: log.error( "No issue to migrate iid, possible causes: " "you already migrated iid or you haven't migrated issues yet.") exit(1) if not args.check: sql_cmd = sql.MIGRATE_IID_ISSUES.format( regex=regex_saved_iid, project_id=gitlab_project_id) out = sql.run_query(sql_cmd) try: m = re.match( r'\s*(\d+)\s*', output, re.DOTALL | re.MULTILINE) migrated_count = int(m.group(1)) log.info('Migrated successfully iid for {} issues'.format( migrated_count)) except (IndexError, AttributeError): raise ValueError( 'Invalid output from postgres command: "{}"'.format(output))
class GitlabprojectTestCase(unittest.TestCase): def setUp(self): self.client = FakeGitlabClient() self.project_1 = GitlabProject( 'http://localhost:3000/diaspora/diaspora-project-site', self.client) self.project_2 = GitlabProject( 'http://localhost:3000/brightbox/puppet', self.client) def test_check_repository_empty(self): self.assertEqual(self.project_1.is_repository_empty(), False) self.assertEqual(self.project_2.is_repository_empty(), True) def test_issues(self): self.assertEqual(len(self.project_1.get_issues()), 2) self.assertEqual(len(self.project_2.get_issues()), 0) def test_members(self): self.assertEqual( self.project_1.has_members(['john_smith', 'jack_smith']), True) self.assertEqual( self.project_1.has_members( ['john_smith', 'jack_smith', 'macha_smith']), False) self.assertEqual(self.project_1.has_members([]), True)
def perform_migrate_issues(args): closed_states = [] if (args.closed_states): closed_states = args.closed_states.split(',') custom_fields = [] if (args.custom_fields): custom_fields = args.custom_fields.split(',') redmine = RedmineClient(args.redmine_key, args.no_verify) gitlab = GitlabClient(args.gitlab_key, args.no_verify) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_instance = gitlab_project.get_instance() gitlab_users_index = gitlab_instance.get_users_index() redmine_users_index = redmine_project.get_users_index() milestones_index = gitlab_project.get_milestones_index() log.debug('GitLab milestones are: {}'.format(', '.join(milestones_index) + ' ')) # get issues log.info('Getting redmine issues') issues = redmine_project.get_all_issues() # convert issues log.info('Converting issues') issues_data = (convert_issue(args.redmine_key, i, redmine_users_index, gitlab_users_index, milestones_index, closed_states, custom_fields) for i in issues) # create issues log.info('Creating gitlab issues') for data, meta in issues_data: if args.check: milestone_id = data.get('milestone_id', None) if milestone_id: try: gitlab_project.get_milestone_by_id(milestone_id) except ValueError: raise CommandError( "issue \"{}\" points to unknown milestone_id \"{}\". " "Check that you already migrated roadmaps".format( data['title'], milestone_id)) log.info('Would create issue "{}" and {} notes.'.format( data['title'], len(meta['notes']))) else: try: created = gitlab_project.create_issue(data, meta) log.info('#{iid} {title}'.format(**created)) except: log.info('create issue "{}" failed'.format(data['title'])) raise
class GitlabprojectTestCase(unittest.TestCase): def setUp(self): self.client = FakeGitlabClient() self.project_1 = GitlabProject( 'http://localhost:3000/diaspora/diaspora-project-site', self.client) self.project_2 = GitlabProject( 'http://localhost:3000/brightbox/puppet', self.client) def test_check_repository_empty(self): self.assertEqual(self.project_1.is_repository_empty(), False) self.assertEqual(self.project_2.is_repository_empty(), True) def test_issues(self): self.assertEqual(len(self.project_1.get_issues()), 2) self.assertEqual(len(self.project_2.get_issues()), 0) def test_members(self): self.assertEqual( self.project_1.has_members(['john_smith', 'jack_smith']), True) self.assertEqual( self.project_1.has_members( ['john_smith', 'jack_smith', 'macha_smith']), False) self.assertEqual( self.project_1.has_members([]), True)
def perform_migrate_labels(args): redmine = RedmineClient(args.redmine_key, args.no_verify) gitlab = GitlabClient(args.gitlab_key, args.no_verify) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) pid = redmine_project.get_id() init_db() trackers, statuses, priorities, categories = project_labels(pid) labels = [] for category in categories: labels.append({ "name": category, 'color': "#44AD8E", 'description': category + ' --' }) for tracker in trackers: labels.append({"name": tracker, 'color': "#428BCA"}) for status in statuses: labels.append({"name": status, 'color': "#7F8C8D"}) for priority in priorities: labels.append({"name": priority, 'color': "#F0AD4E"}) for data in labels: if args.check: log.info("Would create label {}".format(data)) else: created = gitlab_project.create_label(data) log.info("Label {}".format(created['name']))
def perform_migrate_issues(args): redmine = RedmineClient(args.redmine_key) gitlab = GitlabClient(args.gitlab_key) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_instance = gitlab_project.get_instance() gitlab_users_index = gitlab_instance.get_users_index() redmine_users_index = redmine_project.get_users_index() checks = [ (check_users, 'Required users presence'), (check_no_issue, 'Project has no pre-existing issue'), ] for i in checks: check( *i, redmine_project=redmine_project, gitlab_project=gitlab_project) # Get issues issues = redmine_project.get_all_issues() milestones_index = gitlab_project.get_milestones_index() issues_data = ( convert_issue( i, redmine_users_index, gitlab_users_index, milestones_index) for i in issues) for data, meta in issues_data: if args.check: milestone_id = data.get('milestone_id', None) if milestone_id: try: gitlab_project.get_milestone_by_id(milestone_id) except ValueError: raise CommandError( "issue \"{}\" points to unknown milestone_id \"{}\". " "Check that you already migrated roadmaps".format( data['title'], milestone_id)) log.info('Would create issue "{}" and {} notes.'.format( data['title'], len(meta['notes']))) else: created = gitlab_project.create_issue(data, meta) log.info('#{iid} {title}'.format(**created))
def perform_migrate_issues(args): closed_states = [] if (args.closed_states): closed_states = args.closed_states.split(',') custom_fields = [] if (args.custom_fields): custom_fields = args.custom_fields.split(',') if (args.user_dict is not None): load_user_dict(args.user_dict) redmine = RedmineClient(args.redmine_key, args.no_verify) gitlab = GitlabClient(args.gitlab_key, args.no_verify) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_instance = gitlab_project.get_instance() if (args.project_members_only): gitlab_users_index = gitlab_project.get_members_index() else: gitlab_users_index = gitlab_instance.get_users_index() redmine_users_index = redmine_project.get_users_index() milestones_index = gitlab_project.get_milestones_index() textile_converter = TextileConverter() log.debug('GitLab milestones are: {}'.format(', '.join(milestones_index) + ' ')) # get issues log.info('Getting redmine issues') issues = redmine_project.get_all_issues() if args.initial_id: issues = [issue for issue in issues if int(args.initial_id) <= issue['id']] # convert issues log.info('Converting issues') issues_data = ( convert_issue(args.redmine_key, i, redmine_users_index, gitlab_users_index, milestones_index, closed_states, custom_fields, textile_converter, args.keep_id or args.keep_title, args.sudo) for i in issues) # create issues log.info('Creating gitlab issues') last_iid = int(args.initial_id or 1) - 1 for data, meta, redmine_id in issues_data: if args.check: milestone_id = data.get('milestone_id', None) if milestone_id: try: gitlab_project.get_milestone_by_id(milestone_id) except ValueError: raise CommandError( "issue \"{}\" points to unknown milestone_id \"{}\". " "Check that you already migrated roadmaps".format( data['title'], milestone_id)) log.info('Would create issue "{}" and {} notes.'.format( data['title'], len(meta['notes']))) else: if args.keep_id: try: fake_meta = {'uploads': [], 'notes': [], 'must_close': False} if args.sudo: fake_meta['sudo_user'] = meta['sudo_user'] while redmine_id > last_iid + 1: created = gitlab_project.create_issue({'title': 'fake'}, fake_meta) last_iid = created['iid'] gitlab_project.delete_issue(created['iid']) log.info('#{iid} {title}'.format(**created)) except: log.info('create issue "{}" failed'.format('fake')) raise try: created = gitlab_project.create_issue(data, meta) last_iid = created['iid'] log.info('#{iid} {title}'.format(**created)) except: log.info('create issue "{}" failed'.format(data['title'])) raise
def perform_migrate_roadmap(args): redmine = RedmineClient(args.redmine_key, args.no_verify) gitlab = GitlabClient(args.gitlab_key, args.no_verify) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) # auto create milestones milestones_data = [] if args.auto_milestones: match = re.match("^(monthly|yearly):(\d{4}(-[01]\d)?)$", args.auto_milestones) if not match: log.error( "--auto-milestone must be formed as (monthly|yearly):<start_date>" ) exit(1) milestones_type = match.group(1) milestones_date = list(map(lambda x: int(x), match.group(2).split("-"))) if milestones_type == "monthly" and len(milestones_date) == 1: milestones_date.append(1) log.info("Creating %s milestones, from %s" % (milestones_type, "-".join( map(lambda x: str(x).zfill(2), milestones_date)))) day = date(milestones_date[0], milestones_date[1], 1) today = date.today() while day <= today: month_days = monthrange(day.year, day.month)[1] milestone = { "title": day.strftime("%Y-%m"), "description": "\n\n*(auto-generated milestone)*", "start_date": day.strftime("%Y-%m-%d"), "due_date": day.replace(day=month_days).strftime("%Y-%m-%d"), } must_close = True if day > today - timedelta(days=4 * 31): # keep last 4 months open must_close = False meta = { "must_close": must_close, } milestones_data.append((milestone, meta)) # next month day = day + timedelta(days=month_days) checks = [ #(check_no_milestone, 'Gitlab project has no pre-existing milestone'), (check_origin_milestone, 'Redmine project contains versions'), ] for i in checks: check(*i, redmine_project=redmine_project, gitlab_project=gitlab_project) versions = redmine_project.get_versions() versions_data = [convert_version(i) for i in versions] if milestones_data: versions_titles = [] for data, meta in versions_data: versions_titles.append(data['title']) for data, meta in milestones_data: if data['title'] in versions_titles: continue versions_data.append((data, meta)) for data, meta in versions_data: if args.check: log.info("Would create version {}".format(data)) else: created = gitlab_project.create_milestone(data, meta) log.info("Version {}".format(created['title']))
def perform_migrate_issues(args): closed_states = [] if (args.closed_states): closed_states = args.closed_states.split(',') custom_fields = [] if (args.custom_fields): custom_fields = args.custom_fields.split(',') if (args.user_dict is not None): load_user_dict(args.user_dict) redmine = RedmineClient(args.redmine_key, args.no_verify) gitlab = GitlabClient(args.gitlab_key, args.no_verify) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_instance = gitlab_project.get_instance() if (args.project_members_only): gitlab_users_index = gitlab_project.get_members_index() else: gitlab_users_index = gitlab_instance.get_users_index() redmine_users_index = redmine_project.get_users_index() milestones_index = gitlab_project.get_milestones_index() textile_converter = TextileConverter() log.debug('GitLab milestones are: {}'.format(', '.join(milestones_index) + ' ')) # get issues log.info('Getting redmine issues') issues = redmine_project.get_all_issues() if args.initial_id: issues = [issue for issue in issues if int(args.initial_id) <= issue['id']] if args.filter: filters = args.filter.split(",") filters = [filter.split("=") for filter in filters] filters = [[field.split("."), re.compile(value)] for field, value in filters] issues = [issue for issue in issues if all( value.match(reduce( lambda object, key: object.get(key) if object is not None else None, field, issue )) for field, value in filters)] # convert issues log.info('Converting issues') issues_data = ( convert_issue(args.redmine_key, i, redmine_users_index, gitlab_users_index, milestones_index, closed_states, custom_fields, textile_converter, args.keep_id or args.keep_title, args.sudo, args.archive_acc, args.migrate_version_as_label) for i in issues) # create issues log.info('Creating gitlab issues') last_iid = int(args.initial_id or 1) - 1 for data, meta, redmine_id in issues_data: if args.check: milestone_id = data.get('milestone_id', None) if milestone_id: try: gitlab_project.get_milestone_by_id(milestone_id) except ValueError: raise CommandError( "issue \"{}\" points to unknown milestone_id \"{}\". " "Check that you already migrated roadmaps".format( data['title'], milestone_id)) log.info('Would create issue "{}" and {} notes.'.format( data['title'], len(meta['notes']))) else: if args.keep_id: try: fake_meta = {'uploads': [], 'notes': [], 'must_close': False} if args.sudo: fake_meta['sudo_user'] = args.archive_acc while redmine_id > last_iid + 1: created = gitlab_project.create_issue({'title': 'fake'}, fake_meta, gitlab.get_auth_headers()) last_iid = created['iid'] gitlab_project.delete_issue(created['iid']) log.info('#{iid} {title}'.format(**created)) except: log.info('create issue "{}" failed'.format('fake')) raise try: created = gitlab_project.create_issue(data, meta, gitlab.get_auth_headers()) last_iid = created['iid'] log.info('#{iid} {title}'.format(**created)) except: log.info('create issue "{}" failed'.format(data['title'])) raise
def perform_migrate_issues(args): closed_states = [] if (args.closed_states): closed_states = args.closed_states.split(',') custom_fields = [] if (args.custom_fields): custom_fields = args.custom_fields.split(',') if (args.user_dict is not None): load_user_dict(args.user_dict) redmine = RedmineClient(args.redmine_key, args.no_verify) gitlab = GitlabClient(args.gitlab_key, args.no_verify) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_instance = gitlab_project.get_instance() if (args.project_members_only): gitlab_users_index = gitlab_project.get_members_index() else: gitlab_users_index = gitlab_instance.get_users_index() redmine_users_index = redmine_project.get_users_index(args.issue_ids) milestones_index = gitlab_project.get_milestones_index() if args.no_textile: textile_converter = NopConverter() else: textile_converter = TextileConverter() log.debug('GitLab milestones are: {}'.format(', '.join(milestones_index) + ' ')) # get issues log.info('Getting redmine issues') issues = redmine_project.get_issues(args.issue_ids) if args.initial_id: issues = [ issue for issue in issues if int(args.initial_id) <= issue['id'] ] # convert issues log.info('Converting issues') issues_data = (convert_issue(args.redmine_key, i, redmine_users_index, gitlab_users_index, milestones_index, closed_states, custom_fields, textile_converter, args.keep_id or args.keep_title, args.sudo) for i in issues) # create issues log.info('Creating gitlab issues') last_iid = int(args.initial_id or 1) - 1 for data, meta, redmine_id in issues_data: if args.check: milestone_id = data.get('milestone_id', None) if milestone_id: try: gitlab_project.get_milestone_by_id(milestone_id) except ValueError: raise CommandError( "issue \"{}\" points to unknown milestone_id \"{}\". " "Check that you already migrated roadmaps".format( data['title'], milestone_id)) log.info('Would create issue "{}" and {} notes.'.format( data['title'], len(meta['notes']))) else: if args.keep_id: data['iid'] = redmine_id try: created = gitlab_project.create_issue(data, meta) last_iid = created['iid'] log.info('#{iid} {title}'.format(**created)) except: log.info('create issue "{}" failed'.format(data['title'])) raise
def perform_list_gitlab_users(args): gitlab = GitlabClient(args.gitlab_key) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_instance = gitlab_project.get_instance() users = gitlab_instance.get_all_users() print(users)
def perform_migrate_issues(args): init_db() closed_states = [] if (args.closed_states): closed_states = args.closed_states.split(',') custom_fields = [] if (args.custom_fields): custom_fields = args.custom_fields.split(',') if (args.user_dict is not None): load_user_dict(args.user_dict) if (args.user_keys is not None): load_user_keys(args.user_keys) redmine = RedmineClient(args.redmine_key, args.no_verify) gitlab = GitlabClient(args.gitlab_key, args.no_verify) redmine_project = RedmineProject(args.redmine_project_url, redmine) gitlab_project = GitlabProject(args.gitlab_project_url, gitlab) gitlab_instance = gitlab_project.get_instance() if (args.project_members_only): gitlab_users_index = gitlab_project.get_members_index() else: gitlab_users_index = gitlab_instance.get_users_index() redmine_users_index = redmine_project.get_users_index() milestones_index = gitlab_project.get_milestones_index() textile_converter = TextileConverter() log.debug('GitLab milestones are: {}'.format(', '.join(milestones_index) + ' ')) # get issues log.info('Getting redmine issues') issues = redmine_project.get_all_issues() if args.initial_id: issues = [ issue for issue in issues if int(args.initial_id) <= issue['id'] ] if args.max_id: issues = [issue for issue in issues if int(args.max_id) >= issue['id']] # convert issues log.info('Converting issues') issues_data = (convert_issue(args.redmine_key, i, redmine_users_index, gitlab_users_index, milestones_index, closed_states, custom_fields, textile_converter, args.keep_id or args.keep_title, args.sudo) for i in issues) # create issues log.info('Creating gitlab issues') last_iid = int(args.initial_id or 1) - 1 for data, meta, redmine_id in issues_data: if args.check: milestone_id = data.get('milestone_id', None) if milestone_id: try: gitlab_project.get_milestone_by_id(milestone_id) except ValueError: raise CommandError( "issue \"{}\" points to unknown milestone_id \"{}\". " "Check that you already migrated roadmaps".format( data['title'], milestone_id)) log.info('Would create issue "{}" and {} notes.'.format( data['title'], len(meta['notes']))) log.info('Labels %s' % meta.get('labels', [])) log.info('Tags %s' % meta.get('tags', [])) log.info('Watchers %s' % meta.get('watchers', [])) else: if args.keep_id: try: fake_meta = { 'uploads': [], 'notes': [], 'must_close': False } if args.sudo: fake_meta['sudo_user'] = meta['sudo_user'] while redmine_id > last_iid + 1: created = gitlab_project.create_issue( {'title': 'fake'}, fake_meta) last_iid = created['iid'] gitlab_project.delete_issue(created['id']) log.info('#{iid} {title}'.format(**created)) except: log.info('create issue "{}" failed'.format('fake')) raise try: # labels for label in meta.get('labels', []): gitlab_project.create_label(label) # tags for tag in meta.get('tags', []): gitlab_project.create_label(tag) # issue created = gitlab_project.create_issue(data, meta) last_iid = created['iid'] # watchers for watcher in meta.get('watchers', []): created_watcher = gitlab_project.create_watcher( watcher.get('data', {}), watcher, created['iid']) log.info('#{iid} {title}'.format(**created)) except: log.info('create issue "{}" failed'.format(data['title'])) raise