def main(argv=None): ''' Process the command line arguments and create the JSON dump. :param argv: List of arguments, as if specified on the command-line. If None, ``sys.argv[1:]`` is used instead. :type argv: list of str ''' # Get command line arguments parser = argparse.ArgumentParser( description="Export all users/issues from GitLab to JIRA JSON format.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, conflict_handler='resolve') parser.add_argument('gitlab_url', help='The full URL to your GitLab instance.') parser.add_argument('-d', '--date_filter', help='Only include issues, notes, etc. created after\ the specified date. Expected format is \ YYYY-MM-DD', type=get_datetime, default='1970-01-01') parser.add_argument('-e', '--include_empty', help='Include projects in output that do not have any\ issues.', action='store_true') parser.add_argument('-i', '--ignore_list', help='List of project names to exclude from dump.', type=argparse.FileType('r')) parser.add_argument('-p', '--password', help='The password to use to authenticate if token is \ not specified. If password and token are both \ unspecified, you will be prompted to enter a \ password.') parser.add_argument('-P', '--page_size', help='When retrieving result from GitLab, how many \ results should be included in a given page?.', type=int, default=20) parser.add_argument('-s', '--verify_ssl', help='Enable SSL certificate verification', action='store_true') parser.add_argument('-t', '--token', help='The private GitLab API token to use for \ authentication. Either this or username and \ password must be set.') parser.add_argument('-u', '--username', help='The username to use for authentication, if token\ is unspecified.') parser.add_argument('-v', '--verbose', help='Print more status information. For every ' + 'additional time this flag is specified, ' + 'output gets more verbose.', default=0, action='count') parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(__version__)) args = parser.parse_args(argv) args.page_size = max(100, args.page_size) # Convert verbose flag to actually logging level log_levels = [logging.WARNING, logging.INFO, logging.DEBUG] log_level = log_levels[min(args.verbose, 2)] # Make warnings from built-in warnings module get formatted more nicely logging.captureWarnings(True) logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' + '%(message)s'), level=log_level) # Setup authenticated GitLab instance if args.token: git = GitLab(args.gitlab_url, token=args.token, verify_ssl=args.verify_ssl) else: if not args.username: print('Username: '******'').strip() if not args.password: args.password = getpass.getpass('Password: '******'Creating project entries...', end="", file=sys.stderr) sys.stderr.flush() key_set = set() mentioned_users = set() if args.ignore_list is not None: ignore_list = {line.strip().lower() for line in args.ignore_list} else: ignore_list = {} for project in gen_all_results(git.getallprojects, per_page=args.page_size): proj_name_lower = project['name'].lower() if proj_name_lower not in ignore_list and project['issues_enabled']: project_issues = [] for issue in gen_all_results(git.getprojectissues, project['id'], per_page=args.page_size): if args.date_filter < datetime.strptime( issue['updated_at'], TIME_FORMAT): project_issues.append(issue) else: for note in git.getissuewallnotes(project['id'], issue['id']): if (args.date_filter < datetime.strptime( note['created_at'], TIME_FORMAT)): project_issues.append(issue) break if project_issues or args.include_empty: jira_project = {} jira_project['name'] = project['name_with_namespace'] key = project['name'] if key.islower(): key = key.title() key = re.sub(r'[^A-Z]', '', key) if len(key) < 2: key = re.sub(r'[^A-Za-z]', '', project['name'])[0:2].upper() added = False suffix = 65 while key in key_set: if not added: key += 'A' else: suffix += 1 key = key[:-1] + chr(suffix) key_set.add(key) jira_project['key'] = key jira_project['description'] = md_to_wiki( project['description']) # jira_project['created'] = project['created_at'] jira_project['issues'] = [] for issue in project_issues: jira_issue = {} jira_issue['externalId'] = issue['iid'] if issue['state'] == 'closed': jira_issue['status'] = 'Closed' jira_issue['resolution'] = 'Resolved' else: jira_issue['status'] = 'Open' jira_issue['description'] = md_to_wiki( issue['description']) jira_issue['reporter'] = issue['author']['username'] mentioned_users.add(jira_issue['reporter']) jira_issue['labels'] = issue['labels'] jira_issue['summary'] = issue['title'] if issue['assignee']: jira_issue['assignee'] = issue['assignee']['username'] mentioned_users.add(jira_issue['assignee']) jira_issue['issueType'] = 'Bug' jira_issue['comments'] = [] # Get all comments/notes for note in git.getissuewallnotes(project['id'], issue['id']): jira_note = {} jira_note['body'] = md_to_wiki(note['body']) jira_note['author'] = note['author']['username'] mentioned_users.add(jira_note['author']) jira_note['created'] = note['created_at'] jira_issue['comments'].append(jira_note) jira_project['issues'].append(jira_issue) output_dict['projects'].append(jira_project) print('.', end="", file=sys.stderr) sys.stderr.flush() print('\nCreating user entries...', end="", file=sys.stderr) sys.stderr.flush() for user in gen_all_results(git.getusers, per_page=args.page_size): # Only add users who are actually referenced in issues if user['username'] in mentioned_users: jira_user = {} jira_user['name'] = user['username'] jira_user['fullname'] = user['name'] jira_user['email'] = user['email'] jira_user['groups'] = ['gitlab-users'] jira_user['active'] = (user['state'] == 'active') output_dict['users'].append(jira_user) print('.', end="", file=sys.stderr) sys.stderr.flush() print('\nPrinting JSON output...', file=sys.stderr) sys.stderr.flush() print(json.dumps(output_dict, indent=4))
def main(argv=None): ''' Process the command line arguments and create the JSON dump. :param argv: List of arguments, as if specified on the command-line. If None, ``sys.argv[1:]`` is used instead. :type argv: list of str ''' # Get command line arguments parser = argparse.ArgumentParser( description="Transfer all projects/repositories from GitLab to Stash. \ Note: This script assumes you have your SSH key \ registered with both GitLab and Stash.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, conflict_handler='resolve') parser.add_argument('gitlab_url', help='The full URL to your GitLab instance.') parser.add_argument('stash_url', help='The full URL to your Stash instance.') parser.add_argument('-p', '--password', help='The password to use to authenticate if token is \ not specified. If password and token are both \ unspecified, you will be prompted to enter a \ password.') parser.add_argument('-P', '--page_size', help='When retrieving result from GitLab, how many \ results should be included in a given page?.', type=int, default=20) parser.add_argument('-s', '--verify_ssl', help='Enable SSL certificate verification', action='store_true') parser.add_argument('-S', '--skip_existing', help='Do not update existing repositories and just \ skip them.', action='store_true') parser.add_argument('-t', '--token', help='The private GitLab API token to use for \ authentication. Either this or username and \ password must be set.') parser.add_argument('-u', '--username', help='The username to use for authentication, if token\ is unspecified.') parser.add_argument('-v', '--verbose', help='Print more status information. For every ' + 'additional time this flag is specified, ' + 'output gets more verbose.', default=0, action='count') parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(__version__)) args = parser.parse_args(argv) args.page_size = max(100, args.page_size) # Convert verbose flag to actually logging level log_levels = [logging.WARNING, logging.INFO, logging.DEBUG] log_level = log_levels[min(args.verbose, 2)] # Make warnings from built-in warnings module get formatted more nicely logging.captureWarnings(True) logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' + '%(message)s'), level=log_level) # Setup authenticated GitLab and Stash instances if args.token: git = GitLab(args.gitlab_url, token=args.token, verify_ssl=args.verify_ssl) else: git = None if not args.username: print('Username: '******'').strip() if not args.password: args.password = getpass.getpass('Password: '******'Retrieving existing Stash projects...', end="", file=sys.stderr) sys.stderr.flush() key_set = {proj['key'] for proj in stash.projects} stash_project_names = {proj['name'] for proj in stash.projects} names_to_keys = {proj['name']: proj['key'] for proj in stash.projects} print('done', file=sys.stderr) sys.stderr.flush() updated_projects = set() repo_to_slugs = {} failed_to_clone = set() cwd = os.getcwd() transfer_count = 0 skipped_count = 0 print('Processing GitLab projects...', file=sys.stderr) sys.stderr.flush() for project in gen_all_results(git.getallprojects, per_page=args.page_size): print('\n' + ('=' * 80) + '\n', file=sys.stderr) sys.stderr.flush() proj_name = project['namespace']['name'] # Create Stash project if it doesn't already exist if proj_name not in stash_project_names: # Create Stash project key key = proj_name if key.islower(): key = key.title() key = re.sub(r'[^A-Z]', '', key) if len(key) < 2: key = re.sub(r'[^A-Za-z]', '', proj_name)[0:2].upper() added = False suffix = 65 while key in key_set: if not added: key += 'A' else: suffix += 1 key = key[:-1] + chr(suffix) key_set.add(key) # Actually add the project to Stash print('Creating Stash project "%s" with key %s...' % (proj_name, key), end="", file=sys.stderr) sys.stderr.flush() stash.projects.create(key, proj_name) names_to_keys[proj_name] = key stash_project_names.add(proj_name) print('done', file=sys.stderr) sys.stderr.flush() else: key = names_to_keys[proj_name] stash_project = stash.projects[key] # Initialize maping from repository names to slugs for later if key not in repo_to_slugs: repo_to_slugs[key] = { repo['name']: repo['slug'] for repo in stash_project.repos } # Create Stash-compatible name for repository # Repository names are limited to 128 characters. # They must start with a letter or number and may contain spaces, # hyphens, underscores and periods repo_name = project['name'] if not repo_name[0].isalnum(): repo_name = 'A ' + repo_name repo_name = re.sub(r'[^A-Za-z0-9 _.-]', ' ', repo_name) if len(repo_name) > 128: repo_name = repo_name[0:128] # Add repository to Stash project if it's not already there if repo_name not in repo_to_slugs[key]: print('Creating Stash repository "%s" in project "%s"...' % (repo_name, proj_name), end="", file=sys.stderr) sys.stderr.flush() stash_repo = stash_project.repos.create(repo_name) repo_to_slugs[key][repo_name] = stash_repo['slug'] print('done', file=sys.stderr) sys.stderr.flush() elif args.skip_existing: print('Skipping existing Stash repository "%s" in project "%s"' % (repo_name, proj_name), file=sys.stderr) sys.stderr.flush() skipped_count += 1 continue else: print('Updating existing Stash repository "%s" in project "%s"' % (repo_name, proj_name), file=sys.stderr) sys.stderr.flush() repo_slug = repo_to_slugs[key][repo_name] stash_repo = stash_project.repos[repo_slug].get() for clone_link in stash_repo['links']['clone']: if clone_link['name'] == 'ssh': stash_repo_url = clone_link['href'] break with tempfile.TemporaryDirectory() as temp_dir: # Clone repository to temporary directory print('\nCloning GitLab repository...', file=sys.stderr) sys.stderr.flush() try: subprocess.check_call([ 'git', 'clone', '--mirror', project['ssh_url_to_repo'], temp_dir ]) except subprocess.CalledProcessError: print('Failed to clone GitLab repository. This usually when ' + 'it does not exist.', file=sys.stderr) failed_to_clone.add(project['name_with_namespace']) skipped_count += 1 continue os.chdir(temp_dir) # Check that repository is not empty try: subprocess.check_call(['git', 'log', '--format=oneline', '-1'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except subprocess.CalledProcessError: print('Repository is empty, so skipping push to Stash.', file=sys.stderr) skipped_count += 1 else: # Change remote to Stash and push print('\nPushing repository to Stash...', file=sys.stderr) sys.stderr.flush() subprocess.check_call( ['git', 'remote', 'set-url', 'origin', stash_repo_url]) subprocess.check_call(['git', 'push', '--mirror']) transfer_count += 1 os.chdir(cwd) updated_projects.add(proj_name) print('\n' + ('=' * 35) + 'SUMMARY' + ('=' * 35), file=sys.stderr) print('{} repositories transferred.\n'.format(transfer_count), file=sys.stderr) print('{} repositories skipped.\n'.format(skipped_count), file=sys.stderr) print('Projects created/updated:', file=sys.stderr) for proj in sorted(updated_projects): print('\t' + proj, file=sys.stderr) print('Repositories that we could not clone:', file=sys.stderr) for repo_name in sorted(failed_to_clone): print('\t' + repo_name, file=sys.stderr)