gh-issues-import.py

#!/usr/bin/env python3

import urllib.request, urllib.error, urllib.parse
import json
import base64
import sys, os
import datetime
import argparse, configparser
import time

import query

__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
default_config_file = os.path.join(__location__, 'config.ini')
config = configparser.RawConfigParser()

class state:
	current = ""
	INITIALIZING         = "script-initializing"
	LOADING_CONFIG       = "loading-config"
	FETCHING_ISSUES      = "fetching-issues"
	GENERATING           = "generating"
	IMPORT_CONFIRMATION  = "import-confirmation"
	IMPORTING            = "importing"
	IMPORT_COMPLETE      = "import-complete"
	COMPLETE             = "script-complete"
	
state.current = state.INITIALIZING

http_error_messages = {}
http_error_messages[401] = "ERROR: There was a problem during authentication.\nDouble check that your username and password are correct, and that you have permission to read from or write to the specified repositories."
http_error_messages[404] = "ERROR: Unable to find the specified repository.\nDouble check the spelling for the source and target repositories. If either repository is private, make sure the specified user is allowed access to it."


def init_config():
	
	config.add_section('login')
	config.add_section('source')
	config.add_section('target')
	config.add_section('format')
	config.add_section('settings')
	
	arg_parser = argparse.ArgumentParser(description="Import issues from one GitHub repository into another.")
	
	config_group = arg_parser.add_mutually_exclusive_group(required=False)
	config_group.add_argument('--config', help="The location of the config file (either absolute, or relative to the current working directory). Defaults to `config.ini` found in the same folder as this script.")
	config_group.add_argument('--no-config', dest='no_config',  action='store_true', help="No config file will be used, and the default `config.ini` will be ignored. Instead, all settings are either passed as arguments, or (where possible) requested from the user as a prompt.")
	
	arg_parser.add_argument('-u', '--username', help="The username of the account that will create the new issues. The username will not be stored anywhere if passed in as an argument.")
	arg_parser.add_argument('-p', '--password', help="The password (in plaintext) of the account that will create the new issues. The password will not be stored anywhere if passed in as an argument.")
	arg_parser.add_argument('-s', '--source', help="The source repository which the issues should be copied from. Should be in the format `user/repository`.")
	arg_parser.add_argument('-t', '--target', help="The destination repository which the issues should be copied to. Should be in the format `user/repository`.")
	
	arg_parser.add_argument('--ignore-comments',  dest='ignore_comments',  action='store_true', help="Do not import comments in the issue.")		
	arg_parser.add_argument('--ignore-milestone', dest='ignore_milestone', action='store_true', help="Do not import the milestone attached to the issue.")
	arg_parser.add_argument('--ignore-labels',    dest='ignore_labels',    action='store_true', help="Do not import labels attached to the issue.")
	arg_parser.add_argument('--ignore-pullrequests',    dest='ignore_pullrequests',    action='store_true', help="Do not import issues that are pull requests.")
	arg_parser.add_argument('--use-github-import-api', dest='use_github_import_api', action='store_true', help="Use the new github import api")
	
	arg_parser.add_argument('--issue-template', help="Specify a template file for use with issues.")
	arg_parser.add_argument('--comment-template', help="Specify a template file for use with comments.")
	arg_parser.add_argument('--pull-request-template', help="Specify a template file for use with pull requests.")
		
	include_group = arg_parser.add_mutually_exclusive_group(required=True)
	include_group.add_argument("--all", dest='import_all', action='store_true', help="Import all issues, regardless of state.")
	include_group.add_argument("--open", dest='import_open', action='store_true', help="Import only open issues.")
	include_group.add_argument("--closed", dest='import_closed', action='store_true', help="Import only closed issues.")
	include_group.add_argument("-i", "--issues", type=int, nargs='+', help="The list of issues to import.")

	args = arg_parser.parse_args()
	
	def load_config_file(config_file_name):
		try:
			config_file = open(config_file_name)
			config.read_file(config_file)
			return True
		except (FileNotFoundError, IOError):
			return False
	
	if args.no_config:
		print("Ignoring default config file. You may be prompted for some missing settings.")
	elif args.config:
		config_file_name = args.config
		if load_config_file(config_file_name):
			print("Loaded config options from '%s'" % config_file_name)
		else:
			sys.exit("ERROR: Unable to find or open config file '%s'" % config_file_name)
	else:
		config_file_name = default_config_file
		if load_config_file(config_file_name):
			print("Loaded options from default config file in '%s'" % config_file_name)
		else:
			print("Default config file not found in '%s'" % config_file_name)
			print("You may be prompted for some missing settings.")

	
	if args.username: config.set('login', 'username', args.username)
	if args.password: config.set('login', 'password', args.password)
	
	if args.source: config.set('source', 'repository', args.source)
	if args.target: config.set('target', 'repository', args.target)
	
	if args.issue_template: config.set('format', 'issue_template', args.issue_template)
	if args.comment_template: config.set('format', 'comment_template', args.comment_template)
	if args.pull_request_template: config.set('format', 'pull_request_template', args.pull_request_template)
	
	config.set('settings', 'import-comments',  str(not args.ignore_comments))
	config.set('settings', 'import-milestone', str(not args.ignore_milestone))
	config.set('settings', 'import-labels',    str(not args.ignore_labels))
	
	config.set('settings', 'import-open-issues',   str(args.import_all or args.import_open))
	config.set('settings', 'import-closed-issues', str(args.import_all or args.import_closed))
	config.set('settings', 'import-pullrequests',  str(not args.ignore_pullrequests))
	config.set('settings', 'use-github-import-api', str(args.use_github_import_api))
	
	
	# Make sure no required config values are missing
	if not config.has_option('source', 'repository') :
		sys.exit("ERROR: There is no source repository specified either in the config file, or as an argument.")
	if not config.has_option('target', 'repository') :
		sys.exit("ERROR: There is no target repository specified either in the config file, or as an argument.")
	
	
	def get_server_for(which):
		# Default to 'github.com' if no server is specified
		if (not config.has_option(which, 'server')):
			config.set(which, 'server', "github.com")
		
		# if SOURCE server is not github.com, then assume ENTERPRISE github (yourdomain.com/api/v3...)
		if (config.get(which, 'server') == "github.com") :
			api_url = "https://api.github.com"
		else:
			api_url = "https://%s/api/v3" % config.get(which, 'server')
		
		config.set(which, 'url', "%s/repos/%s" % (api_url, config.get(which, 'repository')))
	
	get_server_for('source')
	get_server_for('target')
	
	
	# Prompt for username/password if none is provided in either the config or an argument
	def get_credentials_for(which):
		if not config.has_option(which, 'username'):
			if config.has_option('login', 'username'):
				config.set(which, 'username', config.get('login', 'username'))
			elif ( (which == 'target') and query.yes_no("Do you wish to use the same credentials for the target repository?") ):
				config.set('target', 'username', config.get('source', 'username'))
			else:
				query_str = "Enter your username for '%s' at '%s': " % (config.get(which, 'repository'), config.get(which, 'server'))
				config.set(which, 'username', query.username(query_str))
		
		if not config.has_option(which, 'password'):
			if config.has_option('login', 'password'):
				config.set(which, 'password', config.get('login', 'password'))
			elif ( (which == 'target') and config.get('source', 'username') == config.get('target', 'username') and config.get('source', 'server') == config.get('target', 'server') ):
				config.set('target', 'password', config.get('source', 'password'))
			else:
				query_str = "Enter your password for '%s' at '%s': " % (config.get(which, 'repository'), config.get(which, 'server'))
				config.set(which, 'password', query.password(query_str))
	
	get_credentials_for('source')
	get_credentials_for('target')
	
	# Everything is here! Continue on our merry way...
	return args.issues or []

def format_date(datestring):
	# The date comes from the API in ISO-8601 format
	date = datetime.datetime.strptime(datestring, "%Y-%m-%dT%H:%M:%SZ")
	date_format = config.get('format', 'date', fallback='%A %b %d, %Y at %H:%M GMT', raw=True);
	return date.strftime(date_format)
	
def format_from_template(template_filename, template_data):
	from string import Template
	template_file = open(template_filename, 'r')
	template = Template(template_file.read())
	return template.substitute(template_data)

def format_issue(template_data):
	default_template = os.path.join(__location__, 'templates', 'issue.md')
	template = config.get('format', 'issue_template', fallback=default_template)
	return format_from_template(template, template_data)

def format_pull_request(template_data):
	default_template = os.path.join(__location__, 'templates', 'pull_request.md')
	template = config.get('format', 'pull_request_template', fallback=default_template)
	return format_from_template(template, template_data)

def format_comment(template_data):
	default_template = os.path.join(__location__, 'templates', 'comment.md')
	template = config.get('format', 'comment_template', fallback=default_template)
	return format_from_template(template, template_data)

def send_request(which, url, post_data=None, method=None, accept=None):

	if post_data is not None:
		post_data = json.dumps(post_data).encode("utf-8")
	
	full_url = "%s/%s" % (config.get(which, 'url'), url)
	req = urllib.request.Request(full_url, post_data, method=method)
	
	username = config.get(which, 'username')
	password = config.get(which, 'password')
	req.add_header("Authorization", b"Basic " + base64.urlsafe_b64encode(username.encode("utf-8") + b":" + password.encode("utf-8")))
	
	req.add_header("Content-Type", "application/json")
	if accept is not None:
		req.add_header("Accept", accept)
	else:
		req.add_header("Accept", "application/json")
	req.add_header("User-Agent", "IQAndreas/github-issues-import")

	retry = True
	while retry:
		retry = False
		try:
			response = urllib.request.urlopen(req)
			json_data = response.read()
		except urllib.error.HTTPError as error:

			error_details = error.read();
			error_details = json.loads(error_details.decode("utf-8"))

			if error.code == 403:
				print("Got 403 [%s], assuming rate limit error and waiting for 1 minute..." % error_details['message'])
				time.sleep(60)
				retry = True
			elif error.code in http_error_messages:
				sys.exit(http_error_messages[error.code])
			else:
				error_message = "ERROR: There was a problem importing the issues.\n%s %s" % (error.code, error.reason)
				if 'message' in error_details:
					error_message += "\nDETAILS: " + error_details['message']
				sys.exit(error_message)
	
	return json.loads(json_data.decode("utf-8"))

def get_milestones(which):
	return send_request(which, "milestones?state=open")

def get_labels(which):
	page = 1
	labels = []
	while True:
		next_labels = send_request(which, "labels?page=%s" % page)
		if next_labels:
			labels.extend(next_labels)
			page += 1
		else:
			return labels
	
def get_issue_by_id(which, issue_id):
	return send_request(which, "issues/%d" % issue_id)

def get_issues_by_id(which, issue_ids):
	# Populate issues based on issue IDs
	issues = []
	for issue_id in issue_ids:
		issues.append(get_issue_by_id(which, int(issue_id)))
	
	return issues

# Allowed values for state are 'open' and 'closed'
def get_issues_by_state(which, state):
	issues = []
	page = 1
	import_pullrequests = config.getboolean('settings', 'import-pullrequests')
	while True:
		new_issues = send_request(which, "issues?state=%s&direction=asc&page=%d" % (state, page))
		if not new_issues:
			break
		if import_pullrequests:
			issues.extend(new_issues)
		else:
			issues.extend(filter(lambda issue:'pull_request' not in issue ,new_issues))
		page += 1
	return issues

def get_comments_on_issue(which, issue):
	if issue['comments'] != 0:
		return send_request(which, "issues/%s/comments" % issue['number'])
	else :
		return []

def import_milestone(source):
	data = {
		"title": source['title'],
		"state": "open",
		"description": source['description'],
		"due_on": source['due_on']
	}
	
	result_milestone = send_request('target', "milestones", data)
	print("Successfully created milestone '%s'" % result_milestone['title'])
	return result_milestone

def import_label(source):
	data = {
		"name": source['name'],
		"color": source['color']
	}
	
	result_label = send_request('target', "labels", data)
	print("Successfully created label '%s'" % result_label['name'])
	return result_label

def import_comments(comments, issue_number):
	result_comments = []
	for comment in comments:
	
		template_data = {}
		template_data['user_name'] = comment['user']['login']
		template_data['user_url'] = comment['user']['html_url']
		template_data['user_avatar'] = comment['user']['avatar_url']
		template_data['date'] = format_date(comment['created_at'])
		template_data['url'] =  comment['html_url']
		template_data['body'] = comment['body']
		
		comment['body'] = format_comment(template_data)

		result_comment = send_request('target', "issues/%s/comments" % issue_number, comment)
		result_comments.append(result_comment)
		
	return result_comments

def import_issues_golden_comet(issues):
	state.current = state.GENERATING

	known_milestones = get_milestones('target')
	def get_milestone_by_title(title):
		for milestone in known_milestones:
			if milestone['title'] == title : return milestone
		return None
	
	known_labels = get_labels('target')
	def get_label_by_name(name):
		for label in known_labels:
			if label['name'] == name : return label
		return None

	issue_migrations = []
	new_milestones = []
	num_new_comments = 0
	new_labels = []
	
	for issue in issues:
		issue_migration = {}
		new_issue = {}
		new_issue['title'] = issue['title']
		if issue['closed_at'] is not None:
			new_issue['closed_at'] = issue['closed_at']
			new_issue['closed'] = True
		new_issue['created_at'] = issue['created_at']
		new_issue['updated_at'] = issue['updated_at']
		# TODO new_issue['assignee'] = issue['assignee']

		if config.getboolean('settings', 'import-milestone') and 'milestone' in issue and issue['milestone'] is not None:
			# Since the milestones' ids are going to differ, we will compare them by title instead
			found_milestone = get_milestone_by_title(issue['milestone']['title'])
			if found_milestone:
				new_issue['milestone_object'] = found_milestone
			else:
				new_milestone = issue['milestone']
				new_issue['milestone_object'] = new_milestone
				known_milestones.append(new_milestone) # Allow it to be found next time
				new_milestones.append(new_milestone)   # Put it in a queue to add it later
		
		if config.getboolean('settings', 'import-labels') and 'labels' in issue and issue['labels'] is not None:
			new_issue['label_objects'] = []
			for issue_label in issue['labels']:
				found_label = get_label_by_name(issue_label['name'])
				if found_label:
					new_issue['label_objects'].append(found_label)
				else:
					new_issue['label_objects'].append(issue_label)
					known_labels.append(issue_label) # Allow it to be found next time
					new_labels.append(issue_label)   # Put it in a queue to add it later
		
		comments = []
		if config.getboolean('settings', 'import-comments') and 'comments' in issue and issue['comments'] != 0:
			num_new_comments += int(issue['comments'])
			original_comments = get_comments_on_issue('source', issue)
			for original_comment in original_comments:
				comment = {}
				ctemplate_data = {}
				ctemplate_data['user_name'] = original_comment['user']['login']
				ctemplate_data['user_url'] = original_comment['user']['html_url']
				ctemplate_data['user_avatar'] = original_comment['user']['avatar_url']
				ctemplate_data['date'] = format_date(original_comment['created_at'])
				ctemplate_data['url'] =  original_comment['html_url']
				ctemplate_data['body'] = original_comment['body']
				
				comment['body'] = format_comment(ctemplate_data)
				comment['created_at'] = original_comment['created_at']
				comments.append(comment)
				
		template_data = {}
		template_data['user_name'] = issue['user']['login']
		template_data['user_url'] = issue['user']['html_url']
		template_data['user_avatar'] = issue['user']['avatar_url']
		template_data['date'] = format_date(issue['created_at'])
		template_data['url'] =  issue['html_url']
		template_data['body'] = issue['body']

		if "pull_request" in issue and issue['pull_request']['html_url'] is not None:
			new_issue['body'] = format_pull_request(template_data)
		else:
			new_issue['body'] = format_issue(template_data)

		issue_migration['issue'] = new_issue
		issue_migration['comments'] = comments

		issue_migrations.append(issue_migration)

	state.current = state.IMPORT_CONFIRMATION

	print("You are about to add to '" + config.get('target', 'repository') + "':")
	print(" *", len(issue_migrations), "new issues") 
	print(" *", num_new_comments, "new comments") 
	print(" *", len(new_milestones), "new milestones") 
	print(" *", len(new_labels), "new labels") 
	if not query.yes_no("Are you sure you wish to continue?"):
		sys.exit()

	state.current = state.IMPORTING
	
	for milestone in new_milestones:
		result_milestone = import_milestone(milestone)
		milestone['number'] = result_milestone['number']
		milestone['url'] = result_milestone['url']
	
	for label in new_labels:
		result_label = import_label(label)
	
	migration_results = []
	for issue_migration in issue_migrations:
		issue = issue_migration['issue']
		if 'milestone_object' in issue:
			issue['milestone'] = issue['milestone_object']['number']
			del issue['milestone_object']
		
		if 'label_objects' in issue:
			issue_labels = []
			for label in issue['label_objects']:
				issue_labels.append(label['name'])
			issue['labels'] = issue_labels
			del issue['label_objects']
		
		migration_result = send_request('target', "import/issues", issue_migration, "POST", "application/vnd.github.golden-comet-preview+json")
		print("Sent import request. Status: '%s'. Status url: '%s'" % (migration_result['status'], migration_result['url']))
		
		migration_results.append(migration_result)
	
	state.current = state.IMPORT_COMPLETE
	
	# TODO wait for import requests to complete
	
	return issue_migrations


# Will only import milestones and issues that are in use by the imported issues, and do not exist in the target repository
def import_issues(issues):

	state.current = state.GENERATING
	
	known_milestones = get_milestones('target')
	def get_milestone_by_title(title):
		for milestone in known_milestones:
			if milestone['title'] == title : return milestone
		return None
	
	known_labels = get_labels('target')
	def get_label_by_name(name):
		for label in known_labels:
			if label['name'] == name : return label
		return None
	
	new_issues = []
	num_new_comments = 0
	new_milestones = []
	new_labels = []
	
	for issue in issues:
		
		new_issue = {}
		new_issue['title'] = issue['title']
		
		# Temporary fix for marking closed issues
		if issue['closed_at']:
			new_issue['state'] = "closed"
		
		if config.getboolean('settings', 'import-comments') and 'comments' in issue and issue['comments'] != 0:
			num_new_comments += int(issue['comments'])
			new_issue['comments'] = get_comments_on_issue('source', issue)
		
		if config.getboolean('settings', 'import-milestone') and 'milestone' in issue and issue['milestone'] is not None:
			# Since the milestones' ids are going to differ, we will compare them by title instead
			found_milestone = get_milestone_by_title(issue['milestone']['title'])
			if found_milestone:
				new_issue['milestone_object'] = found_milestone
			else:
				new_milestone = issue['milestone']
				new_issue['milestone_object'] = new_milestone
				known_milestones.append(new_milestone) # Allow it to be found next time
				new_milestones.append(new_milestone)   # Put it in a queue to add it later
		
		if config.getboolean('settings', 'import-labels') and 'labels' in issue and issue['labels'] is not None:
			new_issue['label_objects'] = []
			for issue_label in issue['labels']:
				found_label = get_label_by_name(issue_label['name'])
				if found_label:
					new_issue['label_objects'].append(found_label)
				else:
					new_issue['label_objects'].append(issue_label)
					known_labels.append(issue_label) # Allow it to be found next time
					new_labels.append(issue_label)   # Put it in a queue to add it later
		
		template_data = {}
		template_data['user_name'] = issue['user']['login']
		template_data['user_url'] = issue['user']['html_url']
		template_data['user_avatar'] = issue['user']['avatar_url']
		template_data['date'] = format_date(issue['created_at'])
		template_data['url'] =  issue['html_url']
		template_data['body'] = issue['body']
		
		if "pull_request" in issue and issue['pull_request']['html_url'] is not None:
			new_issue['body'] = format_pull_request(template_data)
		else:
			new_issue['body'] = format_issue(template_data)
		
		new_issues.append(new_issue)
	
	state.current = state.IMPORT_CONFIRMATION
	
	print("You are about to add to '" + config.get('target', 'repository') + "':")
	print(" *", len(new_issues), "new issues") 
	print(" *", num_new_comments, "new comments") 
	print(" *", len(new_milestones), "new milestones") 
	print(" *", len(new_labels), "new labels") 
	if not query.yes_no("Are you sure you wish to continue?"):
		sys.exit()
	
	state.current = state.IMPORTING
	
	for milestone in new_milestones:
		result_milestone = import_milestone(milestone)
		milestone['number'] = result_milestone['number']
		milestone['url'] = result_milestone['url']
	
	for label in new_labels:
		result_label = import_label(label)
	
	result_issues = []
	for issue in new_issues:
		
		if 'milestone_object' in issue:
			issue['milestone'] = issue['milestone_object']['number']
			del issue['milestone_object']
		
		if 'label_objects' in issue:
			issue_labels = []
			for label in issue['label_objects']:
				issue_labels.append(label['name'])
			issue['labels'] = issue_labels
			del issue['label_objects']
		
		result_issue = send_request('target', "issues", issue)
		print("Successfully created issue '%s'" % result_issue['title'])
		
		if 'comments' in issue:
			result_comments = import_comments(issue['comments'], result_issue['number'])		
			print(" > Successfully added", len(result_comments), "comments.")
		
		result_issues.append(result_issue)

		# add issue closing
		if 'state' in issue and issue['state'] == 'closed':
			number = result_issue['number']
			result_issue = send_request('target', 'issues/%s' % number, issue, 'PATCH')
			print(" > Successfully closed issue '%s'" % result_issue['title'])
	
	state.current = state.IMPORT_COMPLETE
	
	return result_issues


if __name__ == '__main__':
	
	state.current = state.LOADING_CONFIG
	
	issue_ids = init_config()	
	issues = []
	
	state.current = state.FETCHING_ISSUES
	
	# Argparser will prevent us from getting both issue ids and specifying issue state, so no duplicates will be added
	if (len(issue_ids) > 0):
		issues += get_issues_by_id('source', issue_ids)
	
	if config.getboolean('settings', 'import-open-issues'):
		issues += get_issues_by_state('source', 'open')
	
	if config.getboolean('settings', 'import-closed-issues'):
		issues += get_issues_by_state('source', 'closed')
	
	# Sort issues based on their original `id` field
	# Confusing, but taken from http://stackoverflow.com/a/2878123/617937
	issues.sort(key=lambda x:x['number'])
	
	# Further states defined within the function
	# Finally, add these issues to the target repository
	if config.getboolean('settings', 'use-github-import-api'):
		import_issues_golden_comet(issues)
	else:
		import_issues(issues)
	
	state.current = state.COMPLETE