def perform_migrate_issues(args):
closed_states = []
if (args.closed_states):
closed_states = args.closed_states.split(',')
custom_fields = []
if (args.custom_fields):
custom_fields = args.custom_fields.split(',')
redmine = RedmineClient(args.redmine_key, args.no_verify)
gitlab = GitlabClient(args.gitlab_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
gitlab_project = GitlabProject(args.gitlab_project_url, gitlab)
gitlab_instance = gitlab_project.get_instance()
gitlab_users_index = gitlab_instance.get_users_index()
redmine_users_index = redmine_project.get_users_index()
milestones_index = gitlab_project.get_milestones_index()
log.debug('GitLab milestones are: {}'.format(', '.join(milestones_index) +
' '))
# get issues
log.info('Getting redmine issues')
issues = redmine_project.get_all_issues()
# convert issues
log.info('Converting issues')
issues_data = (convert_issue(args.redmine_key, i, redmine_users_index,
gitlab_users_index, milestones_index,
closed_states, custom_fields) for i in issues)
# create issues
log.info('Creating gitlab issues')
for data, meta in issues_data:
if args.check:
milestone_id = data.get('milestone_id', None)
if milestone_id:
try:
gitlab_project.get_milestone_by_id(milestone_id)
except ValueError:
raise CommandError(
"issue \"{}\" points to unknown milestone_id \"{}\". "
"Check that you already migrated roadmaps".format(
data['title'], milestone_id))
log.info('Would create issue "{}" and {} notes.'.format(
data['title'], len(meta['notes'])))
else:
try:
created = gitlab_project.create_issue(data, meta)
log.info('#{iid} {title}'.format(**created))
except:
log.info('create issue "{}" failed'.format(data['title']))
raise
def perform_migrate_roadmap(args):
redmine = RedmineClient(args.redmine_key, args.no_verify)
gitlab = GitlabClient(args.gitlab_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
gitlab_project = GitlabProject(args.gitlab_project_url, gitlab)
checks = [
#(check_no_milestone, 'Gitlab project has no pre-existing milestone'),
(check_origin_milestone, 'Redmine project contains versions'),
]
for i in checks:
check(
*i, redmine_project=redmine_project,
gitlab_project=gitlab_project)
versions = redmine_project.get_versions()
if args.filter:
name_filter = re.compile(args.filter)
versions = [version for version in versions if name_filter.match(version['name'])]
versions_data = (convert_version(i) for i in versions)
for data, meta in versions_data:
if args.check:
log.info("Would create version {}".format(data))
else:
created = gitlab_project.create_milestone(data, meta)
log.info("Version {}".format(created['title']))
def perform_migrate_pages(args):
redmine = RedmineClient(args.redmine_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
# Get copy of GitLab wiki repository
wiki = WikiPageConverter(args.gitlab_wiki)
# convert all pages including history
pages = []
for page in redmine_project.get_all_pages():
print("Collecting " + page["title"])
start_version = page["version"] if args.no_history else 1
for version in range(start_version, page["version"] + 1):
try:
full_page = redmine_project.get_page(page["title"], version)
pages.append(full_page)
except:
log.error("Error when retrieving " + page["title"] +
", version " + str(version))
# sort everything by date and convert
pages.sort(key=lambda page: page["updated_on"])
for page in pages:
wiki.convert(page)
def perform_redirect(args):
redmine = RedmineClient(args.redmine_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
# get issues
redmine_issues = redmine_project.get_all_issues()
print('# uncomment next line to enable RewriteEngine')
print('# RewriteEngine On')
print('# Redirects from {} to {}'.format(args.redmine_project_url, args.gitlab_project_url))
for issue in redmine_issues:
print('RedirectMatch 301 ^/issues/{}$ {}/issues/{}'.format(issue['id'], args.gitlab_project_url, issue['id']))
def perform_migrate_issues(args):
redmine = RedmineClient(args.redmine_key)
gitlab = GitlabClient(args.gitlab_key)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
gitlab_project = GitlabProject(args.gitlab_project_url, gitlab)
gitlab_instance = gitlab_project.get_instance()
gitlab_users_index = gitlab_instance.get_users_index()
redmine_users_index = redmine_project.get_users_index()
checks = [
(check_users, 'Required users presence'),
(check_no_issue, 'Project has no pre-existing issue'),
]
for i in checks:
check(
*i, redmine_project=redmine_project, gitlab_project=gitlab_project)
# Get issues
issues = redmine_project.get_all_issues()
milestones_index = gitlab_project.get_milestones_index()
issues_data = (
convert_issue(
i, redmine_users_index, gitlab_users_index, milestones_index)
for i in issues)
for data, meta in issues_data:
if args.check:
milestone_id = data.get('milestone_id', None)
if milestone_id:
try:
gitlab_project.get_milestone_by_id(milestone_id)
except ValueError:
raise CommandError(
"issue \"{}\" points to unknown milestone_id \"{}\". "
"Check that you already migrated roadmaps".format(
data['title'], milestone_id))
log.info('Would create issue "{}" and {} notes.'.format(
data['title'],
len(meta['notes'])))
else:
created = gitlab_project.create_issue(data, meta)
log.info('#{iid} {title}'.format(**created))
def perform_migrate_pages(args):
redmine = RedmineClient(args.redmine_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
if args.no_textile:
textile_converter = NopConverter()
else:
textile_converter = TextileConverter()
# Get copy of GitLab wiki repository
wiki = WikiPageConverter(args.gitlab_wiki, textile_converter)
# convert all pages including history
pages = []
for page in redmine_project.get_all_pages():
print("Collecting " + page["title"])
start_version = page["version"] if args.no_history else 1
for version in range(start_version, page["version"] + 1):
try:
for i in range(5):
try:
full_page = redmine_project.get_page(
page["title"], version)
break
except:
sleep(1)
else:
full_page = redmine_project.get_page(
page["title"], version)
except Exception as e:
log.error("Error when retrieving " + page["title"] +
", version " + str(version))
continue
for attachment_key, attachment in enumerate(
full_page['attachments']):
full_page['attachments'][attachment_key] = wiki_attachment(
attachment, args.redmine_key, wiki.repo_path)
pages.append(full_page)
# sort everything by date and convert
pages.sort(key=lambda page: page["updated_on"])
for page in pages:
wiki.convert(page)
def perform_migrate_labels(args):
redmine = RedmineClient(args.redmine_key, args.no_verify)
gitlab = GitlabClient(args.gitlab_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
gitlab_project = GitlabProject(args.gitlab_project_url, gitlab)
pid = redmine_project.get_id()
init_db()
trackers, statuses, priorities, categories = project_labels(pid)
labels = []
for category in categories:
labels.append({
"name": category,
'color': "#44AD8E",
'description': category + ' --'
})
for tracker in trackers:
labels.append({"name": tracker, 'color': "#428BCA"})
for status in statuses:
labels.append({"name": status, 'color': "#7F8C8D"})
for priority in priorities:
labels.append({"name": priority, 'color': "#F0AD4E"})
for data in labels:
if args.check:
log.info("Would create label {}".format(data))
else:
created = gitlab_project.create_label(data)
log.info("Label {}".format(created['name']))
def perform_migrate_issues(args):
closed_states = []
if (args.closed_states):
closed_states = args.closed_states.split(',')
custom_fields = []
if (args.custom_fields):
custom_fields = args.custom_fields.split(',')
if (args.user_dict is not None):
load_user_dict(args.user_dict)
redmine = RedmineClient(args.redmine_key, args.no_verify)
gitlab = GitlabClient(args.gitlab_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
gitlab_project = GitlabProject(args.gitlab_project_url, gitlab)
gitlab_instance = gitlab_project.get_instance()
if (args.project_members_only):
gitlab_users_index = gitlab_project.get_members_index()
else:
gitlab_users_index = gitlab_instance.get_users_index()
redmine_users_index = redmine_project.get_users_index()
milestones_index = gitlab_project.get_milestones_index()
textile_converter = TextileConverter()
log.debug('GitLab milestones are: {}'.format(', '.join(milestones_index) + ' '))
# get issues
log.info('Getting redmine issues')
issues = redmine_project.get_all_issues()
if args.initial_id:
issues = [issue for issue in issues if int(args.initial_id) <= issue['id']]
if args.filter:
filters = args.filter.split(",")
filters = [filter.split("=") for filter in filters]
filters = [[field.split("."), re.compile(value)] for field, value in filters]
issues = [issue for issue in issues
if all(
value.match(reduce(
lambda object, key: object.get(key) if object is not None else None,
field,
issue
)) for field, value in filters)]
# convert issues
log.info('Converting issues')
issues_data = (
convert_issue(args.redmine_key,
i, redmine_users_index, gitlab_users_index, milestones_index, closed_states, custom_fields, textile_converter,
args.keep_id or args.keep_title, args.sudo, args.archive_acc, args.migrate_version_as_label)
for i in issues)
# create issues
log.info('Creating gitlab issues')
last_iid = int(args.initial_id or 1) - 1
for data, meta, redmine_id in issues_data:
if args.check:
milestone_id = data.get('milestone_id', None)
if milestone_id:
try:
gitlab_project.get_milestone_by_id(milestone_id)
except ValueError:
raise CommandError(
"issue \"{}\" points to unknown milestone_id \"{}\". "
"Check that you already migrated roadmaps".format(
data['title'], milestone_id))
log.info('Would create issue "{}" and {} notes.'.format(
data['title'],
len(meta['notes'])))
else:
if args.keep_id:
try:
fake_meta = {'uploads': [], 'notes': [], 'must_close': False}
if args.sudo:
fake_meta['sudo_user'] = args.archive_acc
while redmine_id > last_iid + 1:
created = gitlab_project.create_issue({'title': 'fake'}, fake_meta, gitlab.get_auth_headers())
last_iid = created['iid']
gitlab_project.delete_issue(created['iid'])
log.info('#{iid} {title}'.format(**created))
except:
log.info('create issue "{}" failed'.format('fake'))
raise
try:
created = gitlab_project.create_issue(data, meta, gitlab.get_auth_headers())
last_iid = created['iid']
log.info('#{iid} {title}'.format(**created))
except:
log.info('create issue "{}" failed'.format(data['title']))
raise
def perform_migrate_issues(args):
closed_states = []
if (args.closed_states):
closed_states = args.closed_states.split(',')
custom_fields = []
if (args.custom_fields):
custom_fields = args.custom_fields.split(',')
if (args.user_dict is not None):
load_user_dict(args.user_dict)
redmine = RedmineClient(args.redmine_key, args.no_verify)
gitlab = GitlabClient(args.gitlab_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
gitlab_project = GitlabProject(args.gitlab_project_url, gitlab)
gitlab_instance = gitlab_project.get_instance()
if (args.project_members_only):
gitlab_users_index = gitlab_project.get_members_index()
else:
gitlab_users_index = gitlab_instance.get_users_index()
redmine_users_index = redmine_project.get_users_index(args.issue_ids)
milestones_index = gitlab_project.get_milestones_index()
if args.no_textile:
textile_converter = NopConverter()
else:
textile_converter = TextileConverter()
log.debug('GitLab milestones are: {}'.format(', '.join(milestones_index) +
' '))
# get issues
log.info('Getting redmine issues')
issues = redmine_project.get_issues(args.issue_ids)
if args.initial_id:
issues = [
issue for issue in issues if int(args.initial_id) <= issue['id']
]
# convert issues
log.info('Converting issues')
issues_data = (convert_issue(args.redmine_key, i, redmine_users_index,
gitlab_users_index, milestones_index,
closed_states, custom_fields,
textile_converter, args.keep_id
or args.keep_title, args.sudo)
for i in issues)
# create issues
log.info('Creating gitlab issues')
last_iid = int(args.initial_id or 1) - 1
for data, meta, redmine_id in issues_data:
if args.check:
milestone_id = data.get('milestone_id', None)
if milestone_id:
try:
gitlab_project.get_milestone_by_id(milestone_id)
except ValueError:
raise CommandError(
"issue \"{}\" points to unknown milestone_id \"{}\". "
"Check that you already migrated roadmaps".format(
data['title'], milestone_id))
log.info('Would create issue "{}" and {} notes.'.format(
data['title'], len(meta['notes'])))
else:
if args.keep_id:
data['iid'] = redmine_id
try:
created = gitlab_project.create_issue(data, meta)
last_iid = created['iid']
log.info('#{iid} {title}'.format(**created))
except:
log.info('create issue "{}" failed'.format(data['title']))
raise
def perform_migrate_roadmap(args):
redmine = RedmineClient(args.redmine_key, args.no_verify)
gitlab = GitlabClient(args.gitlab_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
gitlab_project = GitlabProject(args.gitlab_project_url, gitlab)
# auto create milestones
milestones_data = []
if args.auto_milestones:
match = re.match("^(monthly|yearly):(\d{4}(-[01]\d)?)$",
args.auto_milestones)
if not match:
log.error(
"--auto-milestone must be formed as (monthly|yearly):<start_date>"
)
exit(1)
milestones_type = match.group(1)
milestones_date = list(map(lambda x: int(x),
match.group(2).split("-")))
if milestones_type == "monthly" and len(milestones_date) == 1:
milestones_date.append(1)
log.info("Creating %s milestones, from %s" %
(milestones_type, "-".join(
map(lambda x: str(x).zfill(2), milestones_date))))
day = date(milestones_date[0], milestones_date[1], 1)
today = date.today()
while day <= today:
month_days = monthrange(day.year, day.month)[1]
milestone = {
"title": day.strftime("%Y-%m"),
"description": "\n\n*(auto-generated milestone)*",
"start_date": day.strftime("%Y-%m-%d"),
"due_date": day.replace(day=month_days).strftime("%Y-%m-%d"),
}
must_close = True
if day > today - timedelta(days=4 * 31): # keep last 4 months open
must_close = False
meta = {
"must_close": must_close,
}
milestones_data.append((milestone, meta))
# next month
day = day + timedelta(days=month_days)
checks = [
#(check_no_milestone, 'Gitlab project has no pre-existing milestone'),
(check_origin_milestone, 'Redmine project contains versions'),
]
for i in checks:
check(*i,
redmine_project=redmine_project,
gitlab_project=gitlab_project)
versions = redmine_project.get_versions()
versions_data = [convert_version(i) for i in versions]
if milestones_data:
versions_titles = []
for data, meta in versions_data:
versions_titles.append(data['title'])
for data, meta in milestones_data:
if data['title'] in versions_titles:
continue
versions_data.append((data, meta))
for data, meta in versions_data:
if args.check:
log.info("Would create version {}".format(data))
else:
created = gitlab_project.create_milestone(data, meta)
log.info("Version {}".format(created['title']))
def perform_migrate_issues(args):
init_db()
closed_states = []
if (args.closed_states):
closed_states = args.closed_states.split(',')
custom_fields = []
if (args.custom_fields):
custom_fields = args.custom_fields.split(',')
if (args.user_dict is not None):
load_user_dict(args.user_dict)
if (args.user_keys is not None):
load_user_keys(args.user_keys)
redmine = RedmineClient(args.redmine_key, args.no_verify)
gitlab = GitlabClient(args.gitlab_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
gitlab_project = GitlabProject(args.gitlab_project_url, gitlab)
gitlab_instance = gitlab_project.get_instance()
if (args.project_members_only):
gitlab_users_index = gitlab_project.get_members_index()
else:
gitlab_users_index = gitlab_instance.get_users_index()
redmine_users_index = redmine_project.get_users_index()
milestones_index = gitlab_project.get_milestones_index()
textile_converter = TextileConverter()
log.debug('GitLab milestones are: {}'.format(', '.join(milestones_index) +
' '))
# get issues
log.info('Getting redmine issues')
issues = redmine_project.get_all_issues()
if args.initial_id:
issues = [
issue for issue in issues if int(args.initial_id) <= issue['id']
]
if args.max_id:
issues = [issue for issue in issues if int(args.max_id) >= issue['id']]
# convert issues
log.info('Converting issues')
issues_data = (convert_issue(args.redmine_key, i, redmine_users_index,
gitlab_users_index, milestones_index,
closed_states, custom_fields,
textile_converter, args.keep_id
or args.keep_title, args.sudo)
for i in issues)
# create issues
log.info('Creating gitlab issues')
last_iid = int(args.initial_id or 1) - 1
for data, meta, redmine_id in issues_data:
if args.check:
milestone_id = data.get('milestone_id', None)
if milestone_id:
try:
gitlab_project.get_milestone_by_id(milestone_id)
except ValueError:
raise CommandError(
"issue \"{}\" points to unknown milestone_id \"{}\". "
"Check that you already migrated roadmaps".format(
data['title'], milestone_id))
log.info('Would create issue "{}" and {} notes.'.format(
data['title'], len(meta['notes'])))
log.info('Labels %s' % meta.get('labels', []))
log.info('Tags %s' % meta.get('tags', []))
log.info('Watchers %s' % meta.get('watchers', []))
else:
if args.keep_id:
try:
fake_meta = {
'uploads': [],
'notes': [],
'must_close': False
}
if args.sudo:
fake_meta['sudo_user'] = meta['sudo_user']
while redmine_id > last_iid + 1:
created = gitlab_project.create_issue(
{'title': 'fake'}, fake_meta)
last_iid = created['iid']
gitlab_project.delete_issue(created['id'])
log.info('#{iid} {title}'.format(**created))
except:
log.info('create issue "{}" failed'.format('fake'))
raise
try:
# labels
for label in meta.get('labels', []):
gitlab_project.create_label(label)
# tags
for tag in meta.get('tags', []):
gitlab_project.create_label(tag)
# issue
created = gitlab_project.create_issue(data, meta)
last_iid = created['iid']
# watchers
for watcher in meta.get('watchers', []):
created_watcher = gitlab_project.create_watcher(
watcher.get('data', {}), watcher, created['iid'])
log.info('#{iid} {title}'.format(**created))
except:
log.info('create issue "{}" failed'.format(data['title']))
raise
def perform_migrate_pages(args):
redmine = RedmineClient(args.redmine_key, args.no_verify)
redmine_project = RedmineProject(args.redmine_project_url, redmine)
# create work dict
work = {}
for p in redmine_project.get_all_pages():
if 'parent' in p:
work[p['title']] = p['parent']['title']
else:
work[p['title']] = None
# recursive function to parse the child/parent
def parseTree(work, next = None):
ret = {}
for child,parent in work.copy().items():
if parent == next:
work.pop(child)
ret[child] = parseTree(work, child)
if not ret:
ret = None
return ret
wikitree = parseTree(work)
# create a path tree
def printTree(wikitree, ischild = ''):
if wikitree and not len(wikitree) == 0:
ret = {}
for p,child in wikitree.items():
if p == 'WikiStart':
p = 'home'
if ischild != '':
# create the tree folder structure
if not os.path.exists(ischild):
os.makedirs(ischild, exist_ok=True)
ret[p] = os.path.join(ischild, p)
else:
ret[p] = p
# has child
if child and not len(child) == 0:
dirpath = os.path.join(ischild, p)
ret.update(printTree(child, dirpath))
return ret
tree = printTree(wikitree)
# create tree index.md file
idxcont = "# Index\n\n"
for k,e in tree.items():
spaces = ''
indent = len(e.split('/'))
if indent > 1:
spaces = (4 * (indent-1)) * (' ')
idxcont += spaces + "* ["+k+"]("+e+")\n"
# write tree index file
with open(args.gitlab_wiki + "/index.md", "w") as f:
f.write(idxcont)
# add tree index to git repo
repo = Repo(args.gitlab_wiki)
repo.index.add(['index.md'])
repo.index.commit("add new index file")
# convert all pages including history
pages = []
for page in redmine_project.get_all_pages():
print("Collecting " + page["title"])
start_version = page["version"] if args.no_history else 1
for version in range(start_version, page["version"]+1):
try:
full_page = redmine_project.get_page(page["title"], version)
if page["title"] == 'WikiStart':
full_page['path'] = tree['home']
else:
full_page['path'] = tree[page['title']]
pages.append(full_page)
except:
log.error("Error when retrieving " + page["title"] + ", version " + str(version))
# sort everything by date and convert
pages.sort(key=lambda page: page["updated_on"])
# Get copy of GitLab wiki repository
wiki = WikiPageConverter(args.gitlab_wiki, tree=tree, textformat=args.textformat)
for page in pages:
wiki.convert(page)