def convert_wiki(source, dest, dest_project_id):
if overwrite and (method == 'direct'):
dest.clear_wiki_attachments(dest_project_id)
exclude_authors = [a.strip() for a in config.get('wiki', 'exclude_authors').split(',')]
target_directory = config.get('wiki', 'target-directory')
server = xmlrpclib.MultiCall(source)
for name in source.wiki.getAllPages():
info = source.wiki.getPageInfo(name)
if (info['author'] not in exclude_authors):
page = source.wiki.getPage(name)
print("Page %s:%s" % (name, info))
if (name == 'WikiStart'):
name = 'home'
converted = trac2down.convert(page, os.path.dirname('/wikis/%s' % name))
if method == 'direct':
for attachment in source.wiki.listAttachments(name):
print(attachment)
binary_attachment = source.wiki.getAttachment(attachment).data
try:
attachment_path = dest.create_wiki_attachment(dest_project_id, users_map[info['author']], convert_xmlrpc_datetime(info['lastModified']), attachment, binary_attachment)
except KeyError:
attachment_path = dest.create_wiki_attachment(dest_project_id, default_user, convert_xmlrpc_datetime(info['lastModified']), attachment, binary_attachment)
attachment_name = attachment.split('/')[-1]
converted = converted.replace(r'](%s)' % attachment_name, r'](%s)' % os.path.relpath(attachment_path, '/namespace/project/wiki/page'))
trac2down.save_file(converted, name, info['version'], info['lastModified'], info['author'], target_directory)
def convert_wiki(source, dest, dest_project_id):
if overwrite and (method == 'direct'):
dest.clear_wiki_attachments(dest_project_id)
exclude_authors = [a.strip() for a in config.get('wiki', 'exclude_authors').split(',')]
target_directory = config.get('wiki', 'target-directory')
server = xmlrpclib.MultiCall(source)
for name in source.wiki.getAllPages():
info = source.wiki.getPageInfo(name)
if (info['author'] not in exclude_authors):
page = source.wiki.getPage(name)
print "Page %s:%s" % (name, info)
if (name == 'WikiStart'):
name = 'home'
converted = trac2down.convert(page, os.path.dirname('/wikis/%s' % name))
if method == 'direct':
for attachment in source.wiki.listAttachments(name):
print attachment
binary_attachment = source.wiki.getAttachment(attachment).data
try:
attachment_path = dest.create_wiki_attachment(dest_project_id, users_map[info['author']], convert_xmlrpc_datetime(info['lastModified']), attachment, binary_attachment)
except KeyError:
attachment_path = dest.create_wiki_attachment(dest_project_id, default_user, convert_xmlrpc_datetime(info['lastModified']), attachment, binary_attachment)
attachment_name = attachment.split('/')[-1]
converted = converted.replace(r'](%s)' % attachment_name, r'](%s)' % os.path.relpath(attachment_path, '/namespace/project/wiki/page'))
trac2down.save_file(converted, name, info['version'], info['lastModified'], info['author'], target_directory)
def convert_issue_content(text, ticket_mapping):
"""
Convert TracWiki text to GitHub Markdown.
Change the ticket IDs to GitHub URLs according to the mapping.
Ignore included images.
"""
text = text.replace(config.TRAC_TICKET_PREFIX, '#')
ticket_re = '#([0-9]+)'
for match in matches(ticket_re, text):
try:
github_url = ticket_mapping[int(match)]
new_ticket_id = github_url.rsplit('/', 1)[1]
text = sub(f'#{match}', f'[#{new_ticket_id}]({github_url})', text)
except KeyError:
# We don't know this ticket. Leave it alone.
print("Warning: unknown ticket: #" + str(match))
pass
return convert(text, base_path='')
def convert_issue_content(text, ticket_mapping):
"""
Convert TracWiki text to GitHub Markdown.
Change the ticket IDs to GitHub URLs according to the mapping.
Ignore included images.
"""
text = text.replace(config.TRAC_TICKET_PREFIX, '#')
ticket_re = '#([0-9]+)'
text = update_changeset(text)
for match in matches(ticket_re, text):
try:
github_url = ticket_mapping[int(match)]
new_ticket_id = github_url.rsplit('/', 1)[1]
text = sub(f'#{match}', f'[#{new_ticket_id}]({github_url})', text)
except KeyError:
# We don't know this ticket. Warn about it.
print(f"Warning: ticket #{match} not in tickets_expected_gold.tsv"
f" - leaving it as #{match}")
return convert(text, base_path='', wiki_prefix=config.MIGRATED_WIKI_PREFIX)
def convert_issues(source,
dest,
dest_project_id,
only_issues=None,
blacklist_issues=None):
if overwrite and (method == 'direct'):
dest.clear_issues(dest_project_id)
milestone_map_id = {}
if migrate_milestones:
milestone_id = 0
for milestone_name in source.ticket.milestone.getAll():
milestone = source.ticket.milestone.get(milestone_name)
print(milestone)
new_milestone = Milestones(
description=trac2down.convert(
fix_wiki_syntax(milestone['description']), '/milestones/',
False),
title=milestone['name'],
state='active'
if str(milestone['completed']) == '0' else 'closed')
if method == 'direct':
new_milestone.project = dest_project_id
if milestone['due']:
new_milestone.due_date = convert_xmlrpc_datetime(
milestone['due'])
new_milestone = dest.create_milestone(dest_project_id,
new_milestone)
if new_milestone.id:
milestone_map_id[milestone_name] = new_milestone.id
milestone_id = new_milestone.id + 1
else:
milestone_map_id[milestone_name] = milestone_id
milestone_id = milestone_id + 1
get_all_tickets = xmlrpclib.MultiCall(source)
for ticket in source.ticket.query("max=0&order=id"):
get_all_tickets.ticket.get(ticket)
for src_ticket in get_all_tickets():
src_ticket_id = src_ticket[0]
if only_issues and src_ticket_id not in only_issues:
print("SKIP unwanted ticket #%s" % src_ticket_id)
continue
if blacklist_issues and src_ticket_id in blacklist_issues:
print("SKIP blacklisted ticket #%s" % src_ticket_id)
continue
src_ticket_data = src_ticket[3]
src_ticket_reporter = src_ticket_data['reporter']
src_ticket_priority = 'normal'
if 'priority' in src_ticket_data:
src_ticket_priority = src_ticket_data['priority']
src_ticket_resolution = src_ticket_data['resolution']
src_ticket_severity = src_ticket_data.get('severity')
src_ticket_status = src_ticket_data['status']
src_ticket_component = src_ticket_data.get('component', '')
src_ticket_keywords = src_ticket_data['keywords']
if (component_filter and src_ticket_component not in component_filter):
continue
new_labels = []
if src_ticket_priority == 'high':
new_labels.append('high priority')
elif src_ticket_priority == 'medium':
pass
elif src_ticket_priority == 'low':
new_labels.append('low priority')
if src_ticket_resolution == '':
# active ticket
pass
elif src_ticket_resolution == 'fixed':
pass
elif src_ticket_resolution == 'invalid':
new_labels.append('invalid')
elif src_ticket_resolution == 'wontfix':
new_labels.append("won't fix")
elif src_ticket_resolution == 'duplicate':
new_labels.append('duplicate')
elif src_ticket_resolution == 'worksforme':
new_labels.append('works for me')
if src_ticket_severity == 'high':
new_labels.append('critical')
elif src_ticket_severity == 'medium':
pass
elif src_ticket_severity == 'low':
new_labels.append("minor")
# Current ticket types are: enhancement, defect, compilation, performance, style, scientific, task, requirement
# new_labels.append(src_ticket_type)
if add_component_as_label and src_ticket_component != '':
for component in src_ticket_component.split(','):
new_labels.append(component.strip())
if add_label:
new_labels.append(add_label)
if src_ticket_keywords != '' and migrate_keywords:
for keyword in src_ticket_keywords.split(','):
new_labels.append(keyword.strip())
print("new labels: %s" % new_labels)
new_state = ''
if src_ticket_status == 'new':
new_state = 'opened'
elif src_ticket_status == 'assigned':
new_state = 'opened'
elif src_ticket_status == 'reopened':
new_state = 'reopened'
elif src_ticket_status == 'closed':
new_state = 'closed'
else:
print("!!! unknown ticket status: %s" % src_ticket_status)
new_description = (
create_issue_header(author=src_ticket_reporter,
created=src_ticket[1],
updated=src_ticket[2]) +
trac2down.convert(fix_wiki_syntax(src_ticket_data['description']),
'/issues/', False))
# Minimal parameters
new_issue = Issues(
title=(src_ticket_data['summary'][:245] +
'...') if len(src_ticket_data['summary']) > 245 else
src_ticket_data['summary'],
description=new_description,
state=new_state,
labels=",".join(new_labels))
if src_ticket_data['owner'] != '':
try:
new_issue.assignee = dest.get_user_id(
users_map[src_ticket_data['owner']])
except KeyError:
new_issue.assignee = dest.get_user_id(default_user)
# Additional parameters for direct access
if (method == 'direct'):
new_issue.created_at = convert_xmlrpc_datetime(src_ticket[1])
new_issue.updated_at = convert_xmlrpc_datetime(src_ticket[2])
new_issue.project = dest_project_id
new_issue.state = new_state
new_issue.author = dest.get_user_id(
users_map.get(src_ticket_reporter, default_user))
if overwrite:
new_issue.iid = src_ticket_id
else:
new_issue.iid = dest.get_issues_iid(dest_project_id)
# Set correct issue id
new_issue.iid = src_ticket_id
if 'milestone' in src_ticket_data:
milestone = src_ticket_data['milestone']
if milestone and milestone in milestone_map_id:
new_issue.milestone = milestone_map_id[milestone]
new_ticket = dest.create_issue(dest_project_id, new_issue)
changelog = source.ticket.changeLog(src_ticket_id)
is_attachment = False
attachment = None
binary_attachment = None
newowner = None
for change in changelog:
# New line
change_time = str(convert_xmlrpc_datetime(change[0]))
change_type = change[2]
print((" %s by %s (%s -> %s)" %
(change_type, change[1], change[3][:40].replace(
"\n", " "), change[4][:40].replace("\n", " "))).encode(
"ascii", "replace"))
#assert attachment is None or change_type == "comment", "an attachment must be followed by a comment"
author = dest.get_user_id(users_map[change[1]])
if change_type == "attachment":
# The attachment will be described in the next change!
is_attachment = True
attachment = change
if (change_type == "comment"):
desc = change[4]
if (desc == '' and is_attachment == False):
continue
if (desc != ''):
desc = fix_wiki_syntax(change[4])
note = Notes(note=create_issue_header(
author=change[1], created=change[0], is_comment=True) +
trac2down.convert(desc, '/issues/', False))
if attachment is not None:
note.attachment_name = attachment[4] # name of attachment
binary_attachment = source.ticket.getAttachment(
src_ticket_id, attachment[4].encode('utf8')).data
try:
note.author = dest.get_user_id(users_map[change[1]])
if note.author == None:
note.author = dest.get_user_id(default_user)
except KeyError:
note.author = dest.get_user_id(default_user)
if (method == 'direct'):
note.created_at = convert_xmlrpc_datetime(change[0])
note.updated_at = convert_xmlrpc_datetime(change[0])
try:
note.author = dest.get_user_id(users_map[change[1]])
except KeyError:
note.author = dest.get_user_id(default_user)
if (is_attachment):
note.attachment = attachment[4]
binary_attachment = source.ticket.getAttachment(
src_ticket_id, attachment[4].encode('utf8')).data
dest.comment_issue(dest_project_id, new_ticket, note,
binary_attachment)
is_attachment = False
if change_type == "status":
if change[3] == 'vendor':
# remove label 'vendor'
new_ticket.labels.remove('vendor')
# workaround #3 dest.update_issue_property(dest_project_id, issue, author, change_time, 'labels')
# we map here the various statii we have in trac to just 2 statii in gitlab (open or close), so loose some information
if change[4] in [
'new', 'assigned', 'analyzed', 'vendor', 'reopened'
]:
newstate = 'open'
elif change[4] in ['closed']:
newstate = 'closed'
else:
raise (" unknown ticket status: " + change[4])
if new_ticket.state != newstate:
new_ticket.state = newstate
if change[4] == 'vendor':
# add label 'vendor'
new_ticket.labels.append('vendor')
dest.ensure_label(dest_project_id, 'vendor',
labelcolor['vendor'])
if newstate == 'closed':
dest.close_issue(dest_project_id, new_ticket.iid)
dest.comment_issue(
dest_project_id, new_ticket,
Notes(note='Changing status from ' + change[3] + ' to ' +
change[4] + '.',
created_at=change_time,
author=author), binary_attachment)
def convert_issues(source,
dest,
dest_project_id,
only_issues=None,
blacklist_issues=None):
if overwrite and (method == 'direct'):
dest.clear_issues(dest_project_id)
milestone_map_id = {}
if migrate_milestones:
for milestone_name in source.ticket.milestone.getAll():
milestone = source.ticket.milestone.get(milestone_name)
print(milestone)
new_milestone = Milestones(
description=trac2down.convert(
fix_wiki_syntax(milestone['description']), '/milestones/',
False),
title=milestone['name'],
state='active'
if str(milestone['completed']) == '0' else 'closed')
if method == 'direct':
new_milestone.project = dest_project_id
if milestone['due']:
new_milestone.due_date = convert_xmlrpc_datetime(
milestone['due'])
new_milestone = dest.create_milestone(dest_project_id,
new_milestone)
milestone_map_id[milestone_name] = new_milestone.id
get_all_tickets = xmlrpclib.MultiCall(source)
for ticket in source.ticket.query("max=0&order=id"):
get_all_tickets.ticket.get(ticket)
for src_ticket in get_all_tickets():
src_ticket_id = src_ticket[0]
if only_issues and src_ticket_id not in only_issues:
print("SKIP unwanted ticket #%s" % src_ticket_id)
continue
if blacklist_issues and src_ticket_id in blacklist_issues:
print("SKIP blacklisted ticket #%s" % src_ticket_id)
continue
src_ticket_data = src_ticket[3]
src_ticket_priority = 'normal'
if 'priority' in src_ticket_data:
src_ticket_priority = src_ticket_data['priority']
src_ticket_resolution = src_ticket_data['resolution']
src_ticket_severity = src_ticket_data['severity']
src_ticket_status = src_ticket_data['status']
src_ticket_component = src_ticket_data.get('component', '')
src_ticket_keywords = src_ticket_data['keywords']
if (component_filter and src_ticket_component not in component_filter):
continue
new_labels = []
if src_ticket_priority == 'high':
new_labels.append('high priority')
elif src_ticket_priority == 'medium':
pass
elif src_ticket_priority == 'low':
new_labels.append('low priority')
if src_ticket_resolution == '':
# active ticket
pass
elif src_ticket_resolution == 'fixed':
pass
elif src_ticket_resolution == 'invalid':
new_labels.append('invalid')
elif src_ticket_resolution == 'wontfix':
new_labels.append("won't fix")
elif src_ticket_resolution == 'duplicate':
new_labels.append('duplicate')
elif src_ticket_resolution == 'worksforme':
new_labels.append('works for me')
if src_ticket_severity == 'high':
new_labels.append('critical')
elif src_ticket_severity == 'medium':
pass
elif src_ticket_severity == 'low':
new_labels.append("minor")
# Current ticket types are: enhancement, defect, compilation, performance, style, scientific, task, requirement
# new_labels.append(src_ticket_type)
if add_component_as_label and src_ticket_component != '':
for component in src_ticket_component.split(','):
new_labels.append(component.strip())
if add_label:
new_labels.append(add_label)
if src_ticket_keywords != '' and migrate_keywords:
for keyword in src_ticket_keywords.split(','):
new_labels.append(keyword.strip())
print("new labels: %s" % new_labels)
new_state = ''
if src_ticket_status == 'new':
new_state = 'opened'
elif src_ticket_status == 'assigned':
new_state = 'opened'
elif src_ticket_status == 'reopened':
new_state = 'reopened'
elif src_ticket_status == 'closed':
new_state = 'closed'
else:
print("!!! unknown ticket status: %s" % src_ticket_status)
# Minimal parameters
new_issue = Issues(title=src_ticket_data['summary'],
description=trac2down.convert(
fix_wiki_syntax(src_ticket_data['description']),
'/issues/', False),
state=new_state,
labels=",".join(new_labels))
if src_ticket_data['owner'] != '':
try:
new_issue.assignee = dest.get_user_id(
users_map[src_ticket_data['owner']])
except KeyError:
new_issue.assignee = dest.get_user_id(default_user)
# Additional parameters for direct access
if (method == 'direct'):
new_issue.created_at = convert_xmlrpc_datetime(src_ticket[1])
new_issue.updated_at = convert_xmlrpc_datetime(src_ticket[2])
new_issue.project = dest_project_id
new_issue.state = new_state
try:
new_issue.author = dest.get_user_id(
users_map[src_ticket_data['reporter']])
except KeyError:
new_issue.author = dest.get_user_id(default_user)
if overwrite:
new_issue.iid = src_ticket_id
else:
new_issue.iid = dest.get_issues_iid(dest_project_id)
if 'milestone' in src_ticket_data:
milestone = src_ticket_data['milestone']
if milestone and milestone in milestone_map_id:
new_issue.milestone = milestone_map_id[milestone]
new_ticket = dest.create_issue(dest_project_id, new_issue)
# new_ticket_id = new_ticket.id
changelog = source.ticket.changeLog(src_ticket_id)
is_attachment = False
for change in changelog:
change_type = change[2]
if change_type == "attachment":
# The attachment will be described in the next change!
is_attachment = True
attachment = change
if (change_type == "comment"):
desc = change[4]
if (desc == '' and is_attachment == False):
continue
if (desc != ''):
desc = fix_wiki_syntax(change[4])
note = Notes(note=trac2down.convert(desc, '/issues/', False))
binary_attachment = None
if (method == 'direct'):
note.created_at = convert_xmlrpc_datetime(change[0])
note.updated_at = convert_xmlrpc_datetime(change[0])
try:
note.author = dest.get_user_id(users_map[change[1]])
except KeyError:
note.author = dest.get_user_id(default_user)
if (is_attachment):
note.attachment = attachment[4]
binary_attachment = source.ticket.getAttachment(
src_ticket_id, attachment[4].encode('utf8')).data
dest.comment_issue(dest_project_id, new_ticket, note,
binary_attachment)
is_attachment = False
def convert_issues(source, dest, dest_project_id, only_issues=None):
if overwrite and (method == 'direct'):
dest.clear_issues(dest_project_id)
milestone_map_id={}
for milestone_name in source.ticket.milestone.getAll():
milestone = source.ticket.milestone.get(milestone_name)
print milestone
new_milestone = Milestones(
description = milestone['description'],
title = milestone['name'],
state = 'active' if str(milestone['completed']) == '0' else 'closed'
)
if method == 'direct':
new_milestone.project = dest_project_id
if milestone['due']:
new_milestone.due_date = convert_xmlrpc_datetime(milestone['due'])
new_milestone = dest.create_milestone(dest_project_id, new_milestone)
milestone_map_id[milestone_name] = new_milestone.id
get_all_tickets = xmlrpclib.MultiCall(source)
for ticket in source.ticket.query("max=0"):
get_all_tickets.ticket.get(ticket)
for src_ticket in get_all_tickets():
src_ticket_id = src_ticket[0]
if only_issues and src_ticket_id not in only_issues:
print "SKIP unwanted ticket #%s" % src_ticket_id
continue
src_ticket_data = src_ticket[3]
src_ticket_priority = src_ticket_data['priority']
src_ticket_resolution = src_ticket_data['resolution']
# src_ticket_severity = src_ticket_data['severity']
src_ticket_status = src_ticket_data['status']
src_ticket_component = src_ticket_data['component']
new_labels = []
if src_ticket_priority == 'high':
new_labels.append('high priority')
elif src_ticket_priority == 'medium':
pass
elif src_ticket_priority == 'low':
new_labels.append('low priority')
if src_ticket_resolution == '':
# active ticket
pass
elif src_ticket_resolution == 'fixed':
pass
elif src_ticket_resolution == 'invalid':
new_labels.append('invalid')
elif src_ticket_resolution == 'wontfix':
new_labels.append("won't fix")
elif src_ticket_resolution == 'duplicate':
new_labels.append('duplicate')
elif src_ticket_resolution == 'worksforme':
new_labels.append('works for me')
# if src_ticket_severity == 'high':
# new_labels.append('critical')
# elif src_ticket_severity == 'medium':
# pass
# elif src_ticket_severity == 'low':
# new_labels.append("minor")
# Current ticket types are: enhancement, defect, compilation, performance, style, scientific, task, requirement
# new_labels.append(src_ticket_type)
if src_ticket_component != '':
for component in src_ticket_component.split(','):
new_labels.append(component.strip())
print "new labels:", new_labels
new_state = ''
if src_ticket_status == 'new':
new_state = 'opened'
elif src_ticket_status == 'assigned':
new_state = 'opened'
elif src_ticket_status == 'reopened':
new_state = 'reopened'
elif src_ticket_status == 'closed':
new_state = 'closed'
else:
print "!!! unknown ticket status:", src_ticket_status
# Minimal parameters
new_issue = Issues(
title=src_ticket_data['summary'],
description=trac2down.convert(fix_wiki_syntax(src_ticket_data['description']), '/issues/', False),
state=new_state,
labels=",".join(new_labels)
)
if src_ticket_data['owner'] != '':
try:
new_issue.assignee = dest.get_user_id(users_map[src_ticket_data['owner']])
except KeyError:
new_issue.assignee = dest.get_user_id(default_user)
# Additional parameters for direct access
if (method == 'direct'):
new_issue.created_at = convert_xmlrpc_datetime(src_ticket[1])
new_issue.updated_at = convert_xmlrpc_datetime(src_ticket[2])
new_issue.project = dest_project_id
new_issue.state = new_state
try:
new_issue.author = dest.get_user_id(users_map[src_ticket_data['reporter']])
except KeyError:
new_issue.author = dest.get_user_id(default_user)
if overwrite:
new_issue.iid = src_ticket_id
else:
new_issue.iid = dest.get_issues_iid(dest_project_id)
if 'milestone' in src_ticket_data:
milestone = src_ticket_data['milestone']
if milestone and milestone_map_id[milestone]:
new_issue.milestone = milestone_map_id[milestone]
new_ticket = dest.create_issue(dest_project_id, new_issue)
# new_ticket_id = new_ticket.id
changelog = source.ticket.changeLog(src_ticket_id)
is_attachment = False
for change in changelog:
change_type = change[2]
if change_type == "attachment":
# The attachment will be described in the next change!
is_attachment = True
attachment = change
if (change_type == "comment") and change[4] != '':
note = Notes(
note=trac2down.convert(fix_wiki_syntax(change[4]), '/issues/', False)
)
binary_attachment = None
if (method == 'direct'):
note.created_at = convert_xmlrpc_datetime(change[0])
note.updated_at = convert_xmlrpc_datetime(change[0])
try:
note.author = dest.get_user_id(users_map[change[1]])
except KeyError:
note.author = dest.get_user_id(default_user)
if (is_attachment):
note.attachment = attachment[4]
binary_attachment = source.ticket.getAttachment(src_ticket_id, attachment[4].encode('utf8')).data
dest.comment_issue(dest_project_id, new_ticket, note, binary_attachment)
is_attachment = False
"""Add double [[...]] around wikilinks (given as `targets`)
Most prosaic method possible - simply loop over the explicit list
of targets. Tries to be careful in avoiding code blocks, but
will still be fooled by inline code
"""
lines = []
is_code_block = False
for line in text.split('\n'):
# not blockquote?
if not line.startswith(' '):
if line.startswith("````"):
is_code_block = not is_code_block
if not is_code_block:
for target in targets:
line = line.replace(target, f"[[{target}]]")
# line = re.sub(r'\!(([A-Z][a-z0-9]+){2,})', r'[[\1]]', line)
lines.append(line)
return "\n".join(lines)
# Process all the files in the input folder
for p in input_path.glob("*"):
# Open file from the dump of the trac database and convert it
with open(p) as f:
text = trac2down.convert(f.read(), ".")
text = fixup_wikilinks(text, wikinames)
# Save the converted file with a ".md" extension to output folder
# The 3rd, 4th, 5th arguments to this function are unused
trac2down.save_file(text, p.name, None, None, None, save_path)
def convert_wiki(source, dest):
exclude_authors = [a.strip() for a in config.get('wiki', 'exclude_authors').split(',')]
target_directory = config.get('wiki', 'target-directory')
if wiki_override_page:
pages = [wiki_override_page]
else:
pages = source.wiki.getAllPages()
i = 0
for name in pages:
i += 1
info = source.wiki.getPageInfo(name)
if info == 0:
raise Exception("No page named %s could be found" % name)
if info['author'] in exclude_authors:
continue
page = source.wiki.getPage(name)
print("[%d/%d] Page %s:%s" % (i, len(pages), name, info))
if name == 'WikiStart':
name = 'home'
sanitized_name = name.replace('/', '-').lower()
upload_prefix = 'uploads/%s' % sanitized_name
old_attachment_prefix = '/attachment/wiki/%s' % name
old_raw_attachment_prefix = '/raw-attachment/wiki/%s' % name
converted = trac2down.convert(
page,
os.path.dirname('/wikis/%s' % name),
wiki_upload_prefix=upload_prefix,
old_attachment_prefix=old_attachment_prefix,
old_raw_attachment_prefix=old_raw_attachment_prefix
)
if method == 'direct' and not ignore_wiki_attachments:
files_not_linked_to = []
for attachment_filename in source.wiki.listAttachments(name):
binary_attachment = source.wiki.getAttachment(attachment_filename).data
attachment_name = attachment_filename.split('/')[-1]
sanitized_attachment_name = attachment_name \
.replace(' ', '_') \
.replace('(', '') \
.replace(')', '')
attachment_directory = os.path.join(target_directory, 'uploads', sanitized_name)
dest.save_wiki_attachment(attachment_directory, sanitized_attachment_name, binary_attachment)
converted = converted.replace(r'%s/%s)' % (sanitized_name, attachment_filename),
r'%s/%s)' % (sanitized_name, sanitized_attachment_name))
if '%s/%s)' % (upload_prefix, sanitized_attachment_name) not in converted:
files_not_linked_to.append(sanitized_attachment_name)
print(' ' + sanitized_attachment_name)
if len(files_not_linked_to) > 0:
print ' %d non-linked attachments detected, manually adding to generated Markdown' % len(files_not_linked_to)
converted += '\n\n'
converted += '##### Attached files:\n'
for file_name in files_not_linked_to:
converted += '- [%s](uploads/%s/%s)\n' % (file_name, sanitized_name, file_name)
trac2down.save_file(converted, name, info['version'], info['lastModified'], info['author'], target_directory)
def convert_issues(source, dest, dest_project_ids, convert_milestones, only_issues=None,
get_dest_project_id_for_issue=None, issue_mutator=None):
if only_issues is None: only_issues = []
if overwrite and method == 'direct':
for project_id in dest_project_ids:
dest.clear_issues(project_id, only_issues)
milestone_map_id = {}
if convert_milestones:
for dest_project_id in dest_project_ids:
for milestone_name in source.ticket.milestone.getAll():
milestone = source.ticket.milestone.get(milestone_name)
print("migrated milestone: %s" % milestone_name)
new_milestone = Milestones(
description=trac2down.convert(fix_wiki_syntax(milestone['description']), '/milestones/', False),
title=milestone['name'],
state='active' if str(milestone['completed']) == '0' else 'closed'
)
if method == 'direct':
new_milestone.project = dest_project_id
if milestone['due']:
new_milestone.due_date = convert_xmlrpc_datetime(milestone['due'])
new_milestone = dest.create_milestone(dest_project_id, new_milestone)
milestone_map_id[milestone_name] = new_milestone.id
get_all_tickets = xmlrpclib.MultiCall(source)
gitlab_user_cache = {}
if only_issues:
print("getting tickets from trac: %s" % only_issues)
for ticket in only_issues:
get_all_tickets.ticket.get(ticket)
else:
print("getting all tickets from trac")
for ticket in source.ticket.query("max=0&order=id"):
get_all_tickets.ticket.get(ticket)
image_regexp = re.compile(r'\.(jpg|jpeg|png|gif)$')
title_label_regexp = re.compile(r'(\[.+?\]|.+?:)')
for src_ticket in get_all_tickets():
src_ticket_id = src_ticket[0]
if only_issues and src_ticket_id not in only_issues:
print("SKIP unwanted ticket #%s" % src_ticket_id)
continue
print 'migrating ticket %d' % src_ticket_id
src_ticket_data = src_ticket[3]
src_ticket_billable = src_ticket_data.get('billable', '0')
src_ticket_component = src_ticket_data['component']
src_ticket_keywords = re.split(r'[, ]', src_ticket_data['keywords'])
src_ticket_milestone = src_ticket_data['milestone']
src_ticket_priority = src_ticket_data['priority']
src_ticket_resolution = src_ticket_data['resolution']
src_ticket_status = src_ticket_data['status']
src_ticket_version = src_ticket_data['version']
new_labels = CasePreservingSet()
if src_ticket_billable == '1':
new_labels.add('billable')
if src_ticket_milestone:
for label in translate_milestone(src_ticket_milestone):
new_labels.add(label)
if src_ticket_priority == 'high':
new_labels.add('high priority')
elif src_ticket_priority == 'medium':
pass
elif src_ticket_priority == 'low':
new_labels.add('low priority')
if src_ticket_resolution == '':
# active ticket
pass
elif src_ticket_resolution == 'fixed':
pass
elif src_ticket_resolution == 'invalid':
new_labels.add('invalid')
elif src_ticket_resolution == 'wontfix':
new_labels.add("won't fix")
elif src_ticket_resolution == 'duplicate':
new_labels.add('duplicate')
elif src_ticket_resolution == 'worksforme':
new_labels.add('works for me')
if src_ticket_component != '':
for component in src_ticket_component.split(','):
component = component.strip()
translated_component = translate_component(component)
if translated_component:
new_labels.add(translated_component)
else:
print(' WARN: Dropping component %s' % component)
for keyword in src_ticket_keywords:
keyword = keyword.strip()
if not keyword:
continue
translated_keyword = translate_keyword(keyword)
if translated_keyword:
new_labels.add(translated_keyword)
else:
print(' WARN: Dropping keyword %s' % keyword)
new_state = 'opened'
if src_ticket_status == 'new':
new_state = 'opened'
elif src_ticket_status == 'assigned':
new_state = 'opened'
new_labels.add('Do')
elif src_ticket_status == 'reopened':
# There is no 'reopened' state in GitLab.
new_state = 'opened'
elif src_ticket_status == 'closed':
new_state = 'closed'
elif src_ticket_status == 'accepted':
new_state = 'opened'
new_labels.add('Do')
elif src_ticket_status == 'reviewing' or src_ticket_status == 'testing':
new_labels.add('Check')
else:
print("!!! Unknown ticket status: %s, not preserving in migrated data" % src_ticket_status)
summary = src_ticket_data['summary']
sanitized_summary = summary
done = False
while not done:
title_result = title_label_regexp.search(sanitized_summary)
if title_result:
prefix = title_result.group(1)
lowercased_prefix = prefix.lower()
# Awkward way, but prefix.translate() works differently on str and unicode objects so
# this is good enough for now.
mangled_prefix = lowercased_prefix.replace('[', '').replace(']', '').replace(':', '')
translated_prefix = label_prefix_translation_map.get(mangled_prefix, '')
if translated_prefix != '':
if translated_prefix == None:
# None values have a special meaning, indicate: "Remove this prefix, but don't add a label".
print(' !!! Dropping prefix %s' % mangled_prefix)
else:
# Prefix found in whitelist.
new_labels.add(translated_prefix)
sanitized_summary = sanitized_summary[title_result.end():].strip()
else:
# Prefix doesn't exist in mangling map. Leave it as-is, has to be manually handled.
done = True
else:
done = True
# FIXME: Would like to put these in deeply nested folder structure instead of dashes, but
# the GitLab uploads route only supports a single subfolder below uploads:
# https://github.com/gitlabhq/gitlabhq/blob/master/config/routes/uploads.rb#L22-L25
issue_attachment_path = os.path.join('issue-attachment-%d' % src_ticket_id)
new_issue = Issues(
title=sanitized_summary,
description=trac2down.convert(
fix_wiki_syntax(src_ticket_data['description']),
'/issues/',
False,
issue_upload_prefix='/uploads/' + issue_attachment_path
),
state=new_state,
labels=new_labels
)
if get_dest_project_id_for_issue:
dest_project_id = get_dest_project_id_for_issue(dest, new_issue)
else:
# No function defined - assume that we have been provided a single project ID.
dest_project_id = dest_project_ids[0]
if issue_mutator:
issue_mutator(new_issue)
print(" Final set of labels: %s" % ', '.join(new_issue.labels))
if src_ticket_version:
if src_ticket_version == 'trunk' or src_ticket_version == 'dev':
pass
else:
release_milestone_name = 'release-%s' % src_ticket_version
if release_milestone_name not in milestone_map_id:
print(" creating new milestone for %s" % release_milestone_name)
new_milestone = Milestones(
title=release_milestone_name,
description='',
state='closed'
)
if method == 'direct':
new_milestone.project = dest_project_id
new_milestone = dest.create_milestone(dest_project_id, new_milestone)
milestone_map_id[release_milestone_name] = new_milestone.id
new_issue.milestone = milestone_map_id[release_milestone_name]
# Additional parameters for direct access
if method == 'direct':
new_issue.created_at = convert_xmlrpc_datetime(src_ticket[1])
new_issue.updated_at = convert_xmlrpc_datetime(src_ticket[2])
new_issue.project = dest_project_id
new_issue.state = new_state
try:
new_issue.author = get_cached_user_id(dest, gitlab_user_cache, users_map[src_ticket_data['reporter']])
except KeyError:
if default_user:
new_issue.author = get_cached_user_id(dest, gitlab_user_cache, default_user)
else:
raise
if overwrite:
new_issue.iid = src_ticket_id
else:
new_issue.iid = dest.get_issues_iid(dest_project_id)
if 'milestone' in src_ticket_data and not new_issue.milestone:
milestone = src_ticket_data['milestone']
if milestone and milestone_map_id.get(milestone):
new_issue.milestone = milestone_map_id[milestone]
new_ticket = dest.create_issue(dest_project_id, new_issue)
if src_ticket_data['owner'] != '':
try:
mapped_user = users_map[src_ticket_data['owner']]
except KeyError:
if default_user:
mapped_user = default_user
else:
raise
assign_query = IssueAssignees.insert(
issue=new_ticket.id,
user=get_cached_user_id(dest, gitlab_user_cache, mapped_user)
)
dest.assign_issue(assign_query)
changelog = source.ticket.changeLog(src_ticket_id)
is_attachment = False
for change in changelog:
(change_datetime, change_user, change_type, _, change_text, _) = change
if change_type == "attachment":
# The attachment will be described in the next change!
is_attachment = True
attachment_file_name = change_text
if change_type == "comment" and (change_text != '' or is_attachment):
note = Notes(
note=trac2down.convert(
fix_wiki_syntax(change_text),
'/issues/',
False,
issue_upload_prefix=issue_attachment_path
)
)
binary_attachment = None
if method == 'direct':
note.created_at = convert_xmlrpc_datetime(change_datetime)
note.updated_at = convert_xmlrpc_datetime(change_datetime)
try:
user = users_map[change_user]
note.author = get_cached_user_id(dest, gitlab_user_cache, user)
except KeyError:
if default_user:
note.author = get_cached_user_id(dest, gitlab_user_cache, default_user)
else:
raise
if is_attachment:
# Intermediate save needed to make note.id be populated with the real ID of the record.
note.save()
note.attachment = '%s/%s' % (issue_attachment_path, attachment_file_name)
image_prefix = ''
if image_regexp.search(attachment_file_name):
image_prefix = '!'
attachment_label = note.note
if not attachment_label:
attachment_label = attachment_file_name
note.note = '%s[%s](/uploads/%s)' % (image_prefix, attachment_label, note.attachment)
print(" migrating attachment for ticket id %s: %s" % (src_ticket_id, attachment_file_name))
binary_attachment = source.ticket.getAttachment(src_ticket_id,
attachment_file_name.encode('utf8')).data
dest.comment_issue(dest_project_id, new_ticket, note, binary_attachment)
is_attachment = False
