source = Connection(source_url, source_login, source_password)
target = Connection(
target_url, target_login, target_password
) #, proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888)
#reset connections to avoid disconnections
user_importer.resetConnections(source, target)
link_importer.resetConnections(target)
# copy project, subsystems, versions
project = source.getProject(projectId)
link_importer.addAvailableIssuesFrom(projectId)
project_custom_fields = source.getProjectCustomFields(projectId)
# create bundles and additional values
for pcf_ref in project_custom_fields:
pcf = source.getProjectCustomField(projectId, pcf_ref.name)
if hasattr(pcf, "bundle"):
create_bundle_from_bundle(source, target, pcf.bundle,
source.getCustomField(pcf.name).type,
user_importer)
target_project_fields = [
pcf.name for pcf in target.getProjectCustomFields(projectId)
]
for field in project_custom_fields:
if field.name in target_project_fields:
if hasattr(field, 'bundle'):
if field.bundle != target.getProjectCustomField(
projectId, field.name).bundle:
target.deleteProjectCustomField(projectId, field.name)
create_project_custom_field(target, field, projectId)
def youtrack2youtrack(source_url, source_login, source_password, target_url, target_login, target_password,
project_ids, query='', source_token=None, target_token=None, params=None):
if not len(project_ids):
print("You should sign at least one project to import")
return
if params is None:
params = {}
source = Connection(source_url, source_login, source_password) if (source_token is None) else Connection(source_url,
token=source_token)
target = Connection(target_url, target_login, target_password) if (target_token is None) else Connection(target_url,
token=target_token)
# , proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888)
print("Import issue link types")
for ilt in source.getIssueLinkTypes():
try:
print(target.createIssueLinkType(ilt))
except youtrack.YouTrackException as e:
print(e.message)
user_importer = UserImporter(source, target, caching_users=params.get('enable_user_caching', True))
link_importer = LinkImporter(target)
# create all projects with minimum info and project lead set
created_projects = []
for project_id in project_ids:
created = create_project_stub(source, target, project_id, user_importer)
created_projects.append(created)
# save created project ids to create correct group roles afterwards
user_importer.addCreatedProjects([project.id for project in created_projects])
# import project leads with group they are included and roles assigned to these groups
user_importer.importUsersRecursively([target.getUser(project.lead) for project in created_projects])
# afterwards in a script any user import imply recursive import
cf_names_to_import = set([]) # names of cf prototypes that should be imported
for project_id in project_ids:
cf_names_to_import.update([pcf.name.capitalize() for pcf in source.getProjectCustomFields(project_id)])
target_cf_names = [pcf.name.capitalize() for pcf in target.getCustomFields()]
period_cf_names = []
for cf_name in cf_names_to_import:
source_cf = source.getCustomField(cf_name)
if source_cf.type.lower() == 'period':
period_cf_names.append(source_cf.name.lower())
print("Processing custom field '%s'" % utf8encode(cf_name))
if cf_name in target_cf_names:
target_cf = target.getCustomField(cf_name)
if not (target_cf.type == source_cf.type):
print("In your target and source YT instances you have field with name [ %s ]" % utf8encode(cf_name))
print("They have different types. Source field type [ %s ]. Target field type [ %s ]" %
(source_cf.type, target_cf.type))
print("exiting...")
exit()
else:
if hasattr(source_cf, "defaultBundle"):
create_bundle_from_bundle(source, target, source_cf.defaultBundle, source_cf.type, user_importer)
target.createCustomField(source_cf)
failed_commands = []
for projectId in project_ids:
source = Connection(source_url, source_login, source_password) if (source_token is None) else Connection(
source_url, token=source_token)
target = Connection(target_url, target_login, target_password) if (target_token is None) else Connection(
target_url, token=target_token)
# , proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888)
# reset connections to avoid disconnections
user_importer.resetConnections(source, target)
link_importer.resetConnections(target)
# copy project, subsystems, versions
project = source.getProject(projectId)
link_importer.addAvailableIssuesFrom(projectId)
project_custom_fields = source.getProjectCustomFields(projectId)
# create bundles and additional values
for pcf_ref in project_custom_fields:
pcf = source.getProjectCustomField(projectId, pcf_ref.name)
if hasattr(pcf, "bundle"):
try:
create_bundle_from_bundle(source, target, pcf.bundle, source.getCustomField(pcf.name).type,
user_importer)
except youtrack.YouTrackException as e:
if e.response.status != 409:
raise e
else:
print(e)
target_project_fields = [pcf.name.lower() for pcf in target.getProjectCustomFields(projectId)]
for field in project_custom_fields:
if field.name.lower() in target_project_fields:
if hasattr(field, 'bundle'):
if field.bundle != target.getProjectCustomField(projectId, field.name).bundle:
target.deleteProjectCustomField(projectId, field.name)
create_project_custom_field(target, field, projectId)
else:
try:
create_project_custom_field(target, field, projectId)
except youtrack.YouTrackException as e:
if e.response.status != 409:
raise e
else:
print(e)
# copy issues
start = 0
max = 20
sync_workitems = enable_time_tracking(source, target, projectId)
tt_settings = target.getProjectTimeTrackingSettings(projectId)
print("Import issues")
last_created_issue_number = 0
while True:
try:
print("Get issues from " + str(start) + " to " + str(start + max))
issues = source.getIssues(projectId, query, start, max)
if len(issues) <= 0:
break
if convert_period_values and period_cf_names:
for issue in issues:
for pname in period_cf_names:
for fname in issue.__dict__:
if fname.lower() != pname:
continue
issue[fname] = period_to_minutes(issue[fname])
users = set([])
for issue in issues:
print("Collect users for issue [%s]" % issue.id)
users.add(issue.getReporter())
if issue.hasAssignee():
if isinstance(issue.Assignee, (list, tuple)):
users.update(issue.getAssignee())
else:
users.add(issue.getAssignee())
# TODO: http://youtrack.jetbrains.net/issue/JT-6100
users.add(issue.getUpdater())
if issue.hasVoters():
users.update(issue.getVoters())
for comment in issue.getComments():
users.add(comment.getAuthor())
print("Collect links for issue [%s]" % issue.id)
link_importer.collectLinks(issue.getLinks(True))
# links.extend(issue.getLinks(True))
# fix problem with comment.text
for comment in issue.getComments():
if not hasattr(comment, "text") or (len(comment.text.strip()) == 0):
setattr(comment, 'text', 'no text')
user_importer.importUsersRecursively(users)
print("Create issues [" + str(len(issues)) + "]")
if params.get('create_new_issues'):
create_issues(target, issues, last_created_issue_number)
else:
print(target.importIssues(projectId, project.name + ' Assignees', issues))
link_importer.addAvailableIssues(issues)
for issue in issues:
try:
target_issue = target.getIssue(issue.id)
except youtrack.YouTrackException as e:
print("Cannot get target issue")
print(e)
continue
if params.get('sync_tags') and issue.tags:
try:
for tag in issue.tags:
tag = re.sub(r'[,&<>]', '_', tag)
try:
target.executeCommand(issue.id, 'tag ' + tag, disable_notifications=True)
except youtrack.YouTrackException:
tag = re.sub(r'[\s-]', '_', tag)
target.executeCommand(issue.id, 'tag ' + tag, disable_notifications=True)
except youtrack.YouTrackException as e:
print("Cannot sync tags for issue " + issue.id)
print(e)
if params.get('add_new_comments'):
target_comments = dict()
max_id = 0
for c in target_issue.getComments():
target_comments[c.created] = c
if max_id < c.created:
max_id = c.created
for c in issue.getComments():
if c.created > max_id or c.created not in target_comments:
group = None
if hasattr(c, 'permittedGroup'):
group = c.permittedGroup
try:
target.executeCommand(issue.id, 'comment', c.text, group, c.author,
disable_notifications=True)
except youtrack.YouTrackException as e:
print('Cannot add comment to issue')
print(e)
if params.get('sync_custom_fields'):
skip_fields = []
if tt_settings and tt_settings.Enabled and tt_settings.TimeSpentField:
skip_fields.append(tt_settings.TimeSpentField)
skip_fields = [name.lower() for name in skip_fields]
for pcf in [pcf for pcf in project_custom_fields if pcf.name.lower() not in skip_fields]:
target_cf_value = None
if pcf.name in target_issue:
target_cf_value = target_issue[pcf.name]
if isinstance(target_cf_value, (list, tuple)):
target_cf_value = set(target_cf_value)
elif target_cf_value == target.getProjectCustomField(projectId, pcf.name).emptyText:
target_cf_value = None
source_cf_value = None
if pcf.name in issue:
source_cf_value = issue[pcf.name]
if isinstance(source_cf_value, (list, tuple)):
source_cf_value = set(source_cf_value)
elif source_cf_value == source.getProjectCustomField(projectId, pcf.name).emptyText:
source_cf_value = None
if source_cf_value == target_cf_value:
continue
if isinstance(source_cf_value, set) or isinstance(target_cf_value, set):
if source_cf_value is None:
source_cf_value = set([])
elif not isinstance(source_cf_value, set):
source_cf_value = set([source_cf_value])
if target_cf_value is None:
target_cf_value = set([])
elif not isinstance(target_cf_value, set):
target_cf_value = set([target_cf_value])
for v in target_cf_value:
if v not in source_cf_value:
target.executeCommand(issue.id, 'remove %s %s' % (pcf.name, v),
disable_notifications=True)
for v in source_cf_value:
if v not in target_cf_value:
target.executeCommand(issue.id, 'add %s %s' % (pcf.name, v),
disable_notifications=True)
else:
if source_cf_value is None:
source_cf_value = target.getProjectCustomField(projectId, pcf.name).emptyText
if pcf.type.lower() == 'date':
m = re.match(r'(\d{10})(?:\d{3})?', str(source_cf_value))
if m:
source_cf_value = datetime.datetime.fromtimestamp(
int(m.group(1))).strftime('%Y-%m-%d')
elif pcf.type.lower() == 'period':
source_cf_value = '%sm' % source_cf_value
command = '%s %s' % (pcf.name, source_cf_value)
try:
target.executeCommand(issue.id, command, disable_notifications=True)
except youtrack.YouTrackException as e:
if e.response.status == 412 and e.response.reason.find('Precondition Failed') > -1:
print('WARN: Some workflow blocks following command: %s' % command)
failed_commands.append((issue.id, command))
if sync_workitems:
workitems = source.getWorkItems(issue.id)
if workitems:
existing_workitems = dict()
target_workitems = target.getWorkItems(issue.id)
if target_workitems:
for w in target_workitems:
_id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration)
if hasattr(w, 'description'):
_id += '\n%s' % w.description
existing_workitems[_id] = w
new_workitems = []
for w in workitems:
_id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration)
if hasattr(w, 'description'):
_id += '\n%s' % w.description
if _id not in existing_workitems:
new_workitems.append(w)
if new_workitems:
print("Process workitems for issue [ " + issue.id + "]")
try:
user_importer.importUsersRecursively(
[source.getUser(w.authorLogin)
for w in new_workitems])
target.importWorkItems(issue.id, new_workitems)
except youtrack.YouTrackException as e:
if e.response.status == 404:
print("WARN: Target YouTrack doesn't support workitems importing.")
print("WARN: Workitems won't be imported.")
sync_workitems = False
else:
print("ERROR: Skipping workitems because of error:" + str(e))
print("Process attachments for issue [%s]" % issue.id)
existing_attachments = dict()
try:
for a in target.getAttachments(issue.id):
existing_attachments[a.name + '\n' + a.created] = a
except youtrack.YouTrackException as e:
if e.response.status == 404:
print("Skip importing attachments because issue %s doesn't exist" % issue.id)
continue
raise e
attachments = []
users = set([])
for a in issue.getAttachments():
if a.name + '\n' + a.created in existing_attachments and not params.get('replace_attachments'):
a.name = utf8encode(a.name)
try:
print("Skip attachment '%s' (created: %s) because it's already exists"
% (utf8encode(a.name), utf8encode(a.created)))
except Exception:
pass
continue
attachments.append(a)
author = a.getAuthor()
if author is not None:
users.add(author)
user_importer.importUsersRecursively(users)
for a in attachments:
print("Transfer attachment of " + utf8encode(issue.id) + ": " + utf8encode(a.name))
# TODO: add authorLogin to workaround http://youtrack.jetbrains.net/issue/JT-6082
# a.authorLogin = target_login
try:
target.createAttachmentFromAttachment(issue.id, a)
except BaseException as e:
print("Cant import attachment [ %s ]" % utf8encode(a.name))
print(repr(e))
continue
if params.get('replace_attachments'):
try:
old_attachment = existing_attachments.get(a.name + '\n' + a.created)
if old_attachment:
print('Deleting old attachment')
target.deleteAttachment(issue.id, old_attachment.id)
except BaseException as e:
print("Cannot delete attachment '%s' from issue %s" % (
utf8encode(a.name), utf8encode(issue.id)))
print(e)
except Exception as e:
print('Cant process issues from ' + str(start) + ' to ' + str(start + max))
traceback.print_exc()
raise e
start += max
print("Import issue links")
link_importer.importCollectedLinks()
print("Trying to execute failed commands once again")
for issue_id, command in failed_commands:
try:
print('Executing command on issue %s: %s' % (issue_id, command))
target.executeCommand(issue_id, command, disable_notifications=True)
except youtrack.YouTrackException as e:
print('Failed to execute command for issue #%s: %s' % (issue_id, command))
print(e)
for projectId in project_ids:
source = Connection(source_url, source_login, source_password)
target = Connection(target_url, target_login,
target_password) #, proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888)
#reset connections to avoid disconnections
user_importer.resetConnections(source, target)
link_importer.resetConnections(target)
# copy project, subsystems, versions
project = source.getProject(projectId)
link_importer.addAvailableIssuesFrom(projectId)
project_custom_fields = source.getProjectCustomFields(projectId)
# create bundles and additional values
for pcf_ref in project_custom_fields:
pcf = source.getProjectCustomField(projectId, pcf_ref.name)
if hasattr(pcf, "bundle"):
create_bundle_from_bundle(source, target, pcf.bundle, source.getCustomField(pcf.name).type, user_importer)
target_project_fields = [pcf.name for pcf in target.getProjectCustomFields(projectId)]
for field in project_custom_fields:
if field.name in target_project_fields:
if hasattr(field, 'bundle'):
if field.bundle != target.getProjectCustomField(projectId, field.name).bundle:
target.deleteProjectCustomField(projectId, field.name)
create_project_custom_field(target, field, projectId)
else:
create_project_custom_field(target, field, projectId)
# copy issues
start = 0
for projectId in project_ids:
source = Connection(source_url, source_login, source_password)
target = Connection(target_url, target_login,
target_password) #, proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888)
#reset connections to avoid disconnections
user_importer.resetConnections(source, target)
link_importer.resetConnections(target)
# copy project, subsystems, versions
project = source.getProject(projectId)
link_importer.addAvailableIssuesFrom(projectId)
project_custom_fields = source.getProjectCustomFields(projectId)
# create bundles and additional values
for pcf_ref in project_custom_fields:
pcf = source.getProjectCustomField(projectId, pcf_ref.name)
if hasattr(pcf, "bundle"):
create_bundle_from_bundle(source, target, pcf.bundle, source.getCustomField(pcf.name).type, user_importer)
target_project_fields = [pcf.name for pcf in target.getProjectCustomFields(projectId)]
for field in project_custom_fields:
if field.name in target_project_fields:
if hasattr(field, 'bundle'):
if field.bundle != target.getProjectCustomField(projectId, field.name).bundle:
target.deleteProjectCustomField(projectId, field.name)
create_project_custom_field(target, field, projectId)
else:
create_project_custom_field(target, field, projectId)
# copy issues
start = 0
class RedmineImporter(object):
def __init__(self, params):
self._params = params
self._projects = None
self._max_issue_ids = {}
self._issue_ids = {}
self._relations = {}
self._users = {}
self._groups = {}
self._subsystems = {}
self._versions = {}
# Connecting to Redmine
if 'rm_api_key' in params:
self._source = youtrackutils.redmine.RedmineClient(
params['rm_api_key'], params['rm_url'])
elif 'rm_login' in params:
self._source = youtrackutils.redmine.RedmineClient(
None, params['rm_url'], params.get('rm_login'),
params.get('rm_password'))
else:
print("You have to provide Redmine API key or login/password")
sys.exit(1)
# Connecting to YouTrack
token = params.get('token')
if not token and 'token_file' in params:
try:
with open(params['token_file'], 'r') as f:
token = f.read().strip()
except (OSError, IOError) as e:
print("Cannot load token from file: " + str(e))
sys.exit(1)
if token:
self._target = Connection(params['yt_url'], token=token)
elif 'yt_login' in params:
self._target = Connection(params['yt_url'], params.get('yt_login'),
params.get('yt_password'))
else:
print("You have to provide YouTrack token or login/password")
sys.exit(1)
if not params.get('project_lead_login'):
project_lead = params.get('yt_login')
if not project_lead:
for login in ('root', 'admin', 'administrator', 'guest'):
try:
project_lead = self._target.getUser(login).login
break
except youtrack.YouTrackException:
continue
self._params['project_lead_login'] = project_lead
def do_import(self, project_ids):
try:
projects2import = self._get_projects(project_ids)
except youtrackutils.redmine.RedmineException as e:
print('FATAL:', e)
sys.exit(1)
print('===> Import Roles')
self._import_roles()
for project in projects2import.values():
self._import_project(project)
print('===> Apply Relations')
self._apply_relations()
def _get_projects(self, project_ids=None, by_internal_id=False):
if by_internal_id:
by = 'by_iid'
else:
by = 'by_pid'
if self._projects is None:
self._projects = {'by_iid': {}, 'by_pid': {}}
if project_ids:
new_projects = [
pid for pid in project_ids if pid not in self._projects[by]
]
else:
new_projects = None
if new_projects is None or new_projects:
for project in self._source.get_projects(new_projects):
project.identifier = re.sub('\W', '', project.identifier)
self._projects['by_iid'][project.id] = project
self._projects['by_pid'][project.identifier] = project
if project_ids:
for pid in project_ids:
if by == 'by_pid':
pid = re.sub('\W', '', pid)
if pid not in self._projects[by]:
raise youtrackutils.redmine.RedmineException(
"Project '%s' doesn't exist in Redmine" % pid)
return self._projects[by]
def _get_project(self, project_id, by_internal_id=False):
return self._get_projects([project_id], by_internal_id)[project_id]
def _get_project_name(self, project):
name = project.name
while True:
if not hasattr(project, 'parent'):
break
name = project.parent.name + ' :: ' + name
project = self._get_project(project.parent.id, True)
return name
def _import_project(self, project):
project_id = project.identifier
project_name = self._get_project_name(project)
project_desc = ''
project_lead = self._params['project_lead_login']
if hasattr(project, 'description') and project.description is not None:
project_desc = project.description
print("===> Importing Project '%s' (%s)" %
(project_name.encode('utf-8'), project_id.encode('utf-8')))
try:
print('Creating project...')
self._target.getProject(project_id)
print('Project already exists')
except youtrack.YouTrackException:
self._target.createProjectDetailed(project_id, project_name,
project_desc, project_lead)
print('Project successfully created')
print('Import project members...')
self._import_members(project)
print('Import issues...')
self._import_issues(project)
def _to_yt_user(self, redmine_user):
if isinstance(redmine_user, basestring):
user_id = redmine_user
else:
user_id = redmine_user.id
if user_id not in self._users:
try:
redmine_user = self._source.get_user(user_id)
except ResourceNotFound:
redmine_user = None
user = youtrack.User()
if redmine_user != None:
try:
user.email = redmine_user.mail
except AttributeError:
pass
try:
# In some cases redmine user login can be empty or missing.
# So, both cases should be handled.
user.login = redmine_user.login
except AttributeError:
pass
if not hasattr(user, 'login') or not user.login:
if hasattr(user, 'email'):
user.login = user.email
else:
user.login = '******'
print('Cannot get login for user id=%s, set it to "%s"' %
(user_id, user.login))
# user.login = redmine_user.login or 'guest'
# user.email = redmine_user.mail or '*****@*****.**'
if redmine_user != None and user.login != 'guest':
if redmine_user.firstname is None and redmine_user.lastname is None:
user.fullName = user.login
elif redmine_user.firstname is None:
user.fullName = redmine_user.lastname
elif redmine_user.lastname is None:
user.fullName = redmine_user.firstname
else:
user.fullName = redmine_user.firstname + ' ' + redmine_user.lastname
else:
user.created = True
if redmine_user != None and hasattr(redmine_user, 'groups'):
user.groups = [
self._to_yt_group(g) for g in redmine_user.groups
]
self._users[user_id] = user
return self._users[user_id]
def _to_yt_group(self, redmine_group, users=None):
if not isinstance(redmine_group, basestring):
redmine_group = redmine_group.name
if redmine_group not in self._groups:
group = youtrack.Group()
group.name = redmine_group
if users is None:
users = []
group.users = users
self._groups[redmine_group] = group
return self._groups[redmine_group]
def _to_yt_role(self, name, projects=None):
role = youtrack.UserRole()
role.name = name
if projects:
if isinstance(projects, list):
role.projects.extend(projects)
else:
role.projects.append(projects)
return role
def _to_yt_version(self, version):
if isinstance(version, basestring):
vid = version
else:
vid = version.id
if vid not in self._versions:
redmine_version = self._source.get_version(vid)
version = youtrack.Version()
version.name = redmine_version.name
version.description = redmine_version.description
if getattr(redmine_version, 'due_date', None):
version.releaseDate = str(to_unixtime(
redmine_version.due_date))
version.released = str(redmine_version.status == 'closed').lower()
version.archived = 'false'
self._versions[vid] = version
return self._versions[vid]
def _to_yt_subsystem(self, category):
if isinstance(category, basestring):
cid = category
else:
cid = category.id
if cid not in self._subsystems:
redmine_cat = self._source.get_category(cid)
subsystem = youtrack.Subsystem()
subsystem.name = redmine_cat.name
if hasattr(redmine_cat, 'assigned_to'):
subsystem.login = self._create_user(
redmine_cat.assigned_to).login
self._subsystems[cid] = subsystem
return self._subsystems[cid]
def _get_assignee_group_name(self, project_id):
return '%s Assignees' % project_id.upper()
def _get_yt_issue_id(self, issue, as_number_in_project=False):
project_id = self._projects['by_iid'][issue.project.id].identifier
new_id = self._max_issue_ids.get(project_id, 0) + 1
rid = int(issue.id)
if rid not in self._issue_ids:
self._max_issue_ids[project_id] = new_id
self._issue_ids[rid] = {'id': new_id, 'project_id': project_id}
if as_number_in_project:
return self._issue_ids[rid]['id']
return self._to_yt_issue_id(rid)
def _to_yt_issue_id(self, iid):
issue = self._issue_ids[iid]
return '%s-%d' % (issue['project_id'], issue['id'])
def _get_yt_issue_number(self, issue):
return self._get_yt_issue_id(issue, True)
def _import_members(self, project):
project_id = project.identifier
members = self._source.get_project_members(project.id)
users_by_role = {}
groups_by_role = {}
if members:
for member in members:
# Sometimes roles can be duplicated
if hasattr(member, 'roles'):
roles = set([r.name for r in member.roles])
if hasattr(member, 'group'):
group = self._to_yt_group(member.group.name)
for role in roles:
if role not in groups_by_role:
groups_by_role[role] = []
groups_by_role[role].append(group)
else:
user = self._to_yt_user(member.user)
for role in roles:
if role not in users_by_role:
users_by_role[role] = []
users_by_role[role].append(user)
for role_name, users in users_by_role.items():
group = self._to_yt_group('%s %s' %
(project_id.upper(), role_name))
self._create_group(group)
self._target.addUserRoleToGroup(
group, self._to_yt_role(role_name, project_id))
self._target.importUsers(users)
for user in users:
self._target.setUserGroup(user.login, group.name)
for role_name, groups in groups_by_role.items():
for group in groups:
self._create_group(group)
self._target.addUserRoleToGroup(
group, self._to_yt_role(role_name, project_id))
def _import_roles(self):
existed_roles = [role.name for role in self._target.getRoles()]
new_roles = {}
for role in self._source.get_roles():
if role.name in existed_roles:
continue
permissions = None
if hasattr(role, 'permissions'):
permissions = []
for perm in role.permissions:
if isinstance(perm, basestring):
rm_perm = perm
else:
rm_perm = perm.name
yt_perm = \
youtrackutils.redmine.Mapping.PERMISSIONS.get(rm_perm)
if not yt_perm:
continue
if isinstance(yt_perm, list):
permissions.extend(yt_perm)
else:
permissions.append(yt_perm)
new_roles[role.name] = permissions
for role_name, role_permissions in new_roles.items():
role = self._to_yt_role(role_name)
self._target.createRole(role)
if role_permissions:
for perm_name in role_permissions:
perm = youtrack.Permission()
perm.name = perm_name
self._target.addPermissionToRole(role, perm)
def _import_issues(self, project, limit=CHUNK_SIZE):
project_id = project.identifier
offset = 0
assignee_group = self._get_assignee_group_name(project_id)
while True:
issues = self._source.get_project_issues(
project.id, limit, offset,
self._params.get('skip_on_error', False))
if not issues:
break
issues = [
issue for issue in issues if issue.project.id == project.id
]
self._target.importIssues(
project_id, assignee_group,
[self._make_issue(issue, project_id) for issue in issues])
for issue in issues:
self._collect_relations(issue)
self._add_attachments(issue)
if self._params.get('import_time_entries', False):
self._enable_timetracking(project)
self._add_work_items(issue)
offset += limit
def _make_issue(self, redmine_issue, project_id):
issue = youtrack.Issue()
issue['comments'] = []
if self._params.get('use_markdown'):
issue['markdown'] = "true"
try:
if self._params.get('create_redmine_linkage', False):
self._add_field_to_issue(project_id, issue, "redmine_id",
int(redmine_issue.id))
for name, value in redmine_issue.attributes.items():
if name in ('project', 'attachments'):
continue
if name == 'assigned_to' and value.name in self._groups:
continue
if name == 'id':
value = str(self._get_yt_issue_number(redmine_issue))
if name == 'custom_fields':
for field in value:
self._add_field_to_issue(project_id, issue, field.name,
getattr(field, 'value', None))
elif name == 'journals':
self._add_journals(issue, value)
else:
if name == 'category':
value = self._to_yt_subsystem(value)
self._add_field_to_issue(project_id, issue, name, value)
except Exception as e:
print('Failed to process issue:')
print(redmine_issue)
traceback.print_exc()
raise e
return issue
def _convert_value(self, field_name, value):
conv_map = youtrackutils.redmine.Mapping.CONVERSION.get(field_name)
if conv_map:
if hasattr(value, 'value'):
if value.value in conv_map:
value.value = conv_map[value.value]
elif hasattr(value, 'name'):
if value.name in conv_map:
value.name = conv_map[value.name]
return value
def _get_yt_field_name(self, field_name):
return youtrackutils.redmine.Mapping.FIELD_NAMES.get(
field_name, field_name)
def _get_yt_field_type(self, field_name):
return youtrackutils.redmine.Mapping.FIELD_TYPES.get(
field_name, youtrack.EXISTING_FIELD_TYPES.get(field_name))
def _add_field_to_issue(self, project_id, issue, name, value):
if value is None:
return
field_name = self._get_yt_field_name(name)
field_type = self._get_yt_field_type(field_name)
if field_type is None:
return
value = self._convert_value(field_name, value)
if isinstance(value, list):
if not value:
return
issue[field_name] = []
for v in value:
v = self._create_field_value(project_id, field_name,
field_type, v)
issue[field_name].append(
self._get_value_presentation(field_type, v))
else:
value = self._create_field_value(project_id, field_name,
field_type, value)
issue[field_name] = self._get_value_presentation(field_type, value)
def _create_field(self, project_id, field_name, field_type):
project_fields = self._target.getProjectCustomFields(project_id)
if field_name.lower() not in [f.name.lower() for f in project_fields]:
all_fields = self._target.getCustomFields()
if field_name.lower() not in [f.name.lower() for f in all_fields]:
self._target.createCustomFieldDetailed(field_name, field_type,
False, True, False, {})
if field_type in ('string', 'date', 'integer', 'float', 'period'):
self._target.createProjectCustomFieldDetailed(
project_id, field_name, 'No ' + field_name)
else:
bundle_name = field_name + ' bundle'
create_bundle_safe(self._target, bundle_name, field_type)
self._target.createProjectCustomFieldDetailed(
project_id, field_name, 'No ' + field_name,
{'bundle': bundle_name})
def _create_field_value(self, project_id, field_name, field_type, value):
if field_type.startswith('user'):
if hasattr(value, 'name'):
value.name = self._create_user(value).login
else:
value = self._create_user(value).login
elif field_type.startswith('version'):
value = self._to_yt_version(value)
if field_name == 'Assignee':
return value
if field_name in youtrack.EXISTING_FIELDS:
return value
self._create_field(project_id, field_name, field_type)
if field_type in ('string', 'date', 'integer', 'float', 'period'):
return value
field = self._target.getProjectCustomField(project_id, field_name)
bundle = self._target.getBundle(field_type, field.bundle)
try:
if hasattr(value, 'value'):
value = value.value
elif hasattr(value, 'name'):
if not (field_type.startswith('version')
or field_type.startswith('ownedField')):
value = value.name
self._target.addValueToBundle(bundle, value)
except youtrack.YouTrackException as e:
if e.response.status != 409 or e.response.reason.lower(
) != 'conflict':
print(e)
return value
def _get_value_presentation(self, field_type, value):
if field_type == 'date':
return str(to_unixtime(value))
if field_type == 'integer':
return '%d' % int(float(value))
if field_type == 'float':
return '%.5f' % float(value)
if field_type == 'string':
return value
if field_type == 'period':
return '%d' % int(float(value) * 60)
if hasattr(value, 'value'):
return value.value
elif hasattr(value, 'name'):
return value.name
return value
def _create_user(self, user):
user = self._to_yt_user(user)
if not hasattr(user, 'created'):
self._target.createUser(user)
user.created = True
if hasattr(user, 'groups'):
for group in user.groups:
self._create_group(group)
self._target.setUserGroup(user.login, group.name)
return user
def _create_group(self, group):
if isinstance(group, basestring):
group = self._to_yt_group(group)
if not hasattr(group, 'created'):
try:
self._target.getGroup(group.name)
except youtrack.YouTrackException:
self._target.createGroup(group)
group.created = True
return group
def _add_journals(self, issue, journals):
if not journals:
return
for rec in journals:
if hasattr(rec,
'notes') and rec.notes is not None and rec.notes != '':
comment = youtrack.Comment()
if self._params.get('use_markdown'):
comment.markdown = "true"
comment.text = rec.notes
comment.author = self._create_user(rec.user).login
comment.created = str(to_unixtime(rec.created_on))
issue['comments'].append(comment)
def _enable_timetracking(self, project):
self._target.setProjectTimeTrackingSettings(project.identifier,
enabled=True)
def _add_work_items(self, issue):
import_data = []
work_items = self._source.get_time_entries(issue.id)
flatten_work_items = []
for item in work_items:
if 'time_entries' in item.attributes:
time_entries = item.time_entries
for time_entry in time_entries:
flatten_work_items.append(time_entry)
else:
flatten_work_items.append(item)
for t in sorted(flatten_work_items, key=lambda tt: tt.spent_on):
work_item = youtrack.WorkItem()
work_item.authorLogin = self._create_user(t.user).login
work_item.date = str(to_unixtime(t.spent_on))
work_item.description = t.comments
work_item.duration = int(float(t.hours) * 60)
import_data.append(work_item)
if import_data:
self._target.importWorkItems(self._get_yt_issue_id(issue),
import_data)
def _add_attachments(self, issue):
if not hasattr(issue, 'attachments'):
return
max_attempts = 5
for attach in issue.attachments:
attach.author.login = self._create_user(attach.author).login
if not attach.author.login:
attach.author.login = '******'
attempts = max_attempts
while attempts:
attempts -= 1
try:
self._target.createAttachmentFromAttachment(
self._get_yt_issue_id(issue),
RedmineAttachment(attach, self._source))
break
except Exception as e:
print(e)
if attempts:
delay = 30 + (max_attempts - attempts - 1) * 10
print("Can't import attachment: %s. Retry in %d s." %
(attach.filename, delay))
time.sleep(delay)
else:
print('Failed to import attachment: %s. Skipped.' %
attach.filename)
def _collect_relations(self, issue):
link_types = {
'duplicates': 'duplicate',
'relates': 'relates',
'blocks': 'depend',
'precedes': 'depend'
}
if hasattr(issue, 'relations'):
for rel in issue.relations:
if rel.relation_type not in link_types:
print('Unsuitable link type: %s. Skipped' %
rel.relation_type)
continue
from_id = rel.issue_id
to_id = rel.issue_to_id
if rel.relation_type == 'duplicates':
from_id, to_id = to_id, from_id
self._push_relation(from_id, to_id,
link_types[rel.relation_type])
if hasattr(issue, 'children'):
for child in issue.children:
self._push_relation(issue.id, child.id, 'subtask')
def _push_relation(self, from_iid, to_iid, relation_type):
from_iid = int(from_iid)
to_iid = int(to_iid)
if relation_type not in self._relations:
self._relations[relation_type] = {}
relations = self._relations[relation_type]
if from_iid not in relations:
relations[from_iid] = {}
relations[from_iid][to_iid] = None
def _apply_relations(self, limit=CHUNK_SIZE):
links = []
for link_type, ids in self._relations.items():
for from_id, to_ids in ids.items():
for to_id in to_ids:
link = youtrack.Link()
link.typeName = link_type
try:
link.source = self._to_yt_issue_id(from_id)
link.target = self._to_yt_issue_id(to_id)
except KeyError:
print("Cannot apply link (%s) to issues: %d and %d" %
(link_type, from_id, to_id))
if hasattr(link, 'source') and link.source:
print("The second issue was not imported")
continue
else:
print("The first issues was not imported")
break
links.append(link)
if len(links) >= limit:
self._target.importLinks(links)
del links[0:]
if links:
self._target.importLinks(links)
source = Connection(source_url, source_login, source_password)
target = Connection(
target_url, target_login, target_password
) #, proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888)
#reset connections to avoid disconnections
user_importer.resetConnections(source, target)
link_importer.resetConnections(target)
# copy project, subsystems, versions
project = source.getProject(projectId)
link_importer.addAvailableIssuesFrom(projectId)
project_custom_fields = source.getProjectCustomFields(projectId)
# create bundles and additional values
for pcf_ref in project_custom_fields:
pcf = source.getProjectCustomField(projectId, pcf_ref.name)
if hasattr(pcf, "bundle"):
create_bundle_from_bundle(source, target, pcf.bundle,
source.getCustomField(pcf.name).type,
user_importer)
target_project_fields = [
pcf.name.lower()
for pcf in target.getProjectCustomFields(projectId)
]
for field in project_custom_fields:
if field.name.lower() in target_project_fields:
if hasattr(field, 'bundle'):
if field.bundle != target.getProjectCustomField(
projectId, field.name).bundle:
target.deleteProjectCustomField(projectId, field.name)
def main(a_pat, yt_url, yt_login, yt_pass):
""" Creates connections to Asana and your YouTrack site, then migrates tasks """
a_conn = asana.Client.access_token(a_pat)
yt_conn = Connection(yt_url, yt_login, yt_pass)
print 'Logged in with Asana User {} and YouTrack User {}'.format(
a_conn.users.me()['name'],
yt_conn.getUser(yt_login).login)
# Get list of Asana Workspaces to migrate - all except Personal Projects
a_workspaces = [
w for w in a_conn.workspaces.find_all()
if w['name'] != "Personal Projects"
]
print "Found Asana Workspaces (excluding Personal Projects): {}".format(
[a['name'] for a in a_workspaces])
yt_projects = [yt_conn.getProject(p) for p in yt_conn.getProjects()]
print "Found existing YouTrack Projects: {}".format(
[y.name for y in yt_projects])
for a_work in a_workspaces:
if a_work['name'] not in [p.name for p in yt_projects]:
print "Creating YouTrack Project from {} Workspace".format(
a_work['name'])
yt_proj = None
for p in yt_projects:
if p.name == a_work['name']:
yt_proj = p
break
if yt_proj is None:
yt_proj = yt.Project()
yt_proj.name = a_work['name']
yt_proj.id = a_work['name'].replace(' ', '').upper()
yt_proj.lead = yt_login
print yt_conn.createProject(yt_proj)
field_name = 'AsanaID'
cf = yt.CustomField()
cf.name = field_name
cf.type = 'string'
cf.isPrivate = False
cf.visibleByDefault = False
# print "Existing Project Fields: {}".format(yt_conn.getProjectCustomFields(yt_proj.id))
try:
asana_id_exists = yt_conn.getCustomField(field_name)
except:
asana_id_exists = False
if not asana_id_exists:
print 'Creating YouTrack Custom Field to save our Asana ID in'
yt_conn.createCustomField(cf)
try:
asana_id_exists = yt_conn.getProjectCustomField(
yt_proj.id, 'Due Date')
except:
asana_id_exists = None
if not asana_id_exists:
print 'Adding YouTrack Due Date Field to {} Project'.format(
yt_proj.id)
yt_conn.createProjectCustomFieldDetailed(yt_proj.id, 'Due Date',
'')
try:
asana_id_exists = yt_conn.getProjectCustomField(
yt_proj.id, field_name)
except:
asana_id_exists = False
if not asana_id_exists:
print 'Adding YouTrack Custom Field {} to {} Project'.format(
field_name, yt_proj.id)
yt_conn.createProjectCustomFieldDetailed(yt_proj.id, field_name,
'')
# Migrate users and save our list for later so we can assign people
migrate_workspace_users(a_conn, yt_conn, a_work)
migrate_projects_to_subsystems(a_conn, yt_conn, a_work, yt_proj)
migrate_tasks_to_issues(a_conn, yt_conn, a_work, yt_proj, yt_login)
