def __init__(self, source, target, import_config):
super(CsvYouTrackImporter, self).__init__(source['issues'], target,
import_config)
self._after = 0
self._comments = dict()
self._attachments = dict()
if 'comments' in source:
for c in source['comments'].get_rows():
issue_id = '%s-%s' % (c[0], c[1])
if issue_id not in self._comments:
self._comments[issue_id] = []
self._comments[issue_id].append(c[2:])
if 'attachments' in source:
for a in source['attachments'].get_rows():
issue_id = '%s-%s' % (a[0], a[1])
if issue_id not in self._attachments:
self._attachments[issue_id] = []
self._attachments[issue_id].append(a[2:])
self._link_importer = LinkImporter(target)
def youtrack2youtrack(source_url, source_login, source_password, target_url, target_login, target_password,
project_ids, query='', source_token=None, target_token=None, params=None):
if not len(project_ids):
print("You should sign at least one project to import")
return
if params is None:
params = {}
source = Connection(source_url, source_login, source_password) if (source_token is None) else Connection(source_url,
token=source_token)
target = Connection(target_url, target_login, target_password) if (target_token is None) else Connection(target_url,
token=target_token)
# , proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888)
print("Import issue link types")
for ilt in source.getIssueLinkTypes():
try:
print(target.createIssueLinkType(ilt))
except youtrack.YouTrackException as e:
print(e.message)
user_importer = UserImporter(source, target, caching_users=params.get('enable_user_caching', True))
link_importer = LinkImporter(target)
# create all projects with minimum info and project lead set
created_projects = []
for project_id in project_ids:
created = create_project_stub(source, target, project_id, user_importer)
created_projects.append(created)
# save created project ids to create correct group roles afterwards
user_importer.addCreatedProjects([project.id for project in created_projects])
# import project leads with group they are included and roles assigned to these groups
user_importer.importUsersRecursively([target.getUser(project.lead) for project in created_projects])
# afterwards in a script any user import imply recursive import
cf_names_to_import = set([]) # names of cf prototypes that should be imported
for project_id in project_ids:
cf_names_to_import.update([pcf.name.capitalize() for pcf in source.getProjectCustomFields(project_id)])
target_cf_names = [pcf.name.capitalize() for pcf in target.getCustomFields()]
period_cf_names = []
for cf_name in cf_names_to_import:
source_cf = source.getCustomField(cf_name)
if source_cf.type.lower() == 'period':
period_cf_names.append(source_cf.name.lower())
print("Processing custom field '%s'" % utf8encode(cf_name))
if cf_name in target_cf_names:
target_cf = target.getCustomField(cf_name)
if not (target_cf.type == source_cf.type):
print("In your target and source YT instances you have field with name [ %s ]" % utf8encode(cf_name))
print("They have different types. Source field type [ %s ]. Target field type [ %s ]" %
(source_cf.type, target_cf.type))
print("exiting...")
exit()
else:
if hasattr(source_cf, "defaultBundle"):
create_bundle_from_bundle(source, target, source_cf.defaultBundle, source_cf.type, user_importer)
target.createCustomField(source_cf)
failed_commands = []
for projectId in project_ids:
source = Connection(source_url, source_login, source_password) if (source_token is None) else Connection(
source_url, token=source_token)
target = Connection(target_url, target_login, target_password) if (target_token is None) else Connection(
target_url, token=target_token)
# , proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888)
# reset connections to avoid disconnections
user_importer.resetConnections(source, target)
link_importer.resetConnections(target)
# copy project, subsystems, versions
project = source.getProject(projectId)
link_importer.addAvailableIssuesFrom(projectId)
project_custom_fields = source.getProjectCustomFields(projectId)
# create bundles and additional values
for pcf_ref in project_custom_fields:
pcf = source.getProjectCustomField(projectId, pcf_ref.name)
if hasattr(pcf, "bundle"):
try:
create_bundle_from_bundle(source, target, pcf.bundle, source.getCustomField(pcf.name).type,
user_importer)
except youtrack.YouTrackException as e:
if e.response.status != 409:
raise e
else:
print(e)
target_project_fields = [pcf.name.lower() for pcf in target.getProjectCustomFields(projectId)]
for field in project_custom_fields:
if field.name.lower() in target_project_fields:
if hasattr(field, 'bundle'):
if field.bundle != target.getProjectCustomField(projectId, field.name).bundle:
target.deleteProjectCustomField(projectId, field.name)
create_project_custom_field(target, field, projectId)
else:
try:
create_project_custom_field(target, field, projectId)
except youtrack.YouTrackException as e:
if e.response.status != 409:
raise e
else:
print(e)
# copy issues
start = 0
max = 20
sync_workitems = enable_time_tracking(source, target, projectId)
tt_settings = target.getProjectTimeTrackingSettings(projectId)
print("Import issues")
last_created_issue_number = 0
while True:
try:
print("Get issues from " + str(start) + " to " + str(start + max))
issues = source.getIssues(projectId, query, start, max)
if len(issues) <= 0:
break
if convert_period_values and period_cf_names:
for issue in issues:
for pname in period_cf_names:
for fname in issue.__dict__:
if fname.lower() != pname:
continue
issue[fname] = period_to_minutes(issue[fname])
users = set([])
for issue in issues:
print("Collect users for issue [%s]" % issue.id)
users.add(issue.getReporter())
if issue.hasAssignee():
if isinstance(issue.Assignee, (list, tuple)):
users.update(issue.getAssignee())
else:
users.add(issue.getAssignee())
# TODO: http://youtrack.jetbrains.net/issue/JT-6100
users.add(issue.getUpdater())
if issue.hasVoters():
users.update(issue.getVoters())
for comment in issue.getComments():
users.add(comment.getAuthor())
print("Collect links for issue [%s]" % issue.id)
link_importer.collectLinks(issue.getLinks(True))
# links.extend(issue.getLinks(True))
# fix problem with comment.text
for comment in issue.getComments():
if not hasattr(comment, "text") or (len(comment.text.strip()) == 0):
setattr(comment, 'text', 'no text')
user_importer.importUsersRecursively(users)
print("Create issues [" + str(len(issues)) + "]")
if params.get('create_new_issues'):
create_issues(target, issues, last_created_issue_number)
else:
print(target.importIssues(projectId, project.name + ' Assignees', issues))
link_importer.addAvailableIssues(issues)
for issue in issues:
try:
target_issue = target.getIssue(issue.id)
except youtrack.YouTrackException as e:
print("Cannot get target issue")
print(e)
continue
if params.get('sync_tags') and issue.tags:
try:
for tag in issue.tags:
tag = re.sub(r'[,&<>]', '_', tag)
try:
target.executeCommand(issue.id, 'tag ' + tag, disable_notifications=True)
except youtrack.YouTrackException:
tag = re.sub(r'[\s-]', '_', tag)
target.executeCommand(issue.id, 'tag ' + tag, disable_notifications=True)
except youtrack.YouTrackException as e:
print("Cannot sync tags for issue " + issue.id)
print(e)
if params.get('add_new_comments'):
target_comments = dict()
max_id = 0
for c in target_issue.getComments():
target_comments[c.created] = c
if max_id < c.created:
max_id = c.created
for c in issue.getComments():
if c.created > max_id or c.created not in target_comments:
group = None
if hasattr(c, 'permittedGroup'):
group = c.permittedGroup
try:
target.executeCommand(issue.id, 'comment', c.text, group, c.author,
disable_notifications=True)
except youtrack.YouTrackException as e:
print('Cannot add comment to issue')
print(e)
if params.get('sync_custom_fields'):
skip_fields = []
if tt_settings and tt_settings.Enabled and tt_settings.TimeSpentField:
skip_fields.append(tt_settings.TimeSpentField)
skip_fields = [name.lower() for name in skip_fields]
for pcf in [pcf for pcf in project_custom_fields if pcf.name.lower() not in skip_fields]:
target_cf_value = None
if pcf.name in target_issue:
target_cf_value = target_issue[pcf.name]
if isinstance(target_cf_value, (list, tuple)):
target_cf_value = set(target_cf_value)
elif target_cf_value == target.getProjectCustomField(projectId, pcf.name).emptyText:
target_cf_value = None
source_cf_value = None
if pcf.name in issue:
source_cf_value = issue[pcf.name]
if isinstance(source_cf_value, (list, tuple)):
source_cf_value = set(source_cf_value)
elif source_cf_value == source.getProjectCustomField(projectId, pcf.name).emptyText:
source_cf_value = None
if source_cf_value == target_cf_value:
continue
if isinstance(source_cf_value, set) or isinstance(target_cf_value, set):
if source_cf_value is None:
source_cf_value = set([])
elif not isinstance(source_cf_value, set):
source_cf_value = set([source_cf_value])
if target_cf_value is None:
target_cf_value = set([])
elif not isinstance(target_cf_value, set):
target_cf_value = set([target_cf_value])
for v in target_cf_value:
if v not in source_cf_value:
target.executeCommand(issue.id, 'remove %s %s' % (pcf.name, v),
disable_notifications=True)
for v in source_cf_value:
if v not in target_cf_value:
target.executeCommand(issue.id, 'add %s %s' % (pcf.name, v),
disable_notifications=True)
else:
if source_cf_value is None:
source_cf_value = target.getProjectCustomField(projectId, pcf.name).emptyText
if pcf.type.lower() == 'date':
m = re.match(r'(\d{10})(?:\d{3})?', str(source_cf_value))
if m:
source_cf_value = datetime.datetime.fromtimestamp(
int(m.group(1))).strftime('%Y-%m-%d')
elif pcf.type.lower() == 'period':
source_cf_value = '%sm' % source_cf_value
command = '%s %s' % (pcf.name, source_cf_value)
try:
target.executeCommand(issue.id, command, disable_notifications=True)
except youtrack.YouTrackException as e:
if e.response.status == 412 and e.response.reason.find('Precondition Failed') > -1:
print('WARN: Some workflow blocks following command: %s' % command)
failed_commands.append((issue.id, command))
if sync_workitems:
workitems = source.getWorkItems(issue.id)
if workitems:
existing_workitems = dict()
target_workitems = target.getWorkItems(issue.id)
if target_workitems:
for w in target_workitems:
_id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration)
if hasattr(w, 'description'):
_id += '\n%s' % w.description
existing_workitems[_id] = w
new_workitems = []
for w in workitems:
_id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration)
if hasattr(w, 'description'):
_id += '\n%s' % w.description
if _id not in existing_workitems:
new_workitems.append(w)
if new_workitems:
print("Process workitems for issue [ " + issue.id + "]")
try:
user_importer.importUsersRecursively(
[source.getUser(w.authorLogin)
for w in new_workitems])
target.importWorkItems(issue.id, new_workitems)
except youtrack.YouTrackException as e:
if e.response.status == 404:
print("WARN: Target YouTrack doesn't support workitems importing.")
print("WARN: Workitems won't be imported.")
sync_workitems = False
else:
print("ERROR: Skipping workitems because of error:" + str(e))
print("Process attachments for issue [%s]" % issue.id)
existing_attachments = dict()
try:
for a in target.getAttachments(issue.id):
existing_attachments[a.name + '\n' + a.created] = a
except youtrack.YouTrackException as e:
if e.response.status == 404:
print("Skip importing attachments because issue %s doesn't exist" % issue.id)
continue
raise e
attachments = []
users = set([])
for a in issue.getAttachments():
if a.name + '\n' + a.created in existing_attachments and not params.get('replace_attachments'):
a.name = utf8encode(a.name)
try:
print("Skip attachment '%s' (created: %s) because it's already exists"
% (utf8encode(a.name), utf8encode(a.created)))
except Exception:
pass
continue
attachments.append(a)
author = a.getAuthor()
if author is not None:
users.add(author)
user_importer.importUsersRecursively(users)
for a in attachments:
print("Transfer attachment of " + utf8encode(issue.id) + ": " + utf8encode(a.name))
# TODO: add authorLogin to workaround http://youtrack.jetbrains.net/issue/JT-6082
# a.authorLogin = target_login
try:
target.createAttachmentFromAttachment(issue.id, a)
except BaseException as e:
print("Cant import attachment [ %s ]" % utf8encode(a.name))
print(repr(e))
continue
if params.get('replace_attachments'):
try:
old_attachment = existing_attachments.get(a.name + '\n' + a.created)
if old_attachment:
print('Deleting old attachment')
target.deleteAttachment(issue.id, old_attachment.id)
except BaseException as e:
print("Cannot delete attachment '%s' from issue %s" % (
utf8encode(a.name), utf8encode(issue.id)))
print(e)
except Exception as e:
print('Cant process issues from ' + str(start) + ' to ' + str(start + max))
traceback.print_exc()
raise e
start += max
print("Import issue links")
link_importer.importCollectedLinks()
print("Trying to execute failed commands once again")
for issue_id, command in failed_commands:
try:
print('Executing command on issue %s: %s' % (issue_id, command))
target.executeCommand(issue_id, command, disable_notifications=True)
except youtrack.YouTrackException as e:
print('Failed to execute command for issue #%s: %s' % (issue_id, command))
print(e)
class CsvYouTrackImporter(YouTrackImporter):
def __init__(self, source, target, import_config):
super(CsvYouTrackImporter, self).__init__(source['issues'],
target,
import_config)
self._after = 0
self._comments = dict()
self._attachments = dict()
if 'comments' in source:
for c in source['comments'].get_rows():
issue_id = '%s-%s' % (c[0], c[1])
if issue_id not in self._comments:
self._comments[issue_id] = []
self._comments[issue_id].append(c[2:])
if 'attachments' in source:
for a in source['attachments'].get_rows():
issue_id = '%s-%s' % (a[0], a[1])
if issue_id not in self._attachments:
self._attachments[issue_id] = []
self._attachments[issue_id].append(a[2:])
self._link_importer = LinkImporter(target)
def import_csv(self, new_projects_owner_login=u'root'):
projects = self._get_projects()
self._source.reset()
self.do_import(projects, new_projects_owner_login)
def _to_yt_comment(self, comment):
result = None
if isinstance(comment, basestring):
result = Comment()
result.author = u'guest'
result.text = comment
result.created = str(int(time.time() * 1000))
elif isinstance(comment, list):
yt_user = self._to_yt_user(comment[0])
self._import_user(yt_user)
result = Comment()
result.author = yt_user.login
result.created = self._import_config.to_unix_date(comment[1])
result.text = comment[2]
if result and getattr(csvClient, 'USE_MARKDOWN', False):
result.markdown = "true"
return result
def get_field_value(self, field_name, field_type, value):
if (field_name == self._import_config.get_project_name_key()) or (
field_name == self._import_config.get_project_id_key()):
return None
if field_type == u'date':
return self._import_config.to_unix_date(value)
if re.match(r'^\s*(enum|version|build|ownedfield|user|group)\[\*\]s*$',
field_type, re.IGNORECASE):
delimiter = getattr(csvClient,
'VALUE_DELIMITER',
csvClient.CSV_DELIMITER)
values = re.split(re.escape(delimiter), value)
if len(values) > 1:
value = values
return super(CsvYouTrackImporter, self).get_field_value(field_name,
field_type,
value)
def _to_yt_user(self, value):
users_mapping = dict()
yt_user = User()
user = value.split(';')
yt_user.login = user[0].replace(' ', '_')
if yt_user.login in users_mapping:
user = users_mapping[yt_user.login]
yt_user.login = user[0]
try:
yt_user.fullName = user[1] or yt_user.login
except IndexError:
yt_user.fullName = yt_user.login
try:
yt_user.email = user[2].strip() or yt_user.login + '@fake.com'
except IndexError:
yt_user.email = yt_user.login + '@fake.com'
return yt_user
def _get_issue_id(self, issue):
number_regex = re.compile("\d+")
idf = self._import_config.get_key_for_field_name(u'numberInProject')
match_result = number_regex.search(issue[idf])
return match_result.group()
def _get_yt_issue_id(self, issue):
number_in_project = self._get_issue_id(issue)
pf = self._import_config.get_key_for_field_name(
self._import_config.get_project_id_key()
)
project_id = issue[pf]
return '%s-%s' % (project_id, number_in_project)
def _get_issues(self, project_id):
issues = self._source.get_issues()
if getattr(csvClient, 'USE_MARKDOWN', False):
for issue in issues:
if self._import_config.get_project(issue)[0] == project_id:
issue['markdown'] = "true"
yield issue
else:
for issue in issues:
if self._import_config.get_project(issue)[0] == project_id:
yield issue
def _get_comments(self, issue):
if self._comments:
return self._comments.get(self._get_yt_issue_id(issue), [])
return issue[self._import_config.get_key_for_field_name(u'comments')]
def _get_attachments(self, issue):
if self._attachments:
return self._attachments.get(self._get_yt_issue_id(issue), [])
return []
def _import_attachments(self, issue_id, issue_attachments):
for attach in issue_attachments:
yt_user = self._to_yt_user(attach[0])
self._import_user(yt_user)
author = yt_user.login
created = self._import_config.to_unix_date(attach[1])
name = os.path.basename(attach[2])
content = open(attach[2], 'rb')
self._target.importAttachment(
issue_id, name, content, author, None, None, created, '')
def _import_issue_links(self, project_ids):
for project_id in project_ids:
self._link_importer.importLinks(
self._get_issue_links(project_id, 0, 0))
def _get_issue_links(self, project_id, after=0, limit=0):
key = self._import_config.get_key_for_field_name(u'Links')
delimiter = getattr(csvClient,
'VALUE_DELIMITER',
csvClient.CSV_DELIMITER)
links = []
for issue in self._get_issues(project_id):
source_id = self._get_yt_issue_id(issue)
self._link_importer.created_issue_ids.add(source_id)
if key not in issue:
continue
link_groups = issue[key].split(delimiter)
for group in link_groups:
ids = group.split(csvClient.CSV_DELIMITER)
if len(ids) < 2:
# Bad format.
# There should be at least link type and one issue id.
continue
link_type = ids.pop(0)
for i in ids:
try:
i = "%s-%d" % (project_id, int(i))
except ValueError:
pass
self._link_importer.created_issue_ids.add(i)
link = Link()
link.typeName = link_type
link.source = source_id
link.target = i
links.append(link)
return links
def _get_issue_tags(self, project_id):
key = self._import_config.get_key_for_field_name(u'Tags')
delimiter = getattr(csvClient,
'VALUE_DELIMITER',
csvClient.CSV_DELIMITER)
return ((self._get_issue_id(issue), issue[key].split(delimiter))
for issue in self._get_issues(project_id)
if (key in issue) and len(issue[key]))
def _get_custom_field_names(self):
project_name_key = self._import_config.get_key_for_field_name(
self._import_config.get_project_name_key())
project_id_key = self._import_config.get_key_for_field_name(
self._import_config.get_project_id_key())
return [key for key in self._source.get_header()
if (key not in [project_name_key, project_id_key])]
def _get_projects(self):
result = {}
for issue in self._source.get_issues():
project_id, project_name = self._import_config.get_project(issue)
if project_id not in result:
result[project_id] = project_name
return result
def _get_custom_fields_for_projects(self, project_ids):
fields = [self._import_config.get_field_info(field_name)
for field_name in self._get_custom_field_names()]
return [f for f in fields if f is not None]
def do_move(source_url, source_login, source_password,
target_url, target_login, target_password, source_issue_id, target):
print("source_url : " + source_url)
print("source_login : "******"source_password : "******"target_url : " + target_url)
print("target_login : "******"target_password : "******"source_id : " + source_issue_id)
if target.find('-') > -1:
print("target_id : " + target)
target_project_id, target_issue_number = target.split('-')
else:
print("target_project_id: " + target)
target_project_id = target
target_issue_number = None
# connecting
try:
target = Connection(target_url, target_login, target_password)
print("Connected to target url [%s]" % target_url)
except Exception as ex:
print("Failed to connect to target url [%s] with login/password [%s/%s]"
% (target_url, target_login, target_password))
raise ex
try:
source = Connection(source_url, source_login, source_password)
print("Connected to source url [%s]" % source_url)
except Exception as ex:
print("Failed to connect to source url [%s] with login/password [%s/%s]"
% (source_url, source_login, source_password))
raise ex
try:
target.getProject(target_project_id)
except Exception as ex:
print("Can't connect to target project [%s]" % target_project_id)
raise ex
# twin issues
try:
source_issue = source.getIssue(source_issue_id)
except Exception as ex:
print("Failed to get issue [%s]" % source_issue_id)
raise ex
target_issue = Issue()
# import users if needed
name_fields = ["reporterName", "assigneeName", "updaterName"]
for field in name_fields:
if field in source_issue:
check_user(source_issue[field], source, target)
if not target_issue_number:
target_issue_number = str(get_new_issue_id(target_project_id, target))
target_issue.numberInProject = target_issue_number
# check subsystem
target_subsystem = None
try:
target.getSubsystem(target_project_id, source_issue.subsystem)
target_subsystem = source_issue.subsystem
except (YouTrackException, AttributeError):
pass
target_issue.subsystem = target_subsystem
for field in PREDEFINED_FIELDS:
if field in source_issue:
target_issue[field] = source_issue[field]
if "Type" in source_issue:
target_issue.type = source_issue["Type"]
elif "type" in source_issue:
target_issue.type = source_issue["type"]
else:
target_issue.type = "Bug"
# convert custom field
target_cfs = target.getProjectCustomFields(target_project_id)
for cf in target_cfs:
cf_name = cf.name
if cf_name in source_issue:
target_issue[cf_name] = source_issue[cf_name]
# comments
target_issue.comments = source_issue.getComments()
for comment in target_issue.comments:
check_user(comment.author, source, target)
# import issue
print(target.importIssues(
target_project_id,
"",
[target_issue]))
# attachments
for attachment in source_issue.getAttachments():
check_user(attachment.authorLogin, source, target)
attachment.url = attachment.url.replace(source_url, "")
target.createAttachmentFromAttachment(
"%s-%s" % (target_project_id, target_issue.numberInProject),
attachment)
# work items
if get_time_tracking_state(
source, target, source_issue_id.split('-')[0],
target_project_id):
workitems = source.getWorkItems(source_issue_id)
if workitems:
existing_workitems = dict()
target_workitems = target.getWorkItems(
target_project_id + '-' + target_issue_number)
if target_workitems:
for w in target_workitems:
_id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration)
if hasattr(w, 'description'):
_id += '\n%s' % w.description
existing_workitems[_id] = w
new_workitems = []
for w in workitems:
_id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration)
if hasattr(w, 'description'):
_id += '\n%s' % w.description
if _id not in existing_workitems:
new_workitems.append(w)
if new_workitems:
print("Process workitems for issue [ " + source_issue_id + "]")
try:
for w in new_workitems:
check_user(w.authorLogin, source, target)
target.importWorkItems(
target_project_id + '-' + target_issue_number,
new_workitems)
except YouTrackException as e:
print("Failed to import workitems: " + str(e))
# links
link_importer = LinkImporter(target)
links2import = source_issue.getLinks()
link_importer.collectLinks(links2import)
link_importer.addAvailableIssue(source_issue)
for l in links2import:
link_importer.addAvailableIssue(source.getIssue(l.source))
link_importer.addAvailableIssue(source.getIssue(l.target))
link_importer.importCollectedLinks()