# fix problem with comment.text for comment in issue.getComments(): if not hasattr(comment, "text") or (len( comment.text.strip()) == 0): setattr(comment, 'text', 'no text') user_importer.importUsersRecursively(users) print "Create issues [" + str(len(issues)) + "]" print target.importIssues(projectId, project.name + ' Assignees', issues) link_importer.addAvailableIssues(issues) for issue in issues: if sync_workitems: workitems = source.getWorkItems(issue.id) if workitems: enable_time_tracking(source, target, project_id) print "Process workitems for issue [ " + issue.id + "]" try: target.importWorkItems(issue.id, workitems) except youtrack.YouTrackException, e: if e.response.status == 404: print "WARN: Target YouTrack doesn't support workitems importing." print "WARN: Workitems won't be imported." sync_workitems = False else: print "ERROR: Skipping workitems because of error:" + str( e) print "Process attachments for issue [ " + issue.id + "]"
def youtrack2youtrack(source_url, source_login, source_password, target_url, target_login, target_password, project_ids, query='', source_token=None, target_token=None, params=None): if not len(project_ids): print("You should sign at least one project to import") return if params is None: params = {} source = Connection(source_url, source_login, source_password) if (source_token is None) else Connection(source_url, token=source_token) target = Connection(target_url, target_login, target_password) if (target_token is None) else Connection(target_url, token=target_token) # , proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888) print("Import issue link types") for ilt in source.getIssueLinkTypes(): try: print(target.createIssueLinkType(ilt)) except youtrack.YouTrackException as e: print(e.message) user_importer = UserImporter(source, target, caching_users=params.get('enable_user_caching', True)) link_importer = LinkImporter(target) # create all projects with minimum info and project lead set created_projects = [] for project_id in project_ids: created = create_project_stub(source, target, project_id, user_importer) created_projects.append(created) # save created project ids to create correct group roles afterwards user_importer.addCreatedProjects([project.id for project in created_projects]) # import project leads with group they are included and roles assigned to these groups user_importer.importUsersRecursively([target.getUser(project.lead) for project in created_projects]) # afterwards in a script any user import imply recursive import cf_names_to_import = set([]) # names of cf prototypes that should be imported for project_id in project_ids: cf_names_to_import.update([pcf.name.capitalize() for pcf in source.getProjectCustomFields(project_id)]) target_cf_names = [pcf.name.capitalize() for pcf in target.getCustomFields()] period_cf_names = [] for cf_name in cf_names_to_import: source_cf = source.getCustomField(cf_name) if source_cf.type.lower() == 'period': period_cf_names.append(source_cf.name.lower()) print("Processing custom field '%s'" % utf8encode(cf_name)) if cf_name in target_cf_names: target_cf = target.getCustomField(cf_name) if not (target_cf.type == source_cf.type): print("In your target and source YT instances you have field with name [ %s ]" % utf8encode(cf_name)) print("They have different types. Source field type [ %s ]. Target field type [ %s ]" % (source_cf.type, target_cf.type)) print("exiting...") exit() else: if hasattr(source_cf, "defaultBundle"): create_bundle_from_bundle(source, target, source_cf.defaultBundle, source_cf.type, user_importer) target.createCustomField(source_cf) failed_commands = [] for projectId in project_ids: source = Connection(source_url, source_login, source_password) if (source_token is None) else Connection( source_url, token=source_token) target = Connection(target_url, target_login, target_password) if (target_token is None) else Connection( target_url, token=target_token) # , proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8888) # reset connections to avoid disconnections user_importer.resetConnections(source, target) link_importer.resetConnections(target) # copy project, subsystems, versions project = source.getProject(projectId) link_importer.addAvailableIssuesFrom(projectId) project_custom_fields = source.getProjectCustomFields(projectId) # create bundles and additional values for pcf_ref in project_custom_fields: pcf = source.getProjectCustomField(projectId, pcf_ref.name) if hasattr(pcf, "bundle"): try: create_bundle_from_bundle(source, target, pcf.bundle, source.getCustomField(pcf.name).type, user_importer) except youtrack.YouTrackException as e: if e.response.status != 409: raise e else: print(e) target_project_fields = [pcf.name.lower() for pcf in target.getProjectCustomFields(projectId)] for field in project_custom_fields: if field.name.lower() in target_project_fields: if hasattr(field, 'bundle'): if field.bundle != target.getProjectCustomField(projectId, field.name).bundle: target.deleteProjectCustomField(projectId, field.name) create_project_custom_field(target, field, projectId) else: try: create_project_custom_field(target, field, projectId) except youtrack.YouTrackException as e: if e.response.status != 409: raise e else: print(e) # copy issues start = 0 max = 20 sync_workitems = enable_time_tracking(source, target, projectId) tt_settings = target.getProjectTimeTrackingSettings(projectId) print("Import issues") last_created_issue_number = 0 while True: try: print("Get issues from " + str(start) + " to " + str(start + max)) issues = source.getIssues(projectId, query, start, max) if len(issues) <= 0: break if convert_period_values and period_cf_names: for issue in issues: for pname in period_cf_names: for fname in issue.__dict__: if fname.lower() != pname: continue issue[fname] = period_to_minutes(issue[fname]) users = set([]) for issue in issues: print("Collect users for issue [%s]" % issue.id) users.add(issue.getReporter()) if issue.hasAssignee(): if isinstance(issue.Assignee, (list, tuple)): users.update(issue.getAssignee()) else: users.add(issue.getAssignee()) # TODO: http://youtrack.jetbrains.net/issue/JT-6100 users.add(issue.getUpdater()) if issue.hasVoters(): users.update(issue.getVoters()) for comment in issue.getComments(): users.add(comment.getAuthor()) print("Collect links for issue [%s]" % issue.id) link_importer.collectLinks(issue.getLinks(True)) # links.extend(issue.getLinks(True)) # fix problem with comment.text for comment in issue.getComments(): if not hasattr(comment, "text") or (len(comment.text.strip()) == 0): setattr(comment, 'text', 'no text') user_importer.importUsersRecursively(users) print("Create issues [" + str(len(issues)) + "]") if params.get('create_new_issues'): create_issues(target, issues, last_created_issue_number) else: print(target.importIssues(projectId, project.name + ' Assignees', issues)) link_importer.addAvailableIssues(issues) for issue in issues: try: target_issue = target.getIssue(issue.id) except youtrack.YouTrackException as e: print("Cannot get target issue") print(e) continue if params.get('sync_tags') and issue.tags: try: for tag in issue.tags: tag = re.sub(r'[,&<>]', '_', tag) try: target.executeCommand(issue.id, 'tag ' + tag, disable_notifications=True) except youtrack.YouTrackException: tag = re.sub(r'[\s-]', '_', tag) target.executeCommand(issue.id, 'tag ' + tag, disable_notifications=True) except youtrack.YouTrackException as e: print("Cannot sync tags for issue " + issue.id) print(e) if params.get('add_new_comments'): target_comments = dict() max_id = 0 for c in target_issue.getComments(): target_comments[c.created] = c if max_id < c.created: max_id = c.created for c in issue.getComments(): if c.created > max_id or c.created not in target_comments: group = None if hasattr(c, 'permittedGroup'): group = c.permittedGroup try: target.executeCommand(issue.id, 'comment', c.text, group, c.author, disable_notifications=True) except youtrack.YouTrackException as e: print('Cannot add comment to issue') print(e) if params.get('sync_custom_fields'): skip_fields = [] if tt_settings and tt_settings.Enabled and tt_settings.TimeSpentField: skip_fields.append(tt_settings.TimeSpentField) skip_fields = [name.lower() for name in skip_fields] for pcf in [pcf for pcf in project_custom_fields if pcf.name.lower() not in skip_fields]: target_cf_value = None if pcf.name in target_issue: target_cf_value = target_issue[pcf.name] if isinstance(target_cf_value, (list, tuple)): target_cf_value = set(target_cf_value) elif target_cf_value == target.getProjectCustomField(projectId, pcf.name).emptyText: target_cf_value = None source_cf_value = None if pcf.name in issue: source_cf_value = issue[pcf.name] if isinstance(source_cf_value, (list, tuple)): source_cf_value = set(source_cf_value) elif source_cf_value == source.getProjectCustomField(projectId, pcf.name).emptyText: source_cf_value = None if source_cf_value == target_cf_value: continue if isinstance(source_cf_value, set) or isinstance(target_cf_value, set): if source_cf_value is None: source_cf_value = set([]) elif not isinstance(source_cf_value, set): source_cf_value = set([source_cf_value]) if target_cf_value is None: target_cf_value = set([]) elif not isinstance(target_cf_value, set): target_cf_value = set([target_cf_value]) for v in target_cf_value: if v not in source_cf_value: target.executeCommand(issue.id, 'remove %s %s' % (pcf.name, v), disable_notifications=True) for v in source_cf_value: if v not in target_cf_value: target.executeCommand(issue.id, 'add %s %s' % (pcf.name, v), disable_notifications=True) else: if source_cf_value is None: source_cf_value = target.getProjectCustomField(projectId, pcf.name).emptyText if pcf.type.lower() == 'date': m = re.match(r'(\d{10})(?:\d{3})?', str(source_cf_value)) if m: source_cf_value = datetime.datetime.fromtimestamp( int(m.group(1))).strftime('%Y-%m-%d') elif pcf.type.lower() == 'period': source_cf_value = '%sm' % source_cf_value command = '%s %s' % (pcf.name, source_cf_value) try: target.executeCommand(issue.id, command, disable_notifications=True) except youtrack.YouTrackException as e: if e.response.status == 412 and e.response.reason.find('Precondition Failed') > -1: print('WARN: Some workflow blocks following command: %s' % command) failed_commands.append((issue.id, command)) if sync_workitems: workitems = source.getWorkItems(issue.id) if workitems: existing_workitems = dict() target_workitems = target.getWorkItems(issue.id) if target_workitems: for w in target_workitems: _id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration) if hasattr(w, 'description'): _id += '\n%s' % w.description existing_workitems[_id] = w new_workitems = [] for w in workitems: _id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration) if hasattr(w, 'description'): _id += '\n%s' % w.description if _id not in existing_workitems: new_workitems.append(w) if new_workitems: print("Process workitems for issue [ " + issue.id + "]") try: user_importer.importUsersRecursively( [source.getUser(w.authorLogin) for w in new_workitems]) target.importWorkItems(issue.id, new_workitems) except youtrack.YouTrackException as e: if e.response.status == 404: print("WARN: Target YouTrack doesn't support workitems importing.") print("WARN: Workitems won't be imported.") sync_workitems = False else: print("ERROR: Skipping workitems because of error:" + str(e)) print("Process attachments for issue [%s]" % issue.id) existing_attachments = dict() try: for a in target.getAttachments(issue.id): existing_attachments[a.name + '\n' + a.created] = a except youtrack.YouTrackException as e: if e.response.status == 404: print("Skip importing attachments because issue %s doesn't exist" % issue.id) continue raise e attachments = [] users = set([]) for a in issue.getAttachments(): if a.name + '\n' + a.created in existing_attachments and not params.get('replace_attachments'): a.name = utf8encode(a.name) try: print("Skip attachment '%s' (created: %s) because it's already exists" % (utf8encode(a.name), utf8encode(a.created))) except Exception: pass continue attachments.append(a) author = a.getAuthor() if author is not None: users.add(author) user_importer.importUsersRecursively(users) for a in attachments: print("Transfer attachment of " + utf8encode(issue.id) + ": " + utf8encode(a.name)) # TODO: add authorLogin to workaround http://youtrack.jetbrains.net/issue/JT-6082 # a.authorLogin = target_login try: target.createAttachmentFromAttachment(issue.id, a) except BaseException as e: print("Cant import attachment [ %s ]" % utf8encode(a.name)) print(repr(e)) continue if params.get('replace_attachments'): try: old_attachment = existing_attachments.get(a.name + '\n' + a.created) if old_attachment: print('Deleting old attachment') target.deleteAttachment(issue.id, old_attachment.id) except BaseException as e: print("Cannot delete attachment '%s' from issue %s" % ( utf8encode(a.name), utf8encode(issue.id))) print(e) except Exception as e: print('Cant process issues from ' + str(start) + ' to ' + str(start + max)) traceback.print_exc() raise e start += max print("Import issue links") link_importer.importCollectedLinks() print("Trying to execute failed commands once again") for issue_id, command in failed_commands: try: print('Executing command on issue %s: %s' % (issue_id, command)) target.executeCommand(issue_id, command, disable_notifications=True) except youtrack.YouTrackException as e: print('Failed to execute command for issue #%s: %s' % (issue_id, command)) print(e)
#links.extend(issue.getLinks(True)) # fix problem with comment.text for comment in issue.getComments(): if not hasattr(comment, "text") or (len(comment.text.strip()) == 0): setattr(comment, 'text', 'no text') user_importer.importUsersRecursively(users) print "Create issues [" + str(len(issues)) + "]" print target.importIssues(projectId, project.name + ' Assignees', issues) link_importer.addAvailableIssues(issues) for issue in issues: if sync_workitems: workitems = source.getWorkItems(issue.id) if workitems: print "Process workitems for issue [ " + issue.id + "]" try: target.importWorkItems(issue.id, workitems) except youtrack.YouTrackException, e: if e.response.status == 404: print "WARN: Target YouTrack doesn't support workitems importing." print "WARN: Workitems won't be imported." sync_workitems = False else: print "ERROR: Skipping workitems because of error:" + str(e) print "Process attachments for issue [ " + issue.id + "]" attachments = issue.getAttachments() users = set([])
m = re.match(r'(\d{10})(?:\d{3})?', str(source_cf_value)) if m: source_cf_value = datetime.datetime.fromtimestamp( int(m.group(1))).strftime('%Y-%m-%d') elif pcf.type.lower() == 'period': source_cf_value = '%sm' % source_cf_value command = '%s %s' % (pcf.name, source_cf_value) try: target.executeCommand(issue.id, command) except youtrack.YouTrackException, e: if e.response.status == 412 and e.response.reason.find('Precondition Failed') > -1: print 'WARN: Some workflow blocks following command: %s' % command failed_commands.append((issue.id, command)) if sync_workitems: workitems = source.getWorkItems(issue.id) if workitems: existing_workitems = dict() target_workitems = target.getWorkItems(issue.id) if target_workitems: for w in target_workitems: _id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration) if hasattr(w, 'description'): _id += '\n%s' % w.description existing_workitems[_id] = w new_workitems = [] for w in workitems: _id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration) if hasattr(w, 'description'): _id += '\n%s' % w.description if _id not in existing_workitems:
if not hasattr(comment, "text") or (len( comment.text.strip()) == 0): setattr(comment, 'text', 'no text') user_importer.importUsersRecursively(users) print "Create issues [" + str(len(issues)) + "]" print target.importIssues(projectId, project.name + ' Assignees', issues) link_importer.addAvailableIssues(issues) for issue in issues: if tt_settings.Enabled: print "Process work items for issue [ " + issue.id + "]" target.importWorkItems(issue.id, source.getWorkItems(issue.id)) print "Process attachments for issue [ " + issue.id + "]" attachments = issue.getAttachments() users = set([]) for a in attachments: author = a.getAuthor() if author is not None: users.add(author) user_importer.importUsersRecursively(users) for a in attachments: print "Transfer attachment of " + issue.id.encode( 'utf-8') + ": " + a.name.encode('utf-8') # TODO: add authorLogin to workaround http://youtrack.jetbrains.net/issue/JT-6082
def do_move(source_url, source_login, source_password, target_url, target_login, target_password, source_issue_id, target): print("source_url : " + source_url) print("source_login : "******"source_password : "******"target_url : " + target_url) print("target_login : "******"target_password : "******"source_id : " + source_issue_id) if target.find('-') > -1: print("target_id : " + target) target_project_id, target_issue_number = target.split('-') else: print("target_project_id: " + target) target_project_id = target target_issue_number = None # connecting try: target = Connection(target_url, target_login, target_password) print("Connected to target url [%s]" % target_url) except Exception as ex: print("Failed to connect to target url [%s] with login/password [%s/%s]" % (target_url, target_login, target_password)) raise ex try: source = Connection(source_url, source_login, source_password) print("Connected to source url [%s]" % source_url) except Exception as ex: print("Failed to connect to source url [%s] with login/password [%s/%s]" % (source_url, source_login, source_password)) raise ex try: target.getProject(target_project_id) except Exception as ex: print("Can't connect to target project [%s]" % target_project_id) raise ex # twin issues try: source_issue = source.getIssue(source_issue_id) except Exception as ex: print("Failed to get issue [%s]" % source_issue_id) raise ex target_issue = Issue() # import users if needed name_fields = ["reporterName", "assigneeName", "updaterName"] for field in name_fields: if field in source_issue: check_user(source_issue[field], source, target) if not target_issue_number: target_issue_number = str(get_new_issue_id(target_project_id, target)) target_issue.numberInProject = target_issue_number # check subsystem target_subsystem = None try: target.getSubsystem(target_project_id, source_issue.subsystem) target_subsystem = source_issue.subsystem except (YouTrackException, AttributeError): pass target_issue.subsystem = target_subsystem for field in PREDEFINED_FIELDS: if field in source_issue: target_issue[field] = source_issue[field] if "Type" in source_issue: target_issue.type = source_issue["Type"] elif "type" in source_issue: target_issue.type = source_issue["type"] else: target_issue.type = "Bug" # convert custom field target_cfs = target.getProjectCustomFields(target_project_id) for cf in target_cfs: cf_name = cf.name if cf_name in source_issue: target_issue[cf_name] = source_issue[cf_name] # comments target_issue.comments = source_issue.getComments() for comment in target_issue.comments: check_user(comment.author, source, target) # import issue print(target.importIssues( target_project_id, "", [target_issue])) # attachments for attachment in source_issue.getAttachments(): check_user(attachment.authorLogin, source, target) attachment.url = attachment.url.replace(source_url, "") target.createAttachmentFromAttachment( "%s-%s" % (target_project_id, target_issue.numberInProject), attachment) # work items if get_time_tracking_state( source, target, source_issue_id.split('-')[0], target_project_id): workitems = source.getWorkItems(source_issue_id) if workitems: existing_workitems = dict() target_workitems = target.getWorkItems( target_project_id + '-' + target_issue_number) if target_workitems: for w in target_workitems: _id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration) if hasattr(w, 'description'): _id += '\n%s' % w.description existing_workitems[_id] = w new_workitems = [] for w in workitems: _id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration) if hasattr(w, 'description'): _id += '\n%s' % w.description if _id not in existing_workitems: new_workitems.append(w) if new_workitems: print("Process workitems for issue [ " + source_issue_id + "]") try: for w in new_workitems: check_user(w.authorLogin, source, target) target.importWorkItems( target_project_id + '-' + target_issue_number, new_workitems) except YouTrackException as e: print("Failed to import workitems: " + str(e)) # links link_importer = LinkImporter(target) links2import = source_issue.getLinks() link_importer.collectLinks(links2import) link_importer.addAvailableIssue(source_issue) for l in links2import: link_importer.addAvailableIssue(source.getIssue(l.source)) link_importer.addAvailableIssue(source.getIssue(l.target)) link_importer.importCollectedLinks()
source_cf_value = '%sm' % source_cf_value command = '%s %s' % (pcf.name, source_cf_value) try: target.executeCommand( issue.id, command, disable_notifications=True) except youtrack.YouTrackException, e: if e.response.status == 412 and e.response.reason.find( 'Precondition Failed') > -1: print 'WARN: Some workflow blocks following command: %s' % command failed_commands.append( (issue.id, command)) if sync_workitems: workitems = source.getWorkItems(issue.id) if workitems: existing_workitems = dict() target_workitems = target.getWorkItems(issue.id) if target_workitems: for w in target_workitems: _id = '%s\n%s\n%s' % ( w.date, w.authorLogin, w.duration) if hasattr(w, 'description'): _id += '\n%s' % w.description existing_workitems[_id] = w new_workitems = [] for w in workitems: _id = '%s\n%s\n%s' % (w.date, w.authorLogin, w.duration) if hasattr(w, 'description'):
class YouTrackExporter(object): def __init__(self, yt, influx): self.yt = YouTrack(**yt) self.influx = InfluxDBClient(**influx) self.logger = logging.getLogger('worktime_reporter') self.project_blacklist = ['RRS', 'RR_INS'] def get_all_issues(self, project): def _gen(): skip = 0 while True: issues = self.yt.getIssues(project.id, 'Spent time: -?', skip, 50) if not issues: break skip += len(issues) yield issues return itertools.chain.from_iterable(_gen()) def issue_to_measurement(self, project, issue): self.logger.info(f'Looking into issue {issue.id}') lead_time = getattr(issue, 'Lead time', None) stream = getattr(issue, 'Stream', None) target = getattr(issue, 'Target', 'Core') _type = getattr(issue, 'Type', None) resolved = getattr(issue, 'resolved', None) estimation = getattr(issue, 'Estimation', None) tags = { 'project': project.id, 'issue': issue.id, 'stream': stream, 'target': target, 'type': _type } counter = defaultdict(lambda: 0) for work_item in self.yt.getWorkItems(issue.id): _tags = dict(tags) _tags['author'] = getattr(work_item, 'authorLogin', None) created = getattr(work_item, 'created', None) work_type = getattr(work_item, 'worktype', None) date = getattr(work_item, 'date', created) duration = int(work_item.duration) if not created: continue created_dt = datetime.datetime.fromtimestamp(float(created)/1000) date_dt = datetime.datetime.fromtimestamp(float(date)/1000) time_dt = datetime.datetime.combine(date_dt.date(), created_dt.time(), date_dt.tzinfo) time_ts = int(datetime.datetime.timestamp(time_dt)*1000) counter[work_type] += duration yield { 'measurement': 'work_item', 'tags': _tags, 'time': time_ts, 'fields': { work_type: duration } } cycle_time = counter['Analytics'] + counter['Development'] + counter['Testing'] created = getattr(issue, 'created', None) if estimation: yield { 'measurement': 'issue', 'tags': tags, 'time': int(created), 'fields': { 'estimation': int(estimation) } } if lead_time and resolved: yield { 'measurement': 'issue', 'tags': tags, 'time': int(resolved), 'fields': { 'lead_time': float(lead_time), 'cycle_time': cycle_time, } } def process_project(self, project): if project.id in self.project_blacklist: self.logger.info(f'Skipping project {project.id}, blacklisted') return ts = self.yt.getProjectTimeTrackingSettings(project['id']) if not ts.Enabled: self.logger.info( f'Skipping project {project.id}, no time tracking') return self.logger.info(f'Looking into project {project.id}') issues = self.get_all_issues(project) measurements = (self.issue_to_measurement(project, issue) for issue in issues) return itertools.chain.from_iterable(measurements) def export(self): def _gen(): projects = self.yt.getProjects() for project_id in projects: project = self.yt.getProject(project_id) measurements = self.process_project(project) if measurements: yield measurements self.influx.drop_measurement('issue') self.influx.drop_measurement('work_item') measurements = itertools.chain.from_iterable(_gen()) for chunk in grouper(measurements, 50): filtered = filter(lambda x: x is not None, chunk) self.influx.write_points(filtered, time_precision='ms')
#links.extend(issue.getLinks(True)) # fix problem with comment.text for comment in issue.getComments(): if not hasattr(comment, "text") or (len(comment.text.strip()) == 0): setattr(comment, 'text', 'no text') user_importer.importUsersRecursively(users) print "Create issues [" + str(len(issues)) + "]" print target.importIssues(projectId, project.name + ' Assignees', issues) link_importer.addAvailableIssues(issues) for issue in issues: print "Process work items for issue [ " + issue.id + "]" target.importWorkItems(issue.id, source.getWorkItems(issue.id)) print "Process attachments for issue [ " + issue.id + "]" attachments = issue.getAttachments() users = set([]) for a in attachments: author = a.getAuthor() if author is not None: users.add(author) user_importer.importUsersRecursively(users) for a in attachments: print "Transfer attachment of " + issue.id.encode('utf-8') + ": " + a.name.encode('utf-8') # TODO: add authorLogin to workaround http://youtrack.jetbrains.net/issue/JT-6082 a.authorLogin = target_login