def _make_page(self, text, filename, commit, renamed_to=None): orig_name = self._format_supported(filename) renamed_orig_name = self._format_supported( renamed_to) if renamed_to else None if not orig_name: return if renamed_to and not renamed_orig_name: return mod_date = datetime.utcfromtimestamp(commit.committed_date) wiki_page = WM.Page.upsert(self._convert_page_name(orig_name)) wiki_page.timestamp = wiki_page.mod_date = mod_date if renamed_orig_name and renamed_to in commit.tree: wiki_page.title = self._convert_page_name(renamed_orig_name) wiki_page.text = self.convert_markup(h.really_unicode(text), renamed_to) elif filename in commit.tree: wiki_page.text = self.convert_markup(h.really_unicode(text), filename) else: wiki_page.delete() import_id_name = renamed_orig_name if renamed_orig_name else orig_name wiki_page.import_id = ImportIdConverter.get().expand( import_id_name, self.app) wiki_page.commit() return wiki_page
def _make_page(self, text, filename, commit, renamed_to=None): orig_name = self._format_supported(filename) renamed_orig_name = self._format_supported( renamed_to) if renamed_to else None if not orig_name: return if renamed_to and not renamed_orig_name: return mod_date = datetime.utcfromtimestamp(commit.committed_date) wiki_page = WM.Page.upsert(self._convert_page_name(orig_name)) wiki_page.timestamp = wiki_page.mod_date = mod_date wiki_page.viewable_by = ['all'] if renamed_orig_name and renamed_to in commit.tree: wiki_page.title = self._convert_page_name(renamed_orig_name) wiki_page.text = self.convert_markup( h.really_unicode(text), renamed_to) elif filename in commit.tree: wiki_page.text = self.convert_markup( h.really_unicode(text), filename) else: wiki_page.delete() import_id_name = renamed_orig_name if renamed_orig_name else orig_name wiki_page.import_id = ImportIdConverter.get().expand( import_id_name, self.app) wiki_page.commit() return wiki_page
def import_tool(self, project, user, project_name, mount_point=None, mount_label=None, **kw): import_id_converter = ImportIdConverter.get() app = project.install_app( 'tickets', mount_point, mount_label, EnableVoting=True, open_status_names='New Accepted Started', closed_status_names='Fixed Verified Invalid Duplicate WontFix Done', import_id={ 'source': self.source, 'project_name': project_name, }, ) ThreadLocalORMSession.flush_all() try: M.session.artifact_orm_session._get().skip_mod_date = True with h.push_config(c, user=M.User.anonymous(), app=app): for ticket_num, issue in GoogleCodeProjectExtractor.iter_issues( project_name): self.max_ticket_num = max(ticket_num, self.max_ticket_num) ticket = TM.Ticket(app_config_id=app.config._id, custom_fields=dict(), ticket_num=ticket_num, import_id=import_id_converter.expand( ticket_num, app)) self.process_fields(ticket, issue) self.process_labels(ticket, issue) self.process_comments(ticket, issue) session(ticket).flush(ticket) session(ticket).expunge(ticket) app.globals.custom_fields = self.postprocess_custom_fields() app.globals.last_ticket_num = self.max_ticket_num ThreadLocalORMSession.flush_all() M.AuditLog.log( 'import tool %s from %s on %s' % ( app.config.options.mount_point, project_name, self.source, ), project=project, user=user, url=app.url, ) g.post_event('project_updated') app.globals.invalidate_bin_counts() return app except Exception as e: h.make_app_admin_only(app) raise finally: M.session.artifact_orm_session._get().skip_mod_date = False
def import_tool(self, project, user, project_name, mount_point=None, mount_label=None, **kw): import_id_converter = ImportIdConverter.get() project_name = '%s/%s' % (kw['user_name'], project_name) extractor = GitHubProjectExtractor(project_name, user=user) if not extractor.has_tracker(): return app = project.install_app('tickets', mount_point, mount_label, EnableVoting=False, open_status_names='open', closed_status_names='closed', import_id={ 'source': self.source, 'project_name': project_name, }) self.github_markdown_converter = GitHubMarkdownConverter( kw['user_name'], project_name) ThreadLocalORMSession.flush_all() try: M.session.artifact_orm_session._get().skip_mod_date = True with h.push_config(c, user=M.User.anonymous(), app=app): for ticket_num, issue in extractor.iter_issues(): self.max_ticket_num = max(ticket_num, self.max_ticket_num) ticket = TM.Ticket(app_config_id=app.config._id, custom_fields=dict(), ticket_num=ticket_num, import_id=import_id_converter.expand( ticket_num, app)) self.process_fields(extractor, ticket, issue) self.process_comments(extractor, ticket, issue) self.process_events(extractor, ticket, issue) self.process_milestones(ticket, issue) session(ticket).flush(ticket) session(ticket).expunge(ticket) app.globals.custom_fields = self.postprocess_milestones() app.globals.last_ticket_num = self.max_ticket_num ThreadLocalORMSession.flush_all() M.AuditLog.log( 'import tool %s from %s on %s' % (app.config.options.mount_point, project_name, self.source), project=project, user=user, url=app.url) g.post_event('project_updated') app.globals.invalidate_bin_counts() return app finally: M.session.artifact_orm_session._get().skip_mod_date = False
def make_artifact(self, ticket_dict): remapped = {} for f, v in six.iteritems(ticket_dict): transform = self.FIELD_MAP.get(f, ()) if transform is None: continue elif transform is True: remapped[f] = v elif callable(transform): transform(remapped, f, v) elif transform is (): self.custom(remapped, f, v, ticket_dict.get('status')) else: new_f, conv = transform remapped[new_f] = conv(v) description = h.really_unicode( self.description_processing(remapped['description'])) creator = owner = '' if ticket_dict.get('submitter') and not remapped.get('reported_by_id'): creator = '*Originally created by:* {0}\n'.format( h.really_unicode(ticket_dict['submitter'])) if ticket_dict.get( 'assigned_to') and not remapped.get('assigned_to_id'): owner = '*Originally owned by:* {0}\n'.format( h.really_unicode(ticket_dict['assigned_to'])) remapped['description'] = '{0}{1}{2}{3}'.format( creator, owner, '\n' if creator or owner else '', description) ticket_num = ticket_dict['id'] existing_ticket = TM.Ticket.query.get(app_config_id=c.app.config._id, ticket_num=ticket_num) if existing_ticket: ticket_num = c.app.globals.next_ticket_num() self.warnings.append( 'Ticket #%s: Ticket with this id already exists, using next available id: %s' % (ticket_dict['id'], ticket_num)) else: if c.app.globals.last_ticket_num < ticket_num: c.app.globals.last_ticket_num = ticket_num ThreadLocalORMSession.flush_all() ticket = TM.Ticket(app_config_id=c.app.config._id, custom_fields=dict(), ticket_num=ticket_num, import_id=ImportIdConverter.get().expand( ticket_dict['id'], c.app)) ticket.update(remapped) return ticket
def import_tool(self, project, user, project_name, mount_point=None, mount_label=None, **kw): import_id_converter = ImportIdConverter.get() project_name = '%s/%s' % (kw['user_name'], project_name) extractor = GitHubProjectExtractor(project_name, user=user) if not extractor.has_tracker(): return app = project.install_app('tickets', mount_point, mount_label, EnableVoting=False, open_status_names='open', closed_status_names='closed', import_id={ 'source': self.source, 'project_name': project_name, } ) self.github_markdown_converter = GitHubMarkdownConverter( kw['user_name'], project_name) ThreadLocalORMSession.flush_all() try: M.session.artifact_orm_session._get().skip_mod_date = True with h.push_config(c, user=M.User.anonymous(), app=app): for ticket_num, issue in extractor.iter_issues(): self.max_ticket_num = max(ticket_num, self.max_ticket_num) ticket = TM.Ticket( app_config_id=app.config._id, custom_fields=dict(), ticket_num=ticket_num, import_id=import_id_converter.expand(ticket_num, app) ) self.process_fields(extractor, ticket, issue) self.process_comments(extractor, ticket, issue) self.process_events(extractor, ticket, issue) self.process_milestones(ticket, issue) session(ticket).flush(ticket) session(ticket).expunge(ticket) app.globals.custom_fields = self.postprocess_milestones() app.globals.last_ticket_num = self.max_ticket_num ThreadLocalORMSession.flush_all() M.AuditLog.log( 'import tool %s from %s on %s' % ( app.config.options.mount_point, project_name, self.source), project=project, user=user, url=app.url) g.post_event('project_updated') app.globals.invalidate_bin_counts() return app finally: M.session.artifact_orm_session._get().skip_mod_date = False
def make_artifact(self, ticket_dict): remapped = {} for f, v in ticket_dict.iteritems(): transform = self.FIELD_MAP.get(f, ()) if transform is None: continue elif transform is True: remapped[f] = v elif callable(transform): transform(remapped, f, v) elif transform is (): self.custom(remapped, f, v, ticket_dict.get('status')) else: new_f, conv = transform remapped[new_f] = conv(v) description = h.really_unicode( self.description_processing(remapped['description'])) creator = owner = '' if ticket_dict.get('submitter') and not remapped.get('reported_by_id'): creator = u'*Originally created by:* {0}\n'.format( h.really_unicode(ticket_dict['submitter'])) if ticket_dict.get('assigned_to') and not remapped.get('assigned_to_id'): owner = u'*Originally owned by:* {0}\n'.format( h.really_unicode(ticket_dict['assigned_to'])) remapped['description'] = u'{0}{1}{2}{3}'.format(creator, owner, '\n' if creator or owner else '', description) ticket_num = ticket_dict['id'] existing_ticket = TM.Ticket.query.get(app_config_id=c.app.config._id, ticket_num=ticket_num) if existing_ticket: ticket_num = c.app.globals.next_ticket_num() self.warnings.append( 'Ticket #%s: Ticket with this id already exists, using next available id: %s' % (ticket_dict['id'], ticket_num)) else: if c.app.globals.last_ticket_num < ticket_num: c.app.globals.last_ticket_num = ticket_num ThreadLocalORMSession.flush_all() ticket = TM.Ticket( app_config_id=c.app.config._id, custom_fields=dict(), ticket_num=ticket_num, import_id=ImportIdConverter.get().expand(ticket_dict['id'], c.app)) ticket.update(remapped) return ticket
def import_tool(self, project, user, project_name, mount_point=None, mount_label=None, **kw): import_id_converter = ImportIdConverter.get() app = project.install_app('tickets', mount_point, mount_label, EnableVoting=True, open_status_names='New Accepted Started', closed_status_names='Fixed Verified Invalid Duplicate WontFix Done', import_id={ 'source': self.source, 'project_name': project_name, }, ) ThreadLocalORMSession.flush_all() try: M.session.artifact_orm_session._get().skip_mod_date = True with h.push_config(c, user=M.User.anonymous(), app=app): for ticket_num, issue in GoogleCodeProjectExtractor.iter_issues(project_name): self.max_ticket_num = max(ticket_num, self.max_ticket_num) ticket = TM.Ticket( app_config_id=app.config._id, custom_fields=dict(), ticket_num=ticket_num, import_id=import_id_converter.expand(ticket_num, app)) self.process_fields(ticket, issue) self.process_labels(ticket, issue) self.process_comments(ticket, issue) session(ticket).flush(ticket) session(ticket).expunge(ticket) app.globals.custom_fields = self.postprocess_custom_fields() app.globals.last_ticket_num = self.max_ticket_num ThreadLocalORMSession.flush_all() M.AuditLog.log( 'import tool %s from %s on %s' % ( app.config.options.mount_point, project_name, self.source, ), project=project, user=user, url=app.url, ) g.post_event('project_updated') app.globals.invalidate_bin_counts() return app except Exception: h.make_app_admin_only(app) raise finally: M.session.artifact_orm_session._get().skip_mod_date = False
def index(self): result = VersionedArtifact.index(self) result.update( title='Ticket %s' % self.ticket_num, version_i=self.version, type_s=self.type_s, created_date_dt=self.created_date, ticket_num_i=self.ticket_num, summary_t=self.summary, milestone_s=self.milestone, status_s=self.status, text=self.description, snippet_s=self.summary, private_b=self.private, votes_up_i=self.votes_up, votes_down_i=self.votes_down, votes_total_i=(self.votes_up - self.votes_down), import_id_s=ImportIdConverter.get().simplify(self.import_id) ) for k, v in self.custom_fields.iteritems(): # Pre solr-4.2.1 code expects all custom fields to be indexed # as strings. if not config.get_bool('new_solr'): result[k + '_s'] = unicode(v) # Now let's also index with proper Solr types. solr_type = self.app.globals.get_custom_field_solr_type(k) if solr_type: result[k + solr_type] = (v or get_default_for_solr_type(solr_type)) if self.reported_by: result['reported_by_s'] = self.reported_by.username if self.assigned_to: result['assigned_to_s'] = self.assigned_to.username # Tracker uses search with default solr parser. It would match only on # `text`, so we're appending all other field values into `text`, to # match on it too. result['text'] += pformat(result.values()) return result
def index(self): result = VersionedArtifact.index(self) result.update(title='Ticket %s' % self.ticket_num, version_i=self.version, type_s=self.type_s, created_date_dt=self.created_date, ticket_num_i=self.ticket_num, summary_t=self.summary, milestone_s=self.milestone, status_s=self.status, text=self.description, snippet_s=self.summary, private_b=self.private, votes_up_i=self.votes_up, votes_down_i=self.votes_down, votes_total_i=(self.votes_up - self.votes_down), import_id_s=ImportIdConverter.get().simplify( self.import_id)) for k, v in self.custom_fields.iteritems(): # Pre solr-4.2.1 code expects all custom fields to be indexed # as strings. if not config.get_bool('new_solr'): result[k + '_s'] = unicode(v) # Now let's also index with proper Solr types. solr_type = self.app.globals.get_custom_field_solr_type(k) if solr_type: result[k + solr_type] = (v or get_default_for_solr_type(solr_type)) if self.reported_by: result['reported_by_s'] = self.reported_by.username if self.assigned_to: result['assigned_to_s'] = self.assigned_to.username # Tracker uses search with default solr parser. It would match only on # `text`, so we're appending all other field values into `text`, to # match on it too. result['text'] += pformat(result.values()) return result
def import_tool(self, project, user, mount_point=None, mount_label=None, **kw): import_id_converter = ImportIdConverter.get() tracker_json = self._load_json(project) tracker_json['tracker_config']['options'].pop('ordinal', None) tracker_json['tracker_config']['options'].pop('mount_point', None) tracker_json['tracker_config']['options'].pop('mount_label', None) tracker_json['tracker_config']['options'].pop('import_id', None) app = project.install_app('tickets', mount_point, mount_label, import_id={ 'source': self.source, 'app_config_id': tracker_json['tracker_config']['_id'], }, open_status_names=tracker_json[ 'open_status_names'], closed_status_names=tracker_json[ 'closed_status_names'], **tracker_json['tracker_config']['options'] ) ThreadLocalORMSession.flush_all() try: M.session.artifact_orm_session._get().skip_mod_date = True for ticket_json in tracker_json['tickets']: reporter = self.get_user(ticket_json['reported_by']) owner = self.get_user(ticket_json['assigned_to']) with h.push_config(c, user=reporter, app=app): self.max_ticket_num = max( ticket_json['ticket_num'], self.max_ticket_num) ticket = TM.Ticket( app_config_id=app.config._id, import_id=import_id_converter.expand( ticket_json['ticket_num'], app), description=self.annotate( self.annotate( ticket_json['description'], owner, ticket_json[ 'assigned_to'], label=' owned'), reporter, ticket_json[ 'reported_by'], label=' created'), created_date=dateutil.parser.parse( ticket_json['created_date']), mod_date=dateutil.parser.parse( ticket_json['mod_date']), ticket_num=ticket_json['ticket_num'], summary=ticket_json['summary'], custom_fields=ticket_json['custom_fields'], status=ticket_json['status'], labels=ticket_json['labels'], votes_down=ticket_json['votes_down'], votes_up=ticket_json['votes_up'], votes=ticket_json['votes_up'] - ticket_json['votes_down'], assigned_to_id=owner._id, ) # trigger the private property ticket.private = ticket_json['private'] self.process_comments( ticket, ticket_json['discussion_thread']['posts']) session(ticket).flush(ticket) session(ticket).expunge(ticket) app.globals.custom_fields = tracker_json['custom_fields'] self.process_bins(app, tracker_json['saved_bins']) app.globals.last_ticket_num = self.max_ticket_num M.AuditLog.log( 'import tool %s from exported Allura JSON' % ( app.config.options.mount_point, ), project=project, user=user, url=app.url, ) g.post_event('project_updated') app.globals.invalidate_bin_counts() ThreadLocalORMSession.flush_all() return app except Exception: h.make_app_admin_only(app) raise finally: M.session.artifact_orm_session._get().skip_mod_date = False
def import_tool(self, project, user, mount_point=None, mount_label=None, **kw): import_id_converter = ImportIdConverter.get() tracker_json = self._load_json(project) tracker_json['tracker_config']['options'].pop('ordinal', None) tracker_json['tracker_config']['options'].pop('mount_point', None) tracker_json['tracker_config']['options'].pop('mount_label', None) tracker_json['tracker_config']['options'].pop('import_id', None) app = project.install_app( 'tickets', mount_point, mount_label, import_id={ 'source': self.source, 'app_config_id': tracker_json['tracker_config']['_id'], }, open_status_names=tracker_json['open_status_names'], closed_status_names=tracker_json['closed_status_names'], **tracker_json['tracker_config']['options']) ThreadLocalORMSession.flush_all() try: M.session.artifact_orm_session._get().skip_mod_date = True for ticket_json in tracker_json['tickets']: reporter = self.get_user(ticket_json['reported_by']) owner = self.get_user(ticket_json['assigned_to']) with h.push_config(c, user=reporter, app=app): self.max_ticket_num = max(ticket_json['ticket_num'], self.max_ticket_num) ticket = TM.Ticket( app_config_id=app.config._id, import_id=import_id_converter.expand( ticket_json['ticket_num'], app), description=self.annotate(self.annotate( ticket_json['description'], owner, ticket_json['assigned_to'], label=' owned'), reporter, ticket_json['reported_by'], label=' created'), created_date=dateutil.parser.parse( ticket_json['created_date']), mod_date=dateutil.parser.parse( ticket_json['mod_date']), ticket_num=ticket_json['ticket_num'], summary=ticket_json['summary'], custom_fields=ticket_json['custom_fields'], status=ticket_json['status'], labels=ticket_json['labels'], votes_down=ticket_json['votes_down'], votes_up=ticket_json['votes_up'], votes=ticket_json['votes_up'] - ticket_json['votes_down'], assigned_to_id=owner._id, ) # add an attachment to the ticket ticket.add_multiple_attachments( [File(a['url']) for a in ticket_json['attachments']]) # trigger the private property ticket.private = ticket_json['private'] self.process_comments( ticket, ticket_json['discussion_thread']['posts']) session(ticket).flush(ticket) session(ticket).expunge(ticket) app.globals.custom_fields = tracker_json['custom_fields'] self.process_bins(app, tracker_json['saved_bins']) app.globals.last_ticket_num = self.max_ticket_num M.AuditLog.log( 'import tool %s from exported Allura JSON' % (app.config.options.mount_point, ), project=project, user=user, url=app.url, ) g.post_event('project_updated') app.globals.invalidate_bin_counts() ThreadLocalORMSession.flush_all() return app except Exception: h.make_app_admin_only(app) raise finally: M.session.artifact_orm_session._get().skip_mod_date = False