def _make_page(self, text, filename, commit, renamed_to=None):
orig_name = self._format_supported(filename)
renamed_orig_name = self._format_supported(
renamed_to) if renamed_to else None
if not orig_name:
return
if renamed_to and not renamed_orig_name:
return
mod_date = datetime.utcfromtimestamp(commit.committed_date)
wiki_page = WM.Page.upsert(self._convert_page_name(orig_name))
wiki_page.timestamp = wiki_page.mod_date = mod_date
if renamed_orig_name and renamed_to in commit.tree:
wiki_page.title = self._convert_page_name(renamed_orig_name)
wiki_page.text = self.convert_markup(h.really_unicode(text),
renamed_to)
elif filename in commit.tree:
wiki_page.text = self.convert_markup(h.really_unicode(text),
filename)
else:
wiki_page.delete()
import_id_name = renamed_orig_name if renamed_orig_name else orig_name
wiki_page.import_id = ImportIdConverter.get().expand(
import_id_name, self.app)
wiki_page.commit()
return wiki_page
def _make_page(self, text, filename, commit, renamed_to=None):
orig_name = self._format_supported(filename)
renamed_orig_name = self._format_supported(
renamed_to) if renamed_to else None
if not orig_name:
return
if renamed_to and not renamed_orig_name:
return
mod_date = datetime.utcfromtimestamp(commit.committed_date)
wiki_page = WM.Page.upsert(self._convert_page_name(orig_name))
wiki_page.timestamp = wiki_page.mod_date = mod_date
wiki_page.viewable_by = ['all']
if renamed_orig_name and renamed_to in commit.tree:
wiki_page.title = self._convert_page_name(renamed_orig_name)
wiki_page.text = self.convert_markup(
h.really_unicode(text), renamed_to)
elif filename in commit.tree:
wiki_page.text = self.convert_markup(
h.really_unicode(text), filename)
else:
wiki_page.delete()
import_id_name = renamed_orig_name if renamed_orig_name else orig_name
wiki_page.import_id = ImportIdConverter.get().expand(
import_id_name, self.app)
wiki_page.commit()
return wiki_page
def import_tool(self,
project,
user,
project_name,
mount_point=None,
mount_label=None,
**kw):
import_id_converter = ImportIdConverter.get()
app = project.install_app(
'tickets',
mount_point,
mount_label,
EnableVoting=True,
open_status_names='New Accepted Started',
closed_status_names='Fixed Verified Invalid Duplicate WontFix Done',
import_id={
'source': self.source,
'project_name': project_name,
},
)
ThreadLocalORMSession.flush_all()
try:
M.session.artifact_orm_session._get().skip_mod_date = True
with h.push_config(c, user=M.User.anonymous(), app=app):
for ticket_num, issue in GoogleCodeProjectExtractor.iter_issues(
project_name):
self.max_ticket_num = max(ticket_num, self.max_ticket_num)
ticket = TM.Ticket(app_config_id=app.config._id,
custom_fields=dict(),
ticket_num=ticket_num,
import_id=import_id_converter.expand(
ticket_num, app))
self.process_fields(ticket, issue)
self.process_labels(ticket, issue)
self.process_comments(ticket, issue)
session(ticket).flush(ticket)
session(ticket).expunge(ticket)
app.globals.custom_fields = self.postprocess_custom_fields()
app.globals.last_ticket_num = self.max_ticket_num
ThreadLocalORMSession.flush_all()
M.AuditLog.log(
'import tool %s from %s on %s' % (
app.config.options.mount_point,
project_name,
self.source,
),
project=project,
user=user,
url=app.url,
)
g.post_event('project_updated')
app.globals.invalidate_bin_counts()
return app
except Exception as e:
h.make_app_admin_only(app)
raise
finally:
M.session.artifact_orm_session._get().skip_mod_date = False
def import_tool(self,
project,
user,
project_name,
mount_point=None,
mount_label=None,
**kw):
import_id_converter = ImportIdConverter.get()
project_name = '%s/%s' % (kw['user_name'], project_name)
extractor = GitHubProjectExtractor(project_name, user=user)
if not extractor.has_tracker():
return
app = project.install_app('tickets',
mount_point,
mount_label,
EnableVoting=False,
open_status_names='open',
closed_status_names='closed',
import_id={
'source': self.source,
'project_name': project_name,
})
self.github_markdown_converter = GitHubMarkdownConverter(
kw['user_name'], project_name)
ThreadLocalORMSession.flush_all()
try:
M.session.artifact_orm_session._get().skip_mod_date = True
with h.push_config(c, user=M.User.anonymous(), app=app):
for ticket_num, issue in extractor.iter_issues():
self.max_ticket_num = max(ticket_num, self.max_ticket_num)
ticket = TM.Ticket(app_config_id=app.config._id,
custom_fields=dict(),
ticket_num=ticket_num,
import_id=import_id_converter.expand(
ticket_num, app))
self.process_fields(extractor, ticket, issue)
self.process_comments(extractor, ticket, issue)
self.process_events(extractor, ticket, issue)
self.process_milestones(ticket, issue)
session(ticket).flush(ticket)
session(ticket).expunge(ticket)
app.globals.custom_fields = self.postprocess_milestones()
app.globals.last_ticket_num = self.max_ticket_num
ThreadLocalORMSession.flush_all()
M.AuditLog.log(
'import tool %s from %s on %s' %
(app.config.options.mount_point, project_name, self.source),
project=project,
user=user,
url=app.url)
g.post_event('project_updated')
app.globals.invalidate_bin_counts()
return app
finally:
M.session.artifact_orm_session._get().skip_mod_date = False
def make_artifact(self, ticket_dict):
remapped = {}
for f, v in six.iteritems(ticket_dict):
transform = self.FIELD_MAP.get(f, ())
if transform is None:
continue
elif transform is True:
remapped[f] = v
elif callable(transform):
transform(remapped, f, v)
elif transform is ():
self.custom(remapped, f, v, ticket_dict.get('status'))
else:
new_f, conv = transform
remapped[new_f] = conv(v)
description = h.really_unicode(
self.description_processing(remapped['description']))
creator = owner = ''
if ticket_dict.get('submitter') and not remapped.get('reported_by_id'):
creator = '*Originally created by:* {0}\n'.format(
h.really_unicode(ticket_dict['submitter']))
if ticket_dict.get(
'assigned_to') and not remapped.get('assigned_to_id'):
owner = '*Originally owned by:* {0}\n'.format(
h.really_unicode(ticket_dict['assigned_to']))
remapped['description'] = '{0}{1}{2}{3}'.format(
creator, owner, '\n' if creator or owner else '', description)
ticket_num = ticket_dict['id']
existing_ticket = TM.Ticket.query.get(app_config_id=c.app.config._id,
ticket_num=ticket_num)
if existing_ticket:
ticket_num = c.app.globals.next_ticket_num()
self.warnings.append(
'Ticket #%s: Ticket with this id already exists, using next available id: %s'
% (ticket_dict['id'], ticket_num))
else:
if c.app.globals.last_ticket_num < ticket_num:
c.app.globals.last_ticket_num = ticket_num
ThreadLocalORMSession.flush_all()
ticket = TM.Ticket(app_config_id=c.app.config._id,
custom_fields=dict(),
ticket_num=ticket_num,
import_id=ImportIdConverter.get().expand(
ticket_dict['id'], c.app))
ticket.update(remapped)
return ticket
def import_tool(self, project, user, project_name, mount_point=None,
mount_label=None, **kw):
import_id_converter = ImportIdConverter.get()
project_name = '%s/%s' % (kw['user_name'], project_name)
extractor = GitHubProjectExtractor(project_name, user=user)
if not extractor.has_tracker():
return
app = project.install_app('tickets', mount_point, mount_label,
EnableVoting=False,
open_status_names='open',
closed_status_names='closed',
import_id={
'source': self.source,
'project_name': project_name,
}
)
self.github_markdown_converter = GitHubMarkdownConverter(
kw['user_name'], project_name)
ThreadLocalORMSession.flush_all()
try:
M.session.artifact_orm_session._get().skip_mod_date = True
with h.push_config(c, user=M.User.anonymous(), app=app):
for ticket_num, issue in extractor.iter_issues():
self.max_ticket_num = max(ticket_num, self.max_ticket_num)
ticket = TM.Ticket(
app_config_id=app.config._id,
custom_fields=dict(),
ticket_num=ticket_num,
import_id=import_id_converter.expand(ticket_num, app)
)
self.process_fields(extractor, ticket, issue)
self.process_comments(extractor, ticket, issue)
self.process_events(extractor, ticket, issue)
self.process_milestones(ticket, issue)
session(ticket).flush(ticket)
session(ticket).expunge(ticket)
app.globals.custom_fields = self.postprocess_milestones()
app.globals.last_ticket_num = self.max_ticket_num
ThreadLocalORMSession.flush_all()
M.AuditLog.log(
'import tool %s from %s on %s' % (
app.config.options.mount_point,
project_name, self.source),
project=project, user=user, url=app.url)
g.post_event('project_updated')
app.globals.invalidate_bin_counts()
return app
finally:
M.session.artifact_orm_session._get().skip_mod_date = False
def make_artifact(self, ticket_dict):
remapped = {}
for f, v in ticket_dict.iteritems():
transform = self.FIELD_MAP.get(f, ())
if transform is None:
continue
elif transform is True:
remapped[f] = v
elif callable(transform):
transform(remapped, f, v)
elif transform is ():
self.custom(remapped, f, v, ticket_dict.get('status'))
else:
new_f, conv = transform
remapped[new_f] = conv(v)
description = h.really_unicode(
self.description_processing(remapped['description']))
creator = owner = ''
if ticket_dict.get('submitter') and not remapped.get('reported_by_id'):
creator = u'*Originally created by:* {0}\n'.format(
h.really_unicode(ticket_dict['submitter']))
if ticket_dict.get('assigned_to') and not remapped.get('assigned_to_id'):
owner = u'*Originally owned by:* {0}\n'.format(
h.really_unicode(ticket_dict['assigned_to']))
remapped['description'] = u'{0}{1}{2}{3}'.format(creator, owner,
'\n' if creator or owner else '', description)
ticket_num = ticket_dict['id']
existing_ticket = TM.Ticket.query.get(app_config_id=c.app.config._id,
ticket_num=ticket_num)
if existing_ticket:
ticket_num = c.app.globals.next_ticket_num()
self.warnings.append(
'Ticket #%s: Ticket with this id already exists, using next available id: %s' %
(ticket_dict['id'], ticket_num))
else:
if c.app.globals.last_ticket_num < ticket_num:
c.app.globals.last_ticket_num = ticket_num
ThreadLocalORMSession.flush_all()
ticket = TM.Ticket(
app_config_id=c.app.config._id,
custom_fields=dict(),
ticket_num=ticket_num,
import_id=ImportIdConverter.get().expand(ticket_dict['id'], c.app))
ticket.update(remapped)
return ticket
def import_tool(self, project, user, project_name, mount_point=None,
mount_label=None, **kw):
import_id_converter = ImportIdConverter.get()
app = project.install_app('tickets', mount_point, mount_label,
EnableVoting=True,
open_status_names='New Accepted Started',
closed_status_names='Fixed Verified Invalid Duplicate WontFix Done',
import_id={
'source': self.source,
'project_name': project_name,
},
)
ThreadLocalORMSession.flush_all()
try:
M.session.artifact_orm_session._get().skip_mod_date = True
with h.push_config(c, user=M.User.anonymous(), app=app):
for ticket_num, issue in GoogleCodeProjectExtractor.iter_issues(project_name):
self.max_ticket_num = max(ticket_num, self.max_ticket_num)
ticket = TM.Ticket(
app_config_id=app.config._id,
custom_fields=dict(),
ticket_num=ticket_num,
import_id=import_id_converter.expand(ticket_num, app))
self.process_fields(ticket, issue)
self.process_labels(ticket, issue)
self.process_comments(ticket, issue)
session(ticket).flush(ticket)
session(ticket).expunge(ticket)
app.globals.custom_fields = self.postprocess_custom_fields()
app.globals.last_ticket_num = self.max_ticket_num
ThreadLocalORMSession.flush_all()
M.AuditLog.log(
'import tool %s from %s on %s' % (
app.config.options.mount_point,
project_name, self.source,
),
project=project,
user=user,
url=app.url,
)
g.post_event('project_updated')
app.globals.invalidate_bin_counts()
return app
except Exception:
h.make_app_admin_only(app)
raise
finally:
M.session.artifact_orm_session._get().skip_mod_date = False
def index(self):
result = VersionedArtifact.index(self)
result.update(
title='Ticket %s' % self.ticket_num,
version_i=self.version,
type_s=self.type_s,
created_date_dt=self.created_date,
ticket_num_i=self.ticket_num,
summary_t=self.summary,
milestone_s=self.milestone,
status_s=self.status,
text=self.description,
snippet_s=self.summary,
private_b=self.private,
votes_up_i=self.votes_up,
votes_down_i=self.votes_down,
votes_total_i=(self.votes_up - self.votes_down),
import_id_s=ImportIdConverter.get().simplify(self.import_id)
)
for k, v in self.custom_fields.iteritems():
# Pre solr-4.2.1 code expects all custom fields to be indexed
# as strings.
if not config.get_bool('new_solr'):
result[k + '_s'] = unicode(v)
# Now let's also index with proper Solr types.
solr_type = self.app.globals.get_custom_field_solr_type(k)
if solr_type:
result[k + solr_type] = (v or
get_default_for_solr_type(solr_type))
if self.reported_by:
result['reported_by_s'] = self.reported_by.username
if self.assigned_to:
result['assigned_to_s'] = self.assigned_to.username
# Tracker uses search with default solr parser. It would match only on
# `text`, so we're appending all other field values into `text`, to
# match on it too.
result['text'] += pformat(result.values())
return result
def index(self):
result = VersionedArtifact.index(self)
result.update(title='Ticket %s' % self.ticket_num,
version_i=self.version,
type_s=self.type_s,
created_date_dt=self.created_date,
ticket_num_i=self.ticket_num,
summary_t=self.summary,
milestone_s=self.milestone,
status_s=self.status,
text=self.description,
snippet_s=self.summary,
private_b=self.private,
votes_up_i=self.votes_up,
votes_down_i=self.votes_down,
votes_total_i=(self.votes_up - self.votes_down),
import_id_s=ImportIdConverter.get().simplify(
self.import_id))
for k, v in self.custom_fields.iteritems():
# Pre solr-4.2.1 code expects all custom fields to be indexed
# as strings.
if not config.get_bool('new_solr'):
result[k + '_s'] = unicode(v)
# Now let's also index with proper Solr types.
solr_type = self.app.globals.get_custom_field_solr_type(k)
if solr_type:
result[k +
solr_type] = (v or get_default_for_solr_type(solr_type))
if self.reported_by:
result['reported_by_s'] = self.reported_by.username
if self.assigned_to:
result['assigned_to_s'] = self.assigned_to.username
# Tracker uses search with default solr parser. It would match only on
# `text`, so we're appending all other field values into `text`, to
# match on it too.
result['text'] += pformat(result.values())
return result
def import_tool(self, project, user, mount_point=None,
mount_label=None, **kw):
import_id_converter = ImportIdConverter.get()
tracker_json = self._load_json(project)
tracker_json['tracker_config']['options'].pop('ordinal', None)
tracker_json['tracker_config']['options'].pop('mount_point', None)
tracker_json['tracker_config']['options'].pop('mount_label', None)
tracker_json['tracker_config']['options'].pop('import_id', None)
app = project.install_app('tickets', mount_point, mount_label,
import_id={
'source': self.source,
'app_config_id': tracker_json['tracker_config']['_id'],
},
open_status_names=tracker_json[
'open_status_names'],
closed_status_names=tracker_json[
'closed_status_names'],
**tracker_json['tracker_config']['options']
)
ThreadLocalORMSession.flush_all()
try:
M.session.artifact_orm_session._get().skip_mod_date = True
for ticket_json in tracker_json['tickets']:
reporter = self.get_user(ticket_json['reported_by'])
owner = self.get_user(ticket_json['assigned_to'])
with h.push_config(c, user=reporter, app=app):
self.max_ticket_num = max(
ticket_json['ticket_num'], self.max_ticket_num)
ticket = TM.Ticket(
app_config_id=app.config._id,
import_id=import_id_converter.expand(
ticket_json['ticket_num'], app),
description=self.annotate(
self.annotate(
ticket_json['description'],
owner, ticket_json[
'assigned_to'], label=' owned'),
reporter, ticket_json[
'reported_by'], label=' created'),
created_date=dateutil.parser.parse(
ticket_json['created_date']),
mod_date=dateutil.parser.parse(
ticket_json['mod_date']),
ticket_num=ticket_json['ticket_num'],
summary=ticket_json['summary'],
custom_fields=ticket_json['custom_fields'],
status=ticket_json['status'],
labels=ticket_json['labels'],
votes_down=ticket_json['votes_down'],
votes_up=ticket_json['votes_up'],
votes=ticket_json['votes_up'] -
ticket_json['votes_down'],
assigned_to_id=owner._id,
)
# trigger the private property
ticket.private = ticket_json['private']
self.process_comments(
ticket, ticket_json['discussion_thread']['posts'])
session(ticket).flush(ticket)
session(ticket).expunge(ticket)
app.globals.custom_fields = tracker_json['custom_fields']
self.process_bins(app, tracker_json['saved_bins'])
app.globals.last_ticket_num = self.max_ticket_num
M.AuditLog.log(
'import tool %s from exported Allura JSON' % (
app.config.options.mount_point,
),
project=project,
user=user,
url=app.url,
)
g.post_event('project_updated')
app.globals.invalidate_bin_counts()
ThreadLocalORMSession.flush_all()
return app
except Exception:
h.make_app_admin_only(app)
raise
finally:
M.session.artifact_orm_session._get().skip_mod_date = False
def import_tool(self,
project,
user,
mount_point=None,
mount_label=None,
**kw):
import_id_converter = ImportIdConverter.get()
tracker_json = self._load_json(project)
tracker_json['tracker_config']['options'].pop('ordinal', None)
tracker_json['tracker_config']['options'].pop('mount_point', None)
tracker_json['tracker_config']['options'].pop('mount_label', None)
tracker_json['tracker_config']['options'].pop('import_id', None)
app = project.install_app(
'tickets',
mount_point,
mount_label,
import_id={
'source': self.source,
'app_config_id': tracker_json['tracker_config']['_id'],
},
open_status_names=tracker_json['open_status_names'],
closed_status_names=tracker_json['closed_status_names'],
**tracker_json['tracker_config']['options'])
ThreadLocalORMSession.flush_all()
try:
M.session.artifact_orm_session._get().skip_mod_date = True
for ticket_json in tracker_json['tickets']:
reporter = self.get_user(ticket_json['reported_by'])
owner = self.get_user(ticket_json['assigned_to'])
with h.push_config(c, user=reporter, app=app):
self.max_ticket_num = max(ticket_json['ticket_num'],
self.max_ticket_num)
ticket = TM.Ticket(
app_config_id=app.config._id,
import_id=import_id_converter.expand(
ticket_json['ticket_num'], app),
description=self.annotate(self.annotate(
ticket_json['description'],
owner,
ticket_json['assigned_to'],
label=' owned'),
reporter,
ticket_json['reported_by'],
label=' created'),
created_date=dateutil.parser.parse(
ticket_json['created_date']),
mod_date=dateutil.parser.parse(
ticket_json['mod_date']),
ticket_num=ticket_json['ticket_num'],
summary=ticket_json['summary'],
custom_fields=ticket_json['custom_fields'],
status=ticket_json['status'],
labels=ticket_json['labels'],
votes_down=ticket_json['votes_down'],
votes_up=ticket_json['votes_up'],
votes=ticket_json['votes_up'] -
ticket_json['votes_down'],
assigned_to_id=owner._id,
)
# add an attachment to the ticket
ticket.add_multiple_attachments(
[File(a['url']) for a in ticket_json['attachments']])
# trigger the private property
ticket.private = ticket_json['private']
self.process_comments(
ticket, ticket_json['discussion_thread']['posts'])
session(ticket).flush(ticket)
session(ticket).expunge(ticket)
app.globals.custom_fields = tracker_json['custom_fields']
self.process_bins(app, tracker_json['saved_bins'])
app.globals.last_ticket_num = self.max_ticket_num
M.AuditLog.log(
'import tool %s from exported Allura JSON' %
(app.config.options.mount_point, ),
project=project,
user=user,
url=app.url,
)
g.post_event('project_updated')
app.globals.invalidate_bin_counts()
ThreadLocalORMSession.flush_all()
return app
except Exception:
h.make_app_admin_only(app)
raise
finally:
M.session.artifact_orm_session._get().skip_mod_date = False
