def start_process_row(self, row_idx, ticket_id):
from ticket import PatchedTicket
if ticket_id > 0:
# existing ticket
self.ticket = PatchedTicket(self.env, tkt_id=ticket_id, db=self.db)
# 'Ticket.time_changed' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
try:
from trac.util.datefmt import to_timestamp
time_changed = to_timestamp(self.ticket.time_changed)
except ImportError:
time_changed = int(self.ticket.time_changed)
if time_changed > self.tickettime:
# just in case, verify if it wouldn't be a ticket that has been modified in the future
# (of course, it shouldn't happen... but who know). If it's the case, don't report it as an error
if time_changed < int(time.time()):
# TODO: this is not working yet...
#
#raise TracError("Sorry, can not execute the import. "
#"The ticket #" + str(ticket_id) + " has been modified by someone else "
#"since preview. You must re-upload and preview your file to avoid overwriting the other changes.")
pass
else:
self.ticket = PatchedTicket(self.env, db=self.db)
self.comment = ''
def start_process_row(self, row_idx, ticket_id):
from ticket import PatchedTicket
if ticket_id > 0:
# existing ticket
self.ticket = PatchedTicket(self.env, tkt_id=ticket_id, db=self.db)
# 'Ticket.time_changed' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
try:
from trac.util.datefmt import to_timestamp
time_changed = to_timestamp(self.ticket.time_changed)
except ImportError:
time_changed = int(self.ticket.time_changed)
if time_changed > self.tickettime:
# just in case, verify if it wouldn't be a ticket that has been modified in the future
# (of course, it shouldn't happen... but who know). If it's the case, don't report it as an error
if time_changed < int(time.time()):
# TODO: this is not working yet...
#
#raise TracError("Sorry, can not execute the import. "
#"The ticket #" + str(ticket_id) + " has been modified by someone else "
#"since preview. You must re-upload and preview your file to avoid overwriting the other changes.")
pass
else:
self.ticket = PatchedTicket(self.env, db=self.db)
self.comment = ''
def start_process_row(self, row_idx, ticket_id):
from ticket import PatchedTicket
self.ticket = None
self.row_idx = row_idx
self.temphdf = []
if ticket_id > 0:
# existing ticket. Load the ticket, to see which fields will be modified
self.ticket = PatchedTicket(self.env, ticket_id)
def process_relativeticket_fields(self, rows, relativeticketfields):
from ticket import PatchedTicket
# Find the WBS columns, if any. We never expect to have more
# than one but this is flexible and easy. There's no good
# reason to go to the trouble of ignoring extras.
wbsfields = []
for row in rows:
for f in relativeticketfields:
if row[f].find('.') != -1:
if f not in wbsfields:
wbsfields.append(f)
# If WBS column present, build reverse lookup to find the
# ticket ID from a WBS number.
wbsref = {}
if wbsfields != []:
row_idx = 0
for row in rows:
wbsref[row[wbsfields[0]]] = self.crossref[row_idx]
row_idx += 1
row_idx = 0
for row in rows:
id = self.crossref[row_idx]
# Get the ticket (added or updated in the main loop
ticket = PatchedTicket(self.env, tkt_id=id, db=self.db)
for f in relativeticketfields:
# Get the value of the relative field column (e.g., "2,3")
v = row[f]
# If it's not empty, process the contents
if len(v) > 0:
# Handle WBS numbers
if f in wbsfields:
if row[f].find('.') == -1:
# Top level, no parent
ticket[f] = ''
else:
# Get this task's wbs
wbs = row[f]
# Remove the last dot-delimited field
pwbs = wbs[:wbs.rindex(".")]
# Look up the parent's ticket ID
ticket[f] = str(wbsref[pwbs])
# Handle dependencies
else:
s = []
for r in v.split(","):
# Make the string value an integer
r = int(r)
# The relative ticket dependencies are 1-based,
# array indices are 0-based. Convert and look up
# the new ticket ID of the other ticket.
i = self.crossref[r - 1]
# TODO check that i != id
s.append(str(i))
# Empty or not, use it to update the ticket
ticket[f] = ', '.join(s)
self._save_ticket(ticket, with_comment=False)
row_idx += 1
class ImportProcessor(object):
def __init__(self, env, req, filename, tickettime):
self.env = env
self.req = req
self.filename = filename
self.modified = {}
self.added = {}
# TODO: check that the tickets haven't changed since preview
self.tickettime = tickettime
# Keep the db to commit it all at once at the end
self.db = self.env.get_db_cnx()
self.missingemptyfields = None
self.missingdefaultedfields = None
self.computedfields = None
self.importedfields = None
def start(self, importedfields, reconciliate_by_owner_also, has_comments):
# Index by row index, returns ticket id
self.crossref = []
self.lowercaseimportedfields = [f.lower() for f in importedfields]
def process_missing_fields(self, missingfields, missingemptyfields,
missingdefaultedfields, computedfields):
self.missingemptyfields = missingemptyfields
self.missingdefaultedfields = missingdefaultedfields
self.computedfields = computedfields
def process_notimported_fields(self, notimportedfields):
pass
def process_comment_field(self, comment):
pass
def start_process_row(self, row_idx, ticket_id):
from ticket import PatchedTicket
if ticket_id > 0:
# existing ticket
self.ticket = PatchedTicket(self.env, tkt_id=ticket_id, db=self.db)
# 'Ticket.time_changed' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
try:
from trac.util.datefmt import to_timestamp
time_changed = to_timestamp(self.ticket.time_changed)
except ImportError:
time_changed = int(self.ticket.time_changed)
if time_changed > self.tickettime:
# just in case, verify if it wouldn't be a ticket that has been modified in the future
# (of course, it shouldn't happen... but who know). If it's the case, don't report it as an error
if time_changed < int(time.time()):
# TODO: this is not working yet...
#
#raise TracError("Sorry, can not execute the import. "
#"The ticket #" + str(ticket_id) + " has been modified by someone else "
#"since preview. You must re-upload and preview your file to avoid overwriting the other changes.")
pass
else:
self.ticket = PatchedTicket(self.env, db=self.db)
self.comment = ''
def process_cell(self, column, cell):
cell = unicode(cell)
column = column.lower()
# if status of new ticket is empty, force to use 'new'
if not self.ticket.exists and column == 'status' and not cell:
cell = 'new'
# this will ensure that the changes are logged, see model.py Ticket.__setitem__
self.ticket[column] = cell
def process_comment(self, comment):
self.comment = comment
def _tickettime(self):
try:
# 'when' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
from trac.util.datefmt import to_datetime
return to_datetime(self.tickettime)
except ImportError:
return self.tickettime
def _save_ticket(self, ticket, with_comment=True):
if with_comment:
if self.comment:
comment = "''Batch update from file " + self.filename + ":'' " + self.comment
else:
comment = "''Batch update from file " + self.filename + "''"
else:
comment = None
ticket.save_changes(get_reporter_id(self.req),
comment,
when=self._tickettime(),
db=self.db)
def end_process_row(self):
if self.ticket.id == None:
if self.missingemptyfields:
for f in self.missingemptyfields:
if f in self.ticket.values and self.ticket[f] is None:
self.ticket[f] = ''
if self.comment:
self.ticket['description'] = self.ticket[
'description'] + "\n[[BR]][[BR]]\n''Batch insert from file " + self.filename + ":''\n" + self.comment
if self.computedfields:
for f in self.computedfields:
if f not in self.lowercaseimportedfields and \
self.computedfields[f] is not None and \
self.computedfields[f]['set']:
self.ticket[f] = self.computedfields[f]['value']
self.ticket.insert(when=self._tickettime(), db=self.db)
self.added[self.ticket.id] = 1
else:
if self.ticket.is_modified() or self.comment:
self._save_ticket(self.ticket)
self.modified[self.ticket.id] = 1
self.crossref.append(self.ticket.id)
self.ticket = None
def process_new_lookups(self, newvalues):
for field, names in newvalues.iteritems():
if field == 'status':
continue
LOOKUPS = {
'component': model.Component,
'milestone': model.Milestone,
'version': model.Version,
'type': model.Type,
}
try:
CurrentLookupEnum = LOOKUPS[field]
except KeyError:
class CurrentLookupEnum(model.AbstractEnum):
# here, you shouldn't put 'self.' before the class field.
type = field
for name in names:
lookup = CurrentLookupEnum(self.env, db=self.db)
lookup.name = name
lookup.insert()
def process_new_users(self, newusers):
pass
# Rows is an array of dictionaries.
# Each row is indexed by the field names in relativeticketfields.
def process_relativeticket_fields(self, rows, relativeticketfields):
from ticket import PatchedTicket
# Find the WBS columns, if any. We never expect to have more
# than one but this is flexible and easy. There's no good
# reason to go to the trouble of ignoring extras.
wbsfields = []
for row in rows:
for f in relativeticketfields:
if row[f].find('.') != -1:
if f not in wbsfields:
wbsfields.append(f)
# If WBS column present, build reverse lookup to find the
# ticket ID from a WBS number.
wbsref = {}
if wbsfields != []:
row_idx = 0
for row in rows:
wbsref[row[wbsfields[0]]] = self.crossref[row_idx]
row_idx += 1
row_idx = 0
for row in rows:
id = self.crossref[row_idx]
# Get the ticket (added or updated in the main loop
ticket = PatchedTicket(self.env, tkt_id=id, db=self.db)
for f in relativeticketfields:
# Get the value of the relative field column (e.g., "2,3")
v = row[f]
# If it's not empty, process the contents
if len(v) > 0:
# Handle WBS numbers
if f in wbsfields:
if row[f].find('.') == -1:
# Top level, no parent
ticket[f] = ''
else:
# Get this task's wbs
wbs = row[f]
# Remove the last dot-delimited field
pwbs = wbs[:wbs.rindex(".")]
# Look up the parent's ticket ID
ticket[f] = str(wbsref[pwbs])
# Handle dependencies
else:
s = []
for r in v.split(","):
# Make the string value an integer
r = int(r)
# The relative ticket dependencies are 1-based,
# array indices are 0-based. Convert and look up
# the new ticket ID of the other ticket.
i = self.crossref[r - 1]
# TODO check that i != id
s.append(str(i))
# Empty or not, use it to update the ticket
ticket[f] = ', '.join(s)
self._save_ticket(ticket, with_comment=False)
row_idx += 1
def end_process(self, numrows):
self.db.commit()
data = {}
data['title'] = 'Import completed'
#data['report.title'] = data['title'].lower()
notmodifiedcount = numrows - len(self.added) - len(self.modified)
message = 'Successfully imported ' + str(numrows) + ' tickets (' + str(
len(self.added)) + ' added, ' + str(len(
self.modified)) + ' modified, ' + str(
notmodifiedcount) + ' unchanged).'
data['message'] = Markup(
"<style type=\"text/css\">#report-notfound { display:none; }</style>\n"
) + wiki_to_html(message, self.env, self.req)
return 'import_preview.html', data, None
class ImportProcessor(object):
def __init__(self, env, req, filename, tickettime):
self.env = env
self.req = req
self.filename = filename
self.modifiedcount = 0
self.notmodifiedcount = 0
self.added = 0
self.parent_tid = 0
# TODO: check that the tickets haven't changed since preview
self.tickettime = tickettime
# Keep the db to commit it all at once at the end
self.db = self.env.get_db_cnx()
self.missingemptyfields = None
self.missingdefaultedfields = None
self.computedfields = None
self.importedfields = None
def start(self, importedfields, reconciliate_by_owner_also, has_comments):
self.lowercaseimportedfields = [f.lower() for f in importedfields]
def process_missing_fields(self, missingfields, missingemptyfields, missingdefaultedfields, computedfields):
self.missingemptyfields = missingemptyfields
self.missingdefaultedfields = missingdefaultedfields
self.computedfields = computedfields
def process_notimported_fields(self, notimportedfields):
pass
def process_comment_field(self, comment):
pass
def start_process_row(self, row_idx, ticket_id):
from ticket import PatchedTicket
if ticket_id > 0:
# existing ticket
self.ticket = PatchedTicket(self.env, tkt_id=ticket_id, db=self.db)
# 'Ticket.time_changed' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
try:
from trac.util.datefmt import to_timestamp
time_changed = to_timestamp(self.ticket.time_changed)
except ImportError:
time_changed = int(self.ticket.time_changed)
if time_changed > self.tickettime:
# just in case, verify if it wouldn't be a ticket that has been modified in the future
# (of course, it shouldn't happen... but who know). If it's the case, don't report it as an error
if time_changed < int(time.time()):
# TODO: this is not working yet...
#
#raise TracError("Sorry, can not execute the import. "
#"The ticket #" + str(ticket_id) + " has been modified by someone else "
#"since preview. You must re-upload and preview your file to avoid overwriting the other changes.")
pass
else:
self.ticket = PatchedTicket(self.env, db=self.db)
self.comment = ''
def process_cell(self, column, cell):
cell = unicode(cell)
# this will ensure that the changes are logged, see model.py Ticket.__setitem__
self.ticket[column.lower()] = cell
def process_comment(self, comment):
self.comment = comment
def end_process_row(self, indent):
try:
# 'when' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
from trac.util.datefmt import to_datetime
tickettime = to_datetime(self.tickettime)
except ImportError:
tickettime = self.tickettime
if self.ticket.id == None:
for f in self.missingemptyfields:
if self.ticket.values.has_key(f) and self.ticket[f] == None:
self.ticket[f] = ''
if self.comment:
self.ticket['description'] = self.ticket['description'] + "\n[[BR]][[BR]]\n''Batch insert from file " + self.filename + ":''\n" + self.comment
for f in self.computedfields:
if f not in self.lowercaseimportedfields and self.computedfields[f] != None and self.computedfields[f]['set']:
self.ticket[f] = self.computedfields[f]['value']
if (indent!=0) and (self.parent_tid!=0) and ('parents' in self.env.config['ticket-custom']):
self.ticket['parents'] = str(self.parent_tid)
self.added += 1
self.ticket.insert(when=tickettime, db=self.db)
if indent==0:
self.parent_tid = self.ticket.id
else:
if self.comment:
message = "''Batch update from file " + self.filename + ":'' " + self.comment
else:
message = "''Batch update from file " + self.filename + "''"
if self.ticket.is_modified() or self.comment:
self.modifiedcount += 1
self.ticket.save_changes(get_reporter_id(self.req), message, when=tickettime, db=self.db) # TODO: handle cnum, cnum = ticket.values['cnum'] + 1)
else:
self.notmodifiedcount += 1
self.ticket = None
def process_new_lookups(self, newvalues):
for field, names in newvalues.iteritems():
if field == 'status':
continue
if field == 'component':
class CurrentLookupEnum(model.Component):
pass
elif field == 'milestone':
class CurrentLookupEnum(model.Milestone):
pass
elif field == 'version':
class CurrentLookupEnum(model.Version):
pass
elif field == 'type':
class CurrentLookupEnum(model.Type):
pass
else:
class CurrentLookupEnum(model.AbstractEnum):
# here, you shouldn't put 'self.' before the class field.
type = field
for name in names:
lookup = CurrentLookupEnum(self.env, db=self.db)
lookup.name = name
lookup.insert()
def process_new_users(self, newusers):
pass
def end_process(self, numrows):
self.db.commit()
data = {}
data['title'] = 'Import completed'
#data['report.title'] = data['title'].lower()
message = 'インポートに成功しました。 ' + str(numrows) + ' tickets (' + str(self.added) + ' 追加, ' + str(self.modifiedcount) + ' 更新, ' + str(self.notmodifiedcount) + ' 未更新).'
data['message'] = Markup("<style type=\"text/css\">#report-notfound { display:none; }</style>\n") + wiki_to_html(message, self.env, self.req)
return 'import_preview.html', data, None
def _process(self, filereader, reporter, processor):
tracfields = [
field['name']
for field in TicketSystem(self.env).get_ticket_fields()
]
tracfields = ['ticket', 'id'] + tracfields
customfields = [
field['name']
for field in TicketSystem(self.env).get_custom_fields()
]
columns, rows = filereader.readers()
# defensive: columns could be non-string, make sure they are
columns = map(str, columns)
importedfields = [f for f in columns if f.lower() in tracfields]
notimportedfields = [
f for f in columns if f.lower() not in tracfields
and f.lower() != 'ticket' and f.lower() != 'id'
]
lowercaseimportedfields = [f.lower() for f in importedfields]
idcolumn = None
if 'ticket' in lowercaseimportedfields and 'id' in lowercaseimportedfields:
raise TracError, 'The first line of the worksheet contains both \'ticket\', and an \'id\' field name. Only one of them is needed to perform the import. Please check the file and try again.'
ownercolumn = None
if 'ticket' in lowercaseimportedfields:
idcolumn = self._find_case_insensitive('ticket', importedfields)
elif 'id' in lowercaseimportedfields:
idcolumn = self._find_case_insensitive('id', importedfields)
elif 'summary' in lowercaseimportedfields:
summarycolumn = self._find_case_insensitive(
'summary', importedfields)
ownercolumn = self._reconciliate_by_owner_also(
) and self._find_case_insensitive('owner', importedfields) or None
else:
raise TracError, 'The first line of the worksheet contains neither a \'ticket\', an \'id\' nor a \'summary\' field name. At least one of them is needed to perform the import. Please check the file and try again.'
# start TODO: this is too complex, it should be replaced by a call to TicketSystem(env).get_ticket_fields()
# The fields that we will have to set a value for, if:
# - they are not in the imported fields, and
# - they are not set in the default values of the Ticket class, and
# - they shouldn't be set to empty
# if 'set' is true, this will be the value that will be set by default (even if the default value in the Ticket class is different)
# if 'set' is false, the value is computed by Trac and we don't have anything to do
computedfields = {
'status': {
'value': 'new',
'set': True
},
'resolution': {
'value': "''(None)''",
'set': False
},
'reporter': {
'value': reporter,
'set': True
},
'time': {
'value': "''(now)''",
'set': False
},
'changetime': {
'value': "''(now)''",
'set': False
}
}
if 'owner' not in lowercaseimportedfields and 'component' in lowercaseimportedfields:
computedfields['owner'] = {}
computedfields['owner']['value'] = 'Computed from component'
computedfields['owner']['set'] = False
# to get the compulted default values
from ticket import PatchedTicket
ticket = PatchedTicket(self.env)
for f in [
'type', 'cc', 'url', 'description', 'keywords', 'component',
'severity', 'priority', 'version', 'milestone'
] + customfields:
if f in ticket.values:
computedfields[f] = {}
computedfields[f]['value'] = ticket.values[f]
computedfields[f]['set'] = False
else:
computedfields[f] = None
processor.start(importedfields, ownercolumn != None)
missingfields = [
f for f in computedfields if f not in lowercaseimportedfields
]
missingemptyfields = [
f for f in missingfields
if computedfields[f] == None or computedfields[f]['value'] == ''
]
missingdefaultedfields = [
f for f in missingfields if f not in missingemptyfields
]
if missingfields != []:
processor.process_missing_fields(missingfields, missingemptyfields,
missingdefaultedfields,
computedfields)
# end TODO: this is too complex
if notimportedfields != []:
processor.process_notimported_fields(notimportedfields)
# TODO: test the cases where those fields have empty values. They should be handled as None. (just to test, may be working already :)
selects = [
#Those ones inherit from AbstractEnum
('type', model.Type),
('status', model.Status),
('priority', model.Priority),
('severity', model.Severity),
('resolution', model.Resolution),
#Those don't
('milestone', model.Milestone),
('component', model.Component),
('version', model.Version)
]
existingvalues = {}
newvalues = {}
for name, cls in selects:
if name not in lowercaseimportedfields:
# this field is not present, nothing to do
continue
options = [val.name for val in cls.select(self.env)]
if not options:
# Fields without possible values are treated as if they didn't
# exist
continue
existingvalues[name] = options
newvalues[name] = []
def add_sql_result(db, sql, list):
cursor = db.cursor()
cursor.execute(sql)
for result in cursor:
list += [result]
existingusers = []
db = self.env.get_db_cnx()
add_sql_result(db, "SELECT DISTINCT reporter FROM ticket",
existingusers)
add_sql_result(db, "SELECT DISTINCT owner FROM ticket", existingusers)
add_sql_result(db, "SELECT DISTINCT owner FROM component",
existingusers)
newusers = []
duplicate_summaries = []
row_idx = 0
for row in rows:
if idcolumn:
ticket_id = row[idcolumn]
if ticket_id:
self._check_ticket(db, ticket_id)
else:
# will create a new ticket
ticket_id = 0
else:
summary = row[summarycolumn]
owner = ownercolumn and row[ownercolumn] or None
if self._skip_lines_with_empty_owner(
) and ownercolumn and not owner:
continue
ticket_id = self._find_ticket(db, summary, owner)
if (summary, owner) in duplicate_summaries:
if owner == None:
raise TracError, 'Summary "%s" is duplicated in the spreadsheet. Ticket reconciliation by summary can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % summary
else:
raise TracError, 'Summary "%s" and owner "%s" are duplicated in the spreadsheet. Ticket reconciliation can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % (
summary, owner)
else:
duplicate_summaries += [(summary, owner)]
processor.start_process_row(row_idx, ticket_id)
for column in importedfields:
cell = row[column]
# collect the new lookup values
if column.lower() in existingvalues.keys():
if cell != '' and cell not in existingvalues[column.lower(
)] and cell not in newvalues[column.lower()]:
newvalues[column.lower()] += [cell]
# also collect the new user names
if (column.lower() == 'owner' or column.lower() == 'reporter'):
if cell != '' and cell not in newusers and cell not in existingusers:
newusers += [cell]
# and proces the value.
if column.lower() != 'ticket' and column.lower() != 'id':
processor.process_cell(column, cell)
processor.end_process_row()
row_idx += 1
if newvalues != {} and reduce(lambda x, y: x == [] and y or x,
newvalues.values()) != []:
processor.process_new_lookups(newvalues)
if newusers != []:
processor.process_new_users(newusers)
return processor.end_process(row_idx)
class ImportProcessor(object):
def __init__(self, env, req, filename, tickettime):
self.env = env
self.req = req
self.filename = filename
self.modified = {}
self.added = {}
# TODO: check that the tickets haven't changed since preview
self.tickettime = tickettime
# Keep the db to commit it all at once at the end
self.db = self.env.get_db_cnx()
self.missingemptyfields = None
self.missingdefaultedfields = None
self.computedfields = None
self.importedfields = None
def start(self, importedfields, reconciliate_by_owner_also, has_comments):
# Index by row index, returns ticket id
self.crossref = []
self.lowercaseimportedfields = [f.lower() for f in importedfields]
def process_missing_fields(self, missingfields, missingemptyfields, missingdefaultedfields, computedfields):
self.missingemptyfields = missingemptyfields
self.missingdefaultedfields = missingdefaultedfields
self.computedfields = computedfields
def process_notimported_fields(self, notimportedfields):
pass
def process_comment_field(self, comment):
pass
def start_process_row(self, row_idx, ticket_id):
from ticket import PatchedTicket
if ticket_id > 0:
# existing ticket
self.ticket = PatchedTicket(self.env, tkt_id=ticket_id, db=self.db)
# 'Ticket.time_changed' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
try:
from trac.util.datefmt import to_timestamp
time_changed = to_timestamp(self.ticket.time_changed)
except ImportError:
time_changed = int(self.ticket.time_changed)
if time_changed > self.tickettime:
# just in case, verify if it wouldn't be a ticket that has been modified in the future
# (of course, it shouldn't happen... but who know). If it's the case, don't report it as an error
if time_changed < int(time.time()):
# TODO: this is not working yet...
#
#raise TracError("Sorry, can not execute the import. "
#"The ticket #" + str(ticket_id) + " has been modified by someone else "
#"since preview. You must re-upload and preview your file to avoid overwriting the other changes.")
pass
else:
self.ticket = PatchedTicket(self.env, db=self.db)
self.comment = ''
def process_cell(self, column, cell):
cell = unicode(cell)
column = column.lower()
# if status of new ticket is empty, force to use 'new'
if not self.ticket.exists and column == 'status' and not cell:
cell = 'new'
# this will ensure that the changes are logged, see model.py Ticket.__setitem__
self.ticket[column] = cell
def process_comment(self, comment):
self.comment = comment
def _tickettime(self):
try:
# 'when' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
from trac.util.datefmt import to_datetime
return to_datetime(self.tickettime)
except ImportError:
return self.tickettime
def _save_ticket(self, ticket, with_comment=True):
if with_comment:
if self.comment:
comment = "''Batch update from file " + self.filename + ":'' " + self.comment
else:
comment = "''Batch update from file " + self.filename + "''"
else:
comment=None
ticket.save_changes(get_reporter_id(self.req),
comment,
when=self._tickettime(),
db=self.db)
def end_process_row(self):
if self.ticket.id == None:
if self.missingemptyfields:
for f in self.missingemptyfields:
if f in self.ticket.values and self.ticket[f] is None:
self.ticket[f] = ''
if self.comment:
self.ticket['description'] = self.ticket['description'] + "\n[[BR]][[BR]]\n''Batch insert from file " + self.filename + ":''\n" + self.comment
if self.computedfields:
for f in self.computedfields:
if f not in self.lowercaseimportedfields and \
self.computedfields[f] is not None and \
self.computedfields[f]['set']:
self.ticket[f] = self.computedfields[f]['value']
self.ticket.insert(when=self._tickettime(), db=self.db)
self.added[self.ticket.id] = 1
else:
if self.ticket.is_modified() or self.comment:
self._save_ticket(self.ticket)
self.modified[self.ticket.id] = 1
self.crossref.append(self.ticket.id)
self.ticket = None
def process_new_lookups(self, newvalues):
for field, names in newvalues.iteritems():
if field == 'status':
continue
LOOKUPS = { 'component': model.Component,
'milestone': model.Milestone,
'version': model.Version,
'type': model.Type,
}
try:
CurrentLookupEnum = LOOKUPS[field]
except KeyError:
class CurrentLookupEnum(model.AbstractEnum):
# here, you shouldn't put 'self.' before the class field.
type = field
for name in names:
lookup = CurrentLookupEnum(self.env, db=self.db)
lookup.name = name
lookup.insert()
def process_new_users(self, newusers):
pass
# Rows is an array of dictionaries.
# Each row is indexed by the field names in relativeticketfields.
def process_relativeticket_fields(self, rows, relativeticketfields):
from ticket import PatchedTicket
# Find the WBS columns, if any. We never expect to have more
# than one but this is flexible and easy. There's no good
# reason to go to the trouble of ignoring extras.
wbsfields=[]
for row in rows:
for f in relativeticketfields:
if row[f].find('.') != -1:
if f not in wbsfields:
wbsfields.append(f)
# If WBS column present, build reverse lookup to find the
# ticket ID from a WBS number.
wbsref = {}
if wbsfields != []:
row_idx = 0
for row in rows:
wbsref[row[wbsfields[0]]] = self.crossref[row_idx]
row_idx += 1
row_idx = 0
for row in rows:
id = self.crossref[row_idx]
# Get the ticket (added or updated in the main loop
ticket = PatchedTicket(self.env, tkt_id=id, db=self.db)
for f in relativeticketfields:
# Get the value of the relative field column (e.g., "2,3")
v = row[f]
# If it's not empty, process the contents
if len(v) > 0:
# Handle WBS numbers
if f in wbsfields:
if row[f].find('.') == -1:
# Top level, no parent
ticket[f] = ''
else:
# Get this task's wbs
wbs = row[f]
# Remove the last dot-delimited field
pwbs = wbs[:wbs.rindex(".")]
# Look up the parent's ticket ID
ticket[f] = str(wbsref[pwbs])
# Handle dependencies
else:
s = []
for r in v.split(","):
# Make the string value an integer
r = int(r)
# The relative ticket dependencies are 1-based,
# array indices are 0-based. Convert and look up
# the new ticket ID of the other ticket.
i = self.crossref[r-1]
# TODO check that i != id
s.append(str(i))
# Empty or not, use it to update the ticket
ticket[f] = ', '.join(s)
self._save_ticket(ticket, with_comment=False)
row_idx += 1
def end_process(self, numrows):
self.db.commit()
data = {}
data['title'] = 'Import completed'
#data['report.title'] = data['title'].lower()
notmodifiedcount = numrows - len(self.added) - len(self.modified)
message = 'Successfully imported ' + str(numrows) + ' tickets (' + str(len(self.added)) + ' added, ' + str(len(self.modified)) + ' modified, ' + str(notmodifiedcount) + ' unchanged).'
data['message'] = Markup("<style type=\"text/css\">#report-notfound { display:none; }</style>\n") + wiki_to_html(message, self.env, self.req)
return 'import_preview.html', data, None
def _process(self, filereader, reporter, processor):
tracfields = [field['name'] for field in TicketSystem(self.env).get_ticket_fields()]
tracfields = [ 'ticket', 'id' ] + tracfields
customfields = [field['name'] for field in TicketSystem(self.env).get_custom_fields()]
columns, rows = filereader.readers()
importedfields = [f for f in columns if f.lower() in tracfields]
notimportedfields = [f for f in columns if f and (f.lower() not in tracfields + ['comment']
# relative fields will be added later
and f[0] != '#')]
commentfields = [f for f in columns if f.lower() == 'comment']
if commentfields:
commentfield = commentfields[0]
else:
commentfield = None
lowercaseimportedfields = [f.lower() for f in importedfields]
# Fields which contain relative ticket numbers to update after import
relativeticketfields = []
lowercaserelativeticketfields = []
for f in columns:
if not f.startswith('#'):
continue
if f[1:].lower() in tracfields:
relativeticketfields.append(f)
lowercaserelativeticketfields.append(f[1:].lower())
else:
notimportedfields.append(f)
idcolumn = None
if 'ticket' in lowercaseimportedfields and 'id' in lowercaseimportedfields:
raise TracError, 'The first line of the worksheet contains both \'ticket\', and an \'id\' field name. Only one of them is needed to perform the import. Please check the file and try again.'
ownercolumn = None
if 'ticket' in lowercaseimportedfields:
idcolumn = self._find_case_insensitive('ticket', importedfields)
elif 'id' in lowercaseimportedfields:
idcolumn = self._find_case_insensitive('id', importedfields)
elif 'summary' in lowercaseimportedfields:
summarycolumn = self._find_case_insensitive('summary', importedfields)
ownercolumn = self._reconciliate_by_owner_also() and self._find_case_insensitive('owner', importedfields) or None
else:
raise TracError, 'The first line of the worksheet contains neither a \'ticket\', an \'id\' nor a \'summary\' field name. At least one of them is needed to perform the import. Please check the file and try again.'
# start TODO: this is too complex, it should be replaced by a call to TicketSystem(env).get_ticket_fields()
# The fields that we will have to set a value for, if:
# - they are not in the imported fields, and
# - they are not set in the default values of the Ticket class, and
# - they shouldn't be set to empty
# if 'set' is true, this will be the value that will be set by default (even if the default value in the Ticket class is different)
# if 'set' is false, the value is computed by Trac and we don't have anything to do
computedfields = {'status': { 'value':'new', 'set': True },
'resolution' : { 'value': "''(None)''", 'set': False },
'reporter' : { 'value': reporter, 'set': True },
'time' : { 'value': "''(now)''", 'set': False },
'changetime' : { 'value': "''(now)''", 'set': False } }
if 'owner' not in lowercaseimportedfields and 'component' in lowercaseimportedfields:
computedfields['owner'] = {}
computedfields['owner']['value'] = 'Computed from component'
computedfields['owner']['set'] = False
# to get the computed default values
from ticket import PatchedTicket
ticket = PatchedTicket(self.env)
for f in [ 'type', 'cc' , 'description', 'keywords', 'component' , 'severity' , 'priority' , 'version', 'milestone' ] + customfields:
if f in ticket.values:
computedfields[f] = {}
computedfields[f]['value'] = ticket.values[f]
computedfields[f]['set'] = False
else:
computedfields[f] = None
processor.start(importedfields, ownercolumn != None, commentfield)
missingfields = [f for f in computedfields if f not in lowercaseimportedfields]
if relativeticketfields:
missingfields = [f for f in missingfields if f not in lowercaserelativeticketfields]
missingemptyfields = [ f for f in missingfields if computedfields[f] == None or computedfields[f]['value'] == '']
missingdefaultedfields = [ f for f in missingfields if f not in missingemptyfields]
if missingfields != []:
processor.process_missing_fields(missingfields, missingemptyfields, missingdefaultedfields, computedfields)
# end TODO: this is too complex
if notimportedfields != []:
processor.process_notimported_fields(notimportedfields)
if commentfield:
processor.process_comment_field(commentfield)
# TODO: test the cases where those fields have empty values. They should be handled as None. (just to test, may be working already :)
selects = [
#Those ones inherit from AbstractEnum
('type', model.Type),
('status', model.Status),
('priority', model.Priority),
('severity', model.Severity),
('resolution', model.Resolution),
#Those don't
('milestone', model.Milestone),
('component', model.Component),
('version', model.Version)
]
existingvalues = {}
newvalues = {}
for name, cls in selects:
if name not in lowercaseimportedfields:
# this field is not present, nothing to do
continue
options = [val.name for val in cls.select(self.env)]
if not options:
# Fields without possible values are treated as if they didn't
# exist
continue
existingvalues[name] = options
newvalues[name] = []
def add_sql_result(db, aset, queries):
cursor = db.cursor()
for query in queries:
cursor.execute(query)
aset.update([val for val, in cursor])
existingusers = set()
db = self.env.get_db_cnx()
add_sql_result(
db, existingusers,
[("SELECT DISTINCT reporter FROM ticket"
" WHERE reporter IS NOT NULL AND reporter != ''"),
("SELECT DISTINCT owner FROM ticket"
" WHERE owner IS NOT NULL AND owner != ''"),
("SELECT DISTINCT owner FROM component"
" WHERE owner IS NOT NULL AND owner != ''")])
for username, name, email in self.env.get_known_users(db):
existingusers.add(username)
newusers = []
duplicate_summaries = []
relativeticketvalues = []
row_idx = 0
for row in rows:
if idcolumn:
ticket_id = row[idcolumn].strip()
if ticket_id:
ticket_id = ticket_id.lstrip('#')
if ticket_id:
self._check_ticket(db, ticket_id)
else:
# will create a new ticket
ticket_id = 0
else:
summary = row[summarycolumn]
owner = ownercolumn and row[ownercolumn] or None
if self._skip_lines_with_empty_owner() and ownercolumn and not owner:
continue
ticket_id = self._find_ticket(db, summary, owner)
if (summary, owner) in duplicate_summaries:
if owner == None:
raise TracError, 'Summary "%s" is duplicated in the spreadsheet. Ticket reconciliation by summary can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % summary
else:
raise TracError, 'Summary "%s" and owner "%s" are duplicated in the spreadsheet. Ticket reconciliation can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % (summary, owner)
else:
duplicate_summaries += [ (summary, owner) ]
processor.start_process_row(row_idx, ticket_id)
for column in importedfields:
cell = row[column]
if cell is None:
cell = ''
column_lower = column.lower()
# collect the new lookup values
if column_lower in existingvalues:
if isinstance(cell, basestring):
cell = cell.strip()
if cell != '' and \
cell not in existingvalues[column_lower] and \
cell not in newvalues[column_lower]:
newvalues[column_lower].append(cell)
# also collect the new user names
if column_lower in ('owner', 'reporter'):
if cell != '' and \
cell not in newusers and \
cell not in existingusers:
newusers.append(cell)
# and proces the value.
if column_lower not in ('ticket', 'id'):
processor.process_cell(column, cell)
if commentfield:
processor.process_comment(row[commentfield])
relativeticketvalues.append(dict([(f[1:].lower(), row[f])
for f in relativeticketfields]))
processor.end_process_row()
row_idx += 1
# All the rows have been processed. Handle global stuff
for name in list(newvalues):
if not newvalues[name]:
del newvalues[name]
if newvalues:
processor.process_new_lookups(newvalues)
if newusers:
processor.process_new_users(newusers)
if relativeticketfields:
processor.process_relativeticket_fields(relativeticketvalues, lowercaserelativeticketfields)
return processor.end_process(row_idx)
class ImportProcessor(object):
def __init__(self, env, req, filename, tickettime):
self.env = env
self.req = req
self.filename = filename
self.modifiedcount = 0
self.notmodifiedcount = 0
self.added = 0
# TODO: check that the tickets haven't changed since preview
self.tickettime = tickettime
# Keep the db to commit it all at once at the end
self.db = self.env.get_db_cnx()
self.missingemptyfields = None
self.missingdefaultedfields = None
self.computedfields = None
def start(self, importedfields, reconciliate_by_owner_also):
pass
def process_missing_fields(self, missingfields, missingemptyfields,
missingdefaultedfields, computedfields):
self.missingemptyfields = missingemptyfields
self.missingdefaultedfields = missingdefaultedfields
self.computedfields = computedfields
def process_notimported_fields(self, notimportedfields):
pass
def start_process_row(self, row_idx, ticket_id):
from ticket import PatchedTicket
if ticket_id > 0:
# existing ticket
self.ticket = PatchedTicket(self.env, tkt_id=ticket_id, db=self.db)
# 'Ticket.time_changed' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
try:
from trac.util.datefmt import to_timestamp
time_changed = to_timestamp(self.ticket.time_changed)
except ImportError:
time_changed = int(self.ticket.time_changed)
if time_changed > self.tickettime:
# just in case, verify if it wouldn't be a ticket that has been modified in the future
# (of course, it shouldn't happen... but who know). If it's the case, don't report it as an error
if time_changed < int(time.time()):
# TODO: this is not working yet...
#
#raise TracError("Sorry, can not execute the import. "
#"The ticket #" + str(ticket_id) + " has been modified by someone else "
#"since preview. You must re-upload and preview your file to avoid overwriting the other changes.")
pass
else:
self.ticket = PatchedTicket(self.env, db=self.db)
def process_cell(self, column, cell):
cell = unicode(cell)
# this will ensure that the changes are logged, see model.py Ticket.__setitem__
self.ticket[column.lower()] = cell
def end_process_row(self):
try:
# 'when' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
from trac.util.datefmt import to_datetime
tickettime = to_datetime(self.tickettime)
except ImportError:
tickettime = self.tickettime
if self.ticket.id == None:
for f in self.missingemptyfields:
if self.ticket.values.has_key(f) and self.ticket[f] == None:
self.ticket[f] = ''
for f in self.computedfields:
if self.computedfields[f] != None and self.computedfields[f][
'set']:
self.ticket[f] = self.computedfields[f]['value']
self.added += 1
self.ticket.insert(when=tickettime, db=self.db)
else:
message = "Batch update from file " + self.filename
if self.ticket.is_modified():
self.modifiedcount += 1
self.ticket.save_changes(
get_reporter_id(self.req),
message,
when=tickettime,
db=self.db
) # TODO: handle cnum, cnum = ticket.values['cnum'] + 1)
else:
self.notmodifiedcount += 1
self.ticket = None
def process_new_lookups(self, newvalues):
for field, names in newvalues.iteritems():
if names == []:
continue
if field == 'component':
class CurrentLookupEnum(model.Component):
pass
elif field == 'milestone':
class CurrentLookupEnum(model.Milestone):
pass
elif field == 'version':
class CurrentLookupEnum(model.Version):
pass
else:
class CurrentLookupEnum(model.AbstractEnum):
# here, you shouldn't put 'self.' before the class field.
type = field
for name in names:
lookup = CurrentLookupEnum(self.env, db=self.db)
lookup.name = name
lookup.insert()
def process_new_users(self, newusers):
pass
def end_process(self, numrows):
self.db.commit()
self.req.hdf['title'] = 'Import completed'
self.req.hdf['report.title'] = self.req.hdf['title'].lower()
message = 'Successfully imported ' + str(numrows) + ' tickets (' + str(
self.added) + ' added, ' + str(
self.modifiedcount) + ' modified, ' + str(
self.notmodifiedcount) + ' unchanged).'
self.req.hdf['report.description'] = Markup(
"<style type=\"text/css\">#report-notfound { display:none; }</style>\n"
) + wiki_to_html(message, self.env, self.req)
self.req.hdf['report.numrows'] = 0
self.req.hdf['report.mode'] = 'list'
return 'report.cs', None