def start_process_row(self, row_idx, ticket_id):
from ticket import PatchedTicket
if ticket_id > 0:
# existing ticket
self.ticket = PatchedTicket(self.env, tkt_id=ticket_id, db=self.db)
# 'Ticket.time_changed' is a datetime in 0.11, and an int in 0.10.
# if we have trac.util.datefmt.to_datetime, we're likely with 0.11
try:
from trac.util.datefmt import to_timestamp
time_changed = to_timestamp(self.ticket.time_changed)
except ImportError:
time_changed = int(self.ticket.time_changed)
if time_changed > self.tickettime:
# just in case, verify if it wouldn't be a ticket that has been modified in the future
# (of course, it shouldn't happen... but who know). If it's the case, don't report it as an error
if time_changed < int(time.time()):
# TODO: this is not working yet...
#
#raise TracError("Sorry, can not execute the import. "
#"The ticket #" + str(ticket_id) + " has been modified by someone else "
#"since preview. You must re-upload and preview your file to avoid overwriting the other changes.")
pass
else:
self.ticket = PatchedTicket(self.env, db=self.db)
self.comment = ''
def start_process_row(self, row_idx, ticket_id):
from ticket import PatchedTicket
self.ticket = None
self.row_idx = row_idx
self.temphdf = []
if ticket_id > 0:
# existing ticket. Load the ticket, to see which fields will be modified
self.ticket = PatchedTicket(self.env, ticket_id)
def process_relativeticket_fields(self, rows, relativeticketfields):
from ticket import PatchedTicket
# Find the WBS columns, if any. We never expect to have more
# than one but this is flexible and easy. There's no good
# reason to go to the trouble of ignoring extras.
wbsfields = []
for row in rows:
for f in relativeticketfields:
if row[f].find('.') != -1:
if f not in wbsfields:
wbsfields.append(f)
# If WBS column present, build reverse lookup to find the
# ticket ID from a WBS number.
wbsref = {}
if wbsfields != []:
row_idx = 0
for row in rows:
wbsref[row[wbsfields[0]]] = self.crossref[row_idx]
row_idx += 1
row_idx = 0
for row in rows:
id = self.crossref[row_idx]
# Get the ticket (added or updated in the main loop
ticket = PatchedTicket(self.env, tkt_id=id, db=self.db)
for f in relativeticketfields:
# Get the value of the relative field column (e.g., "2,3")
v = row[f]
# If it's not empty, process the contents
if len(v) > 0:
# Handle WBS numbers
if f in wbsfields:
if row[f].find('.') == -1:
# Top level, no parent
ticket[f] = ''
else:
# Get this task's wbs
wbs = row[f]
# Remove the last dot-delimited field
pwbs = wbs[:wbs.rindex(".")]
# Look up the parent's ticket ID
ticket[f] = str(wbsref[pwbs])
# Handle dependencies
else:
s = []
for r in v.split(","):
# Make the string value an integer
r = int(r)
# The relative ticket dependencies are 1-based,
# array indices are 0-based. Convert and look up
# the new ticket ID of the other ticket.
i = self.crossref[r - 1]
# TODO check that i != id
s.append(str(i))
# Empty or not, use it to update the ticket
ticket[f] = ', '.join(s)
self._save_ticket(ticket, with_comment=False)
row_idx += 1
def _process(self, filereader, reporter, processor):
tracfields = [
field['name']
for field in TicketSystem(self.env).get_ticket_fields()
]
tracfields = ['ticket', 'id'] + tracfields
customfields = [
field['name']
for field in TicketSystem(self.env).get_custom_fields()
]
columns, rows = filereader.readers()
# defensive: columns could be non-string, make sure they are
columns = map(str, columns)
importedfields = [f for f in columns if f.lower() in tracfields]
notimportedfields = [
f for f in columns if f.lower() not in tracfields
and f.lower() != 'ticket' and f.lower() != 'id'
]
lowercaseimportedfields = [f.lower() for f in importedfields]
idcolumn = None
if 'ticket' in lowercaseimportedfields and 'id' in lowercaseimportedfields:
raise TracError, 'The first line of the worksheet contains both \'ticket\', and an \'id\' field name. Only one of them is needed to perform the import. Please check the file and try again.'
ownercolumn = None
if 'ticket' in lowercaseimportedfields:
idcolumn = self._find_case_insensitive('ticket', importedfields)
elif 'id' in lowercaseimportedfields:
idcolumn = self._find_case_insensitive('id', importedfields)
elif 'summary' in lowercaseimportedfields:
summarycolumn = self._find_case_insensitive(
'summary', importedfields)
ownercolumn = self._reconciliate_by_owner_also(
) and self._find_case_insensitive('owner', importedfields) or None
else:
raise TracError, 'The first line of the worksheet contains neither a \'ticket\', an \'id\' nor a \'summary\' field name. At least one of them is needed to perform the import. Please check the file and try again.'
# start TODO: this is too complex, it should be replaced by a call to TicketSystem(env).get_ticket_fields()
# The fields that we will have to set a value for, if:
# - they are not in the imported fields, and
# - they are not set in the default values of the Ticket class, and
# - they shouldn't be set to empty
# if 'set' is true, this will be the value that will be set by default (even if the default value in the Ticket class is different)
# if 'set' is false, the value is computed by Trac and we don't have anything to do
computedfields = {
'status': {
'value': 'new',
'set': True
},
'resolution': {
'value': "''(None)''",
'set': False
},
'reporter': {
'value': reporter,
'set': True
},
'time': {
'value': "''(now)''",
'set': False
},
'changetime': {
'value': "''(now)''",
'set': False
}
}
if 'owner' not in lowercaseimportedfields and 'component' in lowercaseimportedfields:
computedfields['owner'] = {}
computedfields['owner']['value'] = 'Computed from component'
computedfields['owner']['set'] = False
# to get the compulted default values
from ticket import PatchedTicket
ticket = PatchedTicket(self.env)
for f in [
'type', 'cc', 'url', 'description', 'keywords', 'component',
'severity', 'priority', 'version', 'milestone'
] + customfields:
if f in ticket.values:
computedfields[f] = {}
computedfields[f]['value'] = ticket.values[f]
computedfields[f]['set'] = False
else:
computedfields[f] = None
processor.start(importedfields, ownercolumn != None)
missingfields = [
f for f in computedfields if f not in lowercaseimportedfields
]
missingemptyfields = [
f for f in missingfields
if computedfields[f] == None or computedfields[f]['value'] == ''
]
missingdefaultedfields = [
f for f in missingfields if f not in missingemptyfields
]
if missingfields != []:
processor.process_missing_fields(missingfields, missingemptyfields,
missingdefaultedfields,
computedfields)
# end TODO: this is too complex
if notimportedfields != []:
processor.process_notimported_fields(notimportedfields)
# TODO: test the cases where those fields have empty values. They should be handled as None. (just to test, may be working already :)
selects = [
#Those ones inherit from AbstractEnum
('type', model.Type),
('status', model.Status),
('priority', model.Priority),
('severity', model.Severity),
('resolution', model.Resolution),
#Those don't
('milestone', model.Milestone),
('component', model.Component),
('version', model.Version)
]
existingvalues = {}
newvalues = {}
for name, cls in selects:
if name not in lowercaseimportedfields:
# this field is not present, nothing to do
continue
options = [val.name for val in cls.select(self.env)]
if not options:
# Fields without possible values are treated as if they didn't
# exist
continue
existingvalues[name] = options
newvalues[name] = []
def add_sql_result(db, sql, list):
cursor = db.cursor()
cursor.execute(sql)
for result in cursor:
list += [result]
existingusers = []
db = self.env.get_db_cnx()
add_sql_result(db, "SELECT DISTINCT reporter FROM ticket",
existingusers)
add_sql_result(db, "SELECT DISTINCT owner FROM ticket", existingusers)
add_sql_result(db, "SELECT DISTINCT owner FROM component",
existingusers)
newusers = []
duplicate_summaries = []
row_idx = 0
for row in rows:
if idcolumn:
ticket_id = row[idcolumn]
if ticket_id:
self._check_ticket(db, ticket_id)
else:
# will create a new ticket
ticket_id = 0
else:
summary = row[summarycolumn]
owner = ownercolumn and row[ownercolumn] or None
if self._skip_lines_with_empty_owner(
) and ownercolumn and not owner:
continue
ticket_id = self._find_ticket(db, summary, owner)
if (summary, owner) in duplicate_summaries:
if owner == None:
raise TracError, 'Summary "%s" is duplicated in the spreadsheet. Ticket reconciliation by summary can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % summary
else:
raise TracError, 'Summary "%s" and owner "%s" are duplicated in the spreadsheet. Ticket reconciliation can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % (
summary, owner)
else:
duplicate_summaries += [(summary, owner)]
processor.start_process_row(row_idx, ticket_id)
for column in importedfields:
cell = row[column]
# collect the new lookup values
if column.lower() in existingvalues.keys():
if cell != '' and cell not in existingvalues[column.lower(
)] and cell not in newvalues[column.lower()]:
newvalues[column.lower()] += [cell]
# also collect the new user names
if (column.lower() == 'owner' or column.lower() == 'reporter'):
if cell != '' and cell not in newusers and cell not in existingusers:
newusers += [cell]
# and proces the value.
if column.lower() != 'ticket' and column.lower() != 'id':
processor.process_cell(column, cell)
processor.end_process_row()
row_idx += 1
if newvalues != {} and reduce(lambda x, y: x == [] and y or x,
newvalues.values()) != []:
processor.process_new_lookups(newvalues)
if newusers != []:
processor.process_new_users(newusers)
return processor.end_process(row_idx)
def _process(self, filereader, reporter, processor):
tracfields = [field['name'] for field in TicketSystem(self.env).get_ticket_fields()]
tracfields = [ 'ticket', 'id' ] + tracfields
customfields = [field['name'] for field in TicketSystem(self.env).get_custom_fields()]
columns, rows = filereader.readers()
importedfields = [f for f in columns if f.lower() in tracfields]
notimportedfields = [f for f in columns if f and (f.lower() not in tracfields + ['comment']
# relative fields will be added later
and f[0] != '#')]
commentfields = [f for f in columns if f.lower() == 'comment']
if commentfields:
commentfield = commentfields[0]
else:
commentfield = None
lowercaseimportedfields = [f.lower() for f in importedfields]
# Fields which contain relative ticket numbers to update after import
relativeticketfields = []
lowercaserelativeticketfields = []
for f in columns:
if not f.startswith('#'):
continue
if f[1:].lower() in tracfields:
relativeticketfields.append(f)
lowercaserelativeticketfields.append(f[1:].lower())
else:
notimportedfields.append(f)
idcolumn = None
if 'ticket' in lowercaseimportedfields and 'id' in lowercaseimportedfields:
raise TracError, 'The first line of the worksheet contains both \'ticket\', and an \'id\' field name. Only one of them is needed to perform the import. Please check the file and try again.'
ownercolumn = None
if 'ticket' in lowercaseimportedfields:
idcolumn = self._find_case_insensitive('ticket', importedfields)
elif 'id' in lowercaseimportedfields:
idcolumn = self._find_case_insensitive('id', importedfields)
elif 'summary' in lowercaseimportedfields:
summarycolumn = self._find_case_insensitive('summary', importedfields)
ownercolumn = self._reconciliate_by_owner_also() and self._find_case_insensitive('owner', importedfields) or None
else:
raise TracError, 'The first line of the worksheet contains neither a \'ticket\', an \'id\' nor a \'summary\' field name. At least one of them is needed to perform the import. Please check the file and try again.'
# start TODO: this is too complex, it should be replaced by a call to TicketSystem(env).get_ticket_fields()
# The fields that we will have to set a value for, if:
# - they are not in the imported fields, and
# - they are not set in the default values of the Ticket class, and
# - they shouldn't be set to empty
# if 'set' is true, this will be the value that will be set by default (even if the default value in the Ticket class is different)
# if 'set' is false, the value is computed by Trac and we don't have anything to do
computedfields = {'status': { 'value':'new', 'set': True },
'resolution' : { 'value': "''(None)''", 'set': False },
'reporter' : { 'value': reporter, 'set': True },
'time' : { 'value': "''(now)''", 'set': False },
'changetime' : { 'value': "''(now)''", 'set': False } }
if 'owner' not in lowercaseimportedfields and 'component' in lowercaseimportedfields:
computedfields['owner'] = {}
computedfields['owner']['value'] = 'Computed from component'
computedfields['owner']['set'] = False
# to get the computed default values
from ticket import PatchedTicket
ticket = PatchedTicket(self.env)
for f in [ 'type', 'cc' , 'description', 'keywords', 'component' , 'severity' , 'priority' , 'version', 'milestone' ] + customfields:
if f in ticket.values:
computedfields[f] = {}
computedfields[f]['value'] = ticket.values[f]
computedfields[f]['set'] = False
else:
computedfields[f] = None
processor.start(importedfields, ownercolumn != None, commentfield)
missingfields = [f for f in computedfields if f not in lowercaseimportedfields]
if relativeticketfields:
missingfields = [f for f in missingfields if f not in lowercaserelativeticketfields]
missingemptyfields = [ f for f in missingfields if computedfields[f] == None or computedfields[f]['value'] == '']
missingdefaultedfields = [ f for f in missingfields if f not in missingemptyfields]
if missingfields != []:
processor.process_missing_fields(missingfields, missingemptyfields, missingdefaultedfields, computedfields)
# end TODO: this is too complex
if notimportedfields != []:
processor.process_notimported_fields(notimportedfields)
if commentfield:
processor.process_comment_field(commentfield)
# TODO: test the cases where those fields have empty values. They should be handled as None. (just to test, may be working already :)
selects = [
#Those ones inherit from AbstractEnum
('type', model.Type),
('status', model.Status),
('priority', model.Priority),
('severity', model.Severity),
('resolution', model.Resolution),
#Those don't
('milestone', model.Milestone),
('component', model.Component),
('version', model.Version)
]
existingvalues = {}
newvalues = {}
for name, cls in selects:
if name not in lowercaseimportedfields:
# this field is not present, nothing to do
continue
options = [val.name for val in cls.select(self.env)]
if not options:
# Fields without possible values are treated as if they didn't
# exist
continue
existingvalues[name] = options
newvalues[name] = []
def add_sql_result(db, aset, queries):
cursor = db.cursor()
for query in queries:
cursor.execute(query)
aset.update([val for val, in cursor])
existingusers = set()
db = self.env.get_db_cnx()
add_sql_result(
db, existingusers,
[("SELECT DISTINCT reporter FROM ticket"
" WHERE reporter IS NOT NULL AND reporter != ''"),
("SELECT DISTINCT owner FROM ticket"
" WHERE owner IS NOT NULL AND owner != ''"),
("SELECT DISTINCT owner FROM component"
" WHERE owner IS NOT NULL AND owner != ''")])
for username, name, email in self.env.get_known_users(db):
existingusers.add(username)
newusers = []
duplicate_summaries = []
relativeticketvalues = []
row_idx = 0
for row in rows:
if idcolumn:
ticket_id = row[idcolumn].strip()
if ticket_id:
ticket_id = ticket_id.lstrip('#')
if ticket_id:
self._check_ticket(db, ticket_id)
else:
# will create a new ticket
ticket_id = 0
else:
summary = row[summarycolumn]
owner = ownercolumn and row[ownercolumn] or None
if self._skip_lines_with_empty_owner() and ownercolumn and not owner:
continue
ticket_id = self._find_ticket(db, summary, owner)
if (summary, owner) in duplicate_summaries:
if owner == None:
raise TracError, 'Summary "%s" is duplicated in the spreadsheet. Ticket reconciliation by summary can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % summary
else:
raise TracError, 'Summary "%s" and owner "%s" are duplicated in the spreadsheet. Ticket reconciliation can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % (summary, owner)
else:
duplicate_summaries += [ (summary, owner) ]
processor.start_process_row(row_idx, ticket_id)
for column in importedfields:
cell = row[column]
if cell is None:
cell = ''
column_lower = column.lower()
# collect the new lookup values
if column_lower in existingvalues:
if isinstance(cell, basestring):
cell = cell.strip()
if cell != '' and \
cell not in existingvalues[column_lower] and \
cell not in newvalues[column_lower]:
newvalues[column_lower].append(cell)
# also collect the new user names
if column_lower in ('owner', 'reporter'):
if cell != '' and \
cell not in newusers and \
cell not in existingusers:
newusers.append(cell)
# and proces the value.
if column_lower not in ('ticket', 'id'):
processor.process_cell(column, cell)
if commentfield:
processor.process_comment(row[commentfield])
relativeticketvalues.append(dict([(f[1:].lower(), row[f])
for f in relativeticketfields]))
processor.end_process_row()
row_idx += 1
# All the rows have been processed. Handle global stuff
for name in list(newvalues):
if not newvalues[name]:
del newvalues[name]
if newvalues:
processor.process_new_lookups(newvalues)
if newusers:
processor.process_new_users(newusers)
if relativeticketfields:
processor.process_relativeticket_fields(relativeticketvalues, lowercaserelativeticketfields)
return processor.end_process(row_idx)
