def row_insert(self, row, model_instance, loglist):
""" Insert a row - separate function for transaction wrapping """
msg = ''
if model_instance:
if self.defaults:
for (field, value, foreignkey) in self.defaults:
value = self.type_clean(field, value, loglist)
try:
done = model_instance.getattr(field)
except:
done = False
if not done:
if foreignkey:
value = self.insert_fkey(foreignkey, value)
if value:
model_instance.__setattr__(field, value)
if self.deduplicate:
matchdict = {}
for (column, field, foreignkey) in self.mappings:
matchdict[field + '__exact'] = getattr(
model_instance, field, None)
dup = True
try:
self.model.objects.get(**matchdict)
except ObjectDoesNotExist:
dup = False
if dup:
return
try:
importing_csv.send(sender=model_instance,
row=dict(zip(self.csvfile[:1][0], row)))
model_instance.save()
imported_csv.send(sender=model_instance,
row=dict(zip(self.csvfile[:1][0], row)))
except DatabaseError as err:
try:
error_number, error_message = err
except:
error_message = err
error_number = 0
# Catch duplicate key error.
if error_number != 1062:
loglist.append('Database Error: %s, Number: %d' %
(error_message, error_number))
except ValueError as err:
# Usually only occurs if clean=False
loglist.append(str(err))
# except OverflowError:
# pass
if CSVIMPORT_LOG == 'logger':
for line in loglist:
logger.info(line)
self.loglist.extend(loglist)
loglist = []
def run(self, logid=0):
""" Run the csvimport """
loglist = []
if self.nameindexes:
indexes = self.csvfile.pop(0)
counter = 0
if logid:
csvimportid = logid
else:
csvimportid = 0
if self.mappings:
self.start = 0
loglist.append('Using manually entered (or default) mapping list')
else:
mappingstr = self.parse_header(self.csvfile[0])
if mappingstr:
loglist.append('Using mapping from first row of CSV file')
self.mappings = self.set_mappings(mappingstr)
if not self.mappings:
if not self.model:
loglist.append('Outputting setup message')
else:
warn = 'No fields in the CSV file match ' + self.model._meta.app_label + '.' + self.model.__name__
warn += ' - you must add a header field name row to the CSV file or supply a mapping list'
loglist.append(warn)
return loglist
# count before import
rowcount = self.model.objects.count()
for i, row in enumerate(self.csvfile[self.start:]):
if CSVIMPORT_LOG == 'logger':
logger.info("Import %s %i", self.model.__name__, counter)
counter += 1
model_instance = self.model()
model_instance.csvimport_id = csvimportid
for (column, field, foreignkey) in self.mappings:
if self.nameindexes:
column = indexes.index(column)
else:
column = int(column)-1
if foreignkey:
row[column] = self.insert_fkey(foreignkey, row[column])
if self.debug:
loglist.append('%s.%s = "%s"' % (self.model.__name__,
field, row[column]))
try:
row[column] = self.type_clean(field, row[column], loglist, i)
except:
pass
try:
model_instance.__setattr__(field, row[column])
except:
try:
field = getattr(model_instance, field)
if field:
value = field.to_python(row[column])
except:
msg = 'row %s: Column %s = %s couldnt be set for row' % (i, field, row[column])
loglist.append(msg)
if self.defaults:
for (field, value, foreignkey) in self.defaults:
value = self.type_clean(field, value, loglist)
try:
done = model_instance.getattr(field)
except:
done = False
if not done:
if foreignkey:
value = self.insert_fkey(foreignkey, value)
if value:
model_instance.__setattr__(field, value)
if self.deduplicate:
matchdict = {}
for (column, field, foreignkey) in self.mappings:
matchdict[field + '__exact'] = getattr(model_instance,
field, None)
try:
self.model.objects.get(**matchdict)
continue
except:
pass
try:
importing_csv.send(sender=model_instance,
row=dict(zip(self.csvfile[:1][0], row)))
model_instance.save()
imported_csv.send(sender=model_instance,
row=dict(zip(self.csvfile[:1][0], row)))
except DatabaseError as err:
try:
error_number, error_message = err
except:
error_message = err
error_number = 0
# Catch duplicate key error.
if error_number != 1062:
loglist.append(
'Database Error: %s, Number: %d' % (error_message,
error_number))
#except OverflowError:
# pass
if CSVIMPORT_LOG == 'logger':
for line in loglist:
logger.info(line)
self.loglist.extend(loglist)
loglist = []
# count after import
rowcount = self.model.objects.count() - rowcount
countmsg = 'Imported %s rows to %s' % (rowcount, self.model.__name__)
if CSVIMPORT_LOG == 'logger':
logger.info(countmsg)
if self.loglist:
self.loglist.append(countmsg)
self.props = {'file_name':self.file_name,
'import_user':'******',
'upload_method':'cronjob',
'error_log':'\n'.join(loglist),
'import_date':datetime.now()}
return self.loglist
else:
return ['No logging', ]
def run(self, logid=0):
""" Run the csvimport """
loglist = []
if self.nameindexes:
indexes = self.csvfile.pop(0)
counter = 0
if logid:
csvimportid = logid
else:
csvimportid = 0
if self.mappings:
self.start = 0
loglist.append('Using manually entered (or default) mapping list')
else:
mappingstr = self.parse_header(self.csvfile[0])
if mappingstr:
loglist.append('Using mapping from first row of CSV file')
self.mappings = self.set_mappings(mappingstr)
if not self.mappings:
if not self.model:
loglist.append('Outputting setup message')
else:
warn = 'No fields in the CSV file match ' + self.model._meta.app_label + '.' + self.model.__name__
warn += ' - you must add a header field name row to the CSV file or supply a mapping list'
loglist.append(warn)
return loglist
# count before import
rowcount = self.model.objects.count()
for i, row in enumerate(self.csvfile[self.start:]):
if CSVIMPORT_LOG == 'logger':
logger.info("Import %s %i", self.model.__name__, counter)
counter += 1
model_instance = self.model()
model_instance.csvimport_id = csvimportid
for (column, field, foreignkey) in self.mappings:
if self.nameindexes:
column = indexes.index(column)
else:
column = int(column) - 1
if foreignkey:
row[column] = self.insert_fkey(foreignkey, row[column])
if self.debug:
loglist.append('%s.%s = "%s"' %
(self.model.__name__, field, row[column]))
try:
row[column] = self.type_clean(field, row[column], loglist,
i)
except:
pass
try:
model_instance.__setattr__(field, row[column])
except:
try:
field = getattr(model_instance, field)
if field:
value = field.to_python(row[column])
except:
msg = 'row %s: Column %s = %s couldnt be set for row' % (
i, field, row[column])
loglist.append(msg)
if self.defaults:
for (field, value, foreignkey) in self.defaults:
value = self.type_clean(field, value, loglist)
try:
done = model_instance.getattr(field)
except:
done = False
if not done:
if foreignkey:
value = self.insert_fkey(foreignkey, value)
if value:
model_instance.__setattr__(field, value)
if self.deduplicate:
matchdict = {}
for (column, field, foreignkey) in self.mappings:
matchdict[field + '__exact'] = getattr(
model_instance, field, None)
try:
self.model.objects.get(**matchdict)
continue
except:
pass
try:
importing_csv.send(sender=model_instance,
row=dict(zip(self.csvfile[:1][0], row)))
model_instance.save()
imported_csv.send(sender=model_instance,
row=dict(zip(self.csvfile[:1][0], row)))
except DatabaseError as err:
try:
error_number, error_message = err
except:
error_message = err
error_number = 0
# Catch duplicate key error.
if error_number != 1062:
loglist.append('Database Error: %s, Number: %d' %
(error_message, error_number))
#except OverflowError:
# pass
if CSVIMPORT_LOG == 'logger':
for line in loglist:
logger.info(line)
self.loglist.extend(loglist)
loglist = []
# count after import
rowcount = self.model.objects.count() - rowcount
countmsg = 'Imported %s rows to %s' % (rowcount, self.model.__name__)
if CSVIMPORT_LOG == 'logger':
logger.info(countmsg)
if self.loglist:
self.loglist.append(countmsg)
self.props = {
'file_name': self.file_name,
'import_user': '******',
'upload_method': 'cronjob',
'error_log': '\n'.join(loglist),
'import_date': datetime.now()
}
return self.loglist
else:
return [
'No logging',
]