def test_action_flag_choices(self):
tests = ((1, 'Addition'), (2, 'Change'), (3, 'Deletion'))
for action_flag, display_name in tests:
with self.subTest(action_flag=action_flag):
log = LogEntry(action_flag=action_flag)
self.assertEqual(log.get_action_flag_display(), display_name)
def test_logentry_change_message_not_json(self):
"""LogEntry.change_message was a string before Django 1.10."""
logentry = LogEntry(change_message='non-JSON string')
self.assertEqual(logentry.get_change_message(),
logentry.change_message)
def parseCSVloadfromfolder(self, model, file):
'''
This method reads CSV data from a file and creates or updates
the database records.
The data must follow the following format:
- the first row contains a header, listing all field names
- a first character # marks a comment line
- empty rows are skipped
'''
# Init
headers = []
rownumber = 0
changed = 0
added = 0
content_type_id = ContentType.objects.get_for_model(model).pk
errorcount = 0
admin_log = []
# Handle the complete upload as a single database transaction
with transaction.atomic(using=self.database):
errors = False
has_pk_field = False
for row in self.EncodedCSVReader(file, delimiter=self.delimiter):
rownumber += 1
### Case 1: The first line is read as a header line
if rownumber == 1:
# Collect required fields
required_fields = set()
for i in model._meta.fields:
if not i.blank and i.default == NOT_PROVIDED and not isinstance(
i, AutoField):
required_fields.add(i.name)
# Validate all columns
for col in row:
col = col.strip().strip('#').lower()
if col == "":
headers.append(None)
continue
ok = False
for i in model._meta.fields:
if col == i.name.lower(
) or col == i.verbose_name.lower():
if i.editable is True:
headers.append(i)
else:
headers.append(None)
required_fields.discard(i.name)
ok = True
break
if not ok:
headers.append(None)
print('%s Warning: Skipping field %s' %
(datetime.now(), col),
file=self.logfile,
flush=True)
if col == model._meta.pk.name.lower() or \
col == model._meta.pk.verbose_name.lower():
has_pk_field = True
if required_fields:
# We are missing some required fields
errors = True
print('%s Error: Some keys were missing: %s' %
(datetime.now(), ', '.join(required_fields)),
file=self.logfile,
flush=True)
errorcount += 1
# Abort when there are errors
if errors:
break
header_count = len(headers)
# Create a form class that will be used to validate the data
fields = [i.name for i in headers if i]
UploadForm = modelform_factory(
model,
fields=tuple(fields),
formfield_callback=lambda f:
(isinstance(f, RelatedField) and
BulkForeignKeyFormField(field=f, using=self.database)
) or f.formfield(localize=True))
# Get natural keys for the class
natural_key = None
if hasattr(model.objects, 'get_by_natural_key'):
if model._meta.unique_together:
natural_key = model._meta.unique_together[0]
elif hasattr(model, 'natural_key'):
natural_key = model.natural_key
### Case 2: Skip empty rows and comments rows
elif len(row) == 0 or row[0].startswith('#'):
continue
### Case 3: Process a data row
else:
try:
# Step 1: Build a dictionary with all data fields
d = {}
colnum = 0
for col in row:
# More fields in data row than headers. Move on to the next row.
if colnum >= header_count:
break
if headers[colnum]:
d[headers[colnum].name] = col.strip()
colnum += 1
# Step 2: Fill the form with data, either updating an existing
# instance or creating a new one.
if has_pk_field:
# A primary key is part of the input fields
try:
# Try to find an existing record with the same primary key
it = model.objects.using(self.database).only(
*fields).get(pk=d[model._meta.pk.name])
form = UploadForm(d, instance=it)
except model.DoesNotExist:
form = UploadForm(d)
it = None
elif natural_key:
# A natural key exists for this model
try:
# Build the natural key
key = []
for x in natural_key:
key.append(d.get(x, None))
# Try to find an existing record using the natural key
it = model.objects.get_by_natural_key(*key)
form = UploadForm(d, instance=it)
except model.DoesNotExist:
form = UploadForm(d)
it = None
except model.MultipleObjectsReturned:
print(
'%s Error: Row %s: Key fields not unique' %
(datetime.now(), rownumber),
file=self.logfile,
flush=True)
errorcount += 1
continue
else:
# No primary key required for this model
form = UploadForm(d)
it = None
# Step 3: Validate the form and model, and save to the database
if form.has_changed():
if form.is_valid():
# Save the form
obj = form.save(commit=False)
if it:
changed += 1
obj.save(using=self.database,
force_update=True)
else:
added += 1
obj.save(using=self.database,
force_insert=True)
if self.user:
admin_log.append(
LogEntry(
user_id=self.user.id,
content_type_id=content_type_id,
object_id=obj.pk,
object_repr=force_text(obj),
action_flag=it and CHANGE
or ADDITION,
#. Translators: Translation included with Django
change_message='Changed %s.' %
get_text_list(
form.changed_data, 'and')))
else:
# Validation fails
for error in form.non_field_errors():
print('%s Error: Row %s: %s' %
(datetime.now(), rownumber, error),
file=self.logfile,
flush=True)
errorcount += 1
for field in form:
for error in field.errors:
print('%s Error: Row %s field %s: %s' %
(datetime.now(), rownumber,
field.name, error),
file=self.logfile,
flush=True)
errorcount += 1
except Exception as e:
print("%s Error: Exception during upload: %s" %
(datetime.now(), e),
file=self.logfile,
flush=True)
errorcount += 1
# Save all admin log entries
LogEntry.objects.all().using(self.database).bulk_create(
admin_log, batch_size=1000)
# Report all failed records
if not errors:
print(
'%s Uploaded data successfully: changed %d and added %d records'
% (datetime.now(), changed, added),
file=self.logfile,
flush=True)
else:
print('%s Error: Invalid data format - skipping the file \n' %
datetime.now(),
file=self.logfile,
flush=True)
return errorcount
def _parseData(model, data, rowmapper, user, database, ping):
selfReferencing = []
def formfieldCallback(f):
#global selfReferencing
if isinstance(f, RelatedField):
tmp = BulkForeignKeyFormField(field=f, using=database)
if f.remote_field.model == model:
selfReferencing.append(tmp)
return tmp
else:
return f.formfield(localize=True)
# Initialize
headers = []
rownumber = 0
pingcounter = 0
changed = 0
added = 0
content_type_id = ContentType.objects.get_for_model(model).pk
admin_log = []
errors = 0
warnings = 0
has_pk_field = False
rowWrapper = rowmapper()
for row in data:
rownumber += 1
rowWrapper.setData(row)
# Case 1: The first line is read as a header line
if rownumber == 1:
# Collect required fields
required_fields = set()
for i in model._meta.fields:
if not i.blank and i.default == NOT_PROVIDED and not isinstance(
i, AutoField):
required_fields.add(i.name)
# Validate all columns
for col in rowWrapper.values():
col = col.strip().strip('#').lower() if col else ""
if col == "":
headers.append(None)
continue
ok = False
for i in model._meta.fields:
# Try with translated field names
if col == i.name.lower() \
or col == i.verbose_name.lower() \
or col == ("%s - %s" % (model.__name__, i.verbose_name)).lower():
if i.editable is True:
headers.append(i)
else:
headers.append(None)
required_fields.discard(i.name)
ok = True
break
if translation.get_language() != 'en':
# Try with English field names
with translation.override('en'):
if col == i.name.lower() \
or col == i.verbose_name.lower() \
or col == ("%s - %s" % (model.__name__, i.verbose_name)).lower():
if i.editable is True:
headers.append(i)
else:
headers.append(None)
required_fields.discard(i.name)
ok = True
break
if not ok:
headers.append(None)
warnings += 1
yield (WARNING, None, None, None,
force_text(
_('Skipping unknown field %(column)s' %
{'column': col})))
if col == model._meta.pk.name.lower() or \
col == model._meta.pk.verbose_name.lower():
has_pk_field = True
if required_fields:
# We are missing some required fields
errors += 1
#. Translators: Translation included with django
yield (ERROR, None, None, None,
force_text(
_('Some keys were missing: %(keys)s' %
{'keys': ', '.join(required_fields)})))
# Abort when there are errors
if errors:
return
# Create a form class that will be used to validate the data
fields = [i.name for i in headers if i]
UploadForm = modelform_factory(
model,
fields=tuple(fields),
formfield_callback=formfieldCallback)
rowWrapper = rowmapper(headers)
# Get natural keys for the class
natural_key = None
if hasattr(model.objects, 'get_by_natural_key'):
if model._meta.unique_together:
natural_key = model._meta.unique_together[0]
elif hasattr(model, 'natural_key'):
natural_key = model.natural_key
# Case 2: Skip empty rows
elif len(rowWrapper) == 0:
continue
# Case 3: Process a data row
else:
try:
# Step 1: Send a ping-alive message to make the upload interruptable
if ping:
pingcounter += 1
if pingcounter >= 100:
pingcounter = 0
yield (DEBUG, rownumber, None, None, None)
# Step 2: Fill the form with data, either updating an existing
# instance or creating a new one.
if has_pk_field:
# A primary key is part of the input fields
try:
# Try to find an existing record with the same primary key
it = model.objects.using(database).only(*fields).get(
pk=rowWrapper[model._meta.pk.name])
form = UploadForm(rowWrapper, instance=it)
except model.DoesNotExist:
form = UploadForm(rowWrapper)
it = None
elif natural_key:
# A natural key exists for this model
try:
# Build the natural key
key = []
for x in natural_key:
key.append(rowWrapper.get(x, None))
# Try to find an existing record using the natural key
it = model.objects.get_by_natural_key(*key)
form = UploadForm(rowWrapper, instance=it)
except model.DoesNotExist:
form = UploadForm(rowWrapper)
it = None
except model.MultipleObjectsReturned:
yield (ERROR, rownumber, None, None,
force_text(_('Key fields not unique')))
continue
else:
# No primary key required for this model
form = UploadForm(rowWrapper)
it = None
# Step 3: Validate the form and model, and save to the database
if form.has_changed():
if form.is_valid():
# Save the form
obj = form.save(commit=False)
if it:
changed += 1
obj.save(using=database, force_update=True)
else:
added += 1
obj.save(using=database, force_insert=True)
# Add the new object in the cache of available keys
for x in selfReferencing:
if x.cache is not None and obj.pk not in x.cache:
x.cache[obj.pk] = obj
if user:
admin_log.append(
LogEntry(
user_id=user.id,
content_type_id=content_type_id,
object_id=obj.pk,
object_repr=force_text(obj),
action_flag=it and CHANGE or ADDITION,
#. Translators: Translation included with Django
change_message='Changed %s.' %
get_text_list(form.changed_data, 'and')))
if len(admin_log) > 100:
LogEntry.objects.all().using(
database).bulk_create(admin_log)
admin_log = []
else:
# Validation fails
for error in form.non_field_errors():
errors += 1
yield (ERROR, rownumber, None, None, error)
for field in form:
for error in field.errors:
errors += 1
yield (ERROR, rownumber, field.name,
rowWrapper[field.name], error)
except Exception as e:
errors += 1
yield (ERROR, None, None, None,
"Exception during upload: %s" % e)
# Save remaining admin log entries
LogEntry.objects.all().using(database).bulk_create(admin_log)
yield (
INFO, None, None, None,
_('%(rows)d data rows, changed %(changed)d and added %(added)d records, %(errors)d errors, %(warnings)d warnings'
) % {
'rows': rownumber - 1,
'changed': changed,
'added': added,
'errors': errors,
'warnings': warnings
})
def create_log_entry(**kwargs):
user = UserFactory()
ctype = ContentType.objects.get(app_label="users", model="justfixuser")
LogEntry(user=user, object_repr="blargy", content_type=ctype, **kwargs).save()
def batch_update_view(model_admin,
request,
queryset,
field_names=None,
exclude_field_names=None):
# removes fields not included in field_names
def remove_fields(form, field_names):
for field in list(form.base_fields.keys()):
if not field in field_names:
del form.base_fields[field]
return form
# the return value is the form class, not the form class instance
f = model_admin.get_form(request)
# If no field names are given, do them all
if field_names is None:
field_names = list(f.base_fields.keys())
if exclude_field_names is not None:
# should do this with list comprehension
temp_names = []
for n in field_names:
if n not in exclude_field_names:
temp_names.append(n)
field_names = temp_names
form_class = remove_fields(f, field_names)
if request.method == 'POST':
form = form_class()
# for this there is a hidden field 'form-post' in the html template the edit is confirmed
if 'form-post' in request.POST:
form = form_class(request.POST, request.FILES)
if form.is_valid():
for item in queryset.all():
changed_list = []
for field_name in field_names:
if request.POST.get('{}_use'.format(
field_name, )) == 'on':
if item._meta.get_field(
field_name).__class__ is FileField:
setattr(item, field_name,
request.FILES[field_name])
else:
setattr(item, field_name,
form.cleaned_data[field_name])
changed_list.append(field_name)
if len(changed_list) > 0:
l = LogEntry(
user=request.user,
content_type=ContentType.objects.get_for_model(
model_admin.model, for_concrete_model=False),
object_id=item.pk,
object_repr=str(item),
action_flag=CHANGE,
change_message='Changed {}'.format(
', '.join(changed_list), ),
)
l.save()
item.save()
model_admin.message_user(
request,
"Bulk updated {} records".format(queryset.count()))
return HttpResponseRedirect(request.get_full_path())
return render(request,
'admin/batch_editing_intermediary.html',
context={
'form': form,
'items': queryset,
'fieldnames': field_names,
'media': model_admin.media,
})
def log_usagechange(use, addORrem):
logentr = LogEntry(user=request.user)
def auth_log(message, user=None):
if user == None:
user = User.objects.get(pk=1)
entry = LogEntry(user=user, object_repr=message, action_flag=4)
entry.save()
def get_comment(self):
return LogEntry(change_message=self.comment).get_change_message()
