def truncate_in_place_if_too_big(self) -> 'ProcessResult':
"""Truncate dataframe in-place and add to self.error if truncated."""
len_before = len(self.dataframe)
if sanitizedataframe.truncate_table_if_too_big(self.dataframe):
warning = _('Truncated output from %d rows to %d') % (
len_before, len(self.dataframe))
if self.error:
self.error = f'{self.error}\n{warning}'
else:
self.error = warning
def truncate_in_place_if_too_big(self) -> 'ProcessResult':
"""Truncate dataframe in-place and add to self.error if truncated."""
len_before = len(self.dataframe)
if sanitizedataframe.truncate_table_if_too_big(self.dataframe):
warning = ('Truncated output from %d rows to %d' %
(len_before, len(self.dataframe)))
if self.error:
self.error = f'{self.error}\n{warning}'
else:
self.error = warning
# Nix unused categories
for column in self.dataframe:
series = self.dataframe[column]
if hasattr(series, 'cat'):
series.cat.remove_unused_categories(inplace=True)
def upload_to_table(wf_module, uploaded_file):
try:
table = __parse_uploaded_file(uploaded_file)
except Exception as e:
wf_module.set_error(str(e), notify=True)
uploaded_file.delete(
) # delete uploaded file, we probably can't ever use it
return
# Cut this file down to size to prevent reading in the hugest data on every render
nrows = len(table)
if truncate_table_if_too_big(table):
error = _('File has %d rows, truncated to %d' %
(nrows, settings.MAX_ROWS_PER_TABLE))
wf_module.set_error(error, notify=False)
else:
# start of file upload sets module busy status on client side; undo this.
wf_module.set_ready(notify=False)
sanitize_dataframe(table)
# Save the new output, creating and switching to a new data version
version_added = wf_module.store_fetched_table(table)
# set new StoredObject metadata to the json response the client expects, containing filename and uuid
# (see views.UploadedFile.get)
new_so = StoredObject.objects.get(wf_module=wf_module,
stored_at=version_added)
result = [{'uuid': uploaded_file.uuid, 'name': uploaded_file.name}]
new_so.metadata = json.dumps(result)
new_so.save()
ChangeDataVersionCommand.create(wf_module,
version_added) # also notifies client
# don't delete UploadedFile, so that we can reparse later or allow higher row limit or download origina, etc.
return