def uploadfile(request):
request_file = request.FILES['file']
basename, extension = os.path.splitext(request_file.name)
data_file = DataFile(basename, extension)
file_type = request.POST.get('type', None)
message = invalid_request_file_message(request_file, file_type)
if message:
return HttpResponse(message, status=400)
else:
data_file.write(request_file)
if 'files' not in request.session:
request.session['files'] = []
request.session['files'].append(data_file.as_dict())
# https://docs.djangoproject.com/en/2.2/topics/http/sessions/#when-sessions-are-saved
request.session.modified = True
return JsonResponse({
'files': [{
'id': data_file.id,
'name': request_file.name,
'size': request_file.size,
}],
})
def json_response(files, warnings=None):
file = DataFile('result', '.zip')
file.write_json_to_zip(files)
response = {
'url': file.url,
'size': file.size,
}
if warnings:
response['warnings'] = warnings
return JsonResponse(response)
def test_repr_with_folder(self):
data_file = DataFile('result',
'.zip',
id='identifier',
folder='directory')
assert repr(data_file) == 'directory/result-identifier.zip'
def perform_to_json(request):
input_file = next(get_files_from_session(request))
output_dir = DataFile('unflatten', '', input_file.id, input_file.folder)
output_name = output_dir.path + '.json'
extension = os.path.splitext(input_file.path)[1]
if extension == '.xlsx':
input_file_path = input_file.path
input_format = 'xlsx'
else:
input_file_path = output_dir.path
input_format = 'csv'
if extension == '.zip':
with ZipFile(input_file.path) as zipfile:
for name in zipfile.namelist():
zipfile.extract(name, input_file_path)
else:
if extension == '.csv':
os.mkdir(input_file_path)
shutil.copy(input_file.path, input_file_path)
config = LibCoveOCDSConfig().config
flattentool.unflatten(
input_file_path,
input_format=input_format,
output_name=output_name,
root_list_path=config['root_list_path'],
root_id=config['root_id']
)
# Delete the input CSV files, if any.
if extension in ('.csv', '.zip'):
shutil.rmtree(input_file_path)
# Create a ZIP file of the JSON file.
json_zip = DataFile('result', '.zip', id=input_file.id, folder=input_file.folder)
with ZipFile(json_zip.path, 'w', compression=ZIP_DEFLATED) as zipfile:
zipfile.write(output_name, 'result.json')
return JsonResponse({
'url': input_file.url,
'size': json_zip.size,
})
def perform_to_spreadsheet(request):
input_file = next(get_files_from_session(request))
output_dir = DataFile('flatten', '', input_file.id, input_file.folder)
config = LibCoveOCDSConfig().config
with warnings.catch_warnings():
warnings.filterwarnings('ignore') # flattentool uses UserWarning, so we can't set a specific category
flattentool.flatten(
input_file.path,
output_name=output_dir.path,
main_sheet_name=config['root_list_path'],
root_list_path=config['root_list_path'],
root_id=config['root_id'],
schema=config['schema_version_choices']['1.1'][1] + 'release-schema.json',
disable_local_refs=config['flatten_tool']['disable_local_refs'],
remove_empty_schema_columns=config['flatten_tool']['remove_empty_schema_columns'],
root_is_list=False,
)
# Create a ZIP file of the CSV files, and delete the output CSV files.
csv_zip = DataFile('flatten-csv', '.zip', id=input_file.id, folder=input_file.folder)
with ZipFile(csv_zip.path, 'w', compression=ZIP_DEFLATED) as zipfile:
for filename in os.listdir(output_dir.path):
zipfile.write(os.path.join(output_dir.path, filename), filename)
shutil.rmtree(output_dir.path)
return JsonResponse({
'csv': {
'url': input_file.url + 'csv/',
'size': csv_zip.size,
},
'xlsx': {
'url': input_file.url + 'xlsx/',
'size': os.path.getsize(output_dir.path + '.xlsx'),
}
})
def retrieve_result(request, folder, id, format=None):
if format is None:
prefix = 'result'
ext = '.zip'
filename = 'result.zip'
elif format == 'csv':
prefix = 'flatten-csv'
ext = '.zip'
filename = 'result-csv.zip'
elif format == 'xlsx':
prefix = 'flatten'
ext = '.xlsx'
filename = 'result.xlsx'
else:
raise Http404('Invalid option')
file = DataFile(prefix, ext, id=str(id), folder=folder)
return FileResponse(open(file.path, 'rb'), filename=filename, as_attachment=True)
def get_files_from_session(request):
for fileinfo in request.session['files']:
yield DataFile(**fileinfo)
def test_repr_without_folder(self):
data_file = DataFile('result', '.zip', id='identifier')
assert repr(data_file) == '{:%Y-%m-%d}/result-identifier.zip'.format(
date.today())