def main():
"""
Use ``create_parser`` to get the commandline arguments, and pass them to
the appropriate function in __init__.py (create_template, flatten or
unflatten).
"""
parser = create_parser()
# Store the supplied arguments in args
args = parser.parse_args()
if args.subparser_name is None:
parser.print_help()
return
if not args.verbose:
sys.excepthook = non_verbose_error_handler
warnings.formatwarning = non_verbose_warning_formatter
if args.subparser_name == "create-template":
# Pass the arguments to the create_template function
# If the schema file does not exist we catch it in this exception
try:
# Note: Ensures that empty arguments are not passed to the create_template function
create_template(**kwargs_from_parsed_args(args))
except (OSError, IOError) as e:
print(str(e))
return
elif args.subparser_name == "flatten":
flatten(**kwargs_from_parsed_args(args))
elif args.subparser_name == "unflatten":
unflatten(**kwargs_from_parsed_args(args))
def test_roundtrip_360(tmpdir, output_format, use_titles):
input_name = 'flattentool/tests/fixtures/WellcomeTrust-grants_fixed_2_grants.json'
flatten(input_name=input_name,
output_name=tmpdir.join('flattened').strpath + '.' + output_format,
output_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
main_sheet_name='grants',
root_list_path='grants',
root_id='',
use_titles=use_titles)
unflatten(input_name=tmpdir.join('flattened').strpath + '.' +
output_format,
output_name=tmpdir.join('roundtrip.json').strpath,
input_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
main_sheet_name='grants',
root_list_path='grants',
root_id='',
convert_titles=use_titles)
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join('roundtrip.json'))
# Currently not enough information to successfully roundtrip that values
# are numbers, when this is not required by the schema
# for CSV, and for openpyxl under Python 2
if output_format == 'csv' or sys.version_info < (3, 0):
for grant in original_json['grants']:
grant['plannedDates'][0]['duration'] = str(
grant['plannedDates'][0]['duration'])
assert original_json == roundtripped_json
def test_roundtrip_360(tmpdir, output_format, use_titles):
input_name = (
"flattentool/tests/fixtures/fundingproviders-grants_fixed_2_grants.json"
)
flatten(
input_name=input_name,
output_name=tmpdir.join("flattened").strpath + "." + output_format,
output_format=output_format,
schema="flattentool/tests/fixtures/360-giving-schema.json",
root_list_path="grants",
root_id="",
use_titles=use_titles,
main_sheet_name="grants",
)
unflatten(
input_name=tmpdir.join("flattened").strpath + "." + output_format,
output_name=tmpdir.join("roundtrip.json").strpath,
input_format=output_format,
schema="flattentool/tests/fixtures/360-giving-schema.json",
root_list_path="grants",
root_id="",
convert_titles=use_titles,
)
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join("roundtrip.json"))
assert original_json == roundtripped_json
def main():
"""
Use ``create_parser`` to get the commandline arguments, and pass them to
the appropriate function in __init__.py (create_template, flatten or
unflatten).
"""
parser = create_parser()
# Store the supplied arguments in args
args = parser.parse_args()
if args.subparser_name is None:
parser.print_help()
elif args.subparser_name == 'create-template':
# Pass the arguments to the create_template function
# If the schema file does not exist we catch it in this exception
try:
# Note: Ensures that empty arguments are not passed to the create_template function
create_template(**kwargs_from_parsed_args(args))
except (OSError, IOError) as e:
print(text_type(e))
return
elif args.subparser_name == 'flatten':
flatten(**kwargs_from_parsed_args(args))
elif args.subparser_name == 'unflatten':
unflatten(**kwargs_from_parsed_args(args))
def test_roundtrip_360_rollup(tmpdir, use_titles):
input_name = 'flattentool/tests/fixtures/fundingproviders-grants_fixed_2_grants.json'
output_format = 'csv'
output_name = tmpdir.join('flattened').strpath + '.' + output_format
moved_name = tmpdir.mkdir('flattened_main_only').strpath
flatten(input_name=input_name,
output_name=output_name,
output_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
root_list_path='grants',
root_id='',
use_titles=use_titles,
rollup=True,
main_sheet_name='grants')
os.rename(output_name + '/grants.csv', moved_name + '/grants.csv')
unflatten(input_name=moved_name,
output_name=tmpdir.join('roundtrip.json').strpath,
input_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
root_list_path='grants',
root_id='',
convert_titles=use_titles)
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join('roundtrip.json'))
assert original_json == roundtripped_json
def main():
"""
Use ``create_parser`` to get the commandline arguments, and pass them to
the appropriate function in __init__.py (create_template, flatten or
unflatten).
"""
parser = create_parser()
# Store the supplied arguments in args
args = parser.parse_args()
if args.subparser_name is None:
parser.print_help()
elif args.subparser_name == 'create-template':
# Pass the arguments to the create_template function
# If the schema file does not exist we catch it in this exception
try:
# Note: Ensures that empty arguments are not passed to the create_template function
create_template(**kwargs_from_parsed_args(args))
except (OSError, IOError) as e:
print(text_type(e))
return
elif args.subparser_name == 'flatten':
flatten(**kwargs_from_parsed_args(args))
elif args.subparser_name == 'unflatten':
unflatten(**kwargs_from_parsed_args(args))
def test_roundtrip(tmpdir, output_format):
input_name = 'flattentool/tests/fixtures/tenders_releases_2_releases.json'
base_name = 'flattentool/tests/fixtures/tenders_releases_base.json'
flatten(
input_name=input_name,
output_name=tmpdir.join('flattened').strpath+'.'+output_format,
output_format=output_format,
schema='flattentool/tests/fixtures/release-schema.json',
root_list_path='releases',
main_sheet_name='releases')
unflatten(
input_name=tmpdir.join('flattened').strpath+'.'+output_format,
output_name=tmpdir.join('roundtrip.json').strpath,
input_format=output_format,
base_json=base_name,
schema='flattentool/tests/fixtures/release-schema.json',
root_list_path='releases')
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join('roundtrip.json'))
# Not currently possible to roundtrip Nones
# https://github.com/open-contracting/flattening-ocds/issues/35
for release in roundtripped_json['releases']:
release['tender']['awardCriteriaDetails'] = None
assert original_json == roundtripped_json
def test_roundtrip_360_rollup(tmpdir, use_titles):
input_name = (
"flattentool/tests/fixtures/fundingproviders-grants_fixed_2_grants.json"
)
output_format = "csv"
output_name = tmpdir.join("flattened").strpath + "." + output_format
moved_name = tmpdir.mkdir("flattened_main_only").strpath
flatten(
input_name=input_name,
output_name=output_name,
output_format=output_format,
schema="flattentool/tests/fixtures/360-giving-schema.json",
root_list_path="grants",
root_id="",
use_titles=use_titles,
rollup=True,
main_sheet_name="grants",
)
os.rename(output_name + "/grants.csv", moved_name + "/grants.csv")
unflatten(
input_name=moved_name,
output_name=tmpdir.join("roundtrip.json").strpath,
input_format=output_format,
schema="flattentool/tests/fixtures/360-giving-schema.json",
root_list_path="grants",
root_id="",
convert_titles=use_titles,
)
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join("roundtrip.json"))
assert original_json == roundtripped_json
def test_roundtrip(tmpdir, output_format):
input_name = "flattentool/tests/fixtures/tenders_releases_2_releases.json"
base_name = "flattentool/tests/fixtures/tenders_releases_base.json"
flatten(
input_name=input_name,
output_name=tmpdir.join("flattened").strpath + "." + output_format,
output_format=output_format,
schema="flattentool/tests/fixtures/release-schema.json",
root_list_path="releases",
main_sheet_name="releases",
)
unflatten(
input_name=tmpdir.join("flattened").strpath + "." + output_format,
output_name=tmpdir.join("roundtrip.json").strpath,
input_format=output_format,
base_json=base_name,
schema="flattentool/tests/fixtures/release-schema.json",
root_list_path="releases",
)
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join("roundtrip.json"))
# Not currently possible to roundtrip Nones
# https://github.com/open-contracting/flattening-ocds/issues/35
for release in roundtripped_json["releases"]:
release["tender"]["awardCriteriaDetails"] = None
assert original_json == roundtripped_json
def test_roundtrip_xml(tmpdir, output_format):
input_name = "examples/iati/expected.xml"
flatten(
input_name=input_name,
output_name=tmpdir.join("flattened").strpath + "." + output_format,
output_format=output_format,
root_list_path="iati-activity",
id_name="iati-identifier",
xml=True,
)
unflatten(
input_name=tmpdir.join("flattened").strpath + "." + output_format,
output_name=tmpdir.join("roundtrip.xml").strpath,
input_format=output_format,
root_list_path="iati-activity",
id_name="iati-identifier",
xml=True,
)
original_xml = open(input_name, "rb")
roundtripped_xml = tmpdir.join("roundtrip.xml").open("rb")
# Compare without ordering, by using dict_constructor=dict instead of
# OrderedDict
original = xmltodict.parse(original_xml, dict_constructor=dict)
roundtripped = xmltodict.parse(roundtripped_xml, dict_constructor=dict)
assert original == roundtripped
def main():
"""
Takes any command line arguments and then passes them onto
create_template
Defaults are not set here, but rather given in the create_template
function incase that function is called from elsewhere in future.
"""
parser = create_parser()
# Store the supplied arguments in args
args = parser.parse_args()
if args.subparser_name is None:
parser.print_help()
elif args.subparser_name == 'create-template':
# Pass the arguments to the create_template function
# If the schema file does not exist we catch it in this exception
try:
# Note: Ensures that empty arguments are not passed to the create_template function
create_template(**kwargs_from_parsed_args(args))
except (OSError, IOError) as e:
print(text_type(e))
return
elif args.subparser_name == 'flatten':
flatten(**kwargs_from_parsed_args(args))
elif args.subparser_name == 'unflatten':
unflatten(**kwargs_from_parsed_args(args))
def test_roundtrip(tmpdir, output_format):
input_name = 'flattentool/tests/fixtures/tenders_releases_2_releases.json'
base_name = 'flattentool/tests/fixtures/tenders_releases_base.json'
flatten(input_name=input_name,
output_name=tmpdir.join('flattened').strpath + '.' + output_format,
output_format=output_format,
schema='flattentool/tests/fixtures/release-schema.json',
root_list_path='releases',
main_sheet_name='releases')
unflatten(input_name=tmpdir.join('flattened').strpath + '.' +
output_format,
output_name=tmpdir.join('roundtrip.json').strpath,
input_format=output_format,
base_json=base_name,
schema='flattentool/tests/fixtures/release-schema.json',
root_list_path='releases')
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join('roundtrip.json'))
# Not currently possible to roundtrip Nones
# https://github.com/open-contracting/flattening-ocds/issues/35
for release in roundtripped_json['releases']:
release['tender']['awardCriteriaDetails'] = None
assert original_json == roundtripped_json
def test_roundtrip_360_rollup(tmpdir, use_titles):
input_name = 'flattentool/tests/fixtures/fundingproviders-grants_fixed_2_grants.json'
output_format = 'csv'
output_name = tmpdir.join('flattened').strpath+'.'+output_format
moved_name = tmpdir.mkdir('flattened_main_only').strpath
flatten(
input_name=input_name,
output_name=output_name,
output_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
root_list_path='grants',
root_id='',
use_titles=use_titles,
rollup=True,
main_sheet_name='grants')
os.rename(output_name+'/grants.csv', moved_name+'/grants.csv')
unflatten(
input_name=moved_name,
output_name=tmpdir.join('roundtrip.json').strpath,
input_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
root_list_path='grants',
root_id='',
convert_titles=use_titles)
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join('roundtrip.json'))
assert original_json == roundtripped_json
def test_roundtrip_360(tmpdir, output_format, use_titles):
input_name = 'flattentool/tests/fixtures/WellcomeTrust-grants_fixed_2_grants.json'
flatten(
input_name=input_name,
output_name=tmpdir.join('flattened').strpath+'.'+output_format,
output_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
main_sheet_name='grants',
root_list_path='grants',
root_id='',
use_titles=use_titles)
unflatten(
input_name=tmpdir.join('flattened').strpath+'.'+output_format,
output_name=tmpdir.join('roundtrip.json').strpath,
input_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
main_sheet_name='grants',
root_list_path='grants',
root_id='',
convert_titles=use_titles)
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join('roundtrip.json'))
# Currently not enough information to successfully roundtrip that values
# are numbers, when this is not required by the schema
# for CSV, and for openpyxl under Python 2
if output_format == 'csv' or sys.version_info < (3, 0):
for grant in original_json['grants']:
grant['plannedDates'][0]['duration'] = str(grant['plannedDates'][0]['duration'])
assert original_json == roundtripped_json
def convert_json(request, data):
context = {}
converted_path = os.path.join(data.upload_dir(), 'flattened')
flatten_kwargs = dict(
output_name=converted_path,
main_sheet_name=request.cove_config['main_sheet_name'],
root_list_path=request.cove_config['main_sheet_name'],
root_id=request.cove_config['root_id'],
schema=request.cove_config['item_schema_url'],
)
try:
if not os.path.exists(converted_path + '.xlsx'):
if request.POST.get('flatten'):
flattentool.flatten(data.original_file.file.name, **flatten_kwargs)
else:
return {'conversion': 'flattenable'}
context['converted_file_size'] = os.path.getsize(converted_path + '.xlsx')
if request.cove_config['convert_titles']:
flatten_kwargs.update(dict(
output_name=converted_path + '-titles',
use_titles=True
))
if not os.path.exists(converted_path + '-titles.xlsx'):
flattentool.flatten(data.original_file.file.name, **flatten_kwargs)
context['converted_file_size_titles'] = os.path.getsize(converted_path + '-titles.xlsx')
except BadlyFormedJSONError as err:
raise CoveInputDataError(context={
'sub_title': _("Sorry we can't process that data"),
'link': 'cove:index',
'link_text': _('Try Again'),
'msg': _('We think you tried to upload a JSON file, but it is not well formed JSON.\n\nError message: {}'.format(err))
})
except Exception as err:
logger.exception(err, extra={
'request': request,
})
return {
'conversion': 'flatten',
'conversion_error': repr(err)
}
context.update({
'conversion': 'flatten',
'converted_path': converted_path,
'converted_url': '{}/flattened'.format(data.upload_url())
})
return context
def aplanarArchivo(ubicacionArchivo, directorio):
directorio = getRootPath(directorio)
ubicacionArchivo = getRootPath(ubicacionArchivo)
flattentool.flatten(
ubicacionArchivo,
output_name=directorio,
main_sheet_name='releases',
root_list_path='releases',
root_id='ocid',
# schema=carpetaArchivos + 'release-schema.json',
disable_local_refs=True,
remove_empty_schema_columns=True,
root_is_list=False)
with ZipFile(directorio + '.zip', 'w',
compression=ZIP_DEFLATED) as zipfile:
for filename in os.listdir(directorio):
zipfile.write(os.path.join(directorio, filename), filename)
shutil.rmtree(directorio)
def perform_to_spreadsheet(request):
input_file = next(_get_files_from_session(request))
output_dir = DataFile('flatten', '', input_file.id, input_file.folder)
config = LibCoveOCDSConfig().config
flattentool.flatten(
input_file.path,
output_name=output_dir.path,
main_sheet_name=config['root_list_path'],
root_list_path=config['root_list_path'],
root_id=config['root_id'],
schema=config['schema_version_choices']['1.1'][1] +
'release-schema.json',
disable_local_refs=config['flatten_tool']['disable_local_refs'],
remove_empty_schema_columns=config['flatten_tool']
['remove_empty_schema_columns'],
root_is_list=False,
)
# Create a ZIP file of the CSV files, and delete the CSV files.
csv_zip = DataFile('flatten-csv',
'.zip',
id=input_file.id,
folder=input_file.folder)
with ZipFile(csv_zip.path, 'w', compression=ZIP_DEFLATED) as zipfile:
for filename in os.listdir(output_dir.path):
zipfile.write(os.path.join(output_dir.path, filename), filename)
shutil.rmtree(output_dir.path)
return JsonResponse({
'csv': {
'url': input_file.url + 'csv/',
'size': csv_zip.size,
},
'xlsx': {
'url': input_file.url + 'xlsx/',
'size': os.path.getsize(output_dir.path + '.xlsx'),
}
})
def test_roundtrip_360(tmpdir, output_format, use_titles):
input_name = 'flattentool/tests/fixtures/fundingproviders-grants_fixed_2_grants.json'
flatten(input_name=input_name,
output_name=tmpdir.join('flattened').strpath + '.' + output_format,
output_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
root_list_path='grants',
root_id='',
use_titles=use_titles,
main_sheet_name='grants')
unflatten(input_name=tmpdir.join('flattened').strpath + '.' +
output_format,
output_name=tmpdir.join('roundtrip.json').strpath,
input_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
root_list_path='grants',
root_id='',
convert_titles=use_titles)
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join('roundtrip.json'))
assert original_json == roundtripped_json
def test_roundtrip_xml(tmpdir, output_format):
input_name = 'examples/iati/expected.xml'
flatten(input_name=input_name,
output_name=tmpdir.join('flattened').strpath + '.' + output_format,
output_format=output_format,
root_list_path='iati-activity',
id_name='iati-identifier',
xml=True)
unflatten(input_name=tmpdir.join('flattened').strpath + '.' +
output_format,
output_name=tmpdir.join('roundtrip.xml').strpath,
input_format=output_format,
root_list_path='iati-activity',
id_name='iati-identifier',
xml=True)
original_xml = open(input_name, 'rb')
roundtripped_xml = tmpdir.join('roundtrip.xml').open('rb')
# Compare without ordering, by using dict_constructor=dict instead of
# OrderedDict
original = xmltodict.parse(original_xml, dict_constructor=dict)
roundtripped = xmltodict.parse(roundtripped_xml, dict_constructor=dict)
assert original == roundtripped
def test_roundtrip_360(tmpdir, output_format, use_titles):
input_name = 'flattentool/tests/fixtures/fundingproviders-grants_fixed_2_grants.json'
flatten(
input_name=input_name,
output_name=tmpdir.join('flattened').strpath+'.'+output_format,
output_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
root_list_path='grants',
root_id='',
use_titles=use_titles,
main_sheet_name='grants')
unflatten(
input_name=tmpdir.join('flattened').strpath+'.'+output_format,
output_name=tmpdir.join('roundtrip.json').strpath,
input_format=output_format,
schema='flattentool/tests/fixtures/360-giving-schema.json',
root_list_path='grants',
root_id='',
convert_titles=use_titles)
original_json = json.load(open(input_name))
roundtripped_json = json.load(tmpdir.join('roundtrip.json'))
assert original_json == roundtripped_json
def test_roundtrip_xml(tmpdir, output_format):
input_name = 'examples/iati/expected.xml'
flatten(
input_name=input_name,
output_name=tmpdir.join('flattened').strpath+'.'+output_format,
output_format=output_format,
root_list_path='iati-activity',
id_name='iati-identifier',
xml=True)
unflatten(
input_name=tmpdir.join('flattened').strpath+'.'+output_format,
output_name=tmpdir.join('roundtrip.xml').strpath,
input_format=output_format,
root_list_path='iati-activity',
id_name='iati-identifier',
xml=True)
original_xml = open(input_name, 'rb')
roundtripped_xml = tmpdir.join('roundtrip.xml').open('rb')
# Compare without ordering, by using dict_constructor=dict instead of
# OrderedDict
original = xmltodict.parse(original_xml, dict_constructor=dict)
roundtripped = xmltodict.parse(roundtripped_xml, dict_constructor=dict)
assert original == roundtripped
def convert_json(
upload_dir,
upload_url,
file_name,
lib_cove_config,
root_list_path=None,
root_id=None,
schema_url=None,
replace=False,
request=None,
flatten=False,
cache=True,
xml=False,
):
context = {}
converted_path = os.path.join(upload_dir, "flattened")
if root_list_path is None:
root_list_path = lib_cove_config.config["root_list_path"]
if root_id is None:
root_id = lib_cove_config.config["root_id"]
flatten_kwargs = dict(
output_name=converted_path,
main_sheet_name=root_list_path,
root_list_path=root_list_path,
root_id=root_id,
schema=schema_url,
disable_local_refs=lib_cove_config.config["flatten_tool"]["disable_local_refs"],
remove_empty_schema_columns=lib_cove_config.config["flatten_tool"][
"remove_empty_schema_columns"
],
root_is_list=lib_cove_config.config.get("root_is_list", False),
)
if xml:
flatten_kwargs["xml"] = True
flatten_kwargs["id_name"] = lib_cove_config.config.get("id_name", "id")
try:
conversion_warning_cache_path = os.path.join(
upload_dir, "conversion_warning_messages.json"
)
conversion_exists = os.path.exists(converted_path + ".xlsx")
if not conversion_exists or replace:
with warnings.catch_warnings(record=True) as conversion_warnings:
if flatten or (replace and conversion_exists):
flattentool.flatten(file_name, **flatten_kwargs)
else:
return {"conversion": "flattenable"}
context["conversion_warning_messages"] = filter_conversion_warnings(
conversion_warnings
)
if cache:
with open(conversion_warning_cache_path, "w+") as fp:
json.dump(context["conversion_warning_messages"], fp)
elif os.path.exists(conversion_warning_cache_path):
with open(conversion_warning_cache_path) as fp:
context["conversion_warning_messages"] = json.load(fp)
context["converted_file_size"] = os.path.getsize(converted_path + ".xlsx")
conversion_warning_cache_path_titles = os.path.join(
upload_dir, "conversion_warning_messages_titles.json"
)
if lib_cove_config.config["convert_titles"]:
with warnings.catch_warnings(record=True) as conversion_warnings_titles:
flatten_kwargs.update(
dict(output_name=converted_path + "-titles", use_titles=True)
)
if not os.path.exists(converted_path + "-titles.xlsx") or replace:
flattentool.flatten(file_name, **flatten_kwargs)
context[
"conversion_warning_messages_titles"
] = filter_conversion_warnings(conversion_warnings_titles)
with open(conversion_warning_cache_path_titles, "w+") as fp:
json.dump(context["conversion_warning_messages_titles"], fp)
elif os.path.exists(conversion_warning_cache_path_titles):
with open(conversion_warning_cache_path_titles) as fp:
context["conversion_warning_messages_titles"] = json.load(fp)
context["converted_file_size_titles"] = os.path.getsize(
converted_path + "-titles.xlsx"
)
except BadlyFormedJSONError as err:
raise err
except Exception as err:
logger.exception(err, extra={"request": request})
return {"conversion": "flatten", "conversion_error": repr(err)}
context.update(
{
"conversion": "flatten",
"converted_path": converted_path,
"converted_url": "{}{}flattened".format(
upload_url, "" if upload_url.endswith("/") else "/"
),
}
)
return context
def convert_json(upload_dir,
upload_url,
file_name,
schema_url=None,
replace=False,
request=None,
flatten=False,
cache=True,
xml=False):
context = {}
converted_path = os.path.join(upload_dir, 'flattened')
flatten_kwargs = dict(output_name=converted_path,
main_sheet_name=config['root_list_path'],
root_list_path=config['root_list_path'],
root_id=config['root_id'],
schema=schema_url)
if xml:
flatten_kwargs['xml'] = True
flatten_kwargs['id_name'] = config.get('id_name', 'id')
try:
conversion_warning_cache_path = os.path.join(
upload_dir, 'conversion_warning_messages.json')
conversion_exists = os.path.exists(converted_path + '.xlsx')
if not conversion_exists or replace:
with warnings.catch_warnings(record=True) as conversion_warnings:
if flatten or (replace and conversion_exists):
flattentool.flatten(file_name, **flatten_kwargs)
else:
return {'conversion': 'flattenable'}
context[
'conversion_warning_messages'] = filter_conversion_warnings(
conversion_warnings)
if cache:
with open(conversion_warning_cache_path, 'w+') as fp:
json.dump(context['conversion_warning_messages'], fp)
elif os.path.exists(conversion_warning_cache_path):
with open(conversion_warning_cache_path) as fp:
context['conversion_warning_messages'] = json.load(fp)
context['converted_file_size'] = os.path.getsize(converted_path +
'.xlsx')
conversion_warning_cache_path_titles = os.path.join(
upload_dir, 'conversion_warning_messages_titles.json')
if config['convert_titles']:
with warnings.catch_warnings(
record=True) as conversion_warnings_titles:
flatten_kwargs.update(
dict(output_name=converted_path + '-titles',
use_titles=True))
if not os.path.exists(converted_path +
'-titles.xlsx') or replace:
flattentool.flatten(file_name, **flatten_kwargs)
context[
'conversion_warning_messages_titles'] = filter_conversion_warnings(
conversion_warnings_titles)
with open(conversion_warning_cache_path_titles,
'w+') as fp:
json.dump(
context['conversion_warning_messages_titles'], fp)
elif os.path.exists(conversion_warning_cache_path_titles):
with open(conversion_warning_cache_path_titles) as fp:
context[
'conversion_warning_messages_titles'] = json.load(
fp)
context['converted_file_size_titles'] = os.path.getsize(
converted_path + '-titles.xlsx')
except BadlyFormedJSONError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
_('We think you tried to upload a JSON file, but it is not well formed JSON.\n\nError message: {}'
.format(err))
})
except Exception as err:
logger.exception(err, extra={
'request': request,
})
return {'conversion': 'flatten', 'conversion_error': repr(err)}
context.update({
'conversion':
'flatten',
'converted_path':
converted_path,
'converted_url':
'{}{}flattened'.format(upload_url,
'' if upload_url.endswith('/') else '/')
})
return context
