def explore_360(request, pk, template='cove_360/explore.html'):
schema_360 = Schema360()
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_config = LibCoveConfig()
lib_cove_config.config.update(settings.COVE_CONFIG)
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context['file_type']
if file_type == 'json':
# open the data first so we can inspect for record package
with open(file_name, encoding='utf-8') as fp:
try:
json_data = json.load(fp, parse_float=Decimal)
except ValueError as err:
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}', err)),
'error': format(err)
})
if not isinstance(json_data, dict):
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _('360Giving JSON should have an object as the top level, the JSON you supplied does not.'),
})
context.update(convert_json(upload_dir, upload_url, file_name, schema_url=schema_360.release_schema_url,
request=request, flatten=request.POST.get('flatten'),
lib_cove_config=lib_cove_config))
else:
context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_config, schema_360.release_schema_url,
schema_360.release_pkg_schema_url))
with open(context['converted_path'], encoding='utf-8') as fp:
json_data = json.load(fp, parse_float=Decimal)
context = common_checks_360(context, upload_dir, json_data, schema_360)
if hasattr(json_data, 'get') and hasattr(json_data.get('grants'), '__iter__'):
context['grants'] = json_data['grants']
else:
context['grants'] = []
context['first_render'] = not db_data.rendered
if not db_data.rendered:
db_data.rendered = True
db_data.save()
return render(request, template, context)
def explore_data_context_iati(request, context, db_data, error):
lib_cove_config = LibCoveConfig()
lib_cove_config.config.update(settings.COVE_CONFIG)
file_type = context['file_type']
if file_type != 'xml':
schema_iati = SchemaIATI()
context.update(
convert_spreadsheet(db_data.upload_dir(),
db_data.upload_url(),
db_data.original_file.file.name,
file_type,
lib_cove_config,
xml=True,
xml_schemas=[
schema_iati.activity_schema,
schema_iati.organisation_schema,
schema_iati.common_schema,
]))
data_file = context['converted_path']
else:
data_file = db_data.original_file.file.name
tree = get_tree(data_file)
context = common_checks_context_iati(context, db_data.upload_dir(),
data_file, file_type, tree)
context['first_render'] = not db_data.rendered
context['invalid_embedded_codelist_values'] = aggregate_results(
context['invalid_embedded_codelist_values'])
context['invalid_non_embedded_codelist_values'] = aggregate_results(
context['invalid_non_embedded_codelist_values'])
context['iati_identifiers_count'] = iati_identifier_count(tree)
context['organisation_identifier_count'] = organisation_identifier_count(
tree)
if file_type == 'xml':
if context['organisation_identifier_count']:
root_list_path = 'iati-organisation'
root_id = 'organisation-identifier'
else:
root_list_path = 'iati-activity'
root_id = None
context.update(
convert_json(db_data.upload_dir(),
db_data.upload_url(),
db_data.original_file.file.name,
root_list_path=root_list_path,
root_id=root_id,
request=request,
flatten=request.POST.get('flatten'),
xml=True,
lib_cove_config=lib_cove_config))
if not db_data.rendered:
db_data.rendered = True
return context, db_data, error
def test_convert_org_xml_1():
cove_temp_folder = tempfile.mkdtemp(prefix="lib-cove-iati-tests-",
dir=tempfile.gettempdir())
organisation_xml_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"fixtures",
"converters",
"convert_org_1.xml",
)
lib_cove_config = LibCoveConfig()
output = convert_json(
cove_temp_folder,
"",
organisation_xml_filename,
lib_cove_config,
flatten=True,
xml=True,
root_list_path="iati-organisation",
root_id="organisation-identifier",
)
assert output["converted_url"] == "/flattened"
assert len(output["conversion_warning_messages"]) == 0
assert output["conversion"] == "flatten"
conversion_warning_messages_name = os.path.join(
cove_temp_folder, "conversion_warning_messages.json")
assert os.path.isfile(conversion_warning_messages_name)
with open(conversion_warning_messages_name) as fp:
conversion_warning_messages_data = json.load(fp)
assert conversion_warning_messages_data == []
assert os.path.isfile(os.path.join(cove_temp_folder, "flattened.xlsx"))
assert os.path.isfile(
os.path.join(cove_temp_folder, "flattened", "iati-organisation.csv"))
with open(
os.path.join(cove_temp_folder, "flattened",
"iati-organisation.csv"), "r") as csvfile:
csvreader = csv.reader(csvfile)
header = next(csvreader)
assert header[0] == "organisation-identifier"
assert header[4] == "name/narrative"
row1 = next(csvreader)
assert row1[0] == "GB-GOV-1"
assert row1[4] == "UK Department for International Development"
def test_convert_activity_xml_1():
cove_temp_folder = tempfile.mkdtemp(prefix="lib-cove-iati-tests-",
dir=tempfile.gettempdir())
activity_xml_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"fixtures",
"converters",
"convert_activity_1.xml",
)
lib_cove_config = LibCoveConfig()
output = convert_json(
cove_temp_folder,
"",
activity_xml_filename,
lib_cove_config,
flatten=True,
xml=True,
root_list_path="iati-activity",
)
assert output["converted_url"] == "/flattened"
assert len(output["conversion_warning_messages"]) == 0
assert output["conversion"] == "flatten"
conversion_warning_messages_name = os.path.join(
cove_temp_folder, "conversion_warning_messages.json")
assert os.path.isfile(conversion_warning_messages_name)
with open(conversion_warning_messages_name) as fp:
conversion_warning_messages_data = json.load(fp)
assert conversion_warning_messages_data == []
assert os.path.isfile(os.path.join(cove_temp_folder, "flattened.xlsx"))
assert os.path.isfile(
os.path.join(cove_temp_folder, "flattened", "iati-activity.csv"))
with open(os.path.join(cove_temp_folder, "flattened", "iati-activity.csv"),
"r") as csvfile:
csvreader = csv.reader(csvfile)
header = next(csvreader)
assert header[0] == "@default-currency"
assert header[1] == "iati-identifier"
row1 = next(csvreader)
assert row1[0] == "GBP"
assert row1[1] == "GB-TEST-13-example_ODSC_2019"
def test_convert_json_root_is_list_1():
cove_temp_folder = tempfile.mkdtemp(prefix="lib-cove-ocds-tests-",
dir=tempfile.gettempdir())
json_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"fixtures",
"converters",
"convert_json_root_is_list_1.json",
)
lib_cove_config = LibCoveConfig()
lib_cove_config.config["root_is_list"] = True
output = convert_json(cove_temp_folder,
"",
json_filename,
lib_cove_config,
flatten=True)
assert output["converted_url"] == "/flattened"
assert len(output["conversion_warning_messages"]) == 0
assert output["conversion"] == "flatten"
conversion_warning_messages_name = os.path.join(
cove_temp_folder, "conversion_warning_messages.json")
assert os.path.isfile(conversion_warning_messages_name)
with open(conversion_warning_messages_name) as fp:
conversion_warning_messages_data = json.load(fp)
assert conversion_warning_messages_data == []
assert os.path.isfile(
os.path.join(cove_temp_folder, "flattened", "main.csv"))
with open(os.path.join(cove_temp_folder, "flattened", "main.csv"),
"r") as csvfile:
csvreader = csv.reader(csvfile)
header = next(csvreader)
assert header[0] == "id"
assert header[1] == "title"
row1 = next(csvreader)
assert row1[0] == "1"
assert row1[1] == "Cat"
row2 = next(csvreader)
assert row2[0] == "2"
assert row2[1] == "Hat"
def explore_iati(request, pk):
context, db_data, error = explore_data_context(request, pk, get_file_type)
if error:
return error
lib_cove_config = LibCoveConfig()
lib_cove_config.config.update(settings.COVE_CONFIG)
file_type = context['file_type']
if file_type != 'xml':
schema_iati = SchemaIATI()
context.update(convert_spreadsheet(
db_data.upload_dir(), db_data.upload_url(), db_data.original_file.file.name,
file_type, lib_cove_config, xml=True,
xml_schemas=[
schema_iati.activity_schema,
schema_iati.organisation_schema,
schema_iati.common_schema,
]))
data_file = context['converted_path']
else:
data_file = db_data.original_file.file.name
context.update(convert_json(db_data.upload_dir(), db_data.upload_url(), db_data.original_file.file.name,
request=request, flatten=request.POST.get('flatten'), xml=True, lib_cove_config=lib_cove_config))
tree = get_tree(data_file)
context = common_checks_context_iati(context, db_data.upload_dir(), data_file, file_type, tree)
context['first_render'] = not db_data.rendered
context['invalid_embedded_codelist_values'] = aggregate_results(context['invalid_embedded_codelist_values'])
context['invalid_non_embedded_codelist_values'] = aggregate_results(context['invalid_non_embedded_codelist_values'])
context['iati_identifiers_count'] = iati_identifier_count(tree)
context['organisation_identifier_count'] = organisation_identifier_count(tree)
if not db_data.rendered:
db_data.rendered = True
return render(request, 'cove_iati/explore.html', context)
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config["current_language"] = translation.get_language(
)
lib_cove_ocds_config.config[
"schema_version_choices"] = settings.COVE_CONFIG[
"schema_version_choices"]
lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[
"schema_codelists"]
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context["file_type"]
post_version_choice = request.POST.get(
"version", lib_cove_ocds_config.config["schema_version"])
replace = False
validation_errors_path = os.path.join(upload_dir,
"validation_errors-3.json")
if file_type == "json":
# open the data first so we can inspect for record package
with open(file_name, encoding="utf-8") as fp:
try:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
except ValueError as err:
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_(
format_html(
"We think you tried to upload a JSON file, but it is not well formed JSON."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
"</span> <strong>Error message:</strong> {}",
err,
)),
"error":
format(err),
})
if not isinstance(json_data, dict):
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_("OCDS JSON should have an object as the top level, the JSON you supplied does not."
),
})
version_in_data = json_data.get("version", "")
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=json_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile(
"^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = "{} (it must be a string)".format(
str(version_in_data))
context["unrecognized_version_data"] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
if "records" in json_data:
context["conversion"] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, "flattened")
replace_converted = replace and os.path.exists(converted_path +
".xlsx")
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore'
) # flattentool uses UserWarning, so can't set a specific category
convert_json_context = convert_json(
upload_dir,
upload_url,
file_name,
lib_cove_ocds_config,
schema_url=schema_url,
replace=replace_converted,
request=request,
flatten=request.POST.get("flatten"),
)
context.update(convert_json_context)
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(
select_version="1.1",
lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name,
metatab_schema_url, file_type)
if "version" not in metatab_data:
metatab_data["version"] = "1.0"
else:
db_data.data_schema_version = metatab_data["version"]
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=metatab_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get("version")
if re.compile("^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
context["unrecognized_version_data"] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
pkg_url = schema_ocds.release_pkg_schema_url
if file_type != "csv":
# ORIGINAL UNFLATTEN
conversion_context = convert_spreadsheet(
upload_dir,
upload_url,
file_name,
file_type,
lib_cove_ocds_config,
schema_url=schema_url,
pkg_schema_url=pkg_url,
replace=replace,
)
else:
# Convert Simple CSV to flat OCDS and return context
conversion_context = convert_simple_csv_submission(
db_data,
lib_cove_ocds_config,
schema_url,
replace=replace,
)
context.update(conversion_context)
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context,
upload_dir,
json_data,
schema_ocds,
cache=settings.CACHE_VALIDATION_ERRORS)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
schema_version = getattr(schema_ocds, "version", None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
context.update({
"data_schema_version":
db_data.schema_version,
"first_render":
not db_data.rendered,
"validation_errors_grouped":
group_validation_errors(context["validation_errors"]),
})
ocds_show_schema = SchemaOCDS()
ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj(
deref=True)
if "records" in json_data:
template = "cove_ocds/explore_record.html"
if hasattr(json_data, "get") and hasattr(json_data.get("records"),
"__iter__"):
context["records"] = json_data["records"]
else:
context["records"] = []
if isinstance(json_data["records"],
list) and len(json_data["records"]) < 100:
context["ocds_show_data"] = ocds_show_data(json_data,
ocds_show_deref_schema)
else:
template = "silvereye/explore_release.html"
if hasattr(json_data, "get") and hasattr(json_data.get("releases"),
"__iter__"):
context["releases"] = json_data["releases"]
if (isinstance(json_data["releases"], list)
and len(json_data["releases"]) < 100):
context["ocds_show_data"] = ocds_show_data(
json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context["releases"]:
if hasattr(release, "get") and release.get("date"):
if validate_rfc3339(release["date"]):
release["date"] = parser.parse(release["date"])
else:
release["date"] = None
try:
trans_date = release["contracts"][0]["implementation"][
"transactions"][0]["date"]
parsed_trans_date = parser.parse(trans_date)
release["contracts"][0]["implementation"]["transactions"][
0]["date"] = parsed_trans_date
except KeyError:
pass
if context.get("releases_aggregates"):
date_fields = [
"max_award_date",
"max_contract_date",
"max_release_date",
"max_tender_date",
"min_award_date",
"min_contract_date",
"min_release_date",
"min_tender_date",
]
for field in date_fields:
if context["releases_aggregates"].get(field):
if validate_rfc3339(
context["releases_aggregates"][field]):
context["releases_aggregates"][
field] = parser.parse(
context["releases_aggregates"][field])
else:
context["releases_aggregates"][field] = None
else:
context["releases"] = []
# Include field coverage report
original_file_path = context["original_file"]["path"]
mapper = CSVMapper(csv_path=original_file_path)
db_data.notice_type = mapper.release_type
db_data.save()
coverage_context = mapper.get_coverage_context()
context.update({
"field_coverage": coverage_context,
})
ocds_validation_errors, simple_csv_errors = prepare_simple_csv_validation_errors(
context["validation_errors"], mapper,
coverage_context["required_fields_missing"])
context.update({
"ocds_validation_errors": ocds_validation_errors,
"simple_csv_errors": simple_csv_errors,
"csv_mapper": mapper,
})
# Silvereye: Insert OCDS data
releases = context.get("releases")
if releases:
# If we don't have validation errors
validation_errors_grouped = context["validation_errors_grouped"]
if not validation_errors_grouped:
json_string = json.dumps(json_data,
indent=2,
sort_keys=True,
cls=DjangoJSONEncoder)
UpsertDataHelpers().upsert_ocds_data(json_string,
supplied_data=db_data)
average_field_completion = coverage_context.get(
"average_field_completion")
inst, created = FieldCoverage.objects.update_or_create(
file_submission=db_data,
defaults={
"tenders_field_coverage":
average_field_completion
if mapper.release_type == "tender" else None,
"awards_field_coverage":
average_field_completion
if mapper.release_type == "award" else None,
"spend_field_coverage":
average_field_completion
if mapper.release_type == "spend" else None,
})
update_publisher_monthly_counts()
return render(request, template, context)
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config["current_language"] = translation.get_language(
)
lib_cove_ocds_config.config[
"schema_version_choices"] = settings.COVE_CONFIG[
"schema_version_choices"]
lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[
"schema_codelists"]
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context["file_type"]
post_version_choice = request.POST.get("version")
replace = False
validation_errors_path = os.path.join(upload_dir,
"validation_errors-3.json")
if file_type == "json":
# open the data first so we can inspect for record package
with open(file_name, encoding="utf-8") as fp:
try:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
except UnicodeError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
format_html(
_("The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON follows the I-JSON format, which requires UTF-8 encoding. Ensure that your file uses UTF-8 encoding, then try uploading again."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}'),
err),
'error':
format(err)
})
except ValueError as err:
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
format_html(
_(
"We think you tried to upload a JSON file, but it is not well formed JSON."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
"</span> <strong>Error message:</strong> {}",
),
err,
),
"error":
format(err),
})
if not isinstance(json_data, dict):
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_("OCDS JSON should have an object as the top level, the JSON you supplied does not."
),
})
version_in_data = json_data.get("version", "")
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version,
release_data=json_data,
lib_cove_ocds_config=lib_cove_ocds_config,
record_pkg="records" in json_data)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile(
"^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = "{} (it must be a string)".format(
str(version_in_data))
context["unrecognized_version_data"] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
if "records" in json_data:
context["conversion"] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, "flattened")
replace_converted = replace and os.path.exists(converted_path +
".xlsx")
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore'
) # flattentool uses UserWarning, so can't set a specific category
convert_json_context = convert_json(
upload_dir,
upload_url,
file_name,
lib_cove_ocds_config,
schema_url=url,
replace=replace_converted,
request=request,
flatten=request.POST.get("flatten"),
)
context.update(convert_json_context)
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(
select_version="1.1",
lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name,
metatab_schema_url, file_type)
if "version" not in metatab_data:
metatab_data["version"] = "1.0"
else:
db_data.data_schema_version = metatab_data["version"]
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=metatab_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get("version")
if re.compile("^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
context["unrecognized_version_data"] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
pkg_url = schema_ocds.pkg_schema_url
context.update(
convert_spreadsheet(
upload_dir,
upload_url,
file_name,
file_type,
lib_cove_ocds_config,
schema_url=url,
pkg_schema_url=pkg_url,
replace=replace,
))
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context, upload_dir, json_data, schema_ocds)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
context.update({
"data_schema_version":
db_data.data_schema_version,
"first_render":
not db_data.rendered,
"validation_errors_grouped":
group_validation_errors(context["validation_errors"]),
})
schema_version = getattr(schema_ocds, "version", None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
if "records" in json_data:
ocds_show_schema = SchemaOCDS(record_pkg=True)
ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True)
template = "cove_ocds/explore_record.html"
if hasattr(json_data, "get") and hasattr(json_data.get("records"),
"__iter__"):
context["records"] = json_data["records"]
else:
context["records"] = []
if isinstance(json_data["records"],
list) and len(json_data["records"]) < 100:
context["ocds_show_data"] = ocds_show_data(json_data,
ocds_show_deref_schema)
else:
ocds_show_schema = SchemaOCDS(record_pkg=False)
ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True)
template = "cove_ocds/explore_release.html"
if hasattr(json_data, "get") and hasattr(json_data.get("releases"),
"__iter__"):
context["releases"] = json_data["releases"]
if (isinstance(json_data["releases"], list)
and len(json_data["releases"]) < 100):
context["ocds_show_data"] = ocds_show_data(
json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context["releases"]:
if hasattr(release, "get") and release.get("date"):
if validate_rfc3339(release["date"]):
release["date"] = parser.parse(release["date"])
else:
release["date"] = None
if context.get("releases_aggregates"):
date_fields = [
"max_award_date",
"max_contract_date",
"max_release_date",
"max_tender_date",
"min_award_date",
"min_contract_date",
"min_release_date",
"min_tender_date",
]
for field in date_fields:
if context["releases_aggregates"].get(field):
if validate_rfc3339(
context["releases_aggregates"][field]):
context["releases_aggregates"][
field] = parser.parse(
context["releases_aggregates"][field])
else:
context["releases_aggregates"][field] = None
else:
context["releases"] = []
return render(request, template, context)
def explore_bods(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_bods_config = LibCoveBODSConfig()
lib_cove_bods_config.config['root_list_path'] = settings.COVE_CONFIG[
'root_list_path']
lib_cove_bods_config.config['root_id'] = settings.COVE_CONFIG['root_id']
lib_cove_bods_config.config['id_name'] = settings.COVE_CONFIG['id_name']
lib_cove_bods_config.config['root_is_list'] = settings.COVE_CONFIG[
'root_is_list']
lib_cove_bods_config.config[
'bods_additional_checks_person_birthdate_max_year'] = datetime.datetime.now(
).year
lib_cove_bods_config.config['bods_additional_checks_person_birthdate_min_year'] = \
datetime.datetime.now().year - 120
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context['file_type']
if file_type == 'json':
# open the data first so we can inspect for record package
with open(file_name, encoding='utf-8') as fp:
try:
json_data = json.load(fp, parse_float=Decimal)
except ValueError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
_(
format_html(
'We think you tried to upload a JSON file, but it is not well formed JSON.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}',
err)),
'error':
format(err)
})
if not isinstance(json_data, list):
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
_('BODS JSON should have a list as the top level, the JSON you supplied does not.'
),
})
schema_bods = SchemaBODS(json_data=json_data,
lib_cove_bods_config=lib_cove_bods_config)
context.update(
convert_json(upload_dir,
upload_url,
file_name,
lib_cove_bods_config,
schema_url=schema_bods.release_pkg_schema_url,
replace=True,
request=request,
flatten=True))
else:
schema_bods = SchemaBODS(lib_cove_bods_config=lib_cove_bods_config)
context.update(
convert_spreadsheet(upload_dir,
upload_url,
file_name,
file_type,
lib_cove_bods_config,
schema_url=schema_bods.release_pkg_schema_url))
with open(context['converted_path'], encoding='utf-8') as fp:
json_data = json.load(fp, parse_float=Decimal)
context = common_checks_bods(context,
upload_dir,
json_data,
schema_bods,
lib_cove_bods_config=lib_cove_bods_config)
if not db_data.rendered:
db_data.rendered = True
db_data.save()
# Some extra info from the Schema
context['schema_version_used'] = schema_bods.schema_version
# We need to calculate some stats for showing in the view
total_ownership_or_control_interest_statements = 0
for key, count in context['statistics'][
'count_ownership_or_control_statement_interest_statement_types'].items(
):
total_ownership_or_control_interest_statements += count
context['statistics'][
'count_ownership_or_control_interest_statement'] = total_ownership_or_control_interest_statements # noqa
# The use of r_e_type is to stop flake8 complaining about line length
r_e_type = 'registeredEntity'
context['statistics'][
'count_entities_registeredEntity_legalEntity_with_any_identifier'] = (
context['statistics']
['count_entity_statements_types_with_any_identifier'][r_e_type] +
context['statistics']
['count_entity_statements_types_with_any_identifier']
['legalEntity'])
context['statistics'][
'count_entities_registeredEntity_legalEntity_with_any_identifier_with_id_and_scheme'] = (
context['statistics']
['count_entity_statements_types_with_any_identifier_with_id_and_scheme']
[r_e_type] + context['statistics']
['count_entity_statements_types_with_any_identifier_with_id_and_scheme']
['legalEntity'])
context['statistics']['count_entities_registeredEntity_legalEntity'] = (
context['statistics']['count_entity_statements_types'][r_e_type] +
context['statistics']['count_entity_statements_types']['legalEntity'])
template = 'cove_bods/explore.html'
return render(request, template, context)
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config['current_language'] = translation.get_language()
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context['file_type']
post_version_choice = request.POST.get('version')
replace = False
validation_errors_path = os.path.join(upload_dir, 'validation_errors-3.json')
if file_type == 'json':
# open the data first so we can inspect for record package
with open(file_name, encoding='utf-8') as fp:
try:
json_data = json.load(fp, parse_float=Decimal)
except ValueError as err:
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}', err)),
'error': format(err)
})
if not isinstance(json_data, dict):
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _('OCDS JSON should have an object as the top level, the JSON you supplied does not.'),
})
version_in_data = json_data.get('version', '')
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version, release_data=json_data, lib_cove_ocds_config=lib_cove_ocds_config)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile('^\d+\.\d+\.\d+$').match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = '{} (it must be a string)'.format(str(version_in_data))
context['unrecognized_version_data'] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
if 'records' in json_data:
context['conversion'] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, 'flattened')
replace_converted = replace and os.path.exists(converted_path + '.xlsx')
context.update(convert_json(upload_dir, upload_url, file_name, lib_cove_ocds_config,
schema_url=url, replace=replace_converted,
request=request, flatten=request.POST.get('flatten')))
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(select_version='1.1', lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type)
if 'version' not in metatab_data:
metatab_data['version'] = '1.0'
else:
db_data.data_schema_version = metatab_data['version']
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get('version')
if re.compile('^\d+\.\d+\.\d+$').match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(version_in_data)
else:
context['unrecognized_version_data'] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
pkg_url = schema_ocds.release_pkg_schema_url
context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_ocds_config,
schema_url=url,
pkg_schema_url=pkg_url, replace=replace))
with open(context['converted_path'], encoding='utf-8') as fp:
json_data = json.load(fp, parse_float=Decimal)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context, upload_dir, json_data, schema_ocds)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
context.update({
'data_schema_version': db_data.data_schema_version,
'first_render': not db_data.rendered,
'validation_errors_grouped': group_validation_errors(context['validation_errors']),
})
schema_version = getattr(schema_ocds, 'version', None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
ocds_show_schema = SchemaOCDS()
ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj(deref=True)
if 'records' in json_data:
template = 'cove_ocds/explore_record.html'
if hasattr(json_data, 'get') and hasattr(json_data.get('records'), '__iter__'):
context['records'] = json_data['records']
else:
context['records'] = []
if isinstance(json_data['records'], list) and len(json_data['records']) < 100:
context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema)
else:
template = 'cove_ocds/explore_release.html'
if hasattr(json_data, 'get') and hasattr(json_data.get('releases'), '__iter__'):
context['releases'] = json_data['releases']
if isinstance(json_data['releases'], list) and len(json_data['releases']) < 100:
context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context['releases']:
if hasattr(release, 'get') and release.get('date'):
if validate_rfc3339(release['date']):
release['date'] = parser.parse(release['date'])
else:
release['date'] = None
if context.get('releases_aggregates'):
date_fields = ['max_award_date', 'max_contract_date', 'max_release_date', 'max_tender_date', 'min_award_date', 'min_contract_date', 'min_release_date', 'min_tender_date']
for field in date_fields:
if context['releases_aggregates'].get(field):
if(validate_rfc3339(context['releases_aggregates'][field])):
context['releases_aggregates'][field] = parser.parse(context['releases_aggregates'][field])
else:
context['releases_aggregates'][field] = None
else:
context['releases'] = []
return render(request, template, context)
def ocds_json_output(
output_dir,
file,
schema_version,
convert,
cache_schema=False,
file_type=None,
json_data=None,
lib_cove_ocds_config=None,
record_pkg=False,
):
if not lib_cove_ocds_config:
lib_cove_ocds_config = LibCoveOCDSConfig()
# cache_schema is a deprecated option - now set cache_all_requests in the config instead.
if cache_schema:
lib_cove_ocds_config.config["cache_all_requests"] = True
context = {}
if not file_type:
file_type = get_file_type(file)
context = {"file_type": file_type}
if file_type == "json":
if not json_data:
with open(file, encoding="utf-8") as fp:
try:
json_data = json.load(fp, object_pairs_hook=OrderedDict)
except ValueError:
raise APIException("The file looks like invalid json")
schema_ocds = SchemaOCDS(
schema_version, json_data, lib_cove_ocds_config=lib_cove_ocds_config, record_pkg=record_pkg
)
if schema_ocds.invalid_version_data:
msg = "\033[1;31mThe schema version in your data is not valid. Accepted values: {}\033[1;m"
raise APIException(msg.format(str(list(schema_ocds.version_choices.keys()))))
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(output_dir, "")
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
if convert:
with warnings.catch_warnings():
warnings.filterwarnings("ignore") # flattentool uses UserWarning, so we can't set a specific category
context.update(
convert_json(output_dir, "", file, lib_cove_ocds_config, schema_url=url, flatten=True, cache=False)
)
else:
metatab_schema_url = SchemaOCDS(select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url
metatab_data = get_spreadsheet_meta_data(output_dir, file, metatab_schema_url, file_type=file_type)
schema_ocds = SchemaOCDS(schema_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config)
if schema_ocds.invalid_version_data:
msg = "\033[1;31mThe schema version in your data is not valid. Accepted values: {}\033[1;m"
raise APIException(msg.format(str(list(schema_ocds.version_choices.keys()))))
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(output_dir, "")
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
pkg_url = schema_ocds.pkg_schema_url
context.update(
convert_spreadsheet(
output_dir,
"",
file,
file_type,
lib_cove_ocds_config,
schema_url=url,
pkg_schema_url=pkg_url,
cache=False,
)
)
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp, object_pairs_hook=OrderedDict)
context = context_api_transform(
common_checks_ocds(context, output_dir, json_data, schema_ocds, api=True, cache=False)
)
if file_type == "xlsx":
# Remove unwanted files in the output
# TODO: can we do this by no writing the files in the first place?
os.remove(os.path.join(output_dir, "heading_source_map.json"))
os.remove(os.path.join(output_dir, "cell_source_map.json"))
return context
def explore_bods(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_bods_config = LibCoveBODSConfig()
lib_cove_bods_config.config['root_list_path'] = settings.COVE_CONFIG['root_list_path']
lib_cove_bods_config.config['root_id'] = settings.COVE_CONFIG['root_id']
lib_cove_bods_config.config['id_name'] = settings.COVE_CONFIG['id_name']
lib_cove_bods_config.config['root_is_list'] = settings.COVE_CONFIG['root_is_list']
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context['file_type']
if file_type == 'json':
# open the data first so we can inspect for record package
with open(file_name, encoding='utf-8') as fp:
try:
json_data = json.load(fp, parse_float=Decimal)
except ValueError as err:
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}', err)),
'error': format(err)
})
if not isinstance(json_data, list):
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _('BODS JSON should have an list as the top level, the JSON you supplied does not.'),
})
schema_bods = SchemaBODS(lib_cove_bods_config=lib_cove_bods_config)
context.update(convert_json(upload_dir, upload_url, file_name, lib_cove_bods_config,
schema_url=schema_bods.release_pkg_schema_url, replace=True,
request=request, flatten=True))
else:
schema_bods = SchemaBODS(lib_cove_bods_config=lib_cove_bods_config)
context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_bods_config,
schema_url=schema_bods.release_pkg_schema_url))
with open(context['converted_path'], encoding='utf-8') as fp:
json_data = json.load(fp, parse_float=Decimal)
context = common_checks_bods(context, upload_dir, json_data, schema_bods)
if not db_data.rendered:
db_data.rendered = True
db_data.save()
template = 'cove_bods/explore.html'
return render(request, template, context)