def raise_missing_package_error():
raise CoveInputDataError(
context={
"sub_title":
_("Missing OCDS package"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_(
mark_safe(
"We could not detect a package structure at the top-level of your data. "
'OCDS releases and records should be published within a <a href="http://'
'standard.open-contracting.org/latest/en/schema/release_package/">release '
'package </a> or <a href="http://standard.open-contracting.org/latest/en'
'/schema/record_package/"> record package</a> to provide important meta-'
'data. For more information, please refer to the <a href="http://standard.'
'open-contracting.org/latest/en/getting_started/releases_and_records/"> '
"Releases and Records section </a> in the OCDS documentation.\n\n<span "
'class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span> '
"<strong>Error message:</strong> <em>Missing OCDS package</em>"
)),
"error":
_("Missing OCDS package"),
})
def get_tree(data_file):
with open(data_file, 'rb') as fp:
try:
tree = etree.parse(fp)
except lxml.etree.XMLSyntaxError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
_(
format_html(
'We think you tried to upload a XML file, but it is not well formed XML.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}',
err)),
'error':
format(err)
})
except UnicodeDecodeError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
_(
format_html(
'We think you tried to upload a XML file, but the encoding is incorrect.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}',
err)),
'error':
format(err)
})
return tree
def raise_invalid_version_argument(version):
raise CoveInputDataError(
context={
"sub_title": _("Unrecognised version of the schema"),
"link": "index",
"link_text": _("Try Again"),
"msg": _(
format_html(
"We think you tried to run your data against an unrecognised version of "
'the schema.\n\n<span class="glyphicon glyphicon-exclamation-sign" '
'aria-hidden="true"></span> <strong>Error message:</strong> <em>{}</em> is '
"not a recognised choice for the schema version",
version,
)
),
"error": _("%(version)s is not a known schema version", version),
}
)
def raise_json_deref_error(error):
raise CoveInputDataError(
context={
"sub_title": _("JSON reference error"),
"link": "index",
"link_text": _("Try Again"),
"msg": _(
format_html(
"We have detected a JSON reference error in the schema. This <em> may be "
"</em> due to some extension trying to resolve non-existing references. "
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
"</span> <strong>Error message:</strong> <em>{}</em>",
error,
)
),
"error": _("%(error)s", error),
}
)
def raise_invalid_version_data_with_patch(version):
raise CoveInputDataError(
context={
"sub_title": _("Version format does not comply with the schema"),
"link": "index",
"link_text": _("Try Again"),
"msg": _(
format_html(
'The value for the <em>"version"</em> field in your data follows the '
"<em>major.minor.patch</em> pattern but according to the schema the patch digit "
'shouldn\'t be included (e.g. <em>"1.1.0"</em> should appear as <em>"1.1"</em> in '
"your data as this tool always uses the latest patch release for a major.minor "
'version).\n\nPlease get rid of the patch digit and try again.\n\n<span class="glyphicon '
'glyphicon-exclamation-sign" aria-hidden="true"></span> <strong>Error message: '
"</strong> <em>{}</em> format does not comply with the schema",
version,
)
),
"error": _("%(version)s is not a known schema version", version),
}
)
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config["current_language"] = translation.get_language(
)
lib_cove_ocds_config.config[
"schema_version_choices"] = settings.COVE_CONFIG[
"schema_version_choices"]
lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[
"schema_codelists"]
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context["file_type"]
post_version_choice = request.POST.get(
"version", lib_cove_ocds_config.config["schema_version"])
replace = False
validation_errors_path = os.path.join(upload_dir,
"validation_errors-3.json")
if file_type == "json":
# open the data first so we can inspect for record package
with open(file_name, encoding="utf-8") as fp:
try:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
except ValueError as err:
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_(
format_html(
"We think you tried to upload a JSON file, but it is not well formed JSON."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
"</span> <strong>Error message:</strong> {}",
err,
)),
"error":
format(err),
})
if not isinstance(json_data, dict):
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_("OCDS JSON should have an object as the top level, the JSON you supplied does not."
),
})
version_in_data = json_data.get("version", "")
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=json_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile(
"^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = "{} (it must be a string)".format(
str(version_in_data))
context["unrecognized_version_data"] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
if "records" in json_data:
context["conversion"] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, "flattened")
replace_converted = replace and os.path.exists(converted_path +
".xlsx")
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore'
) # flattentool uses UserWarning, so can't set a specific category
convert_json_context = convert_json(
upload_dir,
upload_url,
file_name,
lib_cove_ocds_config,
schema_url=schema_url,
replace=replace_converted,
request=request,
flatten=request.POST.get("flatten"),
)
context.update(convert_json_context)
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(
select_version="1.1",
lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name,
metatab_schema_url, file_type)
if "version" not in metatab_data:
metatab_data["version"] = "1.0"
else:
db_data.data_schema_version = metatab_data["version"]
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=metatab_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get("version")
if re.compile("^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
context["unrecognized_version_data"] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
pkg_url = schema_ocds.release_pkg_schema_url
if file_type != "csv":
# ORIGINAL UNFLATTEN
conversion_context = convert_spreadsheet(
upload_dir,
upload_url,
file_name,
file_type,
lib_cove_ocds_config,
schema_url=schema_url,
pkg_schema_url=pkg_url,
replace=replace,
)
else:
# Convert Simple CSV to flat OCDS and return context
conversion_context = convert_simple_csv_submission(
db_data,
lib_cove_ocds_config,
schema_url,
replace=replace,
)
context.update(conversion_context)
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context,
upload_dir,
json_data,
schema_ocds,
cache=settings.CACHE_VALIDATION_ERRORS)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
schema_version = getattr(schema_ocds, "version", None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
context.update({
"data_schema_version":
db_data.schema_version,
"first_render":
not db_data.rendered,
"validation_errors_grouped":
group_validation_errors(context["validation_errors"]),
})
ocds_show_schema = SchemaOCDS()
ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj(
deref=True)
if "records" in json_data:
template = "cove_ocds/explore_record.html"
if hasattr(json_data, "get") and hasattr(json_data.get("records"),
"__iter__"):
context["records"] = json_data["records"]
else:
context["records"] = []
if isinstance(json_data["records"],
list) and len(json_data["records"]) < 100:
context["ocds_show_data"] = ocds_show_data(json_data,
ocds_show_deref_schema)
else:
template = "silvereye/explore_release.html"
if hasattr(json_data, "get") and hasattr(json_data.get("releases"),
"__iter__"):
context["releases"] = json_data["releases"]
if (isinstance(json_data["releases"], list)
and len(json_data["releases"]) < 100):
context["ocds_show_data"] = ocds_show_data(
json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context["releases"]:
if hasattr(release, "get") and release.get("date"):
if validate_rfc3339(release["date"]):
release["date"] = parser.parse(release["date"])
else:
release["date"] = None
try:
trans_date = release["contracts"][0]["implementation"][
"transactions"][0]["date"]
parsed_trans_date = parser.parse(trans_date)
release["contracts"][0]["implementation"]["transactions"][
0]["date"] = parsed_trans_date
except KeyError:
pass
if context.get("releases_aggregates"):
date_fields = [
"max_award_date",
"max_contract_date",
"max_release_date",
"max_tender_date",
"min_award_date",
"min_contract_date",
"min_release_date",
"min_tender_date",
]
for field in date_fields:
if context["releases_aggregates"].get(field):
if validate_rfc3339(
context["releases_aggregates"][field]):
context["releases_aggregates"][
field] = parser.parse(
context["releases_aggregates"][field])
else:
context["releases_aggregates"][field] = None
else:
context["releases"] = []
# Include field coverage report
original_file_path = context["original_file"]["path"]
mapper = CSVMapper(csv_path=original_file_path)
db_data.notice_type = mapper.release_type
db_data.save()
coverage_context = mapper.get_coverage_context()
context.update({
"field_coverage": coverage_context,
})
ocds_validation_errors, simple_csv_errors = prepare_simple_csv_validation_errors(
context["validation_errors"], mapper,
coverage_context["required_fields_missing"])
context.update({
"ocds_validation_errors": ocds_validation_errors,
"simple_csv_errors": simple_csv_errors,
"csv_mapper": mapper,
})
# Silvereye: Insert OCDS data
releases = context.get("releases")
if releases:
# If we don't have validation errors
validation_errors_grouped = context["validation_errors_grouped"]
if not validation_errors_grouped:
json_string = json.dumps(json_data,
indent=2,
sort_keys=True,
cls=DjangoJSONEncoder)
UpsertDataHelpers().upsert_ocds_data(json_string,
supplied_data=db_data)
average_field_completion = coverage_context.get(
"average_field_completion")
inst, created = FieldCoverage.objects.update_or_create(
file_submission=db_data,
defaults={
"tenders_field_coverage":
average_field_completion
if mapper.release_type == "tender" else None,
"awards_field_coverage":
average_field_completion
if mapper.release_type == "award" else None,
"spend_field_coverage":
average_field_completion
if mapper.release_type == "spend" else None,
})
update_publisher_monthly_counts()
return render(request, template, context)
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config["current_language"] = translation.get_language(
)
lib_cove_ocds_config.config[
"schema_version_choices"] = settings.COVE_CONFIG[
"schema_version_choices"]
lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[
"schema_codelists"]
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context["file_type"]
post_version_choice = request.POST.get("version")
replace = False
validation_errors_path = os.path.join(upload_dir,
"validation_errors-3.json")
if file_type == "json":
# open the data first so we can inspect for record package
with open(file_name, encoding="utf-8") as fp:
try:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
except UnicodeError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
format_html(
_("The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON follows the I-JSON format, which requires UTF-8 encoding. Ensure that your file uses UTF-8 encoding, then try uploading again."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}'),
err),
'error':
format(err)
})
except ValueError as err:
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
format_html(
_(
"We think you tried to upload a JSON file, but it is not well formed JSON."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
"</span> <strong>Error message:</strong> {}",
),
err,
),
"error":
format(err),
})
if not isinstance(json_data, dict):
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_("OCDS JSON should have an object as the top level, the JSON you supplied does not."
),
})
version_in_data = json_data.get("version", "")
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version,
release_data=json_data,
lib_cove_ocds_config=lib_cove_ocds_config,
record_pkg="records" in json_data)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile(
"^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = "{} (it must be a string)".format(
str(version_in_data))
context["unrecognized_version_data"] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
if "records" in json_data:
context["conversion"] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, "flattened")
replace_converted = replace and os.path.exists(converted_path +
".xlsx")
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore'
) # flattentool uses UserWarning, so can't set a specific category
convert_json_context = convert_json(
upload_dir,
upload_url,
file_name,
lib_cove_ocds_config,
schema_url=url,
replace=replace_converted,
request=request,
flatten=request.POST.get("flatten"),
)
context.update(convert_json_context)
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(
select_version="1.1",
lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name,
metatab_schema_url, file_type)
if "version" not in metatab_data:
metatab_data["version"] = "1.0"
else:
db_data.data_schema_version = metatab_data["version"]
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=metatab_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get("version")
if re.compile("^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
context["unrecognized_version_data"] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
pkg_url = schema_ocds.pkg_schema_url
context.update(
convert_spreadsheet(
upload_dir,
upload_url,
file_name,
file_type,
lib_cove_ocds_config,
schema_url=url,
pkg_schema_url=pkg_url,
replace=replace,
))
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context, upload_dir, json_data, schema_ocds)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
context.update({
"data_schema_version":
db_data.data_schema_version,
"first_render":
not db_data.rendered,
"validation_errors_grouped":
group_validation_errors(context["validation_errors"]),
})
schema_version = getattr(schema_ocds, "version", None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
if "records" in json_data:
ocds_show_schema = SchemaOCDS(record_pkg=True)
ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True)
template = "cove_ocds/explore_record.html"
if hasattr(json_data, "get") and hasattr(json_data.get("records"),
"__iter__"):
context["records"] = json_data["records"]
else:
context["records"] = []
if isinstance(json_data["records"],
list) and len(json_data["records"]) < 100:
context["ocds_show_data"] = ocds_show_data(json_data,
ocds_show_deref_schema)
else:
ocds_show_schema = SchemaOCDS(record_pkg=False)
ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True)
template = "cove_ocds/explore_release.html"
if hasattr(json_data, "get") and hasattr(json_data.get("releases"),
"__iter__"):
context["releases"] = json_data["releases"]
if (isinstance(json_data["releases"], list)
and len(json_data["releases"]) < 100):
context["ocds_show_data"] = ocds_show_data(
json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context["releases"]:
if hasattr(release, "get") and release.get("date"):
if validate_rfc3339(release["date"]):
release["date"] = parser.parse(release["date"])
else:
release["date"] = None
if context.get("releases_aggregates"):
date_fields = [
"max_award_date",
"max_contract_date",
"max_release_date",
"max_tender_date",
"min_award_date",
"min_contract_date",
"min_release_date",
"min_tender_date",
]
for field in date_fields:
if context["releases_aggregates"].get(field):
if validate_rfc3339(
context["releases_aggregates"][field]):
context["releases_aggregates"][
field] = parser.parse(
context["releases_aggregates"][field])
else:
context["releases_aggregates"][field] = None
else:
context["releases"] = []
return render(request, template, context)
def explore_bods(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_bods_config = LibCoveBODSConfig()
lib_cove_bods_config.config['root_list_path'] = settings.COVE_CONFIG[
'root_list_path']
lib_cove_bods_config.config['root_id'] = settings.COVE_CONFIG['root_id']
lib_cove_bods_config.config['id_name'] = settings.COVE_CONFIG['id_name']
lib_cove_bods_config.config['root_is_list'] = settings.COVE_CONFIG[
'root_is_list']
lib_cove_bods_config.config[
'bods_additional_checks_person_birthdate_max_year'] = datetime.datetime.now(
).year
lib_cove_bods_config.config['bods_additional_checks_person_birthdate_min_year'] = \
datetime.datetime.now().year - 120
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context['file_type']
if file_type == 'json':
# open the data first so we can inspect for record package
with open(file_name, encoding='utf-8') as fp:
try:
json_data = json.load(fp, parse_float=Decimal)
except ValueError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
_(
format_html(
'We think you tried to upload a JSON file, but it is not well formed JSON.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}',
err)),
'error':
format(err)
})
if not isinstance(json_data, list):
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
_('BODS JSON should have a list as the top level, the JSON you supplied does not.'
),
})
schema_bods = SchemaBODS(json_data=json_data,
lib_cove_bods_config=lib_cove_bods_config)
context.update(
convert_json(upload_dir,
upload_url,
file_name,
lib_cove_bods_config,
schema_url=schema_bods.release_pkg_schema_url,
replace=True,
request=request,
flatten=True))
else:
schema_bods = SchemaBODS(lib_cove_bods_config=lib_cove_bods_config)
context.update(
convert_spreadsheet(upload_dir,
upload_url,
file_name,
file_type,
lib_cove_bods_config,
schema_url=schema_bods.release_pkg_schema_url))
with open(context['converted_path'], encoding='utf-8') as fp:
json_data = json.load(fp, parse_float=Decimal)
context = common_checks_bods(context,
upload_dir,
json_data,
schema_bods,
lib_cove_bods_config=lib_cove_bods_config)
if not db_data.rendered:
db_data.rendered = True
db_data.save()
# Some extra info from the Schema
context['schema_version_used'] = schema_bods.schema_version
# We need to calculate some stats for showing in the view
total_ownership_or_control_interest_statements = 0
for key, count in context['statistics'][
'count_ownership_or_control_statement_interest_statement_types'].items(
):
total_ownership_or_control_interest_statements += count
context['statistics'][
'count_ownership_or_control_interest_statement'] = total_ownership_or_control_interest_statements # noqa
# The use of r_e_type is to stop flake8 complaining about line length
r_e_type = 'registeredEntity'
context['statistics'][
'count_entities_registeredEntity_legalEntity_with_any_identifier'] = (
context['statistics']
['count_entity_statements_types_with_any_identifier'][r_e_type] +
context['statistics']
['count_entity_statements_types_with_any_identifier']
['legalEntity'])
context['statistics'][
'count_entities_registeredEntity_legalEntity_with_any_identifier_with_id_and_scheme'] = (
context['statistics']
['count_entity_statements_types_with_any_identifier_with_id_and_scheme']
[r_e_type] + context['statistics']
['count_entity_statements_types_with_any_identifier_with_id_and_scheme']
['legalEntity'])
context['statistics']['count_entities_registeredEntity_legalEntity'] = (
context['statistics']['count_entity_statements_types'][r_e_type] +
context['statistics']['count_entity_statements_types']['legalEntity'])
template = 'cove_bods/explore.html'
return render(request, template, context)
def explore_bods(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_bods_config = LibCoveBODSConfig()
lib_cove_bods_config.config['root_list_path'] = settings.COVE_CONFIG['root_list_path']
lib_cove_bods_config.config['root_id'] = settings.COVE_CONFIG['root_id']
lib_cove_bods_config.config['id_name'] = settings.COVE_CONFIG['id_name']
lib_cove_bods_config.config['root_is_list'] = settings.COVE_CONFIG['root_is_list']
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context['file_type']
if file_type == 'json':
# open the data first so we can inspect for record package
with open(file_name, encoding='utf-8') as fp:
try:
json_data = json.load(fp, parse_float=Decimal)
except ValueError as err:
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}', err)),
'error': format(err)
})
if not isinstance(json_data, list):
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _('BODS JSON should have an list as the top level, the JSON you supplied does not.'),
})
schema_bods = SchemaBODS(lib_cove_bods_config=lib_cove_bods_config)
context.update(convert_json(upload_dir, upload_url, file_name, lib_cove_bods_config,
schema_url=schema_bods.release_pkg_schema_url, replace=True,
request=request, flatten=True))
else:
schema_bods = SchemaBODS(lib_cove_bods_config=lib_cove_bods_config)
context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_bods_config,
schema_url=schema_bods.release_pkg_schema_url))
with open(context['converted_path'], encoding='utf-8') as fp:
json_data = json.load(fp, parse_float=Decimal)
context = common_checks_bods(context, upload_dir, json_data, schema_bods)
if not db_data.rendered:
db_data.rendered = True
db_data.save()
template = 'cove_bods/explore.html'
return render(request, template, context)