def test_schema_ocds_extended_release_schema_file():
data = SuppliedData.objects.create()
with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_1_release_with_extensions_1_1.json')) as fp:
data.original_file.save('test.json', UploadedFile(fp))
fp.seek(0)
json_data = json.load(fp)
schema = SchemaOCDS(release_data=json_data)
assert not schema.extended
schema.get_release_schema_obj()
assert schema.extended
assert not schema.extended_schema_file
assert not schema.extended_schema_url
schema.create_extended_release_schema_file(data.upload_dir(), data.upload_url())
assert schema.extended_schema_file == os.path.join(data.upload_dir(), 'extended_release_schema.json')
assert schema.extended_schema_url == os.path.join(data.upload_url(), 'extended_release_schema.json')
json_data = json.loads('{"version": "1.1", "extensions": [], "releases": [{"ocid": "xx"}]}')
schema = SchemaOCDS(release_data=json_data)
schema.get_release_schema_obj()
schema.create_extended_release_schema_file(data.upload_dir(), data.upload_url())
assert not schema.extended
assert not schema.extended_schema_file
assert not schema.extended_schema_url
def test_schema_ocds_extensions(release_data, extensions, invalid_extension, extended, extends_schema):
schema = SchemaOCDS(release_data=release_data)
assert schema.extensions == extensions
assert not schema.extended
release_schema_obj = schema.get_release_schema_obj()
assert schema.invalid_extension == invalid_extension
assert schema.extended == extended
if extends_schema:
assert 'Metric' in release_schema_obj['definitions'].keys()
assert release_schema_obj['definitions']['Award']['properties'].get('agreedMetrics')
else:
assert 'Metric' not in release_schema_obj['definitions'].keys()
assert not release_schema_obj['definitions']['Award']['properties'].get('agreedMetrics')
def test_schema_ocds_extensions(release_data, extensions, invalid_extension, extended, extends_schema):
schema = SchemaOCDS(release_data=release_data)
assert schema.extensions == extensions
assert not schema.extended
release_schema_obj = schema.get_release_schema_obj()
assert schema.invalid_extension == invalid_extension
assert schema.extended == extended
if extends_schema:
assert 'Metric' in release_schema_obj['definitions'].keys()
assert release_schema_obj['definitions']['Award']['properties'].get('agreedMetrics')
else:
assert 'Metric' not in release_schema_obj['definitions'].keys()
assert not release_schema_obj['definitions']['Award']['properties'].get('agreedMetrics')
def test_get_additional_codelist_values():
with open(
os.path.join("tests", "fixtures",
"tenders_releases_2_releases_codelists.json")) as fp:
json_data_w_additial_codelists = json.load(fp)
schema_obj = SchemaOCDS(select_version="1.1")
additional_codelist_values = cove_common.get_additional_codelist_values(
schema_obj, json_data_w_additial_codelists)
assert additional_codelist_values == {
("releases/tag"): {
"codelist": "releaseTag.csv",
"codelist_url":
"https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/codelists/releaseTag.csv", # noqa: E501
"codelist_amend_urls": [],
"field": "tag",
"extension_codelist": False,
"isopen": False,
"path": "releases",
"values": ["oh no"],
},
("releases/tender/items/classification/scheme"): {
"codelist": "itemClassificationScheme.csv",
"codelist_url":
"https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/codelists/itemClassificationScheme.csv", # noqa: E501
"codelist_amend_urls": [],
"extension_codelist": False,
"field": "scheme",
"isopen": True,
"path": "releases/tender/items/classification",
"values": ["GSINS"],
},
}
def test_get_additional_codelist_values():
with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_2_releases_codelists.json')) as fp:
json_data_w_additial_codelists = json.load(fp)
schema_obj = SchemaOCDS(select_version='1.1')
additional_codelist_values = cove_common.get_additional_codelist_values(schema_obj, json_data_w_additial_codelists)
assert additional_codelist_values == {
('releases/tag'): {
'codelist': 'releaseTag.csv',
'codelist_url': 'https://raw.githubusercontent.com/open-contracting/standard/1.1/standard/schema/codelists/releaseTag.csv',
'codelist_amend_urls': [],
'field': 'tag',
'extension_codelist': False,
'isopen': False,
'path': 'releases',
'values': ['oh no']
},
('releases/tender/items/classification/scheme'): {
'codelist': 'itemClassificationScheme.csv',
'codelist_url': 'https://raw.githubusercontent.com/open-contracting/standard/1.1/standard/schema/codelists/itemClassificationScheme.csv',
'codelist_amend_urls': [],
'extension_codelist': False,
'field': 'scheme',
'isopen': True,
'path': 'releases/tender/items/classification',
'values': ['GSINS']}
}
def test_get_schema_deprecated_paths():
schema_obj = SchemaOCDS()
schema_obj.schema_host = os.path.join('cove_ocds', 'fixtures/')
schema_obj.release_pkg_schema_name = 'release_package_schema_ref_release_schema_deprecated_fields.json'
schema_obj.release_pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.release_pkg_schema_name)
deprecated_paths = cove_common._get_schema_deprecated_paths(schema_obj)
expected_results = [
(('releases', 'initiationType'), ('1.1', 'Not a useful field as always has to be tender')),
(('releases', 'planning',), ('1.1', "Testing deprecation for objects with '$ref'")),
(('releases', 'tender', 'hasEnquiries'), ('1.1', 'Deprecated just for fun')),
(('releases', 'contracts', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')),
(('releases', 'tender', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')),
(('releases', 'awards', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities'))
]
assert len(deprecated_paths) == 6
for path in expected_results:
assert path in deprecated_paths
def test_get_schema_deprecated_paths():
schema_obj = SchemaOCDS()
schema_obj.schema_host = os.path.join('cove_ocds', 'fixtures/')
schema_obj.release_pkg_schema_name = 'release_package_schema_ref_release_schema_deprecated_fields.json'
schema_obj.release_pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.release_pkg_schema_name)
deprecated_paths = cove_common._get_schema_deprecated_paths(schema_obj)
expected_results = [
(('releases', 'initiationType'), ('1.1', 'Not a useful field as always has to be tender')),
(('releases', 'planning',), ('1.1', "Testing deprecation for objects with '$ref'")),
(('releases', 'tender', 'hasEnquiries'), ('1.1', 'Deprecated just for fun')),
(('releases', 'contracts', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')),
(('releases', 'tender', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')),
(('releases', 'awards', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities'))
]
assert len(deprecated_paths) == 6
for path in expected_results:
assert path in deprecated_paths
def test_get_json_data_deprecated_fields():
with open(
os.path.join(
"tests",
"fixtures",
"tenders_releases_2_releases_with_deprecated_fields.json",
)) as fp:
json_data_w_deprecations = json.load(fp)
schema_obj = SchemaOCDS()
schema_obj.schema_host = os.path.join(
"tests", "fixtures/") # "/" is for urljoin in lib-cove
schema_obj.pkg_schema_name = (
"release_package_schema_ref_release_schema_deprecated_fields.json")
schema_obj.pkg_schema_url = os.path.join(schema_obj.schema_host,
schema_obj.pkg_schema_name)
json_data_paths = cove_common.get_json_data_generic_paths(
json_data_w_deprecations, generic_paths={})
deprecated_data_fields = cove_common.get_json_data_deprecated_fields(
json_data_paths, schema_obj)
expected_result = OrderedDict([
(
"initiationType",
{
"paths": ("releases/0", "releases/1"),
"explanation": (
"1.1",
"Not a useful field as always has to be tender",
),
},
),
(
"quantity",
{
"paths": ("releases/0/tender/items/0", ),
"explanation": ("1.1", "Nobody cares about quantities"),
},
),
])
for field_name in expected_result.keys():
assert field_name in deprecated_data_fields
assert (expected_result[field_name]["paths"] ==
deprecated_data_fields[field_name]["paths"])
assert (expected_result[field_name]["explanation"] ==
deprecated_data_fields[field_name]["explanation"])
def test_schema_ocds_extensions(release_data, extensions, invalid_extension,
extended, extends_schema):
schema = SchemaOCDS(release_data=release_data)
assert schema.extensions == extensions
assert not schema.extended
schema_obj = schema.get_schema_obj()
assert schema.invalid_extension == invalid_extension
assert schema.extended == extended
if extends_schema:
assert "Metric" in schema_obj["definitions"].keys()
assert schema_obj["definitions"]["Award"]["properties"].get(
"agreedMetrics")
else:
assert "Metric" not in schema_obj["definitions"].keys()
assert not schema_obj["definitions"]["Award"]["properties"].get(
"agreedMetrics")
def test_get_schema_validation_errors():
schema_obj = SchemaOCDS(select_version='1.0')
schema_name = schema_obj.release_pkg_schema_name
with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_2_releases.json')) as fp:
error_list = cove_common.get_schema_validation_errors(json.load(fp), schema_obj, schema_name, {}, {})
assert len(error_list) == 0
with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_2_releases_invalid.json')) as fp:
error_list = cove_common.get_schema_validation_errors(json.load(fp), schema_obj, schema_name, {}, {})
assert len(error_list) > 0
def test_corner_cases_for_deprecated_data_fields(json_data):
data = json.loads(json_data)
schema = SchemaOCDS(release_data=data)
json_data_paths = cove_common.get_json_data_generic_paths(data)
deprecated_fields = cove_common.get_json_data_deprecated_fields(json_data_paths, schema)
assert deprecated_fields['additionalIdentifiers']['explanation'][0] == '1.1'
assert 'parties section at the top level of a release' in deprecated_fields['additionalIdentifiers']['explanation'][1]
assert deprecated_fields['additionalIdentifiers']['paths'] == ('releases/buyer',)
assert len(deprecated_fields.keys()) == 1
assert len(deprecated_fields['additionalIdentifiers']['paths']) == 1
def test_get_json_data_deprecated_fields():
with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_2_releases_with_deprecated_fields.json')) as fp:
json_data_w_deprecations = json.load(fp)
schema_obj = SchemaOCDS()
schema_obj.schema_host = os.path.join('cove_ocds', 'fixtures/')
schema_obj.release_pkg_schema_name = 'release_package_schema_ref_release_schema_deprecated_fields.json'
schema_obj.release_pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.release_pkg_schema_name)
json_data_paths = cove_common.get_json_data_generic_paths(json_data_w_deprecations)
deprecated_data_fields = cove_common.get_json_data_deprecated_fields(json_data_paths, schema_obj)
expected_result = OrderedDict([
('initiationType', {"paths": ('releases/0', 'releases/1'),
"explanation": ('1.1', 'Not a useful field as always has to be tender')}),
('quantity', {"paths": ('releases/0/tender/items/0',),
"explanation": ('1.1', 'Nobody cares about quantities')})
])
for field_name in expected_result.keys():
assert field_name in deprecated_data_fields
assert expected_result[field_name]["paths"] == deprecated_data_fields[field_name]["paths"]
assert expected_result[field_name]["explanation"] == deprecated_data_fields[field_name]["explanation"]
def test_get_json_data_deprecated_fields():
with open(os.path.join('cove', 'fixtures', 'tenders_releases_2_releases_with_deprecated_fields.json')) as fp:
json_data_w_deprecations = json.load(fp)
schema_obj = SchemaOCDS()
schema_obj.schema_host = os.path.join('cove_ocds', 'fixtures/')
schema_obj.release_pkg_schema_name = 'release_package_schema_ref_release_schema_deprecated_fields.json'
schema_obj.release_pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.release_pkg_schema_name)
json_data_paths = cove_common.get_json_data_generic_paths(json_data_w_deprecations)
deprecated_data_fields = cove_common.get_json_data_deprecated_fields(json_data_paths, schema_obj)
expected_result = OrderedDict([
('initiationType', {"paths": ('releases/0', 'releases/1'),
"explanation": ('1.1', 'Not a useful field as always has to be tender')}),
('quantity', {"paths": ('releases/0/tender/items/0',),
"explanation": ('1.1', 'Nobody cares about quantities')})
])
for field_name in expected_result.keys():
assert field_name in deprecated_data_fields
assert expected_result[field_name]["paths"] == deprecated_data_fields[field_name]["paths"]
assert expected_result[field_name]["explanation"] == deprecated_data_fields[field_name]["explanation"]
def test_get_schema_deprecated_paths():
schema_obj = SchemaOCDS()
schema_obj.schema_host = os.path.join(
"tests", "fixtures/") # "/" is for urljoin in lib-cove
schema_obj.pkg_schema_name = (
"release_package_schema_ref_release_schema_deprecated_fields.json")
schema_obj.pkg_schema_url = os.path.join(schema_obj.schema_host,
schema_obj.pkg_schema_name)
deprecated_paths = cove_common._get_schema_deprecated_paths(schema_obj)
expected_results = [
(
("releases", "initiationType"),
("1.1", "Not a useful field as always has to be tender"),
),
(
(
"releases",
"planning",
),
("1.1", "Testing deprecation for objects with '$ref'"),
),
(("releases", "tender", "hasEnquiries"), ("1.1",
"Deprecated just for fun")),
(
("releases", "contracts", "items", "quantity"),
("1.1", "Nobody cares about quantities"),
),
(
("releases", "tender", "items", "quantity"),
("1.1", "Nobody cares about quantities"),
),
(
("releases", "awards", "items", "quantity"),
("1.1", "Nobody cares about quantities"),
),
]
assert len(deprecated_paths) == 6
for path in expected_results:
assert path in deprecated_paths
def test_schema_ocds_constructor(select_version, release_data, version, invalid_version_argument,
invalid_version_data, extensions):
schema = SchemaOCDS(select_version=select_version, release_data=release_data)
name = settings.COVE_CONFIG['schema_name']['release']
host = settings.COVE_CONFIG['schema_version_choices'][version][1]
url = host + name
assert schema.version == version
assert schema.release_pkg_schema_name == name
assert schema.schema_host == host
assert schema.release_pkg_schema_url == url
assert schema.invalid_version_argument == invalid_version_argument
assert schema.invalid_version_data == invalid_version_data
assert schema.extensions == extensions
def test_corner_cases_for_deprecated_data_fields(json_data):
data = json.loads(json_data)
schema = SchemaOCDS(release_data=data)
json_data_paths = cove_common.get_json_data_generic_paths(data)
deprecated_fields = cove_common.get_json_data_deprecated_fields(
json_data_paths, schema)
assert deprecated_fields["additionalIdentifiers"]["explanation"][
0] == "1.1"
assert ("parties section at the top level of a release"
in deprecated_fields["additionalIdentifiers"]["explanation"][1])
assert deprecated_fields["additionalIdentifiers"]["paths"] == (
"releases/buyer", )
assert len(deprecated_fields.keys()) == 1
assert len(deprecated_fields["additionalIdentifiers"]["paths"]) == 1
def test_get_schema_non_required_ids():
schema_obj = SchemaOCDS(select_version="1.1")
non_required_ids = cove_common._get_schema_non_required_ids(schema_obj)
results = [
("releases", "awards", "amendments", "id"),
("releases", "awards", "suppliers", "id"),
("releases", "contracts", "amendments", "id"),
("releases", "contracts", "relatedProcesses", "id"),
("releases", "parties", "id"),
("releases", "relatedProcesses", "id"),
("releases", "tender", "amendments", "id"),
("releases", "tender", "tenderers", "id"),
]
assert sorted(non_required_ids) == results
def test_get_schema_non_required_ids():
schema_obj = SchemaOCDS(select_version="1.1")
non_required_ids = cove_common._get_schema_non_required_ids(schema_obj)
results = [
('releases', 'awards', 'amendments', 'id'),
('releases', 'awards', 'suppliers', 'id'),
('releases', 'contracts', 'amendments', 'id'),
('releases', 'contracts', 'relatedProcesses', 'id'),
('releases', 'parties', 'id'),
('releases', 'relatedProcesses', 'id'),
('releases', 'tender', 'amendments', 'id'),
('releases', 'tender', 'tenderers', 'id')
]
assert sorted(non_required_ids) == results
def test_get_schema_validation_errors():
schema_obj = SchemaOCDS(select_version="1.0")
schema_name = schema_obj.pkg_schema_name
with open(
os.path.join("tests", "fixtures",
"tenders_releases_2_releases.json")) as fp:
error_list = cove_common.get_schema_validation_errors(
json.load(fp), schema_obj, schema_name, {}, {})
assert len(error_list) == 0
with open(
os.path.join("tests", "fixtures",
"tenders_releases_2_releases_invalid.json")) as fp:
error_list = cove_common.get_schema_validation_errors(
json.load(fp), schema_obj, schema_name, {}, {})
assert len(error_list) > 0
def test_get_json_data_missing_ids():
file_name = os.path.join(
'cove_ocds', 'fixtures', 'tenders_releases_2_releases_1_1_tenderers_with_missing_ids.json'
)
with open(os.path.join(file_name)) as fp:
user_data = json.load(fp)
schema_obj = SchemaOCDS(release_data=user_data)
results = [
'releases/0/tender/tenderers/1/id',
'releases/0/tender/tenderers/2/id',
'releases/0/tender/tenderers/5/id',
'releases/1/tender/tenderers/1/id',
'releases/1/tender/tenderers/2/id',
'releases/1/tender/tenderers/4/id'
]
user_data_paths = cove_common.get_json_data_generic_paths(user_data)
missin_ids_paths = cove_common.get_json_data_missing_ids(user_data_paths, schema_obj)
assert missin_ids_paths == results
def test_schema_ocds_constructor(
select_version,
release_data,
version,
invalid_version_argument,
invalid_version_data,
extensions,
):
schema = SchemaOCDS(select_version=select_version,
release_data=release_data)
name = settings.COVE_CONFIG["schema_name"]["release"]
host = settings.COVE_CONFIG["schema_version_choices"][version][1].format(
lang="en")
url = host + name
assert schema.version == version
assert schema.pkg_schema_name == name
assert schema.schema_host == host
assert schema.pkg_schema_url == url
assert schema.invalid_version_argument == invalid_version_argument
assert schema.invalid_version_data == invalid_version_data
assert schema.extensions == extensions
def ocds_json_output(
output_dir,
file,
schema_version,
convert,
cache_schema=False,
file_type=None,
json_data=None,
lib_cove_ocds_config=None,
record_pkg=False,
):
if not lib_cove_ocds_config:
lib_cove_ocds_config = LibCoveOCDSConfig()
# cache_schema is a deprecated option - now set cache_all_requests in the config instead.
if cache_schema:
lib_cove_ocds_config.config["cache_all_requests"] = True
context = {}
if not file_type:
file_type = get_file_type(file)
context = {"file_type": file_type}
if file_type == "json":
if not json_data:
with open(file, encoding="utf-8") as fp:
try:
json_data = json.load(fp, object_pairs_hook=OrderedDict)
except ValueError:
raise APIException("The file looks like invalid json")
schema_ocds = SchemaOCDS(
schema_version, json_data, lib_cove_ocds_config=lib_cove_ocds_config, record_pkg=record_pkg
)
if schema_ocds.invalid_version_data:
msg = "\033[1;31mThe schema version in your data is not valid. Accepted values: {}\033[1;m"
raise APIException(msg.format(str(list(schema_ocds.version_choices.keys()))))
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(output_dir, "")
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
if convert:
with warnings.catch_warnings():
warnings.filterwarnings("ignore") # flattentool uses UserWarning, so we can't set a specific category
context.update(
convert_json(output_dir, "", file, lib_cove_ocds_config, schema_url=url, flatten=True, cache=False)
)
else:
metatab_schema_url = SchemaOCDS(select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url
metatab_data = get_spreadsheet_meta_data(output_dir, file, metatab_schema_url, file_type=file_type)
schema_ocds = SchemaOCDS(schema_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config)
if schema_ocds.invalid_version_data:
msg = "\033[1;31mThe schema version in your data is not valid. Accepted values: {}\033[1;m"
raise APIException(msg.format(str(list(schema_ocds.version_choices.keys()))))
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(output_dir, "")
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
pkg_url = schema_ocds.pkg_schema_url
context.update(
convert_spreadsheet(
output_dir,
"",
file,
file_type,
lib_cove_ocds_config,
schema_url=url,
pkg_schema_url=pkg_url,
cache=False,
)
)
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp, object_pairs_hook=OrderedDict)
context = context_api_transform(
common_checks_ocds(context, output_dir, json_data, schema_ocds, api=True, cache=False)
)
if file_type == "xlsx":
# Remove unwanted files in the output
# TODO: can we do this by no writing the files in the first place?
os.remove(os.path.join(output_dir, "heading_source_map.json"))
os.remove(os.path.join(output_dir, "cell_source_map.json"))
return context
def test_schema_ocds_extended_release_schema_file():
data = SuppliedData.objects.create()
with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_1_release_with_extensions_1_1.json')) as fp:
data.original_file.save('test.json', UploadedFile(fp))
fp.seek(0)
json_data = json.load(fp)
schema = SchemaOCDS(release_data=json_data)
assert not schema.extended
schema.get_release_schema_obj()
assert schema.extended
assert not schema.extended_schema_file
assert not schema.extended_schema_url
schema.create_extended_release_schema_file(data.upload_dir(), data.upload_url())
assert schema.extended_schema_file == os.path.join(data.upload_dir(), 'extended_release_schema.json')
assert schema.extended_schema_url == os.path.join(data.upload_url(), 'extended_release_schema.json')
json_data = json.loads('{"version": "1.1", "extensions": [], "releases": [{"ocid": "xx"}]}')
schema = SchemaOCDS(release_data=json_data)
schema.get_release_schema_obj()
schema.create_extended_release_schema_file(data.upload_dir(), data.upload_url())
assert not schema.extended
assert not schema.extended_schema_file
assert not schema.extended_schema_url
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config["current_language"] = translation.get_language(
)
lib_cove_ocds_config.config[
"schema_version_choices"] = settings.COVE_CONFIG[
"schema_version_choices"]
lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[
"schema_codelists"]
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context["file_type"]
post_version_choice = request.POST.get("version")
replace = False
validation_errors_path = os.path.join(upload_dir,
"validation_errors-3.json")
if file_type == "json":
# open the data first so we can inspect for record package
with open(file_name, encoding="utf-8") as fp:
try:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
except UnicodeError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
format_html(
_("The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON follows the I-JSON format, which requires UTF-8 encoding. Ensure that your file uses UTF-8 encoding, then try uploading again."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}'),
err),
'error':
format(err)
})
except ValueError as err:
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
format_html(
_(
"We think you tried to upload a JSON file, but it is not well formed JSON."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
"</span> <strong>Error message:</strong> {}",
),
err,
),
"error":
format(err),
})
if not isinstance(json_data, dict):
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_("OCDS JSON should have an object as the top level, the JSON you supplied does not."
),
})
version_in_data = json_data.get("version", "")
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version,
release_data=json_data,
lib_cove_ocds_config=lib_cove_ocds_config,
record_pkg="records" in json_data)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile(
"^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = "{} (it must be a string)".format(
str(version_in_data))
context["unrecognized_version_data"] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
if "records" in json_data:
context["conversion"] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, "flattened")
replace_converted = replace and os.path.exists(converted_path +
".xlsx")
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore'
) # flattentool uses UserWarning, so can't set a specific category
convert_json_context = convert_json(
upload_dir,
upload_url,
file_name,
lib_cove_ocds_config,
schema_url=url,
replace=replace_converted,
request=request,
flatten=request.POST.get("flatten"),
)
context.update(convert_json_context)
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(
select_version="1.1",
lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name,
metatab_schema_url, file_type)
if "version" not in metatab_data:
metatab_data["version"] = "1.0"
else:
db_data.data_schema_version = metatab_data["version"]
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=metatab_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get("version")
if re.compile("^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
context["unrecognized_version_data"] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
pkg_url = schema_ocds.pkg_schema_url
context.update(
convert_spreadsheet(
upload_dir,
upload_url,
file_name,
file_type,
lib_cove_ocds_config,
schema_url=url,
pkg_schema_url=pkg_url,
replace=replace,
))
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context, upload_dir, json_data, schema_ocds)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
context.update({
"data_schema_version":
db_data.data_schema_version,
"first_render":
not db_data.rendered,
"validation_errors_grouped":
group_validation_errors(context["validation_errors"]),
})
schema_version = getattr(schema_ocds, "version", None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
if "records" in json_data:
ocds_show_schema = SchemaOCDS(record_pkg=True)
ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True)
template = "cove_ocds/explore_record.html"
if hasattr(json_data, "get") and hasattr(json_data.get("records"),
"__iter__"):
context["records"] = json_data["records"]
else:
context["records"] = []
if isinstance(json_data["records"],
list) and len(json_data["records"]) < 100:
context["ocds_show_data"] = ocds_show_data(json_data,
ocds_show_deref_schema)
else:
ocds_show_schema = SchemaOCDS(record_pkg=False)
ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True)
template = "cove_ocds/explore_release.html"
if hasattr(json_data, "get") and hasattr(json_data.get("releases"),
"__iter__"):
context["releases"] = json_data["releases"]
if (isinstance(json_data["releases"], list)
and len(json_data["releases"]) < 100):
context["ocds_show_data"] = ocds_show_data(
json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context["releases"]:
if hasattr(release, "get") and release.get("date"):
if validate_rfc3339(release["date"]):
release["date"] = parser.parse(release["date"])
else:
release["date"] = None
if context.get("releases_aggregates"):
date_fields = [
"max_award_date",
"max_contract_date",
"max_release_date",
"max_tender_date",
"min_award_date",
"min_contract_date",
"min_release_date",
"min_tender_date",
]
for field in date_fields:
if context["releases_aggregates"].get(field):
if validate_rfc3339(
context["releases_aggregates"][field]):
context["releases_aggregates"][
field] = parser.parse(
context["releases_aggregates"][field])
else:
context["releases_aggregates"][field] = None
else:
context["releases"] = []
return render(request, template, context)
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config['current_language'] = translation.get_language()
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context['file_type']
post_version_choice = request.POST.get('version')
replace = False
validation_errors_path = os.path.join(upload_dir, 'validation_errors-3.json')
if file_type == 'json':
# open the data first so we can inspect for record package
with open(file_name, encoding='utf-8') as fp:
try:
json_data = json.load(fp, parse_float=Decimal)
except ValueError as err:
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}', err)),
'error': format(err)
})
if not isinstance(json_data, dict):
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _('OCDS JSON should have an object as the top level, the JSON you supplied does not.'),
})
version_in_data = json_data.get('version', '')
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version, release_data=json_data, lib_cove_ocds_config=lib_cove_ocds_config)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile('^\d+\.\d+\.\d+$').match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = '{} (it must be a string)'.format(str(version_in_data))
context['unrecognized_version_data'] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
if 'records' in json_data:
context['conversion'] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, 'flattened')
replace_converted = replace and os.path.exists(converted_path + '.xlsx')
context.update(convert_json(upload_dir, upload_url, file_name, lib_cove_ocds_config,
schema_url=url, replace=replace_converted,
request=request, flatten=request.POST.get('flatten')))
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(select_version='1.1', lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type)
if 'version' not in metatab_data:
metatab_data['version'] = '1.0'
else:
db_data.data_schema_version = metatab_data['version']
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get('version')
if re.compile('^\d+\.\d+\.\d+$').match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(version_in_data)
else:
context['unrecognized_version_data'] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
pkg_url = schema_ocds.release_pkg_schema_url
context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_ocds_config,
schema_url=url,
pkg_schema_url=pkg_url, replace=replace))
with open(context['converted_path'], encoding='utf-8') as fp:
json_data = json.load(fp, parse_float=Decimal)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context, upload_dir, json_data, schema_ocds)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
context.update({
'data_schema_version': db_data.data_schema_version,
'first_render': not db_data.rendered,
'validation_errors_grouped': group_validation_errors(context['validation_errors']),
})
schema_version = getattr(schema_ocds, 'version', None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
ocds_show_schema = SchemaOCDS()
ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj(deref=True)
if 'records' in json_data:
template = 'cove_ocds/explore_record.html'
if hasattr(json_data, 'get') and hasattr(json_data.get('records'), '__iter__'):
context['records'] = json_data['records']
else:
context['records'] = []
if isinstance(json_data['records'], list) and len(json_data['records']) < 100:
context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema)
else:
template = 'cove_ocds/explore_release.html'
if hasattr(json_data, 'get') and hasattr(json_data.get('releases'), '__iter__'):
context['releases'] = json_data['releases']
if isinstance(json_data['releases'], list) and len(json_data['releases']) < 100:
context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context['releases']:
if hasattr(release, 'get') and release.get('date'):
if validate_rfc3339(release['date']):
release['date'] = parser.parse(release['date'])
else:
release['date'] = None
if context.get('releases_aggregates'):
date_fields = ['max_award_date', 'max_contract_date', 'max_release_date', 'max_tender_date', 'min_award_date', 'min_contract_date', 'min_release_date', 'min_tender_date']
for field in date_fields:
if context['releases_aggregates'].get(field):
if(validate_rfc3339(context['releases_aggregates'][field])):
context['releases_aggregates'][field] = parser.parse(context['releases_aggregates'][field])
else:
context['releases_aggregates'][field] = None
else:
context['releases'] = []
return render(request, template, context)
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config["current_language"] = translation.get_language(
)
lib_cove_ocds_config.config[
"schema_version_choices"] = settings.COVE_CONFIG[
"schema_version_choices"]
lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[
"schema_codelists"]
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context["file_type"]
post_version_choice = request.POST.get(
"version", lib_cove_ocds_config.config["schema_version"])
replace = False
validation_errors_path = os.path.join(upload_dir,
"validation_errors-3.json")
if file_type == "json":
# open the data first so we can inspect for record package
with open(file_name, encoding="utf-8") as fp:
try:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
except ValueError as err:
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_(
format_html(
"We think you tried to upload a JSON file, but it is not well formed JSON."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
"</span> <strong>Error message:</strong> {}",
err,
)),
"error":
format(err),
})
if not isinstance(json_data, dict):
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_("OCDS JSON should have an object as the top level, the JSON you supplied does not."
),
})
version_in_data = json_data.get("version", "")
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=json_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile(
"^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = "{} (it must be a string)".format(
str(version_in_data))
context["unrecognized_version_data"] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
if "records" in json_data:
context["conversion"] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, "flattened")
replace_converted = replace and os.path.exists(converted_path +
".xlsx")
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore'
) # flattentool uses UserWarning, so can't set a specific category
convert_json_context = convert_json(
upload_dir,
upload_url,
file_name,
lib_cove_ocds_config,
schema_url=schema_url,
replace=replace_converted,
request=request,
flatten=request.POST.get("flatten"),
)
context.update(convert_json_context)
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(
select_version="1.1",
lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name,
metatab_schema_url, file_type)
if "version" not in metatab_data:
metatab_data["version"] = "1.0"
else:
db_data.data_schema_version = metatab_data["version"]
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=metatab_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get("version")
if re.compile("^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
context["unrecognized_version_data"] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
pkg_url = schema_ocds.release_pkg_schema_url
if file_type != "csv":
# ORIGINAL UNFLATTEN
conversion_context = convert_spreadsheet(
upload_dir,
upload_url,
file_name,
file_type,
lib_cove_ocds_config,
schema_url=schema_url,
pkg_schema_url=pkg_url,
replace=replace,
)
else:
# Convert Simple CSV to flat OCDS and return context
conversion_context = convert_simple_csv_submission(
db_data,
lib_cove_ocds_config,
schema_url,
replace=replace,
)
context.update(conversion_context)
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context,
upload_dir,
json_data,
schema_ocds,
cache=settings.CACHE_VALIDATION_ERRORS)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
schema_version = getattr(schema_ocds, "version", None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
context.update({
"data_schema_version":
db_data.schema_version,
"first_render":
not db_data.rendered,
"validation_errors_grouped":
group_validation_errors(context["validation_errors"]),
})
ocds_show_schema = SchemaOCDS()
ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj(
deref=True)
if "records" in json_data:
template = "cove_ocds/explore_record.html"
if hasattr(json_data, "get") and hasattr(json_data.get("records"),
"__iter__"):
context["records"] = json_data["records"]
else:
context["records"] = []
if isinstance(json_data["records"],
list) and len(json_data["records"]) < 100:
context["ocds_show_data"] = ocds_show_data(json_data,
ocds_show_deref_schema)
else:
template = "silvereye/explore_release.html"
if hasattr(json_data, "get") and hasattr(json_data.get("releases"),
"__iter__"):
context["releases"] = json_data["releases"]
if (isinstance(json_data["releases"], list)
and len(json_data["releases"]) < 100):
context["ocds_show_data"] = ocds_show_data(
json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context["releases"]:
if hasattr(release, "get") and release.get("date"):
if validate_rfc3339(release["date"]):
release["date"] = parser.parse(release["date"])
else:
release["date"] = None
try:
trans_date = release["contracts"][0]["implementation"][
"transactions"][0]["date"]
parsed_trans_date = parser.parse(trans_date)
release["contracts"][0]["implementation"]["transactions"][
0]["date"] = parsed_trans_date
except KeyError:
pass
if context.get("releases_aggregates"):
date_fields = [
"max_award_date",
"max_contract_date",
"max_release_date",
"max_tender_date",
"min_award_date",
"min_contract_date",
"min_release_date",
"min_tender_date",
]
for field in date_fields:
if context["releases_aggregates"].get(field):
if validate_rfc3339(
context["releases_aggregates"][field]):
context["releases_aggregates"][
field] = parser.parse(
context["releases_aggregates"][field])
else:
context["releases_aggregates"][field] = None
else:
context["releases"] = []
# Include field coverage report
original_file_path = context["original_file"]["path"]
mapper = CSVMapper(csv_path=original_file_path)
db_data.notice_type = mapper.release_type
db_data.save()
coverage_context = mapper.get_coverage_context()
context.update({
"field_coverage": coverage_context,
})
ocds_validation_errors, simple_csv_errors = prepare_simple_csv_validation_errors(
context["validation_errors"], mapper,
coverage_context["required_fields_missing"])
context.update({
"ocds_validation_errors": ocds_validation_errors,
"simple_csv_errors": simple_csv_errors,
"csv_mapper": mapper,
})
# Silvereye: Insert OCDS data
releases = context.get("releases")
if releases:
# If we don't have validation errors
validation_errors_grouped = context["validation_errors_grouped"]
if not validation_errors_grouped:
json_string = json.dumps(json_data,
indent=2,
sort_keys=True,
cls=DjangoJSONEncoder)
UpsertDataHelpers().upsert_ocds_data(json_string,
supplied_data=db_data)
average_field_completion = coverage_context.get(
"average_field_completion")
inst, created = FieldCoverage.objects.update_or_create(
file_submission=db_data,
defaults={
"tenders_field_coverage":
average_field_completion
if mapper.release_type == "tender" else None,
"awards_field_coverage":
average_field_completion
if mapper.release_type == "award" else None,
"spend_field_coverage":
average_field_completion
if mapper.release_type == "spend" else None,
})
update_publisher_monthly_counts()
return render(request, template, context)