def test_schema_ocds_extended_release_schema_file(): data = SuppliedData.objects.create() with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_1_release_with_extensions_1_1.json')) as fp: data.original_file.save('test.json', UploadedFile(fp)) fp.seek(0) json_data = json.load(fp) schema = SchemaOCDS(release_data=json_data) assert not schema.extended schema.get_release_schema_obj() assert schema.extended assert not schema.extended_schema_file assert not schema.extended_schema_url schema.create_extended_release_schema_file(data.upload_dir(), data.upload_url()) assert schema.extended_schema_file == os.path.join(data.upload_dir(), 'extended_release_schema.json') assert schema.extended_schema_url == os.path.join(data.upload_url(), 'extended_release_schema.json') json_data = json.loads('{"version": "1.1", "extensions": [], "releases": [{"ocid": "xx"}]}') schema = SchemaOCDS(release_data=json_data) schema.get_release_schema_obj() schema.create_extended_release_schema_file(data.upload_dir(), data.upload_url()) assert not schema.extended assert not schema.extended_schema_file assert not schema.extended_schema_url
def test_schema_ocds_extensions(release_data, extensions, invalid_extension, extended, extends_schema): schema = SchemaOCDS(release_data=release_data) assert schema.extensions == extensions assert not schema.extended release_schema_obj = schema.get_release_schema_obj() assert schema.invalid_extension == invalid_extension assert schema.extended == extended if extends_schema: assert 'Metric' in release_schema_obj['definitions'].keys() assert release_schema_obj['definitions']['Award']['properties'].get('agreedMetrics') else: assert 'Metric' not in release_schema_obj['definitions'].keys() assert not release_schema_obj['definitions']['Award']['properties'].get('agreedMetrics')
def test_schema_ocds_extensions(release_data, extensions, invalid_extension, extended, extends_schema): schema = SchemaOCDS(release_data=release_data) assert schema.extensions == extensions assert not schema.extended release_schema_obj = schema.get_release_schema_obj() assert schema.invalid_extension == invalid_extension assert schema.extended == extended if extends_schema: assert 'Metric' in release_schema_obj['definitions'].keys() assert release_schema_obj['definitions']['Award']['properties'].get('agreedMetrics') else: assert 'Metric' not in release_schema_obj['definitions'].keys() assert not release_schema_obj['definitions']['Award']['properties'].get('agreedMetrics')
def test_get_additional_codelist_values(): with open( os.path.join("tests", "fixtures", "tenders_releases_2_releases_codelists.json")) as fp: json_data_w_additial_codelists = json.load(fp) schema_obj = SchemaOCDS(select_version="1.1") additional_codelist_values = cove_common.get_additional_codelist_values( schema_obj, json_data_w_additial_codelists) assert additional_codelist_values == { ("releases/tag"): { "codelist": "releaseTag.csv", "codelist_url": "https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/codelists/releaseTag.csv", # noqa: E501 "codelist_amend_urls": [], "field": "tag", "extension_codelist": False, "isopen": False, "path": "releases", "values": ["oh no"], }, ("releases/tender/items/classification/scheme"): { "codelist": "itemClassificationScheme.csv", "codelist_url": "https://raw.githubusercontent.com/open-contracting/standard/1.1/schema/codelists/itemClassificationScheme.csv", # noqa: E501 "codelist_amend_urls": [], "extension_codelist": False, "field": "scheme", "isopen": True, "path": "releases/tender/items/classification", "values": ["GSINS"], }, }
def test_get_additional_codelist_values(): with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_2_releases_codelists.json')) as fp: json_data_w_additial_codelists = json.load(fp) schema_obj = SchemaOCDS(select_version='1.1') additional_codelist_values = cove_common.get_additional_codelist_values(schema_obj, json_data_w_additial_codelists) assert additional_codelist_values == { ('releases/tag'): { 'codelist': 'releaseTag.csv', 'codelist_url': 'https://raw.githubusercontent.com/open-contracting/standard/1.1/standard/schema/codelists/releaseTag.csv', 'codelist_amend_urls': [], 'field': 'tag', 'extension_codelist': False, 'isopen': False, 'path': 'releases', 'values': ['oh no'] }, ('releases/tender/items/classification/scheme'): { 'codelist': 'itemClassificationScheme.csv', 'codelist_url': 'https://raw.githubusercontent.com/open-contracting/standard/1.1/standard/schema/codelists/itemClassificationScheme.csv', 'codelist_amend_urls': [], 'extension_codelist': False, 'field': 'scheme', 'isopen': True, 'path': 'releases/tender/items/classification', 'values': ['GSINS']} }
def test_get_schema_deprecated_paths(): schema_obj = SchemaOCDS() schema_obj.schema_host = os.path.join('cove_ocds', 'fixtures/') schema_obj.release_pkg_schema_name = 'release_package_schema_ref_release_schema_deprecated_fields.json' schema_obj.release_pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.release_pkg_schema_name) deprecated_paths = cove_common._get_schema_deprecated_paths(schema_obj) expected_results = [ (('releases', 'initiationType'), ('1.1', 'Not a useful field as always has to be tender')), (('releases', 'planning',), ('1.1', "Testing deprecation for objects with '$ref'")), (('releases', 'tender', 'hasEnquiries'), ('1.1', 'Deprecated just for fun')), (('releases', 'contracts', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')), (('releases', 'tender', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')), (('releases', 'awards', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')) ] assert len(deprecated_paths) == 6 for path in expected_results: assert path in deprecated_paths
def test_get_schema_deprecated_paths(): schema_obj = SchemaOCDS() schema_obj.schema_host = os.path.join('cove_ocds', 'fixtures/') schema_obj.release_pkg_schema_name = 'release_package_schema_ref_release_schema_deprecated_fields.json' schema_obj.release_pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.release_pkg_schema_name) deprecated_paths = cove_common._get_schema_deprecated_paths(schema_obj) expected_results = [ (('releases', 'initiationType'), ('1.1', 'Not a useful field as always has to be tender')), (('releases', 'planning',), ('1.1', "Testing deprecation for objects with '$ref'")), (('releases', 'tender', 'hasEnquiries'), ('1.1', 'Deprecated just for fun')), (('releases', 'contracts', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')), (('releases', 'tender', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')), (('releases', 'awards', 'items', 'quantity'), ('1.1', 'Nobody cares about quantities')) ] assert len(deprecated_paths) == 6 for path in expected_results: assert path in deprecated_paths
def test_get_json_data_deprecated_fields(): with open( os.path.join( "tests", "fixtures", "tenders_releases_2_releases_with_deprecated_fields.json", )) as fp: json_data_w_deprecations = json.load(fp) schema_obj = SchemaOCDS() schema_obj.schema_host = os.path.join( "tests", "fixtures/") # "/" is for urljoin in lib-cove schema_obj.pkg_schema_name = ( "release_package_schema_ref_release_schema_deprecated_fields.json") schema_obj.pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.pkg_schema_name) json_data_paths = cove_common.get_json_data_generic_paths( json_data_w_deprecations, generic_paths={}) deprecated_data_fields = cove_common.get_json_data_deprecated_fields( json_data_paths, schema_obj) expected_result = OrderedDict([ ( "initiationType", { "paths": ("releases/0", "releases/1"), "explanation": ( "1.1", "Not a useful field as always has to be tender", ), }, ), ( "quantity", { "paths": ("releases/0/tender/items/0", ), "explanation": ("1.1", "Nobody cares about quantities"), }, ), ]) for field_name in expected_result.keys(): assert field_name in deprecated_data_fields assert (expected_result[field_name]["paths"] == deprecated_data_fields[field_name]["paths"]) assert (expected_result[field_name]["explanation"] == deprecated_data_fields[field_name]["explanation"])
def test_schema_ocds_extensions(release_data, extensions, invalid_extension, extended, extends_schema): schema = SchemaOCDS(release_data=release_data) assert schema.extensions == extensions assert not schema.extended schema_obj = schema.get_schema_obj() assert schema.invalid_extension == invalid_extension assert schema.extended == extended if extends_schema: assert "Metric" in schema_obj["definitions"].keys() assert schema_obj["definitions"]["Award"]["properties"].get( "agreedMetrics") else: assert "Metric" not in schema_obj["definitions"].keys() assert not schema_obj["definitions"]["Award"]["properties"].get( "agreedMetrics")
def test_get_schema_validation_errors(): schema_obj = SchemaOCDS(select_version='1.0') schema_name = schema_obj.release_pkg_schema_name with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_2_releases.json')) as fp: error_list = cove_common.get_schema_validation_errors(json.load(fp), schema_obj, schema_name, {}, {}) assert len(error_list) == 0 with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_2_releases_invalid.json')) as fp: error_list = cove_common.get_schema_validation_errors(json.load(fp), schema_obj, schema_name, {}, {}) assert len(error_list) > 0
def test_corner_cases_for_deprecated_data_fields(json_data): data = json.loads(json_data) schema = SchemaOCDS(release_data=data) json_data_paths = cove_common.get_json_data_generic_paths(data) deprecated_fields = cove_common.get_json_data_deprecated_fields(json_data_paths, schema) assert deprecated_fields['additionalIdentifiers']['explanation'][0] == '1.1' assert 'parties section at the top level of a release' in deprecated_fields['additionalIdentifiers']['explanation'][1] assert deprecated_fields['additionalIdentifiers']['paths'] == ('releases/buyer',) assert len(deprecated_fields.keys()) == 1 assert len(deprecated_fields['additionalIdentifiers']['paths']) == 1
def test_get_json_data_deprecated_fields(): with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_2_releases_with_deprecated_fields.json')) as fp: json_data_w_deprecations = json.load(fp) schema_obj = SchemaOCDS() schema_obj.schema_host = os.path.join('cove_ocds', 'fixtures/') schema_obj.release_pkg_schema_name = 'release_package_schema_ref_release_schema_deprecated_fields.json' schema_obj.release_pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.release_pkg_schema_name) json_data_paths = cove_common.get_json_data_generic_paths(json_data_w_deprecations) deprecated_data_fields = cove_common.get_json_data_deprecated_fields(json_data_paths, schema_obj) expected_result = OrderedDict([ ('initiationType', {"paths": ('releases/0', 'releases/1'), "explanation": ('1.1', 'Not a useful field as always has to be tender')}), ('quantity', {"paths": ('releases/0/tender/items/0',), "explanation": ('1.1', 'Nobody cares about quantities')}) ]) for field_name in expected_result.keys(): assert field_name in deprecated_data_fields assert expected_result[field_name]["paths"] == deprecated_data_fields[field_name]["paths"] assert expected_result[field_name]["explanation"] == deprecated_data_fields[field_name]["explanation"]
def test_get_json_data_deprecated_fields(): with open(os.path.join('cove', 'fixtures', 'tenders_releases_2_releases_with_deprecated_fields.json')) as fp: json_data_w_deprecations = json.load(fp) schema_obj = SchemaOCDS() schema_obj.schema_host = os.path.join('cove_ocds', 'fixtures/') schema_obj.release_pkg_schema_name = 'release_package_schema_ref_release_schema_deprecated_fields.json' schema_obj.release_pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.release_pkg_schema_name) json_data_paths = cove_common.get_json_data_generic_paths(json_data_w_deprecations) deprecated_data_fields = cove_common.get_json_data_deprecated_fields(json_data_paths, schema_obj) expected_result = OrderedDict([ ('initiationType', {"paths": ('releases/0', 'releases/1'), "explanation": ('1.1', 'Not a useful field as always has to be tender')}), ('quantity', {"paths": ('releases/0/tender/items/0',), "explanation": ('1.1', 'Nobody cares about quantities')}) ]) for field_name in expected_result.keys(): assert field_name in deprecated_data_fields assert expected_result[field_name]["paths"] == deprecated_data_fields[field_name]["paths"] assert expected_result[field_name]["explanation"] == deprecated_data_fields[field_name]["explanation"]
def test_get_schema_deprecated_paths(): schema_obj = SchemaOCDS() schema_obj.schema_host = os.path.join( "tests", "fixtures/") # "/" is for urljoin in lib-cove schema_obj.pkg_schema_name = ( "release_package_schema_ref_release_schema_deprecated_fields.json") schema_obj.pkg_schema_url = os.path.join(schema_obj.schema_host, schema_obj.pkg_schema_name) deprecated_paths = cove_common._get_schema_deprecated_paths(schema_obj) expected_results = [ ( ("releases", "initiationType"), ("1.1", "Not a useful field as always has to be tender"), ), ( ( "releases", "planning", ), ("1.1", "Testing deprecation for objects with '$ref'"), ), (("releases", "tender", "hasEnquiries"), ("1.1", "Deprecated just for fun")), ( ("releases", "contracts", "items", "quantity"), ("1.1", "Nobody cares about quantities"), ), ( ("releases", "tender", "items", "quantity"), ("1.1", "Nobody cares about quantities"), ), ( ("releases", "awards", "items", "quantity"), ("1.1", "Nobody cares about quantities"), ), ] assert len(deprecated_paths) == 6 for path in expected_results: assert path in deprecated_paths
def test_schema_ocds_constructor(select_version, release_data, version, invalid_version_argument, invalid_version_data, extensions): schema = SchemaOCDS(select_version=select_version, release_data=release_data) name = settings.COVE_CONFIG['schema_name']['release'] host = settings.COVE_CONFIG['schema_version_choices'][version][1] url = host + name assert schema.version == version assert schema.release_pkg_schema_name == name assert schema.schema_host == host assert schema.release_pkg_schema_url == url assert schema.invalid_version_argument == invalid_version_argument assert schema.invalid_version_data == invalid_version_data assert schema.extensions == extensions
def test_corner_cases_for_deprecated_data_fields(json_data): data = json.loads(json_data) schema = SchemaOCDS(release_data=data) json_data_paths = cove_common.get_json_data_generic_paths(data) deprecated_fields = cove_common.get_json_data_deprecated_fields( json_data_paths, schema) assert deprecated_fields["additionalIdentifiers"]["explanation"][ 0] == "1.1" assert ("parties section at the top level of a release" in deprecated_fields["additionalIdentifiers"]["explanation"][1]) assert deprecated_fields["additionalIdentifiers"]["paths"] == ( "releases/buyer", ) assert len(deprecated_fields.keys()) == 1 assert len(deprecated_fields["additionalIdentifiers"]["paths"]) == 1
def test_get_schema_non_required_ids(): schema_obj = SchemaOCDS(select_version="1.1") non_required_ids = cove_common._get_schema_non_required_ids(schema_obj) results = [ ("releases", "awards", "amendments", "id"), ("releases", "awards", "suppliers", "id"), ("releases", "contracts", "amendments", "id"), ("releases", "contracts", "relatedProcesses", "id"), ("releases", "parties", "id"), ("releases", "relatedProcesses", "id"), ("releases", "tender", "amendments", "id"), ("releases", "tender", "tenderers", "id"), ] assert sorted(non_required_ids) == results
def test_get_schema_non_required_ids(): schema_obj = SchemaOCDS(select_version="1.1") non_required_ids = cove_common._get_schema_non_required_ids(schema_obj) results = [ ('releases', 'awards', 'amendments', 'id'), ('releases', 'awards', 'suppliers', 'id'), ('releases', 'contracts', 'amendments', 'id'), ('releases', 'contracts', 'relatedProcesses', 'id'), ('releases', 'parties', 'id'), ('releases', 'relatedProcesses', 'id'), ('releases', 'tender', 'amendments', 'id'), ('releases', 'tender', 'tenderers', 'id') ] assert sorted(non_required_ids) == results
def test_get_schema_validation_errors(): schema_obj = SchemaOCDS(select_version="1.0") schema_name = schema_obj.pkg_schema_name with open( os.path.join("tests", "fixtures", "tenders_releases_2_releases.json")) as fp: error_list = cove_common.get_schema_validation_errors( json.load(fp), schema_obj, schema_name, {}, {}) assert len(error_list) == 0 with open( os.path.join("tests", "fixtures", "tenders_releases_2_releases_invalid.json")) as fp: error_list = cove_common.get_schema_validation_errors( json.load(fp), schema_obj, schema_name, {}, {}) assert len(error_list) > 0
def test_get_json_data_missing_ids(): file_name = os.path.join( 'cove_ocds', 'fixtures', 'tenders_releases_2_releases_1_1_tenderers_with_missing_ids.json' ) with open(os.path.join(file_name)) as fp: user_data = json.load(fp) schema_obj = SchemaOCDS(release_data=user_data) results = [ 'releases/0/tender/tenderers/1/id', 'releases/0/tender/tenderers/2/id', 'releases/0/tender/tenderers/5/id', 'releases/1/tender/tenderers/1/id', 'releases/1/tender/tenderers/2/id', 'releases/1/tender/tenderers/4/id' ] user_data_paths = cove_common.get_json_data_generic_paths(user_data) missin_ids_paths = cove_common.get_json_data_missing_ids(user_data_paths, schema_obj) assert missin_ids_paths == results
def test_schema_ocds_constructor( select_version, release_data, version, invalid_version_argument, invalid_version_data, extensions, ): schema = SchemaOCDS(select_version=select_version, release_data=release_data) name = settings.COVE_CONFIG["schema_name"]["release"] host = settings.COVE_CONFIG["schema_version_choices"][version][1].format( lang="en") url = host + name assert schema.version == version assert schema.pkg_schema_name == name assert schema.schema_host == host assert schema.pkg_schema_url == url assert schema.invalid_version_argument == invalid_version_argument assert schema.invalid_version_data == invalid_version_data assert schema.extensions == extensions
def ocds_json_output( output_dir, file, schema_version, convert, cache_schema=False, file_type=None, json_data=None, lib_cove_ocds_config=None, record_pkg=False, ): if not lib_cove_ocds_config: lib_cove_ocds_config = LibCoveOCDSConfig() # cache_schema is a deprecated option - now set cache_all_requests in the config instead. if cache_schema: lib_cove_ocds_config.config["cache_all_requests"] = True context = {} if not file_type: file_type = get_file_type(file) context = {"file_type": file_type} if file_type == "json": if not json_data: with open(file, encoding="utf-8") as fp: try: json_data = json.load(fp, object_pairs_hook=OrderedDict) except ValueError: raise APIException("The file looks like invalid json") schema_ocds = SchemaOCDS( schema_version, json_data, lib_cove_ocds_config=lib_cove_ocds_config, record_pkg=record_pkg ) if schema_ocds.invalid_version_data: msg = "\033[1;31mThe schema version in your data is not valid. Accepted values: {}\033[1;m" raise APIException(msg.format(str(list(schema_ocds.version_choices.keys())))) if schema_ocds.extensions: schema_ocds.create_extended_schema_file(output_dir, "") url = schema_ocds.extended_schema_file or schema_ocds.schema_url if convert: with warnings.catch_warnings(): warnings.filterwarnings("ignore") # flattentool uses UserWarning, so we can't set a specific category context.update( convert_json(output_dir, "", file, lib_cove_ocds_config, schema_url=url, flatten=True, cache=False) ) else: metatab_schema_url = SchemaOCDS(select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url metatab_data = get_spreadsheet_meta_data(output_dir, file, metatab_schema_url, file_type=file_type) schema_ocds = SchemaOCDS(schema_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config) if schema_ocds.invalid_version_data: msg = "\033[1;31mThe schema version in your data is not valid. Accepted values: {}\033[1;m" raise APIException(msg.format(str(list(schema_ocds.version_choices.keys())))) if schema_ocds.extensions: schema_ocds.create_extended_schema_file(output_dir, "") url = schema_ocds.extended_schema_file or schema_ocds.schema_url pkg_url = schema_ocds.pkg_schema_url context.update( convert_spreadsheet( output_dir, "", file, file_type, lib_cove_ocds_config, schema_url=url, pkg_schema_url=pkg_url, cache=False, ) ) with open(context["converted_path"], encoding="utf-8") as fp: json_data = json.load(fp, object_pairs_hook=OrderedDict) context = context_api_transform( common_checks_ocds(context, output_dir, json_data, schema_ocds, api=True, cache=False) ) if file_type == "xlsx": # Remove unwanted files in the output # TODO: can we do this by no writing the files in the first place? os.remove(os.path.join(output_dir, "heading_source_map.json")) os.remove(os.path.join(output_dir, "cell_source_map.json")) return context
def test_schema_ocds_extended_release_schema_file(): data = SuppliedData.objects.create() with open(os.path.join('cove_ocds', 'fixtures', 'tenders_releases_1_release_with_extensions_1_1.json')) as fp: data.original_file.save('test.json', UploadedFile(fp)) fp.seek(0) json_data = json.load(fp) schema = SchemaOCDS(release_data=json_data) assert not schema.extended schema.get_release_schema_obj() assert schema.extended assert not schema.extended_schema_file assert not schema.extended_schema_url schema.create_extended_release_schema_file(data.upload_dir(), data.upload_url()) assert schema.extended_schema_file == os.path.join(data.upload_dir(), 'extended_release_schema.json') assert schema.extended_schema_url == os.path.join(data.upload_url(), 'extended_release_schema.json') json_data = json.loads('{"version": "1.1", "extensions": [], "releases": [{"ocid": "xx"}]}') schema = SchemaOCDS(release_data=json_data) schema.get_release_schema_obj() schema.create_extended_release_schema_file(data.upload_dir(), data.upload_url()) assert not schema.extended assert not schema.extended_schema_file assert not schema.extended_schema_url
def explore_ocds(request, pk): context, db_data, error = explore_data_context(request, pk) if error: return error lib_cove_ocds_config = LibCoveOCDSConfig() lib_cove_ocds_config.config["current_language"] = translation.get_language( ) lib_cove_ocds_config.config[ "schema_version_choices"] = settings.COVE_CONFIG[ "schema_version_choices"] lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[ "schema_codelists"] upload_dir = db_data.upload_dir() upload_url = db_data.upload_url() file_name = db_data.original_file.file.name file_type = context["file_type"] post_version_choice = request.POST.get("version") replace = False validation_errors_path = os.path.join(upload_dir, "validation_errors-3.json") if file_type == "json": # open the data first so we can inspect for record package with open(file_name, encoding="utf-8") as fp: try: json_data = json.load(fp, parse_float=Decimal, object_pairs_hook=OrderedDict) except UnicodeError as err: raise CoveInputDataError( context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': format_html( _("The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON follows the I-JSON format, which requires UTF-8 encoding. Ensure that your file uses UTF-8 encoding, then try uploading again." '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' '</span> <strong>Error message:</strong> {}'), err), 'error': format(err) }) except ValueError as err: raise CoveInputDataError( context={ "sub_title": _("Sorry, we can't process that data"), "link": "index", "link_text": _("Try Again"), "msg": format_html( _( "We think you tried to upload a JSON file, but it is not well formed JSON." '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' "</span> <strong>Error message:</strong> {}", ), err, ), "error": format(err), }) if not isinstance(json_data, dict): raise CoveInputDataError( context={ "sub_title": _("Sorry, we can't process that data"), "link": "index", "link_text": _("Try Again"), "msg": _("OCDS JSON should have an object as the top level, the JSON you supplied does not." ), }) version_in_data = json_data.get("version", "") db_data.data_schema_version = version_in_data select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS(select_version=select_version, release_data=json_data, lib_cove_ocds_config=lib_cove_ocds_config, record_pkg="records" in json_data) if schema_ocds.missing_package: exceptions.raise_missing_package_error() if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: if isinstance(version_in_data, str) and re.compile( "^\d+\.\d+\.\d+$").match(version_in_data): exceptions.raise_invalid_version_data_with_patch( version_in_data) else: if not isinstance(version_in_data, str): version_in_data = "{} (it must be a string)".format( str(version_in_data)) context["unrecognized_version_data"] = version_in_data if schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_schema_file(upload_dir, upload_url) url = schema_ocds.extended_schema_file or schema_ocds.schema_url if "records" in json_data: context["conversion"] = None else: # Replace the spreadsheet conversion only if it exists already. converted_path = os.path.join(upload_dir, "flattened") replace_converted = replace and os.path.exists(converted_path + ".xlsx") with warnings.catch_warnings(): warnings.filterwarnings( 'ignore' ) # flattentool uses UserWarning, so can't set a specific category convert_json_context = convert_json( upload_dir, upload_url, file_name, lib_cove_ocds_config, schema_url=url, replace=replace_converted, request=request, flatten=request.POST.get("flatten"), ) context.update(convert_json_context) else: # Use the lowest release pkg schema version accepting 'version' field metatab_schema_url = SchemaOCDS( select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type) if "version" not in metatab_data: metatab_data["version"] = "1.0" else: db_data.data_schema_version = metatab_data["version"] select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS( select_version=select_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config, ) # Unlike for JSON data case above, do not check for missing data package if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: version_in_data = metatab_data.get("version") if re.compile("^\d+\.\d+\.\d+$").match(version_in_data): exceptions.raise_invalid_version_data_with_patch( version_in_data) else: context["unrecognized_version_data"] = version_in_data # Replace json conversion when user chooses a different schema version. if db_data.schema_version and schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_schema_file(upload_dir, upload_url) url = schema_ocds.extended_schema_file or schema_ocds.schema_url pkg_url = schema_ocds.pkg_schema_url context.update( convert_spreadsheet( upload_dir, upload_url, file_name, file_type, lib_cove_ocds_config, schema_url=url, pkg_schema_url=pkg_url, replace=replace, )) with open(context["converted_path"], encoding="utf-8") as fp: json_data = json.load(fp, parse_float=Decimal, object_pairs_hook=OrderedDict) if replace: if os.path.exists(validation_errors_path): os.remove(validation_errors_path) context = common_checks_ocds(context, upload_dir, json_data, schema_ocds) if schema_ocds.json_deref_error: exceptions.raise_json_deref_error(schema_ocds.json_deref_error) context.update({ "data_schema_version": db_data.data_schema_version, "first_render": not db_data.rendered, "validation_errors_grouped": group_validation_errors(context["validation_errors"]), }) schema_version = getattr(schema_ocds, "version", None) if schema_version: db_data.schema_version = schema_version if not db_data.rendered: db_data.rendered = True db_data.save() if "records" in json_data: ocds_show_schema = SchemaOCDS(record_pkg=True) ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True) template = "cove_ocds/explore_record.html" if hasattr(json_data, "get") and hasattr(json_data.get("records"), "__iter__"): context["records"] = json_data["records"] else: context["records"] = [] if isinstance(json_data["records"], list) and len(json_data["records"]) < 100: context["ocds_show_data"] = ocds_show_data(json_data, ocds_show_deref_schema) else: ocds_show_schema = SchemaOCDS(record_pkg=False) ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True) template = "cove_ocds/explore_release.html" if hasattr(json_data, "get") and hasattr(json_data.get("releases"), "__iter__"): context["releases"] = json_data["releases"] if (isinstance(json_data["releases"], list) and len(json_data["releases"]) < 100): context["ocds_show_data"] = ocds_show_data( json_data, ocds_show_deref_schema) # Parse release dates into objects so the template can format them. for release in context["releases"]: if hasattr(release, "get") and release.get("date"): if validate_rfc3339(release["date"]): release["date"] = parser.parse(release["date"]) else: release["date"] = None if context.get("releases_aggregates"): date_fields = [ "max_award_date", "max_contract_date", "max_release_date", "max_tender_date", "min_award_date", "min_contract_date", "min_release_date", "min_tender_date", ] for field in date_fields: if context["releases_aggregates"].get(field): if validate_rfc3339( context["releases_aggregates"][field]): context["releases_aggregates"][ field] = parser.parse( context["releases_aggregates"][field]) else: context["releases_aggregates"][field] = None else: context["releases"] = [] return render(request, template, context)
def explore_ocds(request, pk): context, db_data, error = explore_data_context(request, pk) if error: return error lib_cove_ocds_config = LibCoveOCDSConfig() lib_cove_ocds_config.config['current_language'] = translation.get_language() upload_dir = db_data.upload_dir() upload_url = db_data.upload_url() file_name = db_data.original_file.file.name file_type = context['file_type'] post_version_choice = request.POST.get('version') replace = False validation_errors_path = os.path.join(upload_dir, 'validation_errors-3.json') if file_type == 'json': # open the data first so we can inspect for record package with open(file_name, encoding='utf-8') as fp: try: json_data = json.load(fp, parse_float=Decimal) except ValueError as err: raise CoveInputDataError(context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.' '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' '</span> <strong>Error message:</strong> {}', err)), 'error': format(err) }) if not isinstance(json_data, dict): raise CoveInputDataError(context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _('OCDS JSON should have an object as the top level, the JSON you supplied does not.'), }) version_in_data = json_data.get('version', '') db_data.data_schema_version = version_in_data select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS(select_version=select_version, release_data=json_data, lib_cove_ocds_config=lib_cove_ocds_config) if schema_ocds.missing_package: exceptions.raise_missing_package_error() if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: if isinstance(version_in_data, str) and re.compile('^\d+\.\d+\.\d+$').match(version_in_data): exceptions.raise_invalid_version_data_with_patch(version_in_data) else: if not isinstance(version_in_data, str): version_in_data = '{} (it must be a string)'.format(str(version_in_data)) context['unrecognized_version_data'] = version_in_data if schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_release_schema_file(upload_dir, upload_url) url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url if 'records' in json_data: context['conversion'] = None else: # Replace the spreadsheet conversion only if it exists already. converted_path = os.path.join(upload_dir, 'flattened') replace_converted = replace and os.path.exists(converted_path + '.xlsx') context.update(convert_json(upload_dir, upload_url, file_name, lib_cove_ocds_config, schema_url=url, replace=replace_converted, request=request, flatten=request.POST.get('flatten'))) else: # Use the lowest release pkg schema version accepting 'version' field metatab_schema_url = SchemaOCDS(select_version='1.1', lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type) if 'version' not in metatab_data: metatab_data['version'] = '1.0' else: db_data.data_schema_version = metatab_data['version'] select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS(select_version=select_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config) # Unlike for JSON data case above, do not check for missing data package if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: version_in_data = metatab_data.get('version') if re.compile('^\d+\.\d+\.\d+$').match(version_in_data): exceptions.raise_invalid_version_data_with_patch(version_in_data) else: context['unrecognized_version_data'] = version_in_data # Replace json conversion when user chooses a different schema version. if db_data.schema_version and schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_release_schema_file(upload_dir, upload_url) url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url pkg_url = schema_ocds.release_pkg_schema_url context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_ocds_config, schema_url=url, pkg_schema_url=pkg_url, replace=replace)) with open(context['converted_path'], encoding='utf-8') as fp: json_data = json.load(fp, parse_float=Decimal) if replace: if os.path.exists(validation_errors_path): os.remove(validation_errors_path) context = common_checks_ocds(context, upload_dir, json_data, schema_ocds) if schema_ocds.json_deref_error: exceptions.raise_json_deref_error(schema_ocds.json_deref_error) context.update({ 'data_schema_version': db_data.data_schema_version, 'first_render': not db_data.rendered, 'validation_errors_grouped': group_validation_errors(context['validation_errors']), }) schema_version = getattr(schema_ocds, 'version', None) if schema_version: db_data.schema_version = schema_version if not db_data.rendered: db_data.rendered = True db_data.save() ocds_show_schema = SchemaOCDS() ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj(deref=True) if 'records' in json_data: template = 'cove_ocds/explore_record.html' if hasattr(json_data, 'get') and hasattr(json_data.get('records'), '__iter__'): context['records'] = json_data['records'] else: context['records'] = [] if isinstance(json_data['records'], list) and len(json_data['records']) < 100: context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema) else: template = 'cove_ocds/explore_release.html' if hasattr(json_data, 'get') and hasattr(json_data.get('releases'), '__iter__'): context['releases'] = json_data['releases'] if isinstance(json_data['releases'], list) and len(json_data['releases']) < 100: context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema) # Parse release dates into objects so the template can format them. for release in context['releases']: if hasattr(release, 'get') and release.get('date'): if validate_rfc3339(release['date']): release['date'] = parser.parse(release['date']) else: release['date'] = None if context.get('releases_aggregates'): date_fields = ['max_award_date', 'max_contract_date', 'max_release_date', 'max_tender_date', 'min_award_date', 'min_contract_date', 'min_release_date', 'min_tender_date'] for field in date_fields: if context['releases_aggregates'].get(field): if(validate_rfc3339(context['releases_aggregates'][field])): context['releases_aggregates'][field] = parser.parse(context['releases_aggregates'][field]) else: context['releases_aggregates'][field] = None else: context['releases'] = [] return render(request, template, context)
def explore_ocds(request, pk): context, db_data, error = explore_data_context(request, pk) if error: return error lib_cove_ocds_config = LibCoveOCDSConfig() lib_cove_ocds_config.config["current_language"] = translation.get_language( ) lib_cove_ocds_config.config[ "schema_version_choices"] = settings.COVE_CONFIG[ "schema_version_choices"] lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[ "schema_codelists"] upload_dir = db_data.upload_dir() upload_url = db_data.upload_url() file_name = db_data.original_file.file.name file_type = context["file_type"] post_version_choice = request.POST.get( "version", lib_cove_ocds_config.config["schema_version"]) replace = False validation_errors_path = os.path.join(upload_dir, "validation_errors-3.json") if file_type == "json": # open the data first so we can inspect for record package with open(file_name, encoding="utf-8") as fp: try: json_data = json.load(fp, parse_float=Decimal, object_pairs_hook=OrderedDict) except ValueError as err: raise CoveInputDataError( context={ "sub_title": _("Sorry, we can't process that data"), "link": "index", "link_text": _("Try Again"), "msg": _( format_html( "We think you tried to upload a JSON file, but it is not well formed JSON." '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' "</span> <strong>Error message:</strong> {}", err, )), "error": format(err), }) if not isinstance(json_data, dict): raise CoveInputDataError( context={ "sub_title": _("Sorry, we can't process that data"), "link": "index", "link_text": _("Try Again"), "msg": _("OCDS JSON should have an object as the top level, the JSON you supplied does not." ), }) version_in_data = json_data.get("version", "") db_data.data_schema_version = version_in_data select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS( select_version=select_version, release_data=json_data, lib_cove_ocds_config=lib_cove_ocds_config, ) if schema_ocds.missing_package: exceptions.raise_missing_package_error() if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: if isinstance(version_in_data, str) and re.compile( "^\d+\.\d+\.\d+$").match(version_in_data): exceptions.raise_invalid_version_data_with_patch( version_in_data) else: if not isinstance(version_in_data, str): version_in_data = "{} (it must be a string)".format( str(version_in_data)) context["unrecognized_version_data"] = version_in_data if schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_release_schema_file( upload_dir, upload_url) schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url if "records" in json_data: context["conversion"] = None else: # Replace the spreadsheet conversion only if it exists already. converted_path = os.path.join(upload_dir, "flattened") replace_converted = replace and os.path.exists(converted_path + ".xlsx") with warnings.catch_warnings(): warnings.filterwarnings( 'ignore' ) # flattentool uses UserWarning, so can't set a specific category convert_json_context = convert_json( upload_dir, upload_url, file_name, lib_cove_ocds_config, schema_url=schema_url, replace=replace_converted, request=request, flatten=request.POST.get("flatten"), ) context.update(convert_json_context) else: # Use the lowest release pkg schema version accepting 'version' field metatab_schema_url = SchemaOCDS( select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type) if "version" not in metatab_data: metatab_data["version"] = "1.0" else: db_data.data_schema_version = metatab_data["version"] select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS( select_version=select_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config, ) # Unlike for JSON data case above, do not check for missing data package if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: version_in_data = metatab_data.get("version") if re.compile("^\d+\.\d+\.\d+$").match(version_in_data): exceptions.raise_invalid_version_data_with_patch( version_in_data) else: context["unrecognized_version_data"] = version_in_data # Replace json conversion when user chooses a different schema version. if db_data.schema_version and schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_release_schema_file( upload_dir, upload_url) schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url pkg_url = schema_ocds.release_pkg_schema_url if file_type != "csv": # ORIGINAL UNFLATTEN conversion_context = convert_spreadsheet( upload_dir, upload_url, file_name, file_type, lib_cove_ocds_config, schema_url=schema_url, pkg_schema_url=pkg_url, replace=replace, ) else: # Convert Simple CSV to flat OCDS and return context conversion_context = convert_simple_csv_submission( db_data, lib_cove_ocds_config, schema_url, replace=replace, ) context.update(conversion_context) with open(context["converted_path"], encoding="utf-8") as fp: json_data = json.load(fp, parse_float=Decimal, object_pairs_hook=OrderedDict) if replace: if os.path.exists(validation_errors_path): os.remove(validation_errors_path) context = common_checks_ocds(context, upload_dir, json_data, schema_ocds, cache=settings.CACHE_VALIDATION_ERRORS) if schema_ocds.json_deref_error: exceptions.raise_json_deref_error(schema_ocds.json_deref_error) schema_version = getattr(schema_ocds, "version", None) if schema_version: db_data.schema_version = schema_version if not db_data.rendered: db_data.rendered = True db_data.save() context.update({ "data_schema_version": db_data.schema_version, "first_render": not db_data.rendered, "validation_errors_grouped": group_validation_errors(context["validation_errors"]), }) ocds_show_schema = SchemaOCDS() ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj( deref=True) if "records" in json_data: template = "cove_ocds/explore_record.html" if hasattr(json_data, "get") and hasattr(json_data.get("records"), "__iter__"): context["records"] = json_data["records"] else: context["records"] = [] if isinstance(json_data["records"], list) and len(json_data["records"]) < 100: context["ocds_show_data"] = ocds_show_data(json_data, ocds_show_deref_schema) else: template = "silvereye/explore_release.html" if hasattr(json_data, "get") and hasattr(json_data.get("releases"), "__iter__"): context["releases"] = json_data["releases"] if (isinstance(json_data["releases"], list) and len(json_data["releases"]) < 100): context["ocds_show_data"] = ocds_show_data( json_data, ocds_show_deref_schema) # Parse release dates into objects so the template can format them. for release in context["releases"]: if hasattr(release, "get") and release.get("date"): if validate_rfc3339(release["date"]): release["date"] = parser.parse(release["date"]) else: release["date"] = None try: trans_date = release["contracts"][0]["implementation"][ "transactions"][0]["date"] parsed_trans_date = parser.parse(trans_date) release["contracts"][0]["implementation"]["transactions"][ 0]["date"] = parsed_trans_date except KeyError: pass if context.get("releases_aggregates"): date_fields = [ "max_award_date", "max_contract_date", "max_release_date", "max_tender_date", "min_award_date", "min_contract_date", "min_release_date", "min_tender_date", ] for field in date_fields: if context["releases_aggregates"].get(field): if validate_rfc3339( context["releases_aggregates"][field]): context["releases_aggregates"][ field] = parser.parse( context["releases_aggregates"][field]) else: context["releases_aggregates"][field] = None else: context["releases"] = [] # Include field coverage report original_file_path = context["original_file"]["path"] mapper = CSVMapper(csv_path=original_file_path) db_data.notice_type = mapper.release_type db_data.save() coverage_context = mapper.get_coverage_context() context.update({ "field_coverage": coverage_context, }) ocds_validation_errors, simple_csv_errors = prepare_simple_csv_validation_errors( context["validation_errors"], mapper, coverage_context["required_fields_missing"]) context.update({ "ocds_validation_errors": ocds_validation_errors, "simple_csv_errors": simple_csv_errors, "csv_mapper": mapper, }) # Silvereye: Insert OCDS data releases = context.get("releases") if releases: # If we don't have validation errors validation_errors_grouped = context["validation_errors_grouped"] if not validation_errors_grouped: json_string = json.dumps(json_data, indent=2, sort_keys=True, cls=DjangoJSONEncoder) UpsertDataHelpers().upsert_ocds_data(json_string, supplied_data=db_data) average_field_completion = coverage_context.get( "average_field_completion") inst, created = FieldCoverage.objects.update_or_create( file_submission=db_data, defaults={ "tenders_field_coverage": average_field_completion if mapper.release_type == "tender" else None, "awards_field_coverage": average_field_completion if mapper.release_type == "award" else None, "spend_field_coverage": average_field_completion if mapper.release_type == "spend" else None, }) update_publisher_monthly_counts() return render(request, template, context)