def explore_360(request, pk, template='cove_360/explore.html'): schema_360 = Schema360() context, db_data, error = explore_data_context(request, pk) if error: return error lib_cove_config = LibCoveConfig() lib_cove_config.config.update(settings.COVE_CONFIG) upload_dir = db_data.upload_dir() upload_url = db_data.upload_url() file_name = db_data.original_file.file.name file_type = context['file_type'] if file_type == 'json': # open the data first so we can inspect for record package with open(file_name, encoding='utf-8') as fp: try: json_data = json.load(fp, parse_float=Decimal) except ValueError as err: raise CoveInputDataError(context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.' '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' '</span> <strong>Error message:</strong> {}', err)), 'error': format(err) }) if not isinstance(json_data, dict): raise CoveInputDataError(context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _('360Giving JSON should have an object as the top level, the JSON you supplied does not.'), }) context.update(convert_json(upload_dir, upload_url, file_name, schema_url=schema_360.release_schema_url, request=request, flatten=request.POST.get('flatten'), lib_cove_config=lib_cove_config)) else: context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_config, schema_360.release_schema_url, schema_360.release_pkg_schema_url)) with open(context['converted_path'], encoding='utf-8') as fp: json_data = json.load(fp, parse_float=Decimal) context = common_checks_360(context, upload_dir, json_data, schema_360) if hasattr(json_data, 'get') and hasattr(json_data.get('grants'), '__iter__'): context['grants'] = json_data['grants'] else: context['grants'] = [] context['first_render'] = not db_data.rendered if not db_data.rendered: db_data.rendered = True db_data.save() return render(request, template, context)
def explore_data_context_iati(request, context, db_data, error): lib_cove_config = LibCoveConfig() lib_cove_config.config.update(settings.COVE_CONFIG) file_type = context['file_type'] if file_type != 'xml': schema_iati = SchemaIATI() context.update( convert_spreadsheet(db_data.upload_dir(), db_data.upload_url(), db_data.original_file.file.name, file_type, lib_cove_config, xml=True, xml_schemas=[ schema_iati.activity_schema, schema_iati.organisation_schema, schema_iati.common_schema, ])) data_file = context['converted_path'] else: data_file = db_data.original_file.file.name tree = get_tree(data_file) context = common_checks_context_iati(context, db_data.upload_dir(), data_file, file_type, tree) context['first_render'] = not db_data.rendered context['invalid_embedded_codelist_values'] = aggregate_results( context['invalid_embedded_codelist_values']) context['invalid_non_embedded_codelist_values'] = aggregate_results( context['invalid_non_embedded_codelist_values']) context['iati_identifiers_count'] = iati_identifier_count(tree) context['organisation_identifier_count'] = organisation_identifier_count( tree) if file_type == 'xml': if context['organisation_identifier_count']: root_list_path = 'iati-organisation' root_id = 'organisation-identifier' else: root_list_path = 'iati-activity' root_id = None context.update( convert_json(db_data.upload_dir(), db_data.upload_url(), db_data.original_file.file.name, root_list_path=root_list_path, root_id=root_id, request=request, flatten=request.POST.get('flatten'), xml=True, lib_cove_config=lib_cove_config)) if not db_data.rendered: db_data.rendered = True return context, db_data, error
def test_convert_org_xml_1(): cove_temp_folder = tempfile.mkdtemp(prefix="lib-cove-iati-tests-", dir=tempfile.gettempdir()) organisation_xml_filename = os.path.join( os.path.dirname(os.path.realpath(__file__)), "fixtures", "converters", "convert_org_1.xml", ) lib_cove_config = LibCoveConfig() output = convert_json( cove_temp_folder, "", organisation_xml_filename, lib_cove_config, flatten=True, xml=True, root_list_path="iati-organisation", root_id="organisation-identifier", ) assert output["converted_url"] == "/flattened" assert len(output["conversion_warning_messages"]) == 0 assert output["conversion"] == "flatten" conversion_warning_messages_name = os.path.join( cove_temp_folder, "conversion_warning_messages.json") assert os.path.isfile(conversion_warning_messages_name) with open(conversion_warning_messages_name) as fp: conversion_warning_messages_data = json.load(fp) assert conversion_warning_messages_data == [] assert os.path.isfile(os.path.join(cove_temp_folder, "flattened.xlsx")) assert os.path.isfile( os.path.join(cove_temp_folder, "flattened", "iati-organisation.csv")) with open( os.path.join(cove_temp_folder, "flattened", "iati-organisation.csv"), "r") as csvfile: csvreader = csv.reader(csvfile) header = next(csvreader) assert header[0] == "organisation-identifier" assert header[4] == "name/narrative" row1 = next(csvreader) assert row1[0] == "GB-GOV-1" assert row1[4] == "UK Department for International Development"
def test_convert_activity_xml_1(): cove_temp_folder = tempfile.mkdtemp(prefix="lib-cove-iati-tests-", dir=tempfile.gettempdir()) activity_xml_filename = os.path.join( os.path.dirname(os.path.realpath(__file__)), "fixtures", "converters", "convert_activity_1.xml", ) lib_cove_config = LibCoveConfig() output = convert_json( cove_temp_folder, "", activity_xml_filename, lib_cove_config, flatten=True, xml=True, root_list_path="iati-activity", ) assert output["converted_url"] == "/flattened" assert len(output["conversion_warning_messages"]) == 0 assert output["conversion"] == "flatten" conversion_warning_messages_name = os.path.join( cove_temp_folder, "conversion_warning_messages.json") assert os.path.isfile(conversion_warning_messages_name) with open(conversion_warning_messages_name) as fp: conversion_warning_messages_data = json.load(fp) assert conversion_warning_messages_data == [] assert os.path.isfile(os.path.join(cove_temp_folder, "flattened.xlsx")) assert os.path.isfile( os.path.join(cove_temp_folder, "flattened", "iati-activity.csv")) with open(os.path.join(cove_temp_folder, "flattened", "iati-activity.csv"), "r") as csvfile: csvreader = csv.reader(csvfile) header = next(csvreader) assert header[0] == "@default-currency" assert header[1] == "iati-identifier" row1 = next(csvreader) assert row1[0] == "GBP" assert row1[1] == "GB-TEST-13-example_ODSC_2019"
def test_convert_json_root_is_list_1(): cove_temp_folder = tempfile.mkdtemp(prefix="lib-cove-ocds-tests-", dir=tempfile.gettempdir()) json_filename = os.path.join( os.path.dirname(os.path.realpath(__file__)), "fixtures", "converters", "convert_json_root_is_list_1.json", ) lib_cove_config = LibCoveConfig() lib_cove_config.config["root_is_list"] = True output = convert_json(cove_temp_folder, "", json_filename, lib_cove_config, flatten=True) assert output["converted_url"] == "/flattened" assert len(output["conversion_warning_messages"]) == 0 assert output["conversion"] == "flatten" conversion_warning_messages_name = os.path.join( cove_temp_folder, "conversion_warning_messages.json") assert os.path.isfile(conversion_warning_messages_name) with open(conversion_warning_messages_name) as fp: conversion_warning_messages_data = json.load(fp) assert conversion_warning_messages_data == [] assert os.path.isfile( os.path.join(cove_temp_folder, "flattened", "main.csv")) with open(os.path.join(cove_temp_folder, "flattened", "main.csv"), "r") as csvfile: csvreader = csv.reader(csvfile) header = next(csvreader) assert header[0] == "id" assert header[1] == "title" row1 = next(csvreader) assert row1[0] == "1" assert row1[1] == "Cat" row2 = next(csvreader) assert row2[0] == "2" assert row2[1] == "Hat"
def explore_iati(request, pk): context, db_data, error = explore_data_context(request, pk, get_file_type) if error: return error lib_cove_config = LibCoveConfig() lib_cove_config.config.update(settings.COVE_CONFIG) file_type = context['file_type'] if file_type != 'xml': schema_iati = SchemaIATI() context.update(convert_spreadsheet( db_data.upload_dir(), db_data.upload_url(), db_data.original_file.file.name, file_type, lib_cove_config, xml=True, xml_schemas=[ schema_iati.activity_schema, schema_iati.organisation_schema, schema_iati.common_schema, ])) data_file = context['converted_path'] else: data_file = db_data.original_file.file.name context.update(convert_json(db_data.upload_dir(), db_data.upload_url(), db_data.original_file.file.name, request=request, flatten=request.POST.get('flatten'), xml=True, lib_cove_config=lib_cove_config)) tree = get_tree(data_file) context = common_checks_context_iati(context, db_data.upload_dir(), data_file, file_type, tree) context['first_render'] = not db_data.rendered context['invalid_embedded_codelist_values'] = aggregate_results(context['invalid_embedded_codelist_values']) context['invalid_non_embedded_codelist_values'] = aggregate_results(context['invalid_non_embedded_codelist_values']) context['iati_identifiers_count'] = iati_identifier_count(tree) context['organisation_identifier_count'] = organisation_identifier_count(tree) if not db_data.rendered: db_data.rendered = True return render(request, 'cove_iati/explore.html', context)
def explore_ocds(request, pk): context, db_data, error = explore_data_context(request, pk) if error: return error lib_cove_ocds_config = LibCoveOCDSConfig() lib_cove_ocds_config.config["current_language"] = translation.get_language( ) lib_cove_ocds_config.config[ "schema_version_choices"] = settings.COVE_CONFIG[ "schema_version_choices"] lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[ "schema_codelists"] upload_dir = db_data.upload_dir() upload_url = db_data.upload_url() file_name = db_data.original_file.file.name file_type = context["file_type"] post_version_choice = request.POST.get( "version", lib_cove_ocds_config.config["schema_version"]) replace = False validation_errors_path = os.path.join(upload_dir, "validation_errors-3.json") if file_type == "json": # open the data first so we can inspect for record package with open(file_name, encoding="utf-8") as fp: try: json_data = json.load(fp, parse_float=Decimal, object_pairs_hook=OrderedDict) except ValueError as err: raise CoveInputDataError( context={ "sub_title": _("Sorry, we can't process that data"), "link": "index", "link_text": _("Try Again"), "msg": _( format_html( "We think you tried to upload a JSON file, but it is not well formed JSON." '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' "</span> <strong>Error message:</strong> {}", err, )), "error": format(err), }) if not isinstance(json_data, dict): raise CoveInputDataError( context={ "sub_title": _("Sorry, we can't process that data"), "link": "index", "link_text": _("Try Again"), "msg": _("OCDS JSON should have an object as the top level, the JSON you supplied does not." ), }) version_in_data = json_data.get("version", "") db_data.data_schema_version = version_in_data select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS( select_version=select_version, release_data=json_data, lib_cove_ocds_config=lib_cove_ocds_config, ) if schema_ocds.missing_package: exceptions.raise_missing_package_error() if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: if isinstance(version_in_data, str) and re.compile( "^\d+\.\d+\.\d+$").match(version_in_data): exceptions.raise_invalid_version_data_with_patch( version_in_data) else: if not isinstance(version_in_data, str): version_in_data = "{} (it must be a string)".format( str(version_in_data)) context["unrecognized_version_data"] = version_in_data if schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_release_schema_file( upload_dir, upload_url) schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url if "records" in json_data: context["conversion"] = None else: # Replace the spreadsheet conversion only if it exists already. converted_path = os.path.join(upload_dir, "flattened") replace_converted = replace and os.path.exists(converted_path + ".xlsx") with warnings.catch_warnings(): warnings.filterwarnings( 'ignore' ) # flattentool uses UserWarning, so can't set a specific category convert_json_context = convert_json( upload_dir, upload_url, file_name, lib_cove_ocds_config, schema_url=schema_url, replace=replace_converted, request=request, flatten=request.POST.get("flatten"), ) context.update(convert_json_context) else: # Use the lowest release pkg schema version accepting 'version' field metatab_schema_url = SchemaOCDS( select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type) if "version" not in metatab_data: metatab_data["version"] = "1.0" else: db_data.data_schema_version = metatab_data["version"] select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS( select_version=select_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config, ) # Unlike for JSON data case above, do not check for missing data package if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: version_in_data = metatab_data.get("version") if re.compile("^\d+\.\d+\.\d+$").match(version_in_data): exceptions.raise_invalid_version_data_with_patch( version_in_data) else: context["unrecognized_version_data"] = version_in_data # Replace json conversion when user chooses a different schema version. if db_data.schema_version and schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_release_schema_file( upload_dir, upload_url) schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url pkg_url = schema_ocds.release_pkg_schema_url if file_type != "csv": # ORIGINAL UNFLATTEN conversion_context = convert_spreadsheet( upload_dir, upload_url, file_name, file_type, lib_cove_ocds_config, schema_url=schema_url, pkg_schema_url=pkg_url, replace=replace, ) else: # Convert Simple CSV to flat OCDS and return context conversion_context = convert_simple_csv_submission( db_data, lib_cove_ocds_config, schema_url, replace=replace, ) context.update(conversion_context) with open(context["converted_path"], encoding="utf-8") as fp: json_data = json.load(fp, parse_float=Decimal, object_pairs_hook=OrderedDict) if replace: if os.path.exists(validation_errors_path): os.remove(validation_errors_path) context = common_checks_ocds(context, upload_dir, json_data, schema_ocds, cache=settings.CACHE_VALIDATION_ERRORS) if schema_ocds.json_deref_error: exceptions.raise_json_deref_error(schema_ocds.json_deref_error) schema_version = getattr(schema_ocds, "version", None) if schema_version: db_data.schema_version = schema_version if not db_data.rendered: db_data.rendered = True db_data.save() context.update({ "data_schema_version": db_data.schema_version, "first_render": not db_data.rendered, "validation_errors_grouped": group_validation_errors(context["validation_errors"]), }) ocds_show_schema = SchemaOCDS() ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj( deref=True) if "records" in json_data: template = "cove_ocds/explore_record.html" if hasattr(json_data, "get") and hasattr(json_data.get("records"), "__iter__"): context["records"] = json_data["records"] else: context["records"] = [] if isinstance(json_data["records"], list) and len(json_data["records"]) < 100: context["ocds_show_data"] = ocds_show_data(json_data, ocds_show_deref_schema) else: template = "silvereye/explore_release.html" if hasattr(json_data, "get") and hasattr(json_data.get("releases"), "__iter__"): context["releases"] = json_data["releases"] if (isinstance(json_data["releases"], list) and len(json_data["releases"]) < 100): context["ocds_show_data"] = ocds_show_data( json_data, ocds_show_deref_schema) # Parse release dates into objects so the template can format them. for release in context["releases"]: if hasattr(release, "get") and release.get("date"): if validate_rfc3339(release["date"]): release["date"] = parser.parse(release["date"]) else: release["date"] = None try: trans_date = release["contracts"][0]["implementation"][ "transactions"][0]["date"] parsed_trans_date = parser.parse(trans_date) release["contracts"][0]["implementation"]["transactions"][ 0]["date"] = parsed_trans_date except KeyError: pass if context.get("releases_aggregates"): date_fields = [ "max_award_date", "max_contract_date", "max_release_date", "max_tender_date", "min_award_date", "min_contract_date", "min_release_date", "min_tender_date", ] for field in date_fields: if context["releases_aggregates"].get(field): if validate_rfc3339( context["releases_aggregates"][field]): context["releases_aggregates"][ field] = parser.parse( context["releases_aggregates"][field]) else: context["releases_aggregates"][field] = None else: context["releases"] = [] # Include field coverage report original_file_path = context["original_file"]["path"] mapper = CSVMapper(csv_path=original_file_path) db_data.notice_type = mapper.release_type db_data.save() coverage_context = mapper.get_coverage_context() context.update({ "field_coverage": coverage_context, }) ocds_validation_errors, simple_csv_errors = prepare_simple_csv_validation_errors( context["validation_errors"], mapper, coverage_context["required_fields_missing"]) context.update({ "ocds_validation_errors": ocds_validation_errors, "simple_csv_errors": simple_csv_errors, "csv_mapper": mapper, }) # Silvereye: Insert OCDS data releases = context.get("releases") if releases: # If we don't have validation errors validation_errors_grouped = context["validation_errors_grouped"] if not validation_errors_grouped: json_string = json.dumps(json_data, indent=2, sort_keys=True, cls=DjangoJSONEncoder) UpsertDataHelpers().upsert_ocds_data(json_string, supplied_data=db_data) average_field_completion = coverage_context.get( "average_field_completion") inst, created = FieldCoverage.objects.update_or_create( file_submission=db_data, defaults={ "tenders_field_coverage": average_field_completion if mapper.release_type == "tender" else None, "awards_field_coverage": average_field_completion if mapper.release_type == "award" else None, "spend_field_coverage": average_field_completion if mapper.release_type == "spend" else None, }) update_publisher_monthly_counts() return render(request, template, context)
def explore_ocds(request, pk): context, db_data, error = explore_data_context(request, pk) if error: return error lib_cove_ocds_config = LibCoveOCDSConfig() lib_cove_ocds_config.config["current_language"] = translation.get_language( ) lib_cove_ocds_config.config[ "schema_version_choices"] = settings.COVE_CONFIG[ "schema_version_choices"] lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[ "schema_codelists"] upload_dir = db_data.upload_dir() upload_url = db_data.upload_url() file_name = db_data.original_file.file.name file_type = context["file_type"] post_version_choice = request.POST.get("version") replace = False validation_errors_path = os.path.join(upload_dir, "validation_errors-3.json") if file_type == "json": # open the data first so we can inspect for record package with open(file_name, encoding="utf-8") as fp: try: json_data = json.load(fp, parse_float=Decimal, object_pairs_hook=OrderedDict) except UnicodeError as err: raise CoveInputDataError( context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': format_html( _("The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON follows the I-JSON format, which requires UTF-8 encoding. Ensure that your file uses UTF-8 encoding, then try uploading again." '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' '</span> <strong>Error message:</strong> {}'), err), 'error': format(err) }) except ValueError as err: raise CoveInputDataError( context={ "sub_title": _("Sorry, we can't process that data"), "link": "index", "link_text": _("Try Again"), "msg": format_html( _( "We think you tried to upload a JSON file, but it is not well formed JSON." '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' "</span> <strong>Error message:</strong> {}", ), err, ), "error": format(err), }) if not isinstance(json_data, dict): raise CoveInputDataError( context={ "sub_title": _("Sorry, we can't process that data"), "link": "index", "link_text": _("Try Again"), "msg": _("OCDS JSON should have an object as the top level, the JSON you supplied does not." ), }) version_in_data = json_data.get("version", "") db_data.data_schema_version = version_in_data select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS(select_version=select_version, release_data=json_data, lib_cove_ocds_config=lib_cove_ocds_config, record_pkg="records" in json_data) if schema_ocds.missing_package: exceptions.raise_missing_package_error() if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: if isinstance(version_in_data, str) and re.compile( "^\d+\.\d+\.\d+$").match(version_in_data): exceptions.raise_invalid_version_data_with_patch( version_in_data) else: if not isinstance(version_in_data, str): version_in_data = "{} (it must be a string)".format( str(version_in_data)) context["unrecognized_version_data"] = version_in_data if schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_schema_file(upload_dir, upload_url) url = schema_ocds.extended_schema_file or schema_ocds.schema_url if "records" in json_data: context["conversion"] = None else: # Replace the spreadsheet conversion only if it exists already. converted_path = os.path.join(upload_dir, "flattened") replace_converted = replace and os.path.exists(converted_path + ".xlsx") with warnings.catch_warnings(): warnings.filterwarnings( 'ignore' ) # flattentool uses UserWarning, so can't set a specific category convert_json_context = convert_json( upload_dir, upload_url, file_name, lib_cove_ocds_config, schema_url=url, replace=replace_converted, request=request, flatten=request.POST.get("flatten"), ) context.update(convert_json_context) else: # Use the lowest release pkg schema version accepting 'version' field metatab_schema_url = SchemaOCDS( select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type) if "version" not in metatab_data: metatab_data["version"] = "1.0" else: db_data.data_schema_version = metatab_data["version"] select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS( select_version=select_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config, ) # Unlike for JSON data case above, do not check for missing data package if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: version_in_data = metatab_data.get("version") if re.compile("^\d+\.\d+\.\d+$").match(version_in_data): exceptions.raise_invalid_version_data_with_patch( version_in_data) else: context["unrecognized_version_data"] = version_in_data # Replace json conversion when user chooses a different schema version. if db_data.schema_version and schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_schema_file(upload_dir, upload_url) url = schema_ocds.extended_schema_file or schema_ocds.schema_url pkg_url = schema_ocds.pkg_schema_url context.update( convert_spreadsheet( upload_dir, upload_url, file_name, file_type, lib_cove_ocds_config, schema_url=url, pkg_schema_url=pkg_url, replace=replace, )) with open(context["converted_path"], encoding="utf-8") as fp: json_data = json.load(fp, parse_float=Decimal, object_pairs_hook=OrderedDict) if replace: if os.path.exists(validation_errors_path): os.remove(validation_errors_path) context = common_checks_ocds(context, upload_dir, json_data, schema_ocds) if schema_ocds.json_deref_error: exceptions.raise_json_deref_error(schema_ocds.json_deref_error) context.update({ "data_schema_version": db_data.data_schema_version, "first_render": not db_data.rendered, "validation_errors_grouped": group_validation_errors(context["validation_errors"]), }) schema_version = getattr(schema_ocds, "version", None) if schema_version: db_data.schema_version = schema_version if not db_data.rendered: db_data.rendered = True db_data.save() if "records" in json_data: ocds_show_schema = SchemaOCDS(record_pkg=True) ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True) template = "cove_ocds/explore_record.html" if hasattr(json_data, "get") and hasattr(json_data.get("records"), "__iter__"): context["records"] = json_data["records"] else: context["records"] = [] if isinstance(json_data["records"], list) and len(json_data["records"]) < 100: context["ocds_show_data"] = ocds_show_data(json_data, ocds_show_deref_schema) else: ocds_show_schema = SchemaOCDS(record_pkg=False) ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True) template = "cove_ocds/explore_release.html" if hasattr(json_data, "get") and hasattr(json_data.get("releases"), "__iter__"): context["releases"] = json_data["releases"] if (isinstance(json_data["releases"], list) and len(json_data["releases"]) < 100): context["ocds_show_data"] = ocds_show_data( json_data, ocds_show_deref_schema) # Parse release dates into objects so the template can format them. for release in context["releases"]: if hasattr(release, "get") and release.get("date"): if validate_rfc3339(release["date"]): release["date"] = parser.parse(release["date"]) else: release["date"] = None if context.get("releases_aggregates"): date_fields = [ "max_award_date", "max_contract_date", "max_release_date", "max_tender_date", "min_award_date", "min_contract_date", "min_release_date", "min_tender_date", ] for field in date_fields: if context["releases_aggregates"].get(field): if validate_rfc3339( context["releases_aggregates"][field]): context["releases_aggregates"][ field] = parser.parse( context["releases_aggregates"][field]) else: context["releases_aggregates"][field] = None else: context["releases"] = [] return render(request, template, context)
def explore_bods(request, pk): context, db_data, error = explore_data_context(request, pk) if error: return error lib_cove_bods_config = LibCoveBODSConfig() lib_cove_bods_config.config['root_list_path'] = settings.COVE_CONFIG[ 'root_list_path'] lib_cove_bods_config.config['root_id'] = settings.COVE_CONFIG['root_id'] lib_cove_bods_config.config['id_name'] = settings.COVE_CONFIG['id_name'] lib_cove_bods_config.config['root_is_list'] = settings.COVE_CONFIG[ 'root_is_list'] lib_cove_bods_config.config[ 'bods_additional_checks_person_birthdate_max_year'] = datetime.datetime.now( ).year lib_cove_bods_config.config['bods_additional_checks_person_birthdate_min_year'] = \ datetime.datetime.now().year - 120 upload_dir = db_data.upload_dir() upload_url = db_data.upload_url() file_name = db_data.original_file.file.name file_type = context['file_type'] if file_type == 'json': # open the data first so we can inspect for record package with open(file_name, encoding='utf-8') as fp: try: json_data = json.load(fp, parse_float=Decimal) except ValueError as err: raise CoveInputDataError( context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _( format_html( 'We think you tried to upload a JSON file, but it is not well formed JSON.' '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' '</span> <strong>Error message:</strong> {}', err)), 'error': format(err) }) if not isinstance(json_data, list): raise CoveInputDataError( context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _('BODS JSON should have a list as the top level, the JSON you supplied does not.' ), }) schema_bods = SchemaBODS(json_data=json_data, lib_cove_bods_config=lib_cove_bods_config) context.update( convert_json(upload_dir, upload_url, file_name, lib_cove_bods_config, schema_url=schema_bods.release_pkg_schema_url, replace=True, request=request, flatten=True)) else: schema_bods = SchemaBODS(lib_cove_bods_config=lib_cove_bods_config) context.update( convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_bods_config, schema_url=schema_bods.release_pkg_schema_url)) with open(context['converted_path'], encoding='utf-8') as fp: json_data = json.load(fp, parse_float=Decimal) context = common_checks_bods(context, upload_dir, json_data, schema_bods, lib_cove_bods_config=lib_cove_bods_config) if not db_data.rendered: db_data.rendered = True db_data.save() # Some extra info from the Schema context['schema_version_used'] = schema_bods.schema_version # We need to calculate some stats for showing in the view total_ownership_or_control_interest_statements = 0 for key, count in context['statistics'][ 'count_ownership_or_control_statement_interest_statement_types'].items( ): total_ownership_or_control_interest_statements += count context['statistics'][ 'count_ownership_or_control_interest_statement'] = total_ownership_or_control_interest_statements # noqa # The use of r_e_type is to stop flake8 complaining about line length r_e_type = 'registeredEntity' context['statistics'][ 'count_entities_registeredEntity_legalEntity_with_any_identifier'] = ( context['statistics'] ['count_entity_statements_types_with_any_identifier'][r_e_type] + context['statistics'] ['count_entity_statements_types_with_any_identifier'] ['legalEntity']) context['statistics'][ 'count_entities_registeredEntity_legalEntity_with_any_identifier_with_id_and_scheme'] = ( context['statistics'] ['count_entity_statements_types_with_any_identifier_with_id_and_scheme'] [r_e_type] + context['statistics'] ['count_entity_statements_types_with_any_identifier_with_id_and_scheme'] ['legalEntity']) context['statistics']['count_entities_registeredEntity_legalEntity'] = ( context['statistics']['count_entity_statements_types'][r_e_type] + context['statistics']['count_entity_statements_types']['legalEntity']) template = 'cove_bods/explore.html' return render(request, template, context)
def explore_ocds(request, pk): context, db_data, error = explore_data_context(request, pk) if error: return error lib_cove_ocds_config = LibCoveOCDSConfig() lib_cove_ocds_config.config['current_language'] = translation.get_language() upload_dir = db_data.upload_dir() upload_url = db_data.upload_url() file_name = db_data.original_file.file.name file_type = context['file_type'] post_version_choice = request.POST.get('version') replace = False validation_errors_path = os.path.join(upload_dir, 'validation_errors-3.json') if file_type == 'json': # open the data first so we can inspect for record package with open(file_name, encoding='utf-8') as fp: try: json_data = json.load(fp, parse_float=Decimal) except ValueError as err: raise CoveInputDataError(context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.' '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' '</span> <strong>Error message:</strong> {}', err)), 'error': format(err) }) if not isinstance(json_data, dict): raise CoveInputDataError(context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _('OCDS JSON should have an object as the top level, the JSON you supplied does not.'), }) version_in_data = json_data.get('version', '') db_data.data_schema_version = version_in_data select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS(select_version=select_version, release_data=json_data, lib_cove_ocds_config=lib_cove_ocds_config) if schema_ocds.missing_package: exceptions.raise_missing_package_error() if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: if isinstance(version_in_data, str) and re.compile('^\d+\.\d+\.\d+$').match(version_in_data): exceptions.raise_invalid_version_data_with_patch(version_in_data) else: if not isinstance(version_in_data, str): version_in_data = '{} (it must be a string)'.format(str(version_in_data)) context['unrecognized_version_data'] = version_in_data if schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_release_schema_file(upload_dir, upload_url) url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url if 'records' in json_data: context['conversion'] = None else: # Replace the spreadsheet conversion only if it exists already. converted_path = os.path.join(upload_dir, 'flattened') replace_converted = replace and os.path.exists(converted_path + '.xlsx') context.update(convert_json(upload_dir, upload_url, file_name, lib_cove_ocds_config, schema_url=url, replace=replace_converted, request=request, flatten=request.POST.get('flatten'))) else: # Use the lowest release pkg schema version accepting 'version' field metatab_schema_url = SchemaOCDS(select_version='1.1', lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type) if 'version' not in metatab_data: metatab_data['version'] = '1.0' else: db_data.data_schema_version = metatab_data['version'] select_version = post_version_choice or db_data.schema_version schema_ocds = SchemaOCDS(select_version=select_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config) # Unlike for JSON data case above, do not check for missing data package if schema_ocds.invalid_version_argument: # This shouldn't happen unless the user sends random POST data. exceptions.raise_invalid_version_argument(post_version_choice) if schema_ocds.invalid_version_data: version_in_data = metatab_data.get('version') if re.compile('^\d+\.\d+\.\d+$').match(version_in_data): exceptions.raise_invalid_version_data_with_patch(version_in_data) else: context['unrecognized_version_data'] = version_in_data # Replace json conversion when user chooses a different schema version. if db_data.schema_version and schema_ocds.version != db_data.schema_version: replace = True if schema_ocds.extensions: schema_ocds.create_extended_release_schema_file(upload_dir, upload_url) url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url pkg_url = schema_ocds.release_pkg_schema_url context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_ocds_config, schema_url=url, pkg_schema_url=pkg_url, replace=replace)) with open(context['converted_path'], encoding='utf-8') as fp: json_data = json.load(fp, parse_float=Decimal) if replace: if os.path.exists(validation_errors_path): os.remove(validation_errors_path) context = common_checks_ocds(context, upload_dir, json_data, schema_ocds) if schema_ocds.json_deref_error: exceptions.raise_json_deref_error(schema_ocds.json_deref_error) context.update({ 'data_schema_version': db_data.data_schema_version, 'first_render': not db_data.rendered, 'validation_errors_grouped': group_validation_errors(context['validation_errors']), }) schema_version = getattr(schema_ocds, 'version', None) if schema_version: db_data.schema_version = schema_version if not db_data.rendered: db_data.rendered = True db_data.save() ocds_show_schema = SchemaOCDS() ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj(deref=True) if 'records' in json_data: template = 'cove_ocds/explore_record.html' if hasattr(json_data, 'get') and hasattr(json_data.get('records'), '__iter__'): context['records'] = json_data['records'] else: context['records'] = [] if isinstance(json_data['records'], list) and len(json_data['records']) < 100: context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema) else: template = 'cove_ocds/explore_release.html' if hasattr(json_data, 'get') and hasattr(json_data.get('releases'), '__iter__'): context['releases'] = json_data['releases'] if isinstance(json_data['releases'], list) and len(json_data['releases']) < 100: context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema) # Parse release dates into objects so the template can format them. for release in context['releases']: if hasattr(release, 'get') and release.get('date'): if validate_rfc3339(release['date']): release['date'] = parser.parse(release['date']) else: release['date'] = None if context.get('releases_aggregates'): date_fields = ['max_award_date', 'max_contract_date', 'max_release_date', 'max_tender_date', 'min_award_date', 'min_contract_date', 'min_release_date', 'min_tender_date'] for field in date_fields: if context['releases_aggregates'].get(field): if(validate_rfc3339(context['releases_aggregates'][field])): context['releases_aggregates'][field] = parser.parse(context['releases_aggregates'][field]) else: context['releases_aggregates'][field] = None else: context['releases'] = [] return render(request, template, context)
def ocds_json_output( output_dir, file, schema_version, convert, cache_schema=False, file_type=None, json_data=None, lib_cove_ocds_config=None, record_pkg=False, ): if not lib_cove_ocds_config: lib_cove_ocds_config = LibCoveOCDSConfig() # cache_schema is a deprecated option - now set cache_all_requests in the config instead. if cache_schema: lib_cove_ocds_config.config["cache_all_requests"] = True context = {} if not file_type: file_type = get_file_type(file) context = {"file_type": file_type} if file_type == "json": if not json_data: with open(file, encoding="utf-8") as fp: try: json_data = json.load(fp, object_pairs_hook=OrderedDict) except ValueError: raise APIException("The file looks like invalid json") schema_ocds = SchemaOCDS( schema_version, json_data, lib_cove_ocds_config=lib_cove_ocds_config, record_pkg=record_pkg ) if schema_ocds.invalid_version_data: msg = "\033[1;31mThe schema version in your data is not valid. Accepted values: {}\033[1;m" raise APIException(msg.format(str(list(schema_ocds.version_choices.keys())))) if schema_ocds.extensions: schema_ocds.create_extended_schema_file(output_dir, "") url = schema_ocds.extended_schema_file or schema_ocds.schema_url if convert: with warnings.catch_warnings(): warnings.filterwarnings("ignore") # flattentool uses UserWarning, so we can't set a specific category context.update( convert_json(output_dir, "", file, lib_cove_ocds_config, schema_url=url, flatten=True, cache=False) ) else: metatab_schema_url = SchemaOCDS(select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url metatab_data = get_spreadsheet_meta_data(output_dir, file, metatab_schema_url, file_type=file_type) schema_ocds = SchemaOCDS(schema_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config) if schema_ocds.invalid_version_data: msg = "\033[1;31mThe schema version in your data is not valid. Accepted values: {}\033[1;m" raise APIException(msg.format(str(list(schema_ocds.version_choices.keys())))) if schema_ocds.extensions: schema_ocds.create_extended_schema_file(output_dir, "") url = schema_ocds.extended_schema_file or schema_ocds.schema_url pkg_url = schema_ocds.pkg_schema_url context.update( convert_spreadsheet( output_dir, "", file, file_type, lib_cove_ocds_config, schema_url=url, pkg_schema_url=pkg_url, cache=False, ) ) with open(context["converted_path"], encoding="utf-8") as fp: json_data = json.load(fp, object_pairs_hook=OrderedDict) context = context_api_transform( common_checks_ocds(context, output_dir, json_data, schema_ocds, api=True, cache=False) ) if file_type == "xlsx": # Remove unwanted files in the output # TODO: can we do this by no writing the files in the first place? os.remove(os.path.join(output_dir, "heading_source_map.json")) os.remove(os.path.join(output_dir, "cell_source_map.json")) return context
def explore_bods(request, pk): context, db_data, error = explore_data_context(request, pk) if error: return error lib_cove_bods_config = LibCoveBODSConfig() lib_cove_bods_config.config['root_list_path'] = settings.COVE_CONFIG['root_list_path'] lib_cove_bods_config.config['root_id'] = settings.COVE_CONFIG['root_id'] lib_cove_bods_config.config['id_name'] = settings.COVE_CONFIG['id_name'] lib_cove_bods_config.config['root_is_list'] = settings.COVE_CONFIG['root_is_list'] upload_dir = db_data.upload_dir() upload_url = db_data.upload_url() file_name = db_data.original_file.file.name file_type = context['file_type'] if file_type == 'json': # open the data first so we can inspect for record package with open(file_name, encoding='utf-8') as fp: try: json_data = json.load(fp, parse_float=Decimal) except ValueError as err: raise CoveInputDataError(context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.' '\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">' '</span> <strong>Error message:</strong> {}', err)), 'error': format(err) }) if not isinstance(json_data, list): raise CoveInputDataError(context={ 'sub_title': _("Sorry, we can't process that data"), 'link': 'index', 'link_text': _('Try Again'), 'msg': _('BODS JSON should have an list as the top level, the JSON you supplied does not.'), }) schema_bods = SchemaBODS(lib_cove_bods_config=lib_cove_bods_config) context.update(convert_json(upload_dir, upload_url, file_name, lib_cove_bods_config, schema_url=schema_bods.release_pkg_schema_url, replace=True, request=request, flatten=True)) else: schema_bods = SchemaBODS(lib_cove_bods_config=lib_cove_bods_config) context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_bods_config, schema_url=schema_bods.release_pkg_schema_url)) with open(context['converted_path'], encoding='utf-8') as fp: json_data = json.load(fp, parse_float=Decimal) context = common_checks_bods(context, upload_dir, json_data, schema_bods) if not db_data.rendered: db_data.rendered = True db_data.save() template = 'cove_bods/explore.html' return render(request, template, context)