def _validate_formats(data, schema):
"""Go through schema checking the formats date-time, time and base64"""
if 'format' in schema:
format_name = schema['format']
if format_name == "date-time" and not validate_rfc3339(data):
raise ForbiddenError("A date-time was not in the required format.")
elif format_name == "time" and not _validate_timestr(data):
raise ForbiddenError("A time was not in the required format.")
elif format_name == "base64" and not _validate_base64(data):
raise ForbiddenError("A string was not valid base64.")
elif format_name == "timezone" and not _validate_timezone(data):
raise ForbiddenError("A string was not a valid timezone.")
if 'properties' in schema and isinstance(schema['properties'], dict):
for key, value in data.items():
try:
_validate_formats(value, schema['properties'][key])
except (TypeError, KeyError):
pass
if 'additionalProperties' in schema:
if isinstance(schema['additionalProperties'], dict):
for value in data.values():
try:
_validate_formats(value, schema['additionalProperties'])
except TypeError:
pass
if 'items' in schema and isinstance(schema['items'], dict):
for item in data:
try:
_validate_formats(item, schema['items'])
except TypeError:
pass
def datetime_or_date(instance):
if not instance:
raise ValueError
result = strict_rfc3339.validate_rfc3339(instance)
if result:
return result
return datetime.datetime.strptime(instance, "%Y-%m-%d")
def get_date(date):
valid = strict_rfc3339.validate_rfc3339(date)
if not valid:
try:
datetime.datetime.strptime(date, "%Y-%m-%d")
except ValueError:
return date
return date_parser.parse(date).strftime("%d %b %Y")
def get_year(date):
valid = strict_rfc3339.validate_rfc3339(date)
if not valid:
try:
datetime.datetime.strptime(date, "%Y-%m-%d")
except ValueError:
return date
return date.split("-")[0]
def is_date(instance):
print instance
if not isinstance(instance, str_types):
# return True
raise FormatError(
"%r is not a date-time format" % (instance)
)
return strict_rfc3339.validate_rfc3339(instance)
def get_date(date):
valid = strict_rfc3339.validate_rfc3339(date)
if not valid:
try:
datetime.datetime.strptime(date, "%Y-%m-%d")
except ValueError:
return date
return date_parser.parse(date).strftime("%d %b %Y")
def test_parse_string():
for base, seconds_fraction, timezone in itertools.product(
BASE_DATES, SECONDS_FRACTIONS, TIMEZONES):
datetime_string = '{0}{1}{2}'.format(base, seconds_fraction, timezone)
assert strict_rfc3339.validate_rfc3339(datetime_string), \
'Not RFC3339: {}'.format(datetime_string) # for sanity
yield _assert_datetime_parse_equals, datetime_string, \
EXPECTED[datetime_string]
def validate_date(s):
'''
Validate date as defined by "full-date" on http://xml2rfc.ietf.org/public/rfc/html/rfc3339.html#anchor14
>>> validate_date('foo')
False
>>> validate_date('2015-07-31')
True
'''
return strict_rfc3339.validate_rfc3339(s + 'T00:00:00Z')
def validate_date(s):
'''
Validate date as defined by "full-date" on http://xml2rfc.ietf.org/public/rfc/html/rfc3339.html#anchor14
>>> validate_date('foo')
False
>>> validate_date('2015-07-31')
True
'''
return strict_rfc3339.validate_rfc3339(s + 'T00:00:00Z')
def is_datetime(instance):
if not isinstance(instance, (binary_type, text_type)):
return False
if DATETIME_HAS_STRICT_RFC3339:
return strict_rfc3339.validate_rfc3339(instance)
if DATETIME_HAS_ISODATE:
return isodate.parse_datetime(instance)
return True
def is_datetime(checker: TypeChecker, instance) -> bool:
"""
Function to passed in TypeChecker for checking if property's type is custom type "interval"
"""
if validate_rfc3339(instance):
return True
try:
datetime.fromisoformat(instance)
except ValueError:
return False
return True
def is_datetime(instance):
if not isinstance(instance, (bytes, str)):
return False
if DATETIME_HAS_RFC3339_VALIDATOR:
return validate_rfc3339(instance)
if DATETIME_HAS_STRICT_RFC3339:
return strict_rfc3339.validate_rfc3339(instance)
if DATETIME_HAS_ISODATE:
return isodate.parse_datetime(instance)
return True
def is_valid_datetime(json_post):
"""
Determines if the datetime is strictly RFC3339
Args:
json_post: a json message as a python dict
Returns:
True if the value in key=datetime is RFC3339
"""
try:
if not strict_rfc3339.validate_rfc3339(json_post["datetime"]):
return False
else:
return True
except KeyError as e:
print(e)
return False
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config["current_language"] = translation.get_language(
)
lib_cove_ocds_config.config[
"schema_version_choices"] = settings.COVE_CONFIG[
"schema_version_choices"]
lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[
"schema_codelists"]
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context["file_type"]
post_version_choice = request.POST.get(
"version", lib_cove_ocds_config.config["schema_version"])
replace = False
validation_errors_path = os.path.join(upload_dir,
"validation_errors-3.json")
if file_type == "json":
# open the data first so we can inspect for record package
with open(file_name, encoding="utf-8") as fp:
try:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
except ValueError as err:
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_(
format_html(
"We think you tried to upload a JSON file, but it is not well formed JSON."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
"</span> <strong>Error message:</strong> {}",
err,
)),
"error":
format(err),
})
if not isinstance(json_data, dict):
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_("OCDS JSON should have an object as the top level, the JSON you supplied does not."
),
})
version_in_data = json_data.get("version", "")
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=json_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile(
"^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = "{} (it must be a string)".format(
str(version_in_data))
context["unrecognized_version_data"] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
if "records" in json_data:
context["conversion"] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, "flattened")
replace_converted = replace and os.path.exists(converted_path +
".xlsx")
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore'
) # flattentool uses UserWarning, so can't set a specific category
convert_json_context = convert_json(
upload_dir,
upload_url,
file_name,
lib_cove_ocds_config,
schema_url=schema_url,
replace=replace_converted,
request=request,
flatten=request.POST.get("flatten"),
)
context.update(convert_json_context)
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(
select_version="1.1",
lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name,
metatab_schema_url, file_type)
if "version" not in metatab_data:
metatab_data["version"] = "1.0"
else:
db_data.data_schema_version = metatab_data["version"]
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=metatab_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get("version")
if re.compile("^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
context["unrecognized_version_data"] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
schema_url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
pkg_url = schema_ocds.release_pkg_schema_url
if file_type != "csv":
# ORIGINAL UNFLATTEN
conversion_context = convert_spreadsheet(
upload_dir,
upload_url,
file_name,
file_type,
lib_cove_ocds_config,
schema_url=schema_url,
pkg_schema_url=pkg_url,
replace=replace,
)
else:
# Convert Simple CSV to flat OCDS and return context
conversion_context = convert_simple_csv_submission(
db_data,
lib_cove_ocds_config,
schema_url,
replace=replace,
)
context.update(conversion_context)
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context,
upload_dir,
json_data,
schema_ocds,
cache=settings.CACHE_VALIDATION_ERRORS)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
schema_version = getattr(schema_ocds, "version", None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
context.update({
"data_schema_version":
db_data.schema_version,
"first_render":
not db_data.rendered,
"validation_errors_grouped":
group_validation_errors(context["validation_errors"]),
})
ocds_show_schema = SchemaOCDS()
ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj(
deref=True)
if "records" in json_data:
template = "cove_ocds/explore_record.html"
if hasattr(json_data, "get") and hasattr(json_data.get("records"),
"__iter__"):
context["records"] = json_data["records"]
else:
context["records"] = []
if isinstance(json_data["records"],
list) and len(json_data["records"]) < 100:
context["ocds_show_data"] = ocds_show_data(json_data,
ocds_show_deref_schema)
else:
template = "silvereye/explore_release.html"
if hasattr(json_data, "get") and hasattr(json_data.get("releases"),
"__iter__"):
context["releases"] = json_data["releases"]
if (isinstance(json_data["releases"], list)
and len(json_data["releases"]) < 100):
context["ocds_show_data"] = ocds_show_data(
json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context["releases"]:
if hasattr(release, "get") and release.get("date"):
if validate_rfc3339(release["date"]):
release["date"] = parser.parse(release["date"])
else:
release["date"] = None
try:
trans_date = release["contracts"][0]["implementation"][
"transactions"][0]["date"]
parsed_trans_date = parser.parse(trans_date)
release["contracts"][0]["implementation"]["transactions"][
0]["date"] = parsed_trans_date
except KeyError:
pass
if context.get("releases_aggregates"):
date_fields = [
"max_award_date",
"max_contract_date",
"max_release_date",
"max_tender_date",
"min_award_date",
"min_contract_date",
"min_release_date",
"min_tender_date",
]
for field in date_fields:
if context["releases_aggregates"].get(field):
if validate_rfc3339(
context["releases_aggregates"][field]):
context["releases_aggregates"][
field] = parser.parse(
context["releases_aggregates"][field])
else:
context["releases_aggregates"][field] = None
else:
context["releases"] = []
# Include field coverage report
original_file_path = context["original_file"]["path"]
mapper = CSVMapper(csv_path=original_file_path)
db_data.notice_type = mapper.release_type
db_data.save()
coverage_context = mapper.get_coverage_context()
context.update({
"field_coverage": coverage_context,
})
ocds_validation_errors, simple_csv_errors = prepare_simple_csv_validation_errors(
context["validation_errors"], mapper,
coverage_context["required_fields_missing"])
context.update({
"ocds_validation_errors": ocds_validation_errors,
"simple_csv_errors": simple_csv_errors,
"csv_mapper": mapper,
})
# Silvereye: Insert OCDS data
releases = context.get("releases")
if releases:
# If we don't have validation errors
validation_errors_grouped = context["validation_errors_grouped"]
if not validation_errors_grouped:
json_string = json.dumps(json_data,
indent=2,
sort_keys=True,
cls=DjangoJSONEncoder)
UpsertDataHelpers().upsert_ocds_data(json_string,
supplied_data=db_data)
average_field_completion = coverage_context.get(
"average_field_completion")
inst, created = FieldCoverage.objects.update_or_create(
file_submission=db_data,
defaults={
"tenders_field_coverage":
average_field_completion
if mapper.release_type == "tender" else None,
"awards_field_coverage":
average_field_completion
if mapper.release_type == "award" else None,
"spend_field_coverage":
average_field_completion
if mapper.release_type == "spend" else None,
})
update_publisher_monthly_counts()
return render(request, template, context)
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config["current_language"] = translation.get_language(
)
lib_cove_ocds_config.config[
"schema_version_choices"] = settings.COVE_CONFIG[
"schema_version_choices"]
lib_cove_ocds_config.config["schema_codelists"] = settings.COVE_CONFIG[
"schema_codelists"]
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context["file_type"]
post_version_choice = request.POST.get("version")
replace = False
validation_errors_path = os.path.join(upload_dir,
"validation_errors-3.json")
if file_type == "json":
# open the data first so we can inspect for record package
with open(file_name, encoding="utf-8") as fp:
try:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
except UnicodeError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
format_html(
_("The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON follows the I-JSON format, which requires UTF-8 encoding. Ensure that your file uses UTF-8 encoding, then try uploading again."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}'),
err),
'error':
format(err)
})
except ValueError as err:
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
format_html(
_(
"We think you tried to upload a JSON file, but it is not well formed JSON."
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
"</span> <strong>Error message:</strong> {}",
),
err,
),
"error":
format(err),
})
if not isinstance(json_data, dict):
raise CoveInputDataError(
context={
"sub_title":
_("Sorry, we can't process that data"),
"link":
"index",
"link_text":
_("Try Again"),
"msg":
_("OCDS JSON should have an object as the top level, the JSON you supplied does not."
),
})
version_in_data = json_data.get("version", "")
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version,
release_data=json_data,
lib_cove_ocds_config=lib_cove_ocds_config,
record_pkg="records" in json_data)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile(
"^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = "{} (it must be a string)".format(
str(version_in_data))
context["unrecognized_version_data"] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
if "records" in json_data:
context["conversion"] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, "flattened")
replace_converted = replace and os.path.exists(converted_path +
".xlsx")
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore'
) # flattentool uses UserWarning, so can't set a specific category
convert_json_context = convert_json(
upload_dir,
upload_url,
file_name,
lib_cove_ocds_config,
schema_url=url,
replace=replace_converted,
request=request,
flatten=request.POST.get("flatten"),
)
context.update(convert_json_context)
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(
select_version="1.1",
lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name,
metatab_schema_url, file_type)
if "version" not in metatab_data:
metatab_data["version"] = "1.0"
else:
db_data.data_schema_version = metatab_data["version"]
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(
select_version=select_version,
release_data=metatab_data,
lib_cove_ocds_config=lib_cove_ocds_config,
)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get("version")
if re.compile("^\d+\.\d+\.\d+$").match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
context["unrecognized_version_data"] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.schema_url
pkg_url = schema_ocds.pkg_schema_url
context.update(
convert_spreadsheet(
upload_dir,
upload_url,
file_name,
file_type,
lib_cove_ocds_config,
schema_url=url,
pkg_schema_url=pkg_url,
replace=replace,
))
with open(context["converted_path"], encoding="utf-8") as fp:
json_data = json.load(fp,
parse_float=Decimal,
object_pairs_hook=OrderedDict)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context, upload_dir, json_data, schema_ocds)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
context.update({
"data_schema_version":
db_data.data_schema_version,
"first_render":
not db_data.rendered,
"validation_errors_grouped":
group_validation_errors(context["validation_errors"]),
})
schema_version = getattr(schema_ocds, "version", None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
if "records" in json_data:
ocds_show_schema = SchemaOCDS(record_pkg=True)
ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True)
template = "cove_ocds/explore_record.html"
if hasattr(json_data, "get") and hasattr(json_data.get("records"),
"__iter__"):
context["records"] = json_data["records"]
else:
context["records"] = []
if isinstance(json_data["records"],
list) and len(json_data["records"]) < 100:
context["ocds_show_data"] = ocds_show_data(json_data,
ocds_show_deref_schema)
else:
ocds_show_schema = SchemaOCDS(record_pkg=False)
ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True)
template = "cove_ocds/explore_release.html"
if hasattr(json_data, "get") and hasattr(json_data.get("releases"),
"__iter__"):
context["releases"] = json_data["releases"]
if (isinstance(json_data["releases"], list)
and len(json_data["releases"]) < 100):
context["ocds_show_data"] = ocds_show_data(
json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context["releases"]:
if hasattr(release, "get") and release.get("date"):
if validate_rfc3339(release["date"]):
release["date"] = parser.parse(release["date"])
else:
release["date"] = None
if context.get("releases_aggregates"):
date_fields = [
"max_award_date",
"max_contract_date",
"max_release_date",
"max_tender_date",
"min_award_date",
"min_contract_date",
"min_release_date",
"min_tender_date",
]
for field in date_fields:
if context["releases_aggregates"].get(field):
if validate_rfc3339(
context["releases_aggregates"][field]):
context["releases_aggregates"][
field] = parser.parse(
context["releases_aggregates"][field])
else:
context["releases_aggregates"][field] = None
else:
context["releases"] = []
return render(request, template, context)
def is_datetime(instance):
if not isinstance(instance, str_types):
return True
return strict_rfc3339.validate_rfc3339(instance)
def validate(self, raw_value: Union[None, str, bytes, _MissingType],
raw: bool) -> Union[None, str, bytes, _MissingType]:
if isinstance(raw_value, (str, bytes)):
value = raw_value
elif raw_value is None:
if self.nullable:
return None
raise ValidatorError("value should be type of str")
elif isinstance(raw_value, _MissingType):
if self.default is None:
return raw_value
value = self.default
else:
raise ValidatorError("value should be type of str")
if self.minLength is not None and len(value) < self.minLength:
raise ValidatorError(
f"value length should be more than {self.minLength}")
if self.maxLength is not None and len(value) > self.maxLength:
raise ValidatorError(
f"value length should be less than {self.maxLength}")
if self.enum is not None and value not in self.enum:
raise ValidatorError(f"value should be one of {self.enum}")
if self.format not in (
StringFormat.Default,
StringFormat.Password,
StringFormat.Binary,
):
assert isinstance(value, str)
if self.format == StringFormat.Uuid:
try:
uuid.UUID(value)
except ValueError:
raise ValidatorError("value should be uuid")
elif self.format == StringFormat.Datetime:
if not strict_rfc3339.validate_rfc3339(value):
raise ValidatorError("value should be datetime format")
elif self.format == StringFormat.Date:
if not strict_rfc3339.validate_rfc3339(f"{value}T00:00:00Z"):
raise ValidatorError("value should be date format")
elif self.format == StringFormat.Email:
if not _EMAIL_REGEX.match(value):
raise ValidatorError("value should be valid email")
elif self.format == StringFormat.Byte:
try:
base64.b64decode(value)
except ValueError:
raise ValidatorError(
"value should be base64-encoded string")
elif self.format == StringFormat.IPv4:
try:
ipaddress.IPv4Address(value)
except ValueError:
raise ValidatorError("value should be valid ipv4 address")
elif self.format == StringFormat.IPv6:
try:
ipaddress.IPv6Address(value)
except ValueError:
raise ValidatorError("value should be valid ipv6 address")
elif self.format == StringFormat.Hostname:
hostname = value[:-1] if value[-1] == "." else value
if len(hostname) > 255 or not all(
_HOSTNAME_REGEX.match(x) for x in hostname.split(".")):
raise ValidatorError("value should be valid hostname")
if (self.format != StringFormat.Binary and self.pattern
and not self.pattern.search(value)):
raise ValidatorError(
f"value should match regex pattern '{self.pattern.pattern}'")
return value
def _rfc3339_datetime(s):
if validate_rfc3339(s):
return s
raise ValueError
def get_post(start_time, end_time):
"""
Get a single post and convert to json.
Args:
start_time: Beginning time of window of data being queried
end_time: End time of window of data being queried
Returns:
Data for a specific period of time. It will output a list of jsons,
outputting the data if found and outputting 'data not found' if
that time was not found in database.
.. :quickref: Data Window; Get window of data
**Example request**:
Shell command:
*with email/password:*
.. sourcecode:: shell
curl --user <email>:<password> -X GET https://localhost/api/v0.1/posts/2017-09-13T13:01:57Z/2017-09-13T13:01:59Z
*or with token:*
.. sourcecode:: shell
curl --user <token>: -X GET https://localhost/api/v0.1/posts/2017-09-13T13:01:57Z/2017-09-13T13:01:59Z
Command response:
.. sourcecode:: http
GET /api/v0.1/posts/2017-09-13T13:01:57Z/2017-09-13T13:01:59Z HTTP/1.1
localhost
Authorization: Basic <b64 encoded email:password or token:>
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
[
{
"JSON_message":"goes_here"
},
{
"next_JSON_message":"goes_here"
}
]
*(JSON cut for length)*
:query start_time: Beginning time of window of data being queried
:query end_time: End time of window of data being queried
:reqheader Authorization: use cURL tag with <email>:<psswrd>, or <token>:
:resheader Content-Type: application/json
:statuscode 200: Successfully retrieved data
:statuscode 401: Invalid credentials
:statuscode 403: Not signed in
"""
time_format = '%Y-%m-%dT%H:%M:%SZ'
start_time_stripped = datetime.strptime(start_time, time_format)
end = datetime.strptime(end_time, time_format)
time_delta = datetime.strptime(end_time, time_format) \
- datetime.strptime(start_time, time_format)
if not (strict_rfc3339.validate_rfc3339(start_time) and
strict_rfc3339.validate_rfc3339(end_time)):
print("Error: datetimes are not RFC 3339")
return bad_request('Error: Datetimes are not RFC 3339')
start_time_epoch = strict_rfc3339.rfc3339_to_timestamp(start_time)
end_time_epoch = strict_rfc3339.rfc3339_to_timestamp(end_time)
# print("start_time_epoch: {} end_time_epoch: {}".format(start_time_epoch,
# end_time_epoch))
if (end_time_epoch < start_time_epoch):
print("Error: end time is before start time")
return bad_request('Error: End time is before start time')
MAX_API_DATA_S = current_app.config['MAX_API_DATA_PER_REQUEST']
if time_delta > timedelta(seconds=MAX_API_DATA_S):
return too_many_requests(
'Request is above {} seconds of data.'.format(MAX_API_DATA_S))
end = start_time_stripped + time_delta
data = []
while start_time_stripped <= end:
# Until we change to 24hr time
strtime = start_time_stripped.strftime(time_format)
try:
cache.get(strtime)
no_redis_connection = False
except RedisError:
no_redis_connection = True
if not no_redis_connection:
if cache.get(strtime) is None:
data_query = Machine.query.filter_by(
datetime=strtime).first()
if data_query is not None:
try:
raw_data = data_query.to_json()
data.append(raw_data)
cache.set(strtime, raw_data, timeout=0)
except BaseException:
data.append({"Error": "Could not find data."})
else:
data.append(cache.get(strtime))
else:
data_query = Machine.query.filter_by(datetime=strtime).first()
if data_query is not None:
try:
raw_data = data_query.to_json()
data.append(raw_data)
except BaseException:
data.append({"Error": "Could not find data."})
start_time_stripped += timedelta(seconds=1)
return jsonify(data)
def test_catalog(self):
singer.write_message = singer_write_message
log_miner.UPDATE_BOOKMARK_PERIOD = 1
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'LOG_BASED')
cur = conn.cursor()
prev_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
our_date = datetime.date(1996, 6, 6)
our_ts = datetime.datetime(1997, 2, 2, 2, 2, 2, 722184)
nyc_tz = pytz.timezone('America/New_York')
our_ts_tz_edt = nyc_tz.localize(
datetime.datetime(1997, 3, 3, 3, 3, 3, 722184))
our_ts_tz_utc = datetime.datetime(1997, 3, 3, 3, 3, 3, 722184,
pytz.UTC)
auckland_tz = pytz.timezone('Pacific/Auckland')
our_ts_local = auckland_tz.localize(
datetime.datetime(1997, 3, 3, 18, 3, 3, 722184))
crud_up_log_miner_fixtures(
cur, 'CHICKEN', {
'"our_date"': our_date,
'"our_ts"': our_ts,
'"our_ts_tz_edt"': our_ts_tz_edt,
'"our_ts_tz_utc"': our_ts_tz_utc,
'"our_ts_tz_local"': our_ts_local
}, self.update_add_1_day)
post_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
LOGGER.info("post SCN: {}".format(post_scn))
state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn',
prev_scn)
state = write_bookmark(state, chicken_stream.tap_stream_id,
'version', 1)
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
verify_crud_messages(self, CAUGHT_MESSAGES, ['ID'])
#verify message 1 - first insert
insert_rec_1 = CAUGHT_MESSAGES[1].record
self.assertIsNotNone(insert_rec_1.get('scn'))
insert_rec_1.pop('scn')
self.assertIsNone(insert_rec_1.get('_sdc_deleted_at'))
self.assertEqual(insert_rec_1.get('ID'), 1)
insert_rec_1.pop('_sdc_deleted_at')
insert_rec_1.pop('ID')
self.assertEqual(
insert_rec_1, {
'our_ts': '1997-02-02T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-03T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-03T03:03:03.722184+00:00',
'our_date': '1996-06-06T00:00:00.00+00:00',
'our_ts_tz_local': '1997-03-03T05:03:03.722184+00:00'
})
for v in insert_rec_1.values():
self.assertTrue(strict_rfc3339.validate_rfc3339(v))
#verify UPDATE
update_rec = CAUGHT_MESSAGES[5].record
self.assertIsNotNone(update_rec.get('scn'))
update_rec.pop('scn')
self.assertEqual(
update_rec, {
'our_ts': '1997-02-03T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00',
'our_date': '1996-06-07T00:00:00.00+00:00',
'_sdc_deleted_at': None,
'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00',
'ID': 1
})
#verify first DELETE message
delete_rec = CAUGHT_MESSAGES[9].record
self.assertIsNotNone(delete_rec.get('scn'))
self.assertIsNotNone(delete_rec.get('_sdc_deleted_at'))
delete_rec.pop('scn')
delete_rec.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec, {
'our_ts': '1997-02-03T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00',
'our_date': '1996-06-07T00:00:00.00+00:00',
'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00',
'ID': 1
})
#verify second DELETE message
delete_rec_2 = CAUGHT_MESSAGES[11].record
self.assertIsNotNone(delete_rec_2.get('scn'))
self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at'))
delete_rec_2.pop('scn')
delete_rec_2.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec_2, {
'our_ts': '1997-02-03T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00',
'our_date': '1996-06-07T00:00:00.00+00:00',
'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00',
'ID': 2
})
def async_statistics(self, start_time, end_time):
"""
Gets a specific set of statistically analyzed data from the database to be used in Statistical Analysis.
Args:
start_time: Beginning time of window of data being queried
end_time: End time of window of data being queried
Returns:
response: the trimmed blower data, or a status report
This async function will output a primitive dict of shape:
{
result:
status:
}
Where, if successful, 'result' is also a primitive dict containing
start_time, end_time, and a list of trimmed json blower data (or an
empty dict if blower wasn't on). If not successful, 'result' will be a
string containing a status message. 'status' will always be an int,
referring to an HTTP status code.
When returned, the statistics_status endpoint handles the primitive result,
and returns a jsonified output to the user.
**Example request**:
Shell command:
*with email/password:*
.. sourcecode:: shell
curl --user <email>:<password> -X GET https://localhost/api/v0.1/statistics/2017-09-13T13:01:57Z/2017-09-13T13:01:59Z
*or with token:*
.. sourcecode:: shell
curl --user <token>: -X GET https://localhost/api/v0.1/statistics/2017-09-13T13:01:57Z/2017-09-13T13:01:59Z
Command response:
.. sourcecode:: http
GET /api/v0.1/statistics/2017-09-13T13:01:57Z/2017-09-13T13:01:59Z HTTP/1.1
localhost
Authorization: Basic <b64 encoded email:password or token:>
:query start_time: Beginning time of window of data being queried
:query end_time: End time of window of data being queried
:reqheader Authorization: use cURL tag with <email>:<psswrd>, or <token>:
:resheader Content-Type: application/json
:statuscode 200: Successfully retrieved data
:statuscode 204: No content
:statuscode 401: Invalid credentials
:statuscode 403: Not signed in
"""
#pr = cProfile.Profile()
#pr.enable()
self.update_state(state='STARTED',
meta={'result': "Gathering data for statistical analysis.",
'status': 202})
# TODO We should define that you cannot run statistics over a super long
# period since this is not an asynchronous request.
MAX_API_DATA_S = current_app.config['MAX_API_DATA_PER_REQUEST']
# issue with collecting more than a half hour at a time
delta_t_chunk_size = MAX_API_DATA_S
# Checking and setting up the start and end times
if not (strict_rfc3339.validate_rfc3339(start_time) and
strict_rfc3339.validate_rfc3339(end_time)):
print("Error: datetimes are not RFC 3339")
response = {'result': "Bad request; datetimes are not RFC 3339.",
'status': 400}
# API will return bad_request
return response
start_time_epoch = strict_rfc3339.rfc3339_to_timestamp(start_time)
end_time_epoch = strict_rfc3339.rfc3339_to_timestamp(end_time)
if (end_time_epoch < start_time_epoch):
print("Error: end time is before start time")
response = {'result': "Bad request; end time is before start time",
'status': 400}
# API will return bad_request
return response
temp_start_time_epoch = start_time_epoch
# Add chunk time to temp time values only if request time exceeds 30 min
if end_time_epoch - start_time_epoch >= 1800:
temp_end_time_epoch = start_time_epoch + delta_t_chunk_size
else:
temp_end_time_epoch = end_time_epoch
# Do stuff
self.update_state(state='PROGRESS',
meta={'result': "Finished gathering data, running stats.",
'status': 202})
# @TODO update whatever the actual response is
response = {'result': {'start_time' : start_time, 'end_time' : end_time,
'data': 'here is data'},
'status': '200'}
#pr.disable()
#s = io.StringIO()
#ps = pstats.Stats(pr, stream=s).sort_stats('tottime')
#ps.print_stats()
#print(s.getvalue())
if response is None:
# An error occured in Statistics.py, couldn't process request
response = {'result': "Processing Failure; Statistics returned nothing.",
'status': 500}
return response
else:
# successfully returned data
return response
def datetime_or_date(instance):
result = strict_rfc3339.validate_rfc3339(instance)
if result:
return result
return datetime.datetime.strptime(instance, "%Y-%m-%d")
def sf_date_time_validator(value: str) -> None:
if not strict_rfc3339.validate_rfc3339(value):
raise ValidatorError("value should be datetime format")
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context['file_type']
post_version_choice = request.POST.get('version')
replace = False
validation_errors_path = os.path.join(upload_dir,
'validation_errors-3.json')
if file_type == 'json':
# open the data first so we can inspect for record package
with open(file_name, encoding='utf-8') as fp:
try:
json_data = json.load(fp, parse_float=Decimal)
except ValueError as err:
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
_(
format_html(
'We think you tried to upload a JSON file, but it is not well formed JSON.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}',
err)),
'error':
format(err)
})
if not isinstance(json_data, dict):
raise CoveInputDataError(
context={
'sub_title':
_("Sorry, we can't process that data"),
'link':
'index',
'link_text':
_('Try Again'),
'msg':
_('OCDS JSON should have an object as the top level, the JSON you supplied does not.'
),
})
version_in_data = json_data.get('version', '')
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version,
release_data=json_data)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile(
'^\d+\.\d+\.\d+$').match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = '{} (it must be a string)'.format(
str(version_in_data))
context['unrecognized_version_data'] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
if 'records' in json_data:
context['conversion'] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, 'flattened')
replace_converted = replace and os.path.exists(converted_path +
'.xlsx')
context.update(
convert_json(upload_dir,
upload_url,
file_name,
schema_url=url,
replace=replace_converted,
request=request,
flatten=request.POST.get('flatten')))
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(
select_version='1.1').release_pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name,
metatab_schema_url, file_type)
if 'version' not in metatab_data:
metatab_data['version'] = '1.0'
else:
db_data.data_schema_version = metatab_data['version']
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version,
release_data=metatab_data)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get('version')
if re.compile('^\d+\.\d+\.\d+$').match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(
version_in_data)
else:
context['unrecognized_version_data'] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(
upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
pkg_url = schema_ocds.release_pkg_schema_url
context.update(
convert_spreadsheet(upload_dir,
upload_url,
file_name,
file_type,
schema_url=url,
pkg_schema_url=pkg_url,
replace=replace))
with open(context['converted_path'], encoding='utf-8') as fp:
json_data = json.load(fp, parse_float=Decimal)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context, upload_dir, json_data, schema_ocds)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
context.update({
'data_schema_version': db_data.data_schema_version,
'first_render': not db_data.rendered
})
schema_version = getattr(schema_ocds, 'version', None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
if 'records' in json_data:
template = 'cove_ocds/explore_record.html'
if hasattr(json_data, 'get') and hasattr(json_data.get('records'),
'__iter__'):
context['records'] = json_data['records']
else:
context['records'] = []
else:
template = 'cove_ocds/explore_release.html'
if hasattr(json_data, 'get') and hasattr(json_data.get('releases'),
'__iter__'):
context['releases'] = json_data['releases']
# Parse release dates into objects so the template can format them.
for release in context['releases']:
if hasattr(release, 'get') and release.get('date'):
if validate_rfc3339(release['date']):
release['date'] = parser.parse(release['date'])
else:
release['date'] = None
if context.get('releases_aggregates'):
date_fields = [
'max_award_date', 'max_contract_date', 'max_release_date',
'max_tender_date', 'min_award_date', 'min_contract_date',
'min_release_date', 'min_tender_date'
]
for field in date_fields:
if context['releases_aggregates'].get(field):
if (validate_rfc3339(
context['releases_aggregates'][field])):
context['releases_aggregates'][
field] = parser.parse(
context['releases_aggregates'][field])
else:
context['releases_aggregates'][field] = None
else:
context['releases'] = []
return render(request, template, context)
def is_datetime(instance):
if not isinstance(instance, str):
return True
return validate_rfc3339(instance)
def explore_ocds(request, pk):
context, db_data, error = explore_data_context(request, pk)
if error:
return error
lib_cove_ocds_config = LibCoveOCDSConfig()
lib_cove_ocds_config.config['current_language'] = translation.get_language()
upload_dir = db_data.upload_dir()
upload_url = db_data.upload_url()
file_name = db_data.original_file.file.name
file_type = context['file_type']
post_version_choice = request.POST.get('version')
replace = False
validation_errors_path = os.path.join(upload_dir, 'validation_errors-3.json')
if file_type == 'json':
# open the data first so we can inspect for record package
with open(file_name, encoding='utf-8') as fp:
try:
json_data = json.load(fp, parse_float=Decimal)
except ValueError as err:
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _(format_html('We think you tried to upload a JSON file, but it is not well formed JSON.'
'\n\n<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true">'
'</span> <strong>Error message:</strong> {}', err)),
'error': format(err)
})
if not isinstance(json_data, dict):
raise CoveInputDataError(context={
'sub_title': _("Sorry, we can't process that data"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _('OCDS JSON should have an object as the top level, the JSON you supplied does not.'),
})
version_in_data = json_data.get('version', '')
db_data.data_schema_version = version_in_data
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version, release_data=json_data, lib_cove_ocds_config=lib_cove_ocds_config)
if schema_ocds.missing_package:
exceptions.raise_missing_package_error()
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
if isinstance(version_in_data, str) and re.compile('^\d+\.\d+\.\d+$').match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(version_in_data)
else:
if not isinstance(version_in_data, str):
version_in_data = '{} (it must be a string)'.format(str(version_in_data))
context['unrecognized_version_data'] = version_in_data
if schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
if 'records' in json_data:
context['conversion'] = None
else:
# Replace the spreadsheet conversion only if it exists already.
converted_path = os.path.join(upload_dir, 'flattened')
replace_converted = replace and os.path.exists(converted_path + '.xlsx')
context.update(convert_json(upload_dir, upload_url, file_name, lib_cove_ocds_config,
schema_url=url, replace=replace_converted,
request=request, flatten=request.POST.get('flatten')))
else:
# Use the lowest release pkg schema version accepting 'version' field
metatab_schema_url = SchemaOCDS(select_version='1.1', lib_cove_ocds_config=lib_cove_ocds_config).release_pkg_schema_url
metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type)
if 'version' not in metatab_data:
metatab_data['version'] = '1.0'
else:
db_data.data_schema_version = metatab_data['version']
select_version = post_version_choice or db_data.schema_version
schema_ocds = SchemaOCDS(select_version=select_version, release_data=metatab_data, lib_cove_ocds_config=lib_cove_ocds_config)
# Unlike for JSON data case above, do not check for missing data package
if schema_ocds.invalid_version_argument:
# This shouldn't happen unless the user sends random POST data.
exceptions.raise_invalid_version_argument(post_version_choice)
if schema_ocds.invalid_version_data:
version_in_data = metatab_data.get('version')
if re.compile('^\d+\.\d+\.\d+$').match(version_in_data):
exceptions.raise_invalid_version_data_with_patch(version_in_data)
else:
context['unrecognized_version_data'] = version_in_data
# Replace json conversion when user chooses a different schema version.
if db_data.schema_version and schema_ocds.version != db_data.schema_version:
replace = True
if schema_ocds.extensions:
schema_ocds.create_extended_release_schema_file(upload_dir, upload_url)
url = schema_ocds.extended_schema_file or schema_ocds.release_schema_url
pkg_url = schema_ocds.release_pkg_schema_url
context.update(convert_spreadsheet(upload_dir, upload_url, file_name, file_type, lib_cove_ocds_config,
schema_url=url,
pkg_schema_url=pkg_url, replace=replace))
with open(context['converted_path'], encoding='utf-8') as fp:
json_data = json.load(fp, parse_float=Decimal)
if replace:
if os.path.exists(validation_errors_path):
os.remove(validation_errors_path)
context = common_checks_ocds(context, upload_dir, json_data, schema_ocds)
if schema_ocds.json_deref_error:
exceptions.raise_json_deref_error(schema_ocds.json_deref_error)
context.update({
'data_schema_version': db_data.data_schema_version,
'first_render': not db_data.rendered,
'validation_errors_grouped': group_validation_errors(context['validation_errors']),
})
schema_version = getattr(schema_ocds, 'version', None)
if schema_version:
db_data.schema_version = schema_version
if not db_data.rendered:
db_data.rendered = True
db_data.save()
ocds_show_schema = SchemaOCDS()
ocds_show_deref_schema = ocds_show_schema.get_release_schema_obj(deref=True)
if 'records' in json_data:
template = 'cove_ocds/explore_record.html'
if hasattr(json_data, 'get') and hasattr(json_data.get('records'), '__iter__'):
context['records'] = json_data['records']
else:
context['records'] = []
if isinstance(json_data['records'], list) and len(json_data['records']) < 100:
context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema)
else:
template = 'cove_ocds/explore_release.html'
if hasattr(json_data, 'get') and hasattr(json_data.get('releases'), '__iter__'):
context['releases'] = json_data['releases']
if isinstance(json_data['releases'], list) and len(json_data['releases']) < 100:
context['ocds_show_data'] = ocds_show_data(json_data, ocds_show_deref_schema)
# Parse release dates into objects so the template can format them.
for release in context['releases']:
if hasattr(release, 'get') and release.get('date'):
if validate_rfc3339(release['date']):
release['date'] = parser.parse(release['date'])
else:
release['date'] = None
if context.get('releases_aggregates'):
date_fields = ['max_award_date', 'max_contract_date', 'max_release_date', 'max_tender_date', 'min_award_date', 'min_contract_date', 'min_release_date', 'min_tender_date']
for field in date_fields:
if context['releases_aggregates'].get(field):
if(validate_rfc3339(context['releases_aggregates'][field])):
context['releases_aggregates'][field] = parser.parse(context['releases_aggregates'][field])
else:
context['releases_aggregates'][field] = None
else:
context['releases'] = []
return render(request, template, context)
def sf_date_validator(value: str) -> None:
if not strict_rfc3339.validate_rfc3339(f"{value}T00:00:00Z"):
raise ValidatorError("value should be date format")
def test_against_legacy(datetime_str):
legacy_result = strict_rfc3339.validate_rfc3339(datetime_str)
new_result = validate_rfc3339(datetime_str)
assert legacy_result == new_result
def test_generated_rfc3339_datetime_strings_are_valid(datetime_string):
assert strict_rfc3339.validate_rfc3339(datetime_string)
def date_time_format_validator(value, **kwargs):
if not strict_rfc3339.validate_rfc3339(value):
raise ValidationError(MESSAGES['format']['invalid_datetime'].format(value))
def is_datetime(instance):
if not isinstance(instance, str_types):
return True
return strict_rfc3339.validate_rfc3339(instance)
def _rfc3339_datetime(s):
if validate_rfc3339(s):
return s
raise ValueError