def eval(self, dataset: Dataset, library: MessageLibrary):
metadata = dataset.metadata
reference_value = self._extract_value(self._reference_name, metadata)
if reference_value is None:
return Issue(
ISSUE_TYPE_ERROR,
"Requested field not contained in metadata: " +
self._reference_name)
compare_value = self._extract_value(self._compare_name, metadata)
if compare_value is None:
return Issue(
ISSUE_TYPE_ERROR,
"Requested field not contained in metadata: " +
self._compare_name)
if self._operator(reference_value, compare_value):
return None
else:
message_dict = GapAwareDict({
"reference": self._reference_name,
"ref_val": reference_value,
"compare": self._compare_name,
"comp_val": compare_value
})
if self._error is not None:
error_message = library.resolve_error(self._error,
message_dict)
return Issue(ISSUE_TYPE_ERROR, error_message)
if self._warning is not None:
warning_message = library.resolve_warning(
self._warning, message_dict)
return Issue(ISSUE_TYPE_WARNING, warning_message)
def eval(self,
unit: str,
values: List[str],
library: MessageLibrary,
missing_value: float = None) -> Optional[List[Issue]]:
issues = []
if unit not in self._units:
message_dict = GapAwareDict({
"field_name": self._name,
"unit": self._units,
"bad_unit": unit
})
error_message = library.resolve_error(self._unit_error,
message_dict)
issues.append(Issue(ISSUE_TYPE_ERROR, error_message))
index = 0
for value in values:
index += 1
message_dict = GapAwareDict({
"field_name": self._name,
"line": index
})
if not self._time_string_pattern.match(value):
error_message = library.resolve_error("@invalid_time_record",
message_dict)
issues.append(Issue(ISSUE_TYPE_ERROR, error_message))
continue
tokens = value.split(":")
hour = int(tokens[0])
minute = int(tokens[1])
second = int(tokens[2])
if hour not in self._hour_range \
or minute not in self._min_sec_range \
or second not in self._min_sec_range:
error_message = library.resolve_error("@invalid_time_value",
message_dict)
issues.append(Issue(ISSUE_TYPE_ERROR, error_message))
continue
if len(issues) > 0:
return issues
return None
def eval(self, dataset: Dataset, library: MessageLibrary):
if not self._name in dataset.metadata:
return None
value = dataset.metadata[self._name]
if len(value) > 0:
return None
else:
message_dict = GapAwareDict({"reference": self._name})
warning_message = library.resolve_warning(self._warning,
message_dict)
return Issue(ISSUE_TYPE_WARNING, warning_message)
def test_to_ref(self):
dsvr = DatasetValidationResult(
status="ERROR",
issues=[
Issue(type="ERROR", description="Yo, not like that my son")
])
sf = SubmissionFile(index=11,
submission_id="yeswecan",
filename="is_a_secret",
filetype="a cool one",
status=QC_STATUS_SUBMITTED,
result=dsvr)
sfr = sf.to_ref()
self.assertEqual(11, sfr.index)
self.assertEqual("yeswecan", sfr.submission_id)
self.assertEqual("is_a_secret", sfr.filename)
self.assertEqual("a cool one", sfr.filetype)
self.assertEqual(QC_STATUS_SUBMITTED, sfr.status)
def test_to_submission(self):
files = [
SubmissionFile(submission_id="submitme",
index=0,
filename="Hans",
filetype="black",
status=QC_STATUS_SUBMITTED,
result=DatasetValidationResult(status="OK",
issues=[])),
SubmissionFile(submission_id="submitme",
index=1,
filename="Helga",
filetype="green",
status=QC_STATUS_VALIDATED,
result=DatasetValidationResult(
status="WARNING",
issues=[
Issue(type="WARNING",
description="This might be wrong")
]))
]
db_subm = DbSubmission(status="Hellyeah",
user_id='Tom',
submission_id="submitme",
files=files,
qc_status="OK",
path="/root/hell/yeah",
date=datetime(2001, 2, 3, 4, 5, 6),
store_sub_path='Tom_Helge')
subm = db_subm.to_submission()
self.assertEqual(datetime(2001, 2, 3, 4, 5, 6), subm.date)
self.assertEqual(2, len(subm.file_refs))
self.assertEqual("Hans", subm.file_refs[0].filename)
self.assertEqual(0, subm.file_refs[0].index)
def eval(self,
unit: str,
values: List[str],
library: MessageLibrary,
missing_value: float = None) -> Optional[List[Issue]]:
empty = ""
issues = []
index = 1
for value in values:
if value is empty:
message_dict = GapAwareDict({
"field_name": self._name,
"line": index
})
error_message = library.resolve_error(self._error,
message_dict)
issues.append(Issue(ISSUE_TYPE_ERROR, error_message))
index += 1
if len(issues) > 0:
return issues
return None
def test_to_dict(self):
files = [
SubmissionFile(submission_id="submitme",
index=0,
filename="Hans",
filetype="DOC",
status=QC_STATUS_SUBMITTED,
result=DatasetValidationResult(status="OK",
issues=[])),
SubmissionFile(submission_id="submitme",
index=1,
filename="Helga",
filetype="MEAS",
status=QC_STATUS_VALIDATED,
result=DatasetValidationResult(
status="WARNING",
issues=[
Issue(type="WARNING",
description="This might be wrong")
]))
]
subm = DbSubmission(
status="Hellyeah",
user_id='Tom',
submission_id="submitme",
path="/the/archive/root/",
files=files,
qc_status="warning",
date=datetime(2001, 2, 3, 4, 5, 6),
publication_date=datetime(2002, 2, 3, 4, 5, 6),
allow_publication=True,
store_sub_path='Tom_Helge',
)
self.assertEqual(
{
'date':
datetime(2001, 2, 3, 4, 5, 6),
'file_refs': [],
'files': [{
'filename': 'Hans',
'filetype': 'DOC',
'index': 0,
'result': {
'issues': [],
'status': 'OK'
},
'status': QC_STATUS_SUBMITTED,
'submission_id': 'submitme'
}, {
'filename': 'Helga',
'filetype': 'MEAS',
'index': 1,
'result': {
'issues': [{
'description': 'This might be wrong',
'type': 'WARNING'
}],
'status':
'WARNING'
},
'status': QC_STATUS_VALIDATED,
'submission_id': 'submitme'
}],
'id':
None,
'path':
'/the/archive/root/',
'publication_date':
datetime(2002, 2, 3, 4, 5, 6),
'allow_publication':
True,
'qc_status':
'warning',
'status':
'Hellyeah',
'submission_id':
'submitme',
'store_sub_path':
'Tom_Helge',
'user_id':
'Tom'
}, subm.to_dict())
def eval(self,
unit: str,
values: List[float],
library: MessageLibrary,
missing_value: float = None) -> Optional[List[Issue]]:
issues = []
if unit not in self._units:
message_dict = GapAwareDict({
"field_name": self._name,
"unit": self._units,
"bad_unit": unit
})
error_message = library.resolve_error(self._unit_error,
message_dict)
issues.append(Issue(ISSUE_TYPE_ERROR, error_message))
line = 1
message_dict = GapAwareDict({
"field_name": self._name,
"line": line,
"value": 0,
"lower_bound": self._lower_bound,
"upper_bound": self._upper_bound
})
for value in values:
out_of_bounds = False
if not (type(value) is float or type(value) is int):
message_dict["line"] = line
message_dict["value"] = value
error_message = library.resolve_error(
"@field_number_not_a_number", message_dict)
issues.append(Issue(ISSUE_TYPE_ERROR, error_message))
line += 1
continue
if missing_value is not None and value == missing_value:
line += 1
continue
if not math.isnan(self._lower_bound):
if value < self._lower_bound:
out_of_bounds = True
if not math.isnan(self._upper_bound):
if value > self._upper_bound:
out_of_bounds = True
if out_of_bounds:
message_dict["line"] = line
message_dict["value"] = value
error_message = library.resolve_error(self._value_error,
message_dict)
issues.append(Issue(ISSUE_TYPE_ERROR, error_message))
line += 1
if len(issues) > 0:
return issues
return None
def create_error_message(self, library):
message_dict = GapAwareDict({"reference": self._name})
error_message = library.resolve_error(self._error, message_dict)
return Issue(ISSUE_TYPE_ERROR, error_message)
def _append_warning_issue(self, message, index, issues, library):
error_message = self._assemble_error_message(index, library, message)
issues.append(Issue(ISSUE_TYPE_WARNING, error_message))
def _append_error_issue(self, message, index, issues, library):
error_message = self._assemble_error_message(index, library, message)
issues.append(Issue(ISSUE_TYPE_ERROR, error_message))