def test_lossy_serialization_warning(caplog):
caplog.set_level(logging.WARNING, logger="great_expectations.core")
d = (Decimal(7091.17117297555159893818)**Decimal(2) +
Decimal(7118.70008070942458289210)**Decimal(2) +
(Decimal(-1513.67274389594149397453))**Decimal(2))**Decimal(1.5)
f_1 = (7091.17117297555159893818**2 + 7118.70008070942458289210**2 +
(-1513.67274389594149397453)**2)**1.5
f_2 = float(d)
assert not (-1e-55 < Decimal.from_float(f_1) - d < 1e-55)
assert not (-1e-55 < Decimal.from_float(f_2) - d < 1e-55)
convert_to_json_serializable(d)
assert len(caplog.messages) == 1
assert caplog.messages[0].startswith("Using lossy conversion for decimal")
caplog.clear()
d = Decimal(0.1)
f_1 = 0.1
f_2 = float(d)
assert (-1e-55 < Decimal.from_float(f_1) - d < 1e-55)
assert (-1e-55 < Decimal.from_float(f_2) - d < 1e-55)
convert_to_json_serializable(d)
assert len(caplog.messages) == 0
def ge_validation_solid(context, dataset):
data_context = context.resources.ge_data_context
if validation_operator_name is not None:
validation_operator = validation_operator_name
else:
data_context.add_validation_operator(
"ephemeral_validation",
{"class_name": "ActionListValidationOperator", "action_list": []},
)
validation_operator = "ephemeral_validation"
suite = data_context.get_expectation_suite(suite_name)
final_batch_kwargs = batch_kwargs or {"dataset": dataset}
if "datasource" in batch_kwargs:
context.log.warning(
"`datasource` field of `batch_kwargs` will be ignored; use the `datasource_name` "
"parameter of the solid factory instead."
)
final_batch_kwargs["datasource"] = datasource_name
batch = data_context.get_batch(final_batch_kwargs, suite)
run_id = {
"run_name": datasource_name + " run",
"run_time": datetime.datetime.utcnow(),
}
results = data_context.run_validation_operator(
validation_operator, assets_to_validate=[batch], run_id=run_id
)
res = convert_to_json_serializable(results.list_validation_results())[0]
md_str = render_multiple_validation_result_pages_markdown(
validation_operator_result=results, run_info_at_end=True,
)
meta_stats = EventMetadataEntry.md(md_str=md_str, label="Expectation Results")
yield ExpectationResult(
success=res["success"], metadata_entries=[meta_stats,],
)
yield Output(res)
def ge_validation_solid(context, pandas_df):
data_context = context.resources.ge_data_context
suite = data_context.get_expectation_suite(suite_name)
batch_kwargs = {
"dataset": pandas_df,
"datasource": datasource_name,
}
batch = data_context.get_batch(batch_kwargs, suite)
run_id = {
"run_name": datasource_name + " run",
"run_time": datetime.datetime.utcnow(),
}
results = data_context.run_validation_operator(
"action_list_operator", assets_to_validate=[batch], run_id=run_id)
res = convert_to_json_serializable(
results.list_validation_results())[0]
nmeta = EventMetadataEntry.json(
{
'overall': res['statistics'],
'individual': res['results']
},
'constraint-metadata',
)
yield ExpectationResult(success=res["success"],
metadata_entries=[nmeta])
yield Output(res)
def _ge_validation_fn(context, dataset):
data_context = context.resources.ge_data_context
if validation_operator_name is not None:
validation_operator = validation_operator_name
else:
data_context.add_validation_operator(
"ephemeral_validation",
{
"class_name": "ActionListValidationOperator",
"action_list": []
},
)
validation_operator = "ephemeral_validation"
suite = data_context.get_expectation_suite(suite_name)
final_batch_kwargs = batch_kwargs or {"dataset": dataset}
if "datasource" in batch_kwargs:
context.log.warning(
"`datasource` field of `batch_kwargs` will be ignored; use the `datasource_name` "
f"parameter of the {decorator_name} factory instead.")
final_batch_kwargs["datasource"] = datasource_name
batch = data_context.get_batch(final_batch_kwargs, suite)
run_id = {
"run_name": datasource_name + " run",
"run_time": datetime.datetime.utcnow(),
}
results = data_context.run_validation_operator(
validation_operator, assets_to_validate=[batch], run_id=run_id)
res = convert_to_json_serializable(
results.list_validation_results())[0]
validation_results_page_renderer = ValidationResultsPageRenderer(
run_info_at_end=True)
rendered_document_content_list = (
validation_results_page_renderer.render_validation_operator_result(
results))
md_str = " ".join(
DefaultMarkdownPageView().render(rendered_document_content_list))
meta_stats = MetadataEntry("Expectation Results",
value=MetadataValue.md(md_str))
yield ExpectationResult(
success=res["success"],
metadata_entries=[
meta_stats,
],
)
yield Output(res)
def ge_validation_solid(context, pandas_df):
data_context = context.resources.ge_data_context
if validation_operator_name is not None:
validation_operator = validation_operator_name
else:
data_context.add_validation_operator(
"ephemeral_validation",
{
"class_name": "ActionListValidationOperator",
"action_list": []
},
)
validation_operator = "ephemeral_validation"
suite = data_context.get_expectation_suite(suite_name)
batch_kwargs = {
"dataset": pandas_df,
"datasource": datasource_name,
}
batch = data_context.get_batch(batch_kwargs, suite)
run_id = {
"run_name": datasource_name + " run",
"run_time": datetime.datetime.utcnow(),
}
results = data_context.run_validation_operator(
validation_operator, assets_to_validate=[batch], run_id=run_id)
res = convert_to_json_serializable(
results.list_validation_results())[0]
md_str = render_multiple_validation_result_pages_markdown(
validation_operator_result=results,
run_info_at_end=True,
)
meta_stats = EventMetadataEntry.md(md_str=md_str,
label="Expectation Results")
yield ExpectationResult(
success=res["success"],
metadata_entries=[
meta_stats,
],
)
yield Output(res)
def prepare_dump(self, data, **kwargs):
data = deepcopy(data)
data._run_results = convert_to_json_serializable(data.run_results)
return data