def expectation_result_from_data(data):
return ExpectationResult(
success=data['success'],
label=data['label'],
description=data.get('description'), # enforce?
metadata_entries=list(event_metadata_entries(data.get('metadataEntries')) or []),
)
def expectation_result_from_data(data):
return ExpectationResult(
success=data["success"],
label=data["label"],
description=data.get("description"), # enforce?
metadata_entries=list(event_metadata_entries(data.get("metadataEntries")) or []),
)
def dagster_event_from_dict(event_dict, pipeline_name):
check.dict_param(event_dict, 'event_dict', key_type=str)
check.str_param(pipeline_name, 'pipeline_name')
materialization = event_dict.get('intermediateMaterialization') or {}
# Get event_type
event_type = _handled_events().get(event_dict['__typename'])
if not event_type:
raise Exception('unhandled event type %s' % event_dict['__typename'])
# Get event_specific_data
event_specific_data = None
if event_type == DagsterEventType.STEP_OUTPUT:
event_specific_data = StepOutputData(
step_output_handle=StepOutputHandle(event_dict['step']['key'],
event_dict['outputName']),
value_repr=event_dict['valueRepr'],
intermediate_materialization=Materialization(
path=materialization.get('path'),
description=materialization.get('description')),
)
elif event_type == DagsterEventType.STEP_SUCCESS:
event_specific_data = StepSuccessData(0.0)
elif event_type == DagsterEventType.STEP_MATERIALIZATION:
event_specific_data = StepMaterializationData(
materialization=Materialization(path=materialization.get('path'),
description=materialization.get(
'description')))
elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT:
result_metadata = event_dict['expectationResult'][
'resultMetadataJsonString']
expectation_result = ExpectationResult(
event_dict['expectationResult']['success'],
event_dict['expectationResult']['name'],
event_dict['expectationResult']['message'],
json.loads(result_metadata) if result_metadata else None,
)
event_specific_data = StepExpectationResultData(expectation_result)
elif event_type == DagsterEventType.STEP_FAILURE:
error_info = SerializableErrorInfo(event_dict['error']['message'],
stack=None,
cls_name=None)
event_specific_data = StepFailureData(error_info)
return DagsterEvent(
event_type_value=event_type.value,
pipeline_name=pipeline_name,
step_key=event_dict['step']['key'],
solid_handle=SolidHandle(event_dict['step']['solidHandleID'], None,
None),
step_kind_value=event_dict['step']['kind'],
logging_tags=None,
event_specific_data=event_specific_data,
)
def _do_expectation(_info, df):
ge_df = ge.from_pandas(df)
ge_result = ge_callback(ge_df)
check.invariant('success' in ge_result)
return ExpectationResult(
success=ge_result['success'],
metadata_entries=[
EventMetadataEntry.json(label='result', data=ge_result)
],
)
def _file_passes(_info, df):
with open(file_path) as ff:
expt_config = json.load(ff)
# This is necessary because ge ends up coercing a type change
# on the same dataframe instance, changing the type. But a
# ge can't be copied, because it causes an error.
# The error is
# AttributeError: 'PandasDataset' object has no attribute 'discard_subset_failing_expectations'
# See https://github.com/great-expectations/great_expectations/issues/342
df_copy = copy.deepcopy(df)
ge_df = ge.from_pandas(df_copy, expt_config)
validate_result = ge_df.validate()
check.invariant('success' in validate_result)
check.invariant('results' in validate_result)
return ExpectationResult(success=validate_result['success'],
result_context=validate_result)
def _stats_records(run_id):
now = time.time()
return [
_event_record(run_id, 'A', now - 325, DagsterEventType.STEP_START),
_event_record(
run_id,
'A',
now - 225,
DagsterEventType.STEP_SUCCESS,
StepSuccessData(duration_ms=100000.0),
),
_event_record(run_id, 'B', now - 225, DagsterEventType.STEP_START),
_event_record(
run_id,
'B',
now - 175,
DagsterEventType.STEP_FAILURE,
StepFailureData(error=None, user_failure_data=None),
),
_event_record(run_id, 'C', now - 175, DagsterEventType.STEP_START),
_event_record(run_id, 'C', now - 150, DagsterEventType.STEP_SKIPPED),
_event_record(run_id, 'D', now - 150, DagsterEventType.STEP_START),
_event_record(
run_id,
'D',
now - 125,
DagsterEventType.STEP_MATERIALIZATION,
StepMaterializationData(Materialization(label='mat 1')),
),
_event_record(
run_id,
'D',
now - 100,
DagsterEventType.STEP_EXPECTATION_RESULT,
StepExpectationResultData(ExpectationResult(success=True, label='exp 1')),
),
_event_record(
run_id,
'D',
now - 75,
DagsterEventType.STEP_MATERIALIZATION,
StepMaterializationData(Materialization(label='mat 2')),
),
_event_record(
run_id,
'D',
now - 50,
DagsterEventType.STEP_EXPECTATION_RESULT,
StepExpectationResultData(ExpectationResult(success=False, label='exp 2')),
),
_event_record(
run_id,
'D',
now - 25,
DagsterEventType.STEP_MATERIALIZATION,
StepMaterializationData(Materialization(label='mat 3')),
),
_event_record(
run_id, 'D', now, DagsterEventType.STEP_SUCCESS, StepSuccessData(duration_ms=150000.0)
),
]
def _stats_records(run_id):
now = time.time()
return [
_event_record(run_id, "A", now - 325, DagsterEventType.STEP_START),
_event_record(
run_id,
"A",
now - 225,
DagsterEventType.STEP_SUCCESS,
StepSuccessData(duration_ms=100000.0),
),
_event_record(run_id, "B", now - 225, DagsterEventType.STEP_START),
_event_record(
run_id,
"B",
now - 175,
DagsterEventType.STEP_FAILURE,
StepFailureData(error=None, user_failure_data=None),
),
_event_record(run_id, "C", now - 175, DagsterEventType.STEP_START),
_event_record(run_id, "C", now - 150, DagsterEventType.STEP_SKIPPED),
_event_record(run_id, "D", now - 150, DagsterEventType.STEP_START),
_event_record(
run_id,
"D",
now - 125,
DagsterEventType.ASSET_MATERIALIZATION,
StepMaterializationData(AssetMaterialization(asset_key="mat_1")),
),
_event_record(
run_id,
"D",
now - 100,
DagsterEventType.STEP_EXPECTATION_RESULT,
StepExpectationResultData(
ExpectationResult(success=True, label="exp 1")),
),
_event_record(
run_id,
"D",
now - 75,
DagsterEventType.ASSET_MATERIALIZATION,
StepMaterializationData(AssetMaterialization(asset_key="mat_2")),
),
_event_record(
run_id,
"D",
now - 50,
DagsterEventType.STEP_EXPECTATION_RESULT,
StepExpectationResultData(
ExpectationResult(success=False, label="exp 2")),
),
_event_record(
run_id,
"D",
now - 25,
DagsterEventType.ASSET_MATERIALIZATION,
StepMaterializationData(AssetMaterialization(asset_key="mat_3")),
),
_event_record(
run_id,
"D",
now,
DagsterEventType.STEP_SUCCESS,
StepSuccessData(duration_ms=150000.0),
),
]
def _do_expectation(_info, df):
ge_df = ge.from_pandas(df)
ge_result = ge_callback(ge_df)
check.invariant('success' in ge_result)
return ExpectationResult(success=ge_result['success'],
result_context=ge_result)