def process_payroll(_, df):
return len(df)
@solid(input_defs=[
InputDefinition(name='numrows'),
InputDefinition(name='expectation')
])
def postprocess_payroll(_, numrows, expectation):
if expectation["success"]:
return numrows
else:
raise ValueError
payroll_expectations = ge_validation_solid_factory("getest", "basic.warning")
@pipeline(
mode_defs=[
ModeDefinition('basic',
resource_defs={'ge_data_context': ge_data_context})
],
preset_defs=[
PresetDefinition(
'sample_preset_success',
mode='basic',
run_config={
'resources': {
'ge_data_context': {
'config': {
def process_payroll(_, df):
return len(df)
@solid(input_defs=[
InputDefinition(name="numrows"),
InputDefinition(name="expectation")
])
def postprocess_payroll(_, numrows, expectation):
if expectation["success"]:
return numrows
else:
raise ValueError
payroll_expectations = ge_validation_solid_factory(datasource_name="getest",
suite_name="basic.warning")
@pipeline(
mode_defs=[
ModeDefinition("basic",
resource_defs={"ge_data_context": ge_data_context})
],
preset_defs=[
PresetDefinition(
"sample_preset_success",
mode="basic",
run_config={
"resources": {
"ge_data_context": {
"config": {
def hello_world_pandas_pipeline_v2():
return reyielder(
ge_validation_solid_factory("ge_validation_solid", "getest",
"basic.warning")(pandas_yielder()))
def hello_world_pyspark_pipeline():
validate = ge_validation_solid_factory(
"getestspark",
"basic.warning",
input_dagster_type=DagsterPySparkDataFrame)
return reyielder(validate(pyspark_yielder()))
