def test_execute_pipeline_iterator():
with instance_for_test() as instance:
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
pipeline = PipelineDefinition(
name="basic_resource_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={"a": resource_a, "b": resource_b},
logger_defs={"callback": construct_event_logger(event_callback)},
)
],
)
iterator = execute_pipeline_iterator(
pipeline, run_config={"loggers": {"callback": {}}}, instance=instance
)
event_type = None
while event_type != "STEP_START":
event = next(iterator)
event_type = event.event_type_value
iterator.close()
events = [record.dagster_event for record in records if record.is_dagster_event]
messages = [record.user_message for record in records if not record.is_dagster_event]
pipeline_failure_events = [event for event in events if event.is_pipeline_failure]
assert len(pipeline_failure_events) == 1
assert "GeneratorExit" in pipeline_failure_events[0].pipeline_failure_data.error.message
assert len([message for message in messages if message == "CLEANING A"]) > 0
assert len([message for message in messages if message == "CLEANING B"]) > 0
def test_execute_pipeline_iterator():
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
pipeline = PipelineDefinition(
name='basic_resource_pipeline',
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={'a': resource_a, 'b': resource_b},
logger_defs={'callback': construct_event_logger(event_callback)},
)
],
)
iterator = execute_pipeline_iterator(
pipeline,
environment_dict={'loggers': {'callback': {}}},
instance=DagsterInstance.local_temp(),
)
event_type = None
while event_type != 'STEP_START':
event = next(iterator)
event_type = event.event_type_value
iterator.close()
events = [record.dagster_event for record in records if record.is_dagster_event]
messages = [record.user_message for record in records if not record.is_dagster_event]
assert len([event for event in events if event.is_pipeline_failure]) > 0
assert len([message for message in messages if message == 'CLEANING A']) > 0
assert len([message for message in messages if message == 'CLEANING B']) > 0
def execute_pipeline_command(output_file, solid_subset, environment_dict, mode,
**kwargs):
'''
This command might want to take a runId instead of current arguments
1. Should take optional flags to determine where to store the log output
2. Investigate python logging library to see what we can do there
'''
with ipc_write_stream(output_file) as stream:
recon_pipeline = recon_pipeline_for_cli_args(kwargs)
definition = recon_pipeline.get_definition()
if solid_subset:
definition = definition.subset_for_execution(
solid_subset.split(","))
# This can raise a ValueError, but this is caught by the broad-except
# and the exception is serialized as a SerializableErrorInfo
environment_dict = json.loads(environment_dict)
instance = DagsterInstance.get()
for event in execute_pipeline_iterator(
definition,
environment_dict=environment_dict,
mode=mode,
instance=instance,
):
stream.send(event)
def execute_pipeline_command(output_file, solid_subset, environment_dict,
instance_ref, mode, **kwargs):
'''
This command might want to take a runId instead of current arguments
1. Should take optional flags to determine where to store the log output
2. Investigate python logging library to see what we can do there
'''
with ipc_write_stream(output_file) as stream:
recon_pipeline = recon_pipeline_for_cli_args(kwargs)
definition = recon_pipeline.get_definition()
if solid_subset:
definition = definition.subset_for_execution(
solid_subset.split(","))
# This can raise a ValueError, but this is caught by the broad-except
# and the exception is serialized as a SerializableErrorInfo
environment_dict = json.loads(environment_dict)
try:
instance = DagsterInstance.from_ref(
deserialize_json_to_dagster_namedtuple(instance_ref))
except: # pylint: disable=bare-except
stream.send_error(
sys.exc_info(),
message='Could not deserialize {json_string}'.format(
json_string=instance_ref),
)
return
for event in execute_pipeline_iterator(
definition,
environment_dict=environment_dict,
mode=mode,
instance=instance,
):
stream.send(event)