def test_execute_pipeline_iterator(): with instance_for_test() as instance: records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) pipeline = PipelineDefinition( name="basic_resource_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={"a": resource_a, "b": resource_b}, logger_defs={"callback": construct_event_logger(event_callback)}, ) ], ) iterator = execute_pipeline_iterator( pipeline, run_config={"loggers": {"callback": {}}}, instance=instance ) event_type = None while event_type != "STEP_START": event = next(iterator) event_type = event.event_type_value iterator.close() events = [record.dagster_event for record in records if record.is_dagster_event] messages = [record.user_message for record in records if not record.is_dagster_event] pipeline_failure_events = [event for event in events if event.is_pipeline_failure] assert len(pipeline_failure_events) == 1 assert "GeneratorExit" in pipeline_failure_events[0].pipeline_failure_data.error.message assert len([message for message in messages if message == "CLEANING A"]) > 0 assert len([message for message in messages if message == "CLEANING B"]) > 0
def test_execute_pipeline_iterator(): records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) pipeline = PipelineDefinition( name='basic_resource_pipeline', solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={'a': resource_a, 'b': resource_b}, logger_defs={'callback': construct_event_logger(event_callback)}, ) ], ) iterator = execute_pipeline_iterator( pipeline, environment_dict={'loggers': {'callback': {}}}, instance=DagsterInstance.local_temp(), ) event_type = None while event_type != 'STEP_START': event = next(iterator) event_type = event.event_type_value iterator.close() events = [record.dagster_event for record in records if record.is_dagster_event] messages = [record.user_message for record in records if not record.is_dagster_event] assert len([event for event in events if event.is_pipeline_failure]) > 0 assert len([message for message in messages if message == 'CLEANING A']) > 0 assert len([message for message in messages if message == 'CLEANING B']) > 0
def execute_pipeline_command(output_file, solid_subset, environment_dict, mode, **kwargs): ''' This command might want to take a runId instead of current arguments 1. Should take optional flags to determine where to store the log output 2. Investigate python logging library to see what we can do there ''' with ipc_write_stream(output_file) as stream: recon_pipeline = recon_pipeline_for_cli_args(kwargs) definition = recon_pipeline.get_definition() if solid_subset: definition = definition.subset_for_execution( solid_subset.split(",")) # This can raise a ValueError, but this is caught by the broad-except # and the exception is serialized as a SerializableErrorInfo environment_dict = json.loads(environment_dict) instance = DagsterInstance.get() for event in execute_pipeline_iterator( definition, environment_dict=environment_dict, mode=mode, instance=instance, ): stream.send(event)
def execute_pipeline_command(output_file, solid_subset, environment_dict, instance_ref, mode, **kwargs): ''' This command might want to take a runId instead of current arguments 1. Should take optional flags to determine where to store the log output 2. Investigate python logging library to see what we can do there ''' with ipc_write_stream(output_file) as stream: recon_pipeline = recon_pipeline_for_cli_args(kwargs) definition = recon_pipeline.get_definition() if solid_subset: definition = definition.subset_for_execution( solid_subset.split(",")) # This can raise a ValueError, but this is caught by the broad-except # and the exception is serialized as a SerializableErrorInfo environment_dict = json.loads(environment_dict) try: instance = DagsterInstance.from_ref( deserialize_json_to_dagster_namedtuple(instance_ref)) except: # pylint: disable=bare-except stream.send_error( sys.exc_info(), message='Could not deserialize {json_string}'.format( json_string=instance_ref), ) return for event in execute_pipeline_iterator( definition, environment_dict=environment_dict, mode=mode, instance=instance, ): stream.send(event)