def test_fs_stores():
@pipeline
def simple():
@lambda_solid
def easy():
return 'easy'
easy()
with seven.TemporaryDirectory() as temp_dir:
run_store = FilesystemRunStorage(temp_dir)
event_store = FilesystemEventLogStorage(temp_dir)
compute_log_manager = LocalComputeLogManager(temp_dir)
instance = DagsterInstance(
instance_type=InstanceType.LOCAL,
root_storage_dir=temp_dir,
run_storage=run_store,
event_storage=event_store,
compute_log_manager=compute_log_manager,
)
run = RunConfig()
execute_pipeline(simple, run_config=run, instance=instance)
assert run_store.has_run(run.run_id)
assert run_store.get_run_by_id(
run.run_id).status == PipelineRunStatus.SUCCESS
assert DagsterEventType.PIPELINE_SUCCESS in [
event.dagster_event.event_type
for event in event_store.get_logs_for_run(run.run_id)
]
def test_filesystem_event_log_storage_run_corrupted():
with seven.TemporaryDirectory() as tmpdir_path:
storage = FilesystemEventLogStorage(tmpdir_path)
with open(storage.filepath_for_run_id('foo'), 'w') as fd:
fd.write('some nonsense')
with pytest.raises(EventLogInvalidForRun) as exc:
storage.get_logs_for_run('foo')
assert exc.value.run_id == 'foo'
def from_ref(instance_ref, fallback_feature_set=None, watch_external_runs=False):
check.inst_param(instance_ref, 'instance_ref', InstanceRef)
check.opt_set_param(fallback_feature_set, 'fallback_feature_set', str)
if isinstance(instance_ref, LocalInstanceRef):
from dagster.core.storage.event_log import FilesystemEventLogStorage
from dagster.core.storage.runs import FilesystemRunStorage
from dagster.core.storage.local_compute_log_manager import LocalComputeLogManager
feature_set = _dagster_feature_set(instance_ref.home_dir) or fallback_feature_set
return DagsterInstance(
instance_type=InstanceType.LOCAL,
root_storage_dir=instance_ref.home_dir,
run_storage=FilesystemRunStorage(
_runs_directory(instance_ref.home_dir), watch_external_runs=watch_external_runs
),
event_storage=FilesystemEventLogStorage(_runs_directory(instance_ref.home_dir)),
compute_log_manager=LocalComputeLogManager(
_compute_logs_base_directory(instance_ref.home_dir)
),
feature_set=feature_set,
)
else:
check.failed('Unhandled instance type {}'.format(type(instance_ref)))
def test_filesystem_event_log_storage_run_corrupted_bad_data():
with seven.TemporaryDirectory() as tmpdir_path:
storage = FilesystemEventLogStorage(tmpdir_path)
with storage._connect('foo') as conn: # pylint: disable=protected-access
conn.cursor().execute(CREATE_EVENT_LOG_SQL)
conn.cursor().execute(INSERT_EVENT_SQL, ('{bar}', ))
with pytest.raises(EventLogInvalidForRun) as exc:
storage.get_logs_for_run('foo')
assert exc.value.run_id == 'foo'
with storage._connect('bar') as conn: # pylint: disable=protected-access
conn.cursor().execute(CREATE_EVENT_LOG_SQL)
conn.cursor().execute(INSERT_EVENT_SQL, ('3', ))
with pytest.raises(EventLogInvalidForRun) as exc:
storage.get_logs_for_run('bar')
assert exc.value.run_id == 'bar'
def test_filesystem_event_log_storage_store_events_and_wipe():
with seven.TemporaryDirectory() as tmpdir_path:
storage = FilesystemEventLogStorage(tmpdir_path)
assert len(storage.get_logs_for_run('foo')) == 0
storage.store_event(
DagsterEventRecord(
None,
'Message2',
'debug',
'',
'foo',
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
'nonce',
event_specific_data=EngineEventData.in_process(999),
),
))
assert len(storage.get_logs_for_run('foo')) == 1
storage.wipe()
assert len(storage.get_logs_for_run('foo')) == 0
def test_filesystem_event_log_storage_run_not_found():
with seven.TemporaryDirectory() as tmpdir_path:
storage = FilesystemEventLogStorage(tmpdir_path)
assert storage.get_logs_for_run('bar') == []
def test_filesystem_event_log_storage_init():
with seven.TemporaryDirectory() as tmpdir_path:
storage = FilesystemEventLogStorage(tmpdir_path)
assert storage.is_persistent