def test_filesystem_event_log_storage_run_corrupted_bad_data(
self, storage):
SqlEventLogStorageMetadata.create_all(
create_engine(storage.conn_string_for_shard("foo")))
with storage.run_connection("foo") as conn:
event_insert = (
SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter
run_id="foo",
event="{bar}",
dagster_event_type=None,
timestamp=None))
conn.execute(event_insert)
with pytest.raises(DagsterEventLogInvalidForRun):
storage.get_logs_for_run("foo")
SqlEventLogStorageMetadata.create_all(
create_engine(storage.conn_string_for_shard("bar")))
with storage.run_connection("bar") as conn:
event_insert = (
SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter
run_id="bar",
event="3",
dagster_event_type=None,
timestamp=None))
conn.execute(event_insert)
with pytest.raises(DagsterEventLogInvalidForRun):
storage.get_logs_for_run("bar")
def test_filesystem_event_log_storage_run_corrupted_bad_data():
with seven.TemporaryDirectory() as tmpdir_path:
storage = SqliteEventLogStorage(tmpdir_path)
SqlEventLogStorageMetadata.create_all(
create_engine(storage.conn_string_for_run_id('foo')))
with storage.connect('foo') as conn:
event_insert = SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter
run_id='foo',
event='{bar}',
dagster_event_type=None,
timestamp=None)
conn.execute(event_insert)
with pytest.raises(DagsterEventLogInvalidForRun):
storage.get_logs_for_run('foo')
SqlEventLogStorageMetadata.create_all(
create_engine(storage.conn_string_for_run_id('bar')))
with storage.connect('bar') as conn: # pylint: disable=protected-access
event_insert = SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter
run_id='bar',
event='3',
dagster_event_type=None,
timestamp=None)
conn.execute(event_insert)
with pytest.raises(DagsterEventLogInvalidForRun):
storage.get_logs_for_run('bar')
def store_event(self, event):
'''Store an event corresponding to a pipeline run.
Args:
event (EventRecord): The event to store.
'''
check.inst_param(event, 'event', EventRecord)
dagster_event_type = None
if event.is_dagster_event:
dagster_event_type = event.dagster_event.event_type_value
run_id = event.run_id
with self.connect() as conn:
# https://stackoverflow.com/a/54386260/324449
event_insert = SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter
run_id=run_id,
event=serialize_dagster_namedtuple(event),
dagster_event_type=dagster_event_type,
timestamp=datetime.datetime.fromtimestamp(event.timestamp),
)
result_proxy = conn.execute(
event_insert.returning(SqlEventLogStorageTable.c.run_id,
SqlEventLogStorageTable.c.id))
res = result_proxy.fetchone()
result_proxy.close()
conn.execute(
'''NOTIFY {channel}, %s; '''.format(channel=CHANNEL_NAME),
(res[0] + '_' + str(res[1]), ),
)
def upgrade():
# This is our root migration, and we don't have a common base. Before this revision, sqlite- and
# postgres-based event logs had different schemas. The conditionality below is to deal with dev
# databases that might not have been stamped by Alembic.
bind = op.get_context().bind
inspector = reflection.Inspector.from_engine(bind)
if "sqlite" not in inspector.dialect.dialect_description:
raise Exception(
"Bailing: refusing to run a migration for sqlite-backed event log storage against "
"a non-sqlite database of dialect {dialect}".format(
dialect=inspector.dialect.dialect_description))
has_columns = [col["name"] for col in inspector.get_columns("event_logs")]
with op.batch_alter_table("event_logs") as batch_op:
if "row_id" in has_columns:
batch_op.alter_column(column_name="row_id", new_column_name="id")
if "run_id" not in has_columns:
batch_op.add_column(column=sa.Column("run_id", sa.String(255)))
op.execute(
SqlEventLogStorageTable.update(None).where(
SqlEventLogStorageTable.c.run_id == None).values(
{"run_id": context.config.attributes.get("run_id", None)}))