def __init__(self, postgres_url, inst_data=None):
self.postgres_url = check.str_param(postgres_url, 'postgres_url')
self._event_watcher = PostgresEventWatcher(self.postgres_url)
with self.get_engine() as engine:
SqlEventLogStorageMetadata.create_all(engine)
self._inst_data = check.opt_inst_param(inst_data, 'inst_data',
ConfigurableClassData)
def test_filesystem_event_log_storage_run_corrupted_bad_data():
with seven.TemporaryDirectory() as tmpdir_path:
storage = SqliteEventLogStorage(tmpdir_path)
SqlEventLogStorageMetadata.create_all(
create_engine(storage.conn_string_for_run_id('foo')))
with storage.connect('foo') as conn:
event_insert = SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter
run_id='foo',
event='{bar}',
dagster_event_type=None,
timestamp=None)
conn.execute(event_insert)
with pytest.raises(DagsterEventLogInvalidForRun):
storage.get_logs_for_run('foo')
SqlEventLogStorageMetadata.create_all(
create_engine(storage.conn_string_for_run_id('bar')))
with storage.connect('bar') as conn: # pylint: disable=protected-access
event_insert = SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter
run_id='bar',
event='3',
dagster_event_type=None,
timestamp=None)
conn.execute(event_insert)
with pytest.raises(DagsterEventLogInvalidForRun):
storage.get_logs_for_run('bar')
def test_filesystem_event_log_storage_run_corrupted_bad_data(
self, storage):
SqlEventLogStorageMetadata.create_all(
create_engine(storage.conn_string_for_shard("foo")))
with storage.run_connection("foo") as conn:
event_insert = (
SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter
run_id="foo",
event="{bar}",
dagster_event_type=None,
timestamp=None))
conn.execute(event_insert)
with pytest.raises(DagsterEventLogInvalidForRun):
storage.get_logs_for_run("foo")
SqlEventLogStorageMetadata.create_all(
create_engine(storage.conn_string_for_shard("bar")))
with storage.run_connection("bar") as conn:
event_insert = (
SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter
run_id="bar",
event="3",
dagster_event_type=None,
timestamp=None))
conn.execute(event_insert)
with pytest.raises(DagsterEventLogInvalidForRun):
storage.get_logs_for_run("bar")
def wipe_storage(mysql_url):
engine = create_engine(mysql_url,
isolation_level="AUTOCOMMIT",
poolclass=db.pool.NullPool)
try:
SqlEventLogStorageMetadata.drop_all(engine)
finally:
engine.dispose()
def __init__(self, postgres_url, inst_data=None):
self.postgres_url = check.str_param(postgres_url, 'postgres_url')
self._event_watcher = PostgresEventWatcher(self.postgres_url)
self._inst_data = check.opt_inst_param(inst_data, 'inst_data', ConfigurableClassData)
self._engine = create_engine(
self.postgres_url, isolation_level='AUTOCOMMIT', poolclass=db.pool.NullPool
)
SqlEventLogStorageMetadata.create_all(self._engine)
def create_clean_storage(conn_string, should_autocreate_tables=True):
engine = create_engine(
conn_string, isolation_level="AUTOCOMMIT", poolclass=db.pool.NullPool
)
try:
SqlEventLogStorageMetadata.drop_all(engine)
finally:
engine.dispose()
return PostgresEventLogStorage(conn_string, should_autocreate_tables)
def __init__(self, postgres_url, inst_data=None):
self.postgres_url = check.str_param(postgres_url, "postgres_url")
self._event_watcher = PostgresEventWatcher(self.postgres_url)
self._inst_data = check.opt_inst_param(inst_data, "inst_data",
ConfigurableClassData)
self._engine = create_engine(self.postgres_url,
isolation_level="AUTOCOMMIT",
poolclass=db.pool.NullPool)
self._disposed = False
with self.connect() as conn:
SqlEventLogStorageMetadata.create_all(conn)
def __init__(self, postgres_url, inst_data=None):
self._inst_data = check.opt_inst_param(inst_data, "inst_data", ConfigurableClassData)
self.postgres_url = check.str_param(postgres_url, "postgres_url")
self._disposed = False
self._event_watcher = PostgresEventWatcher(self.postgres_url)
# Default to not holding any connections open to prevent accumulating connections per DagsterInstance
self._engine = create_engine(
self.postgres_url, isolation_level="AUTOCOMMIT", poolclass=db.pool.NullPool
)
self._secondary_index_cache = {}
with self.connect() as conn:
SqlEventLogStorageMetadata.create_all(conn)
def __init__(self, postgres_url, inst_data=None):
self._inst_data = check.opt_inst_param(inst_data, "inst_data",
ConfigurableClassData)
self.postgres_url = check.str_param(postgres_url, "postgres_url")
self._disposed = False
self._event_watcher = PostgresEventWatcher(self.postgres_url)
# Default to not holding any connections open to prevent accumulating connections per DagsterInstance
self._engine = create_engine(self.postgres_url,
isolation_level="AUTOCOMMIT",
poolclass=db.pool.NullPool)
self._secondary_index_cache = {}
table_names = retry_pg_connection_fn(
lambda: db.inspect(self._engine).get_table_names())
if "event_logs" not in table_names:
with self.connect() as conn:
alembic_config = get_alembic_config(__file__)
retry_pg_creation_fn(
lambda: SqlEventLogStorageMetadata.create_all(conn))
# This revision may be shared by any other dagster storage classes using the same DB
stamp_alembic_rev(alembic_config, conn)
def __init__(self, mysql_url, inst_data=None):
experimental_class_warning("MySQLEventLogStorage")
self._inst_data = check.opt_inst_param(inst_data, "inst_data", ConfigurableClassData)
self.mysql_url = check.str_param(mysql_url, "mysql_url")
self._disposed = False
self._event_watcher = SqlPollingEventWatcher(self)
# Default to not holding any connections open to prevent accumulating connections per DagsterInstance
self._engine = create_engine(
self.mysql_url, isolation_level="AUTOCOMMIT", poolclass=db.pool.NullPool
)
self._secondary_index_cache = {}
table_names = retry_mysql_connection_fn(db.inspect(self._engine).get_table_names)
if "event_logs" not in table_names:
with self._connect() as conn:
alembic_config = mysql_alembic_config(__file__)
retry_mysql_creation_fn(lambda: SqlEventLogStorageMetadata.create_all(conn))
# This revision may be shared by any other dagster storage classes using the same DB
stamp_alembic_rev(alembic_config, conn)
# mark all secondary indexes to be used
self.reindex()
super().__init__()
def _init_db(self):
with self._connect() as conn:
with conn.begin():
SqlEventLogStorageMetadata.create_all(conn)
stamp_alembic_rev(pg_alembic_config(__file__), conn)