def __init__(self, postgres_url, inst_data=None): self.postgres_url = check.str_param(postgres_url, 'postgres_url') self._event_watcher = PostgresEventWatcher(self.postgres_url) with self.get_engine() as engine: SqlEventLogStorageMetadata.create_all(engine) self._inst_data = check.opt_inst_param(inst_data, 'inst_data', ConfigurableClassData)
def test_filesystem_event_log_storage_run_corrupted_bad_data(): with seven.TemporaryDirectory() as tmpdir_path: storage = SqliteEventLogStorage(tmpdir_path) SqlEventLogStorageMetadata.create_all( create_engine(storage.conn_string_for_run_id('foo'))) with storage.connect('foo') as conn: event_insert = SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter run_id='foo', event='{bar}', dagster_event_type=None, timestamp=None) conn.execute(event_insert) with pytest.raises(DagsterEventLogInvalidForRun): storage.get_logs_for_run('foo') SqlEventLogStorageMetadata.create_all( create_engine(storage.conn_string_for_run_id('bar'))) with storage.connect('bar') as conn: # pylint: disable=protected-access event_insert = SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter run_id='bar', event='3', dagster_event_type=None, timestamp=None) conn.execute(event_insert) with pytest.raises(DagsterEventLogInvalidForRun): storage.get_logs_for_run('bar')
def test_filesystem_event_log_storage_run_corrupted_bad_data( self, storage): SqlEventLogStorageMetadata.create_all( create_engine(storage.conn_string_for_shard("foo"))) with storage.run_connection("foo") as conn: event_insert = ( SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter run_id="foo", event="{bar}", dagster_event_type=None, timestamp=None)) conn.execute(event_insert) with pytest.raises(DagsterEventLogInvalidForRun): storage.get_logs_for_run("foo") SqlEventLogStorageMetadata.create_all( create_engine(storage.conn_string_for_shard("bar"))) with storage.run_connection("bar") as conn: event_insert = ( SqlEventLogStorageTable.insert().values( # pylint: disable=no-value-for-parameter run_id="bar", event="3", dagster_event_type=None, timestamp=None)) conn.execute(event_insert) with pytest.raises(DagsterEventLogInvalidForRun): storage.get_logs_for_run("bar")
def wipe_storage(mysql_url): engine = create_engine(mysql_url, isolation_level="AUTOCOMMIT", poolclass=db.pool.NullPool) try: SqlEventLogStorageMetadata.drop_all(engine) finally: engine.dispose()
def __init__(self, postgres_url, inst_data=None): self.postgres_url = check.str_param(postgres_url, 'postgres_url') self._event_watcher = PostgresEventWatcher(self.postgres_url) self._inst_data = check.opt_inst_param(inst_data, 'inst_data', ConfigurableClassData) self._engine = create_engine( self.postgres_url, isolation_level='AUTOCOMMIT', poolclass=db.pool.NullPool ) SqlEventLogStorageMetadata.create_all(self._engine)
def create_clean_storage(conn_string, should_autocreate_tables=True): engine = create_engine( conn_string, isolation_level="AUTOCOMMIT", poolclass=db.pool.NullPool ) try: SqlEventLogStorageMetadata.drop_all(engine) finally: engine.dispose() return PostgresEventLogStorage(conn_string, should_autocreate_tables)
def __init__(self, postgres_url, inst_data=None): self.postgres_url = check.str_param(postgres_url, "postgres_url") self._event_watcher = PostgresEventWatcher(self.postgres_url) self._inst_data = check.opt_inst_param(inst_data, "inst_data", ConfigurableClassData) self._engine = create_engine(self.postgres_url, isolation_level="AUTOCOMMIT", poolclass=db.pool.NullPool) self._disposed = False with self.connect() as conn: SqlEventLogStorageMetadata.create_all(conn)
def __init__(self, postgres_url, inst_data=None): self._inst_data = check.opt_inst_param(inst_data, "inst_data", ConfigurableClassData) self.postgres_url = check.str_param(postgres_url, "postgres_url") self._disposed = False self._event_watcher = PostgresEventWatcher(self.postgres_url) # Default to not holding any connections open to prevent accumulating connections per DagsterInstance self._engine = create_engine( self.postgres_url, isolation_level="AUTOCOMMIT", poolclass=db.pool.NullPool ) self._secondary_index_cache = {} with self.connect() as conn: SqlEventLogStorageMetadata.create_all(conn)
def __init__(self, postgres_url, inst_data=None): self._inst_data = check.opt_inst_param(inst_data, "inst_data", ConfigurableClassData) self.postgres_url = check.str_param(postgres_url, "postgres_url") self._disposed = False self._event_watcher = PostgresEventWatcher(self.postgres_url) # Default to not holding any connections open to prevent accumulating connections per DagsterInstance self._engine = create_engine(self.postgres_url, isolation_level="AUTOCOMMIT", poolclass=db.pool.NullPool) self._secondary_index_cache = {} table_names = retry_pg_connection_fn( lambda: db.inspect(self._engine).get_table_names()) if "event_logs" not in table_names: with self.connect() as conn: alembic_config = get_alembic_config(__file__) retry_pg_creation_fn( lambda: SqlEventLogStorageMetadata.create_all(conn)) # This revision may be shared by any other dagster storage classes using the same DB stamp_alembic_rev(alembic_config, conn)
def __init__(self, mysql_url, inst_data=None): experimental_class_warning("MySQLEventLogStorage") self._inst_data = check.opt_inst_param(inst_data, "inst_data", ConfigurableClassData) self.mysql_url = check.str_param(mysql_url, "mysql_url") self._disposed = False self._event_watcher = SqlPollingEventWatcher(self) # Default to not holding any connections open to prevent accumulating connections per DagsterInstance self._engine = create_engine( self.mysql_url, isolation_level="AUTOCOMMIT", poolclass=db.pool.NullPool ) self._secondary_index_cache = {} table_names = retry_mysql_connection_fn(db.inspect(self._engine).get_table_names) if "event_logs" not in table_names: with self._connect() as conn: alembic_config = mysql_alembic_config(__file__) retry_mysql_creation_fn(lambda: SqlEventLogStorageMetadata.create_all(conn)) # This revision may be shared by any other dagster storage classes using the same DB stamp_alembic_rev(alembic_config, conn) # mark all secondary indexes to be used self.reindex() super().__init__()
def _init_db(self): with self._connect() as conn: with conn.begin(): SqlEventLogStorageMetadata.create_all(conn) stamp_alembic_rev(pg_alembic_config(__file__), conn)