def from_local(cls, base_dir, inst_data=None):
check.str_param(base_dir, "base_dir")
mkdir_p(base_dir)
conn_string = create_db_conn_string(base_dir, "runs")
engine = create_engine(conn_string, poolclass=NullPool)
alembic_config = get_alembic_config(__file__)
should_mark_indexes = False
with engine.connect() as connection:
db_revision, head_revision = check_alembic_revision(
alembic_config, connection)
if not (db_revision and head_revision):
RunStorageSqlMetadata.create_all(engine)
engine.execute("PRAGMA journal_mode=WAL;")
stamp_alembic_rev(alembic_config, connection)
should_mark_indexes = True
table_names = db.inspect(engine).get_table_names()
if "instance_info" not in table_names:
InstanceInfo.create(engine)
run_storage = cls(conn_string, inst_data)
if should_mark_indexes:
run_storage.migrate()
run_storage.optimize()
return run_storage
def _init_db(self):
mkdir_p(self._base_dir)
engine = create_engine(self._conn_string, poolclass=NullPool)
SqlEventLogStorageMetadata.create_all(engine)
engine.execute('PRAGMA journal_mode=WAL;')
alembic_config = get_alembic_config(__file__)
connection = engine.connect()
db_revision, head_revision = check_alembic_revision(alembic_config, connection)
if not (db_revision and head_revision):
stamp_alembic_rev(alembic_config, engine)
def from_local(base_dir, inst_data=None):
check.str_param(base_dir, 'base_dir')
mkdir_p(base_dir)
conn_string = create_db_conn_string(base_dir, 'schedules')
engine = create_engine(conn_string, poolclass=NullPool)
engine.execute('PRAGMA journal_mode=WAL;')
ScheduleStorageSqlMetadata.create_all(engine)
connection = engine.connect()
alembic_config = get_alembic_config(__file__)
db_revision, head_revision = check_alembic_revision(alembic_config, connection)
if not (db_revision and head_revision):
stamp_alembic_rev(alembic_config, engine)
return SqliteScheduleStorage(conn_string, inst_data)
def from_local(base_dir, inst_data=None):
check.str_param(base_dir, "base_dir")
mkdir_p(base_dir)
conn_string = create_db_conn_string(base_dir, "runs")
engine = create_engine(conn_string, poolclass=NullPool)
alembic_config = get_alembic_config(__file__)
with engine.connect() as connection:
db_revision, head_revision = check_alembic_revision(alembic_config, connection)
if not (db_revision and head_revision):
RunStorageSqlMetadata.create_all(engine)
engine.execute("PRAGMA journal_mode=WAL;")
stamp_alembic_rev(alembic_config, connection)
return SqliteRunStorage(conn_string, inst_data)
def _initdb(self, engine):
alembic_config = get_alembic_config(__file__)
retry_limit = 10
while True:
try:
SqlEventLogStorageMetadata.create_all(engine)
engine.execute("PRAGMA journal_mode=WAL;")
with engine.connect() as connection:
db_revision, head_revision = check_alembic_revision(alembic_config, connection)
if not (db_revision and head_revision):
stamp_alembic_rev(alembic_config, engine)
break
except (db.exc.DatabaseError, sqlite3.DatabaseError, sqlite3.OperationalError) as exc:
# This is SQLite-specific handling for concurrency issues that can arise when
# multiple processes (e.g. the dagit process and user code process) contend with
# each other to init the db. When we hit the following errors, we know that another
# process is on the case and we should retry.
err_msg = str(exc)
if not (
"table asset_keys already exists" in err_msg
or "table secondary_indexes already exists" in err_msg
or "table event_logs already exists" in err_msg
or "database is locked" in err_msg
or "table alembic_version already exists" in err_msg
or "UNIQUE constraint failed: alembic_version.version_num" in err_msg
):
raise
if retry_limit == 0:
raise
else:
logging.info(
"SqliteEventLogStorage._initdb: Encountered apparent concurrent init, "
"retrying ({retry_limit} retries left). Exception: {str_exc}".format(
retry_limit=retry_limit, str_exc=err_msg
)
)
time.sleep(0.2)
retry_limit -= 1
def _init_db(self):
mkdir_p(self._base_dir)
engine = create_engine(self._conn_string, poolclass=NullPool)
alembic_config = get_alembic_config(__file__)
should_mark_indexes = False
with engine.connect() as connection:
db_revision, head_revision = check_alembic_revision(alembic_config, connection)
if not (db_revision and head_revision):
SqlEventLogStorageMetadata.create_all(engine)
engine.execute("PRAGMA journal_mode=WAL;")
stamp_alembic_rev(alembic_config, connection)
should_mark_indexes = True
if should_mark_indexes:
# mark all secondary indexes
self.reindex_events()
self.reindex_assets()
def from_local(base_dir, inst_data=None):
check.str_param(base_dir, "base_dir")
mkdir_p(base_dir)
conn_string = create_db_conn_string(base_dir, "runs")
engine = create_engine(conn_string, poolclass=NullPool)
alembic_config = get_alembic_config(__file__)
should_mark_indexes = False
with engine.connect() as connection:
db_revision, head_revision = check_alembic_revision(alembic_config, connection)
if not (db_revision and head_revision):
RunStorageSqlMetadata.create_all(engine)
engine.execute("PRAGMA journal_mode=WAL;")
stamp_alembic_rev(alembic_config, connection)
should_mark_indexes = True
run_storage = SqliteRunStorage(conn_string, inst_data)
if should_mark_indexes:
# mark all secondary indexes
run_storage.build_missing_indexes()
return run_storage
def from_local(cls, base_dir, inst_data=None):
check.str_param(base_dir, "base_dir")
mkdir_p(base_dir)
conn_string = create_db_conn_string(base_dir, "schedules")
engine = create_engine(conn_string, poolclass=NullPool)
alembic_config = get_alembic_config(__file__)
should_migrate_data = False
with engine.connect() as connection:
db_revision, head_revision = check_alembic_revision(
alembic_config, connection)
if not (db_revision and head_revision):
ScheduleStorageSqlMetadata.create_all(engine)
engine.execute("PRAGMA journal_mode=WAL;")
stamp_alembic_rev(alembic_config, connection)
should_migrate_data = True
schedule_storage = cls(conn_string, inst_data)
if should_migrate_data:
schedule_storage.migrate()
schedule_storage.optimize()
return schedule_storage
