def get_all_asset_keys(self, prefix_path=None):
if not prefix_path:
if self.has_secondary_index(SECONDARY_INDEX_ASSET_KEY):
query = db.select([AssetKeyTable.c.asset_key])
else:
query = (db.select([
SqlEventLogStorageTable.c.asset_key
]).where(
SqlEventLogStorageTable.c.asset_key != None).distinct())
else:
if self.has_secondary_index(SECONDARY_INDEX_ASSET_KEY):
query = db.select([AssetKeyTable.c.asset_key]).where(
AssetKeyTable.c.asset_key.startswith(
AssetKey.get_db_prefix(prefix_path)))
else:
query = (db.select([
SqlEventLogStorageTable.c.asset_key
]).where(SqlEventLogStorageTable.c.asset_key != None).where(
SqlEventLogStorageTable.c.asset_key.startswith(
AssetKey.get_db_prefix(prefix_path))).distinct())
with self.connect() as conn:
results = conn.execute(query).fetchall()
return [
AssetKey.from_db_string(asset_key) for (asset_key, ) in results
if asset_key
]
def get_asset_keys(self, prefix_path=None):
lazy_migrate = False
if not prefix_path:
if self.has_secondary_index(SECONDARY_INDEX_ASSET_KEY):
query = db.select([AssetKeyTable.c.asset_key])
else:
query = (db.select([
SqlEventLogStorageTable.c.asset_key
]).where(
SqlEventLogStorageTable.c.asset_key != None).distinct())
# This is in place to migrate everyone to using the secondary index table for asset
# keys. Performing this migration should result in a big performance boost for
# any asset-catalog reads.
# After a sufficient amount of time (>= 0.11.0?), we can remove the checks
# for has_secondary_index(SECONDARY_INDEX_ASSET_KEY) and always read from the
# AssetKeyTable, since we are already writing to the table. Tracking the conditional
# check removal here: https://github.com/dagster-io/dagster/issues/3507
lazy_migrate = True
else:
if self.has_secondary_index(SECONDARY_INDEX_ASSET_KEY):
query = db.select([AssetKeyTable.c.asset_key]).where(
db.or_(
AssetKeyTable.c.asset_key.startswith(
AssetKey.get_db_prefix(prefix_path)),
AssetKeyTable.c.asset_key.startswith(
AssetKey.get_db_prefix(prefix_path, legacy=True)),
))
else:
query = (db.select([
SqlEventLogStorageTable.c.asset_key
]).where(SqlEventLogStorageTable.c.asset_key != None).where(
db.or_(
SqlEventLogStorageTable.c.asset_key.startswith(
AssetKey.get_db_prefix(prefix_path)),
SqlEventLogStorageTable.c.asset_key.startswith(
AssetKey.get_db_prefix(prefix_path, legacy=True)),
)).distinct())
with self.index_connection() as conn:
results = conn.execute(query).fetchall()
if lazy_migrate:
# This is in place to migrate everyone to using the secondary index table for asset
# keys. Performing this migration should result in a big performance boost for
# any subsequent asset-catalog reads.
self._lazy_migrate_secondary_index_asset_key(
[asset_key for (asset_key, ) in results if asset_key])
return list(
set([
AssetKey.from_db_string(asset_key) for (asset_key, ) in results
if asset_key
]))
def get_all_asset_keys(self, prefix_path=None):
if not prefix_path:
query = db.select([SqlEventLogStorageTable.c.asset_key]).distinct()
else:
query = (db.select([SqlEventLogStorageTable.c.asset_key]).where(
SqlEventLogStorageTable.c.asset_key.startswith(
AssetKey.get_db_prefix(prefix_path))).distinct())
with self.connect() as conn:
results = conn.execute(query).fetchall()
return [
AssetKey.from_db_string(asset_key) for (asset_key, ) in results
if asset_key
]
def get_all_asset_keys(self, prefix_path=None):
asset_records = []
for records in self._logs.values():
asset_records += [
record for record in records if record.is_dagster_event
and record.dagster_event.asset_key and
(not prefix_path or record.dagster_event.asset_key.to_string().
startswith(AssetKey.get_db_prefix(prefix_path)))
]
asset_events = [
record.dagster_event for record in sorted(
asset_records, key=lambda x: x.timestamp, reverse=True)
]
asset_keys = OrderedDict()
for event in asset_events:
asset_keys["/".join(event.asset_key.path)] = event.asset_key
return list(asset_keys.values())