def get_all_asset_keys(self, prefix_path=None): if not prefix_path: if self.has_secondary_index(SECONDARY_INDEX_ASSET_KEY): query = db.select([AssetKeyTable.c.asset_key]) else: query = (db.select([ SqlEventLogStorageTable.c.asset_key ]).where( SqlEventLogStorageTable.c.asset_key != None).distinct()) else: if self.has_secondary_index(SECONDARY_INDEX_ASSET_KEY): query = db.select([AssetKeyTable.c.asset_key]).where( AssetKeyTable.c.asset_key.startswith( AssetKey.get_db_prefix(prefix_path))) else: query = (db.select([ SqlEventLogStorageTable.c.asset_key ]).where(SqlEventLogStorageTable.c.asset_key != None).where( SqlEventLogStorageTable.c.asset_key.startswith( AssetKey.get_db_prefix(prefix_path))).distinct()) with self.connect() as conn: results = conn.execute(query).fetchall() return [ AssetKey.from_db_string(asset_key) for (asset_key, ) in results if asset_key ]
def get_asset_keys(self, prefix_path=None): lazy_migrate = False if not prefix_path: if self.has_secondary_index(SECONDARY_INDEX_ASSET_KEY): query = db.select([AssetKeyTable.c.asset_key]) else: query = (db.select([ SqlEventLogStorageTable.c.asset_key ]).where( SqlEventLogStorageTable.c.asset_key != None).distinct()) # This is in place to migrate everyone to using the secondary index table for asset # keys. Performing this migration should result in a big performance boost for # any asset-catalog reads. # After a sufficient amount of time (>= 0.11.0?), we can remove the checks # for has_secondary_index(SECONDARY_INDEX_ASSET_KEY) and always read from the # AssetKeyTable, since we are already writing to the table. Tracking the conditional # check removal here: https://github.com/dagster-io/dagster/issues/3507 lazy_migrate = True else: if self.has_secondary_index(SECONDARY_INDEX_ASSET_KEY): query = db.select([AssetKeyTable.c.asset_key]).where( db.or_( AssetKeyTable.c.asset_key.startswith( AssetKey.get_db_prefix(prefix_path)), AssetKeyTable.c.asset_key.startswith( AssetKey.get_db_prefix(prefix_path, legacy=True)), )) else: query = (db.select([ SqlEventLogStorageTable.c.asset_key ]).where(SqlEventLogStorageTable.c.asset_key != None).where( db.or_( SqlEventLogStorageTable.c.asset_key.startswith( AssetKey.get_db_prefix(prefix_path)), SqlEventLogStorageTable.c.asset_key.startswith( AssetKey.get_db_prefix(prefix_path, legacy=True)), )).distinct()) with self.index_connection() as conn: results = conn.execute(query).fetchall() if lazy_migrate: # This is in place to migrate everyone to using the secondary index table for asset # keys. Performing this migration should result in a big performance boost for # any subsequent asset-catalog reads. self._lazy_migrate_secondary_index_asset_key( [asset_key for (asset_key, ) in results if asset_key]) return list( set([ AssetKey.from_db_string(asset_key) for (asset_key, ) in results if asset_key ]))
def get_all_asset_keys(self, prefix_path=None): if not prefix_path: query = db.select([SqlEventLogStorageTable.c.asset_key]).distinct() else: query = (db.select([SqlEventLogStorageTable.c.asset_key]).where( SqlEventLogStorageTable.c.asset_key.startswith( AssetKey.get_db_prefix(prefix_path))).distinct()) with self.connect() as conn: results = conn.execute(query).fetchall() return [ AssetKey.from_db_string(asset_key) for (asset_key, ) in results if asset_key ]
def get_all_asset_keys(self, prefix_path=None): asset_records = [] for records in self._logs.values(): asset_records += [ record for record in records if record.is_dagster_event and record.dagster_event.asset_key and (not prefix_path or record.dagster_event.asset_key.to_string(). startswith(AssetKey.get_db_prefix(prefix_path))) ] asset_events = [ record.dagster_event for record in sorted( asset_records, key=lambda x: x.timestamp, reverse=True) ] asset_keys = OrderedDict() for event in asset_events: asset_keys["/".join(event.asset_key.path)] = event.asset_key return list(asset_keys.values())