def _inputs_changed(self, cache_timestamp):
# A feed sync has occurred since the eval was done or the image has been updated/reloaded, so inputs can have changed. Must be stale
db = get_session()
# TODO: zhill - test more
feed_group_updated_list = [
group.last_sync if group.last_sync is not None else
datetime.datetime.utcfromtimestamp(0) for feed in get_all_feeds(db)
for group in feed.groups
]
feed_synced = (max(feed_group_updated_list) > cache_timestamp
if feed_group_updated_list else False)
image_updated = self.image.last_modified > cache_timestamp
return feed_synced or image_updated
def sync_metadata(feed_client: IFeedSource,
to_sync: list = None,
operation_id=None) -> tuple:
"""
Get metadata from source and sync db metadata records to that (e.g. add any new groups or feeds)
Executes as a unit-of-work for db, so will commit result and returns the records found on upstream source.
If a record exists in db but was not found upstream, it is not returned
:param feed_client:
:param to_sync: list of string feed names to sync metadata on
:return: tuple, first element: dict of names mapped to db records post-sync only including records successfully updated by upstream, second element is a list of tuples where each tuple is (failed_feed_name, error_obj)
"""
if not to_sync:
return {}, []
db = get_session()
try:
logger.info(
'Syncing feed and group metadata from upstream source (operation_id={})'
.format(operation_id))
source_resp = feed_client.list_feeds()
if to_sync:
feeds = filter(lambda x: x.name in to_sync, source_resp.feeds)
else:
feeds = []
failed = []
source_feeds = {
x.name: {
'meta': x,
'groups': feed_client.list_feed_groups(x.name).groups
}
for x in feeds
}
logger.debug('Upstream feeds available: %s', source_feeds)
db_feeds = DataFeeds._pivot_and_filter_feeds_by_config(
to_sync, list(source_feeds.keys()), get_all_feeds(db))
for feed_name, feed_api_record in source_feeds.items():
try:
logger.info(
'Syncing metadata for feed: {} (operation_id={})'.
format(feed_name, operation_id))
api_feed = feed_api_record['meta']
db_feed = db_feeds.get(api_feed.name)
# Do this instead of a db.merge() to ensure no timestamps are reset or overwritten
if not db_feed:
logger.debug(
'Adding new feed metadata record to db: {} (operation_id={})'
.format(api_feed.name, operation_id))
db_feed = FeedMetadata(
name=api_feed.name,
description=api_feed.description,
access_tier=api_feed.access_tier,
enabled=True)
db.add(db_feed)
db.flush()
else:
logger.debug(
'Feed metadata already in db: {} (operation_id={})'
.format(api_feed.name, operation_id))
# Check for any update
db_feed.description = api_feed.description
db_feed.access_tier = api_feed.access_tier
db_groups = {x.name: x for x in db_feed.groups}
for api_group in feed_api_record.get('groups', []):
db_group = db_groups.get(api_group.name)
# Do this instead of a db.merge() to ensure no timestamps are reset or overwritten
if not db_group:
logger.debug(
'Adding new feed metadata record to db: {} (operation_id={})'
.format(api_group.name, operation_id))
db_group = FeedGroupMetadata(
name=api_group.name,
description=api_group.description,
access_tier=api_group.access_tier,
feed=db_feed,
enabled=True)
db_group.last_sync = None
db.add(db_group)
else:
logger.debug(
'Feed group metadata already in db: {} (operation_id={})'
.format(api_group.name, operation_id))
db_group.access_tier = api_group.access_tier
db_group.description = api_group.description
except Exception as e:
logger.exception('Error syncing feed {}'.format(feed_name))
logger.warn(
'Could not sync metadata for feed: {} (operation_id={})'
.format(feed_name, operation_id))
failed.append((feed_name, e))
finally:
db.flush()
# Reload
db_feeds = DataFeeds._pivot_and_filter_feeds_by_config(
to_sync, list(source_feeds.keys()), get_all_feeds(db))
db.commit()
logger.info(
'Metadata sync from feeds upstream source complete (operation_id={})'
.format(operation_id))
return db_feeds, failed
except Exception as e:
logger.error(
'Rolling back feed metadata update due to error: {} (operation_id={})'
.format(e, operation_id))
db.rollback()
raise
def sync_metadata(
source_feeds: Dict[
str, Dict[str, Union[FeedAPIRecord, List[FeedAPIGroupRecord]]]
],
to_sync: List[str] = None,
operation_id: Optional[str] = None,
groups: bool = True,
) -> Tuple[Dict[str, FeedMetadata], List[Tuple[str, Union[str, BaseException]]]]:
"""
Get metadata from source and sync db metadata records to that (e.g. add any new groups or feeds)
Executes as a unit-of-work for db, so will commit result and returns the records found on upstream source.
If a record exists in db but was not found upstream, it is not returned
:param source_feeds: mapping containing FeedAPIRecord and FeedAPIGroupRecord
:type source_feeds: Dict[str, Dict[str, Union[FeedAPIRecord, List[FeedAPIGroupRecord]]]]
:param to_sync: list of string feed names to sync metadata on
:type to_sync: List[str]
:param operation_id: UUID4 hexadecimal string
:type operation_id: Optional[str]
:param groups: whether or not to sync group metadata (defaults to True, which will sync group metadata)
:type groups: bool
:return: tuple, first element: dict of names mapped to db records post-sync only including records successfully updated by upstream, second element is a list of tuples where each tuple is (failed_feed_name, error_obj)
:rtype: Tuple[Dict[str, FeedMetadata], List[Tuple[str, Union[str, BaseException]]]
"""
if not to_sync:
return {}, []
db = get_session()
try:
logger.info(
"Syncing feed and group metadata from upstream source (operation_id={})".format(
operation_id
)
)
failed = []
db_feeds = MetadataSyncUtils._pivot_and_filter_feeds_by_config(
to_sync, list(source_feeds.keys()), get_all_feeds(db)
)
for feed_name, feed_api_record in source_feeds.items():
try:
logger.info(
"Syncing metadata for feed: {} (operation_id={})".format(
feed_name, operation_id
)
)
feed_metadata_map = MetadataSyncUtils._sync_feed_metadata(
db, feed_api_record, db_feeds, operation_id
)
if groups:
MetadataSyncUtils._sync_feed_group_metadata(
db, feed_api_record, feed_metadata_map, operation_id
)
except Exception as e:
logger.exception("Error syncing feed {}".format(feed_name))
logger.warn(
"Could not sync metadata for feed: {} (operation_id={})".format(
feed_name, operation_id
)
)
failed.append((feed_name, e))
finally:
db.flush()
# Reload
db_feeds = MetadataSyncUtils._pivot_and_filter_feeds_by_config(
to_sync, list(source_feeds.keys()), get_all_feeds(db)
)
db.commit()
logger.info(
"Metadata sync from feeds upstream source complete (operation_id={})".format(
operation_id
)
)
return db_feeds, failed
except Exception as e:
logger.error(
"Rolling back feed metadata update due to error: {} (operation_id={})".format(
e, operation_id
)
)
db.rollback()
raise