def _get(self, feed_item):
"""Retrieves an item from DCM or the local cache.
Args:
feed_item: The feed item representing the creative asset from the
Bulkdozer feed.
Returns:
Instance of the DCM object either from the API or from the local cache.
"""
result = store.get(self._entity,
feed_item.get(FieldMap.CREATIVE_ASSET_ID, None))
if not result:
result = {
'id': feed_item.get(FieldMap.CREATIVE_ASSET_ID, None),
'assetIdentifier': {
'name': feed_item.get(FieldMap.CREATIVE_ASSET_NAME, None),
'type': feed_item.get(FieldMap.CREATIVE_TYPE, None)
}
}
store.set(self._entity,
[feed_item.get(FieldMap.CREATIVE_ASSET_ID, None)],
result)
return result
def pre_fetch(self, feed):
"""Pre-fetches all required items to be update into the cache.
This increases performance for update operations.
Args:
feed: List of feed items to retrieve
"""
if hasattr(self, '_list_name') and self._list_name and self._id_field:
print 'pre fetching %s' % self._list_name
ids = [
feed_item[self._id_field] for feed_item in feed
if isinstance(feed_item[self._id_field], int)
]
if ids:
for chunk_ids in self._chunk(ids, 500):
result = self._retry(
self._service.list(profileId=self.profile_id,
ids=chunk_ids))
while result.get(self._list_name, []):
for item in result[self._list_name]:
store.set(self._entity, [item['id']], item)
result = self._retry(
self._service.list(
profileId=self.profile_id,
pageToken=result['nextPageToken']))
def get(self, feed_item, required=False):
"""Retrieves an item.
Items could be retrieved from a in memory cache in case it has already been
retrieved within the current execution. Also, this method is capable of
translating 'ext' placeholder IDs with concrete CM ids.
Args:
feed_item: Feed item from the Bulkdozer feed representing the item to
retrieve.
Returns:
The CM object that represents the identified entity.
"""
result = None
keys = []
id_value = feed_item.get(self._id_field, None)
if not id_value and self._search_field and feed_item.get(
self._search_field, None):
store_key = feed_item[self._search_field]
if self._parent_filter_name:
if feed_item.get(self._parent_filter_field_name, None):
store_key = str(
feed_item.get(self._parent_filter_field_name,
None)) + store_key
result = store.get(self._entity, store_key)
if not result:
result, key = self._get_by_name(feed_item)
keys.append(key)
if not result and required:
raise Exception('ERROR: Could not find %s with name %s' %
(self._entity, feed_item[self._search_field]))
elif id_value:
if type(id_value) in (str, unicode) and id_value.startswith('ext'):
keys.append(id_value)
id_value = store.translate(self._entity, id_value)
if id_value:
feed_item[self._id_field] = id_value
if id_value:
keys.append(id_value)
result = store.get(self._entity, id_value)
if not result:
result = self._get(feed_item)
if not result and required:
raise Exception('ERROR: Could not find %s with id %s' %
(self._entity, id_value))
store.set(self._entity, keys, result)
return result
def pre_fetch(self, feed):
"""Pre-fetches all required items to be update into the cache.
This increases performance for update operations.
Args:
feed: List of feed items to retrieve
"""
if hasattr(self, '_list_name') and self._list_name and self._id_field:
print('pre fetching %s' % self._list_name)
ids = [
feed_item[self._id_field] for feed_item in feed
if isinstance(feed_item[self._id_field], int)
]
if ids:
for i in range(0, len(ids), 500):
results = self._api(iterate=True).list(
profileId=self.profile_id,
ids=ids[i:i + 500]).execute()
for item in results:
store.set(self._entity, [item['id']], item)
def process(self, feed_item):
"""Processes a Bulkdozer feed item.
This method identifies if the item needs to be inserted or updated, cleans
it, performs the CM operations required, and update the feed item with newly
created ids and name lookups so that the feed can be updated.
Args:
feed_item: Bulkdozer feed item to process.
Returns:
Newly created or updated CM object.
"""
item = self.get(feed_item)
if item:
self._process_update(item, feed_item)
self._clean(item)
self._update(item, feed_item)
else:
new_item = self._process_new(feed_item)
self._clean(new_item)
item = self._insert(new_item, feed_item)
if self._id_field and feed_item.get(self._id_field,
'').startswith('ext'):
store.map(self._entity, feed_item.get(self._id_field),
item['id'])
store.set(self._entity, [feed_item[self._id_field]], item)
if self._search_field and feed_item.get(self._search_field, ''):
store.map(self._entity, feed_item.get(self._search_field),
item['id'])
store.set(self._entity, [feed_item[self._search_field]], item)
if item:
feed_item[self._id_field] = item['id']
store.set(self._entity, [item['id']], item)
self._post_process(feed_item, item)
return item
