def iter_collect_entities_per_page_post(
job_scope: JobScope) -> Generator[Dict[str, Any], None, None]:
"""
Collects an arbitrary entity for a page post
"""
entity_type = job_scope.report_variant
page_token_manager = PageTokenManager.from_job_scope(job_scope)
with PlatformApiContext(
page_token_manager.get_best_token(
job_scope.ad_account_id)) as fb_ctx:
root_fb_entity = fb_ctx.to_fb_model(job_scope.entity_id,
Entity.PagePost)
entities = iter_native_entities_per_page_post(root_fb_entity, entity_type)
record_id_base_data = job_scope.to_dict()
record_id_base_data.update(entity_type=entity_type, report_variant=None)
del record_id_base_data['entity_id']
with ChunkDumpStore(job_scope, chunk_size=DEFAULT_CHUNK_SIZE) as store:
for entity in entities:
entity_data = entity.export_all_data()
entity_data = add_vendor_data(entity_data,
id=generate_universal_id(
entity_id=entity_data.get('id'),
**record_id_base_data))
entity_data['page_id'] = job_scope.ad_account_id
entity_data['page_post_id'] = job_scope.entity_id
# Store the individual datum, use job context for the cold
# storage thing to divine whatever it needs from the job context
store(entity_data)
yield entity_data
def _from_universal_segmented_entity(breakdown_field: str,
data: Dict[str, Any],
entity_type: str = None,
**kwargs) -> Dict[str, str]:
"""
Generates Universal record ID from data that is differentiated only by entity ID and breakdown fields.
"""
assert entity_type
entity_id = data[_entity_type_id_field_map[entity_type]]
# let's be lazy here and assume we always get single day data
# so we'll ignore date_stop for now.
# The rest of data is in kwargs
return {
'id':
generate_universal_id(
fields=universal_id_fields + ['breakdown_field'],
entity_id=entity_id,
entity_type=entity_type,
range_start=data[_date_start],
breakdown_field=data[
breakdown_field], # generate_universal_id URL-encodes values. Sleep well.
**kwargs,
),
'range_start':
data[_date_start],
'entity_id':
entity_id,
'entity_type':
entity_type,
}
def collect_page(job_scope: JobScope, _job_context: JobContext):
"""
Collect a single facebook page
"""
if job_scope.report_variant != Entity.Page:
raise ValueError(
f"Report level {job_scope.report_variant} specified is not: {Entity.Page}"
)
token = job_scope.token
if not token:
raise ValueError(
f"Job {job_scope.job_id} cannot proceed. No platform tokens provided."
)
# We don't use it for getting a token. Something else that calls us does.
# However, we use it to report usages of the token we got.
token_manager = PlatformTokenManager.from_job_scope(job_scope)
with PlatformApiContext(token) as fb_ctx:
page_inst = page.Page(fbid=job_scope.entity_id, api=fb_ctx.api)
page_fetched = page_inst.api_get(fields=get_default_fields(Page))
report_job_status_task.delay(ExternalPlatformJobStatus.DataFetched,
job_scope)
token_manager.report_usage(token, 2)
record_id_data = job_scope.to_dict()
record_id_data.update(entity_type=Entity.Page,
entity_id=job_scope.entity_id,
report_variant=None)
entity_data = page_fetched.export_all_data()
entity_data = add_vendor_data(
entity_data, id=generate_universal_id(**record_id_data))
store = NormalStore(job_scope)
store.store(entity_data)
def test_datetime_handling_special_zero_hour_handling(self):
ad_account_id = random.gen_string_id()
report_type = 'blah'
range_start_dt = datetime(2000, 1, 31, 0, 0, 0)
# even though id-forming logic shortens the string to last
# non-zero value, this does not apply to hours.
# Whenever value is DateTime type, hours are always
# part of string even when hour is zero.
range_start_should_be = range_start_dt.strftime('%Y-%m-%dT%H')
id_should_be = D.join([
'oprm',
'm',
NS,
ad_account_id,
'', # entity Type
'', # entity ID
report_type,
'', # report variant
range_start_should_be, # Range start
# '', # Range end
])
assert id_should_be == id_tools.generate_universal_id(
ad_account_id=ad_account_id,
report_type=report_type,
range_start=range_start_dt)
def test_datetime_handling_and_seconds(self):
ad_account_id = random.gen_string_id()
report_type = 'blah'
range_start_dt = datetime(2000, 1, 31, 3, 0, 47)
range_start_should_be = quote_plus(
range_start_dt.strftime('%Y-%m-%dT%H:%M:%S'))
id_should_be = D.join([
'oprm',
'm',
NS,
ad_account_id,
'', # entity Type
'', # entity ID
report_type,
'', # report variant
range_start_should_be, # Range start
# '', # Range end
])
assert id_should_be == id_tools.generate_universal_id(
ad_account_id=ad_account_id,
report_type=report_type,
range_start=range_start_dt)
def collect_pages_from_business(job_scope: JobScope,
_job_context: JobContext) -> int:
"""
Collect all facebook pages that are active
"""
if job_scope.report_variant != Entity.Page:
raise ValueError(
f"Report level {job_scope.report_variant} specified is not: {Entity.Page}"
)
token = job_scope.token
if not token:
raise ValueError(
f"Job {job_scope.job_id} cannot proceed. No platform tokens provided."
)
# We don't use it for getting a token. Something else that calls us does.
# However, we use it to report usages of the token we got.
token_manager = PlatformTokenManager.from_job_scope(job_scope)
with PlatformApiContext(token) as fb_ctx:
fb_req = FacebookRequest(node_id="me",
method="GET",
endpoint="/businesses",
api=fb_ctx.api,
api_type='EDGE',
target_class=Business)
businesses = fb_req.execute()
report_job_status_task.delay(ExternalPlatformJobStatus.DataFetched,
job_scope)
token_manager.report_usage(token)
entity_type = Entity.Page
record_id_base_data = job_scope.to_dict()
record_id_base_data.update(entity_type=entity_type, report_variant=None)
cnt = 0
for biz in businesses:
client_pages = list(
biz.get_client_pages(fields=get_default_fields(Page)))
owned_pages = list(
biz.get_owned_pages(fields=get_default_fields(Page)))
pages_list = client_pages + owned_pages
for page_inst in pages_list:
entity_data = page_inst.export_all_data()
record_id_base_data.update(entity_id=entity_data.get('id'))
entity_data = add_vendor_data(
entity_data, id=generate_universal_id(**record_id_base_data))
store = NormalStore(job_scope)
store.store(entity_data)
cnt += 1
report_job_status_task.delay(ExternalPlatformJobStatus.Done, job_scope)
return cnt
def collect_adaccount(job_scope: JobScope) -> Dict[str, Any]:
"""
Collects ad account data for a AA specific JobScope definition.
:param JobScope job_scope: The JobScope as we get it from the task itself
"""
if job_scope.report_variant != Entity.AdAccount:
raise ValueError(
f"Report level {job_scope.report_variant} specified is not: {Entity.AdAccount}"
)
token = job_scope.token
if not token:
raise ValueError(
f"Job {job_scope.job_id} cannot proceed. No platform tokens provided."
)
assert (
job_scope.ad_account_id == job_scope.entity_id
), f'This is an ad account entity job, account_id should be equal to entity_id'
# Used to report token usage by this job
token_manager = PlatformTokenManager.from_job_scope(job_scope)
with PlatformApiContext(token) as fb_ctx:
ad_account = fb_ctx.to_fb_model(job_scope.ad_account_id,
Entity.AdAccount)
fields = get_default_fields(ad_account.__class__)
ad_account_with_selected_fields = ad_account.api_get(
fields=fields) # Read just the fields we need
ad_account_data_dict = ad_account_with_selected_fields.export_all_data(
) # Export the object to a dict
token_manager.report_usage(token)
job_scope_base = {
# Duplicate the job_scope data to avoid mutating it
**job_scope.to_dict(),
'entity_type': Entity.AdAccount,
'report_variant': None,
}
augmented_ad_account_data = add_vendor_data(
# Augment the data returned from the remote API with our vendor data
ad_account_data_dict,
id=generate_universal_id(**job_scope_base),
)
feedback_entity_task.delay(ad_account_data_dict,
job_scope.report_variant)
store = NormalStore(job_scope)
store.store(augmented_ad_account_data)
# TODO: feedback account? this probably wouldn't make sense at the moment
# because ad accounts are discovered from console and their lifecycle is controlled from there.
return ad_account_data_dict
def test_runs_correctly(self):
account_id = random.gen_string_id()
job_scope = JobScope(
ad_account_id=self.ad_account_id,
entity_id=self.ad_account_id,
tokens=['A_REAL_TOKEN'],
report_time=datetime.utcnow(),
report_type='entity',
report_variant=Entity.AdAccount,
sweep_id='1',
)
universal_id_should_be = generate_universal_id(
ad_account_id=self.ad_account_id,
report_type=ReportType.entity,
entity_id=self.ad_account_id,
entity_type=Entity.AdAccount,
)
account_data = AdAccount(fbid=account_id)
# Did not find a better way how to set this data on the inner AbstractCrudObject.
timezone = 'Europe/Prague'
account_data._data['timezone_name'] = timezone
account_data._data['account_id'] = account_id
with mock.patch.object(FB_ADACCOUNT_MODEL,
'api_get',
return_value=account_data), mock.patch.object(
NormalStore, 'store') as store:
collect_adaccount(job_scope)
assert store.called_with(
account_data), 'Data should be stored with the cold store module'
assert store.called
store_args, store_keyword_args = store.call_args
assert not store_keyword_args
assert len(
store_args
) == 1, 'Store method should be called with just 1 parameter'
data_actual = store_args[0]
vendor_data_key = '__oprm'
ad_account_dynamo = AdAccountEntity.get(DEFAULT_SCOPE, account_id)
assert ad_account_dynamo.timezone == timezone
assert ad_account_dynamo.ad_account_id == account_id
assert (vendor_data_key in data_actual
and type(data_actual[vendor_data_key])
== dict), 'Special vendor key is present in the returned data'
assert data_actual[vendor_data_key] == {
'id': universal_id_should_be
}, 'Vendor data is set with the right universal id'
def iter_collect_entities_per_page(
job_scope: JobScope) -> Generator[Dict[str, Any], None, None]:
"""
Collects an arbitrary entity for a page
"""
token, entity_type, root_fb_entity = _extract_token_entity_type_parent_entity(
job_scope, [Entity.PagePost, Entity.PageVideo], Entity.Page,
'ad_account_id')
entities = iter_native_entities_per_page(root_fb_entity, entity_type)
record_id_base_data = job_scope.to_dict()
record_id_base_data.update(entity_type=entity_type, report_variant=None)
token_manager = PlatformTokenManager.from_job_scope(job_scope)
with ChunkDumpStore(
job_scope, chunk_size=DEFAULT_CHUNK_SIZE) as store, ChunkDumpStore(
job_scope,
chunk_size=DEFAULT_CHUNK_SIZE,
bucket_type=ColdStoreBucketType.RAW_BUCKET,
custom_namespace=NAMESPACE_RAW,
) as raw_store:
cnt = 0
for entity in entities:
entity_data = entity.export_all_data()
entity_data = add_vendor_data(entity_data,
id=generate_universal_id(
entity_id=entity_data.get('id'),
**record_id_base_data))
entity_data['page_id'] = job_scope.ad_account_id
if entity_type == Entity.PagePost:
# store raw version of response (just to remain consistent)
raw_store(entity_data)
entity_data = _augment_page_post(entity_data)
# Store the individual datum, use job context for the cold
# storage thing to divine whatever it needs from the job context
store(entity_data)
# Signal to the system the new entity
feedback_entity_task.delay(entity_data, entity_type)
yield entity_data
cnt += 1
if cnt % 1000 == 0:
# default paging size for entities per parent
# is typically around 200. So, each 200 results
# means about 5 hits to FB
token_manager.report_usage(token, 5)
token_manager.report_usage(token)
def _from_dma_segmented_entity(data: Dict[str, Any],
entity_type: str = None,
**kwargs) -> Dict[str, str]:
"""
Generates Universal record ID from data that is
differentiated only by entity ID
"""
assert entity_type
# ...
# "account_id": "2034428216844013",
# "ad_id": "23842698250300224",
# "adset_id": "23842698250720224",
# "campaign_id": "23842698250110224",
# "clicks": "0",
# "cpc": "0",
# "cpm": "5",
# "ctr": "0",
# "date_start": "2017-12-31",
# "date_stop": "2017-12-31",
# "dma": "Macon", <--------------------- id
# "impressions": "2",
# "reach": "2",
# "spend": "0.01"
# ...
entity_id = data[_entity_type_id_field_map[entity_type]]
# let's be lazy here and assume we always get single day data
# so we'll ignore date_stop for now.
# The rest of data is in kwargs
return {
'id':
generate_universal_id(
fields=universal_id_fields + ['dma'],
entity_id=entity_id,
entity_type=entity_type,
range_start=data[_date_start],
dma=data[
'dma'], # generate_universal_id URL-encodes values. Sleep well.
**kwargs,
),
'range_start':
data[_date_start],
'entity_id':
entity_id,
'entity_type':
entity_type,
}
def _from_hour_segmented_entity(timezone_name: str,
data: Dict[str, Any],
entity_type: str = None,
**kwargs) -> Dict[str, str]:
"""
Generates Universal record ID from data that is
differentiated only by entity ID
"""
assert timezone_name
assert entity_type
# ...
# "ctr": "0",
# "date_start": "2017-12-31",
# "date_stop": "2017-12-31",
# "frequency": "0",
# "hourly_stats_aggregated_by_advertiser_time_zone": "00:00:00 - 00:59:59",
# "impressions": "371",
# ...
entity_id = data[_entity_type_id_field_map[entity_type]]
# let's be lazy here and assume we always get single day data
# so we'll ignore date_stop for now.
date_str = data[_date_start]
hour_str = data[_hourly_stats_aggregated_by_advertiser_time_zone].split(
'-', 1)[0].strip()
dt = datetime.strptime(date_str + 'T' + hour_str, '%Y-%m-%dT%H:%M:%S')
dt_as_utc = dt_to_other_timezone(dt, 'UTC', timezone_name)
# The rest of data is in kwargs
return {
'id':
generate_universal_id(entity_id=entity_id,
entity_type=entity_type,
range_start=dt_as_utc,
**kwargs),
# Note that we communicate UTC-based range_start
# which will be different in date and time from
# data['date_start'] value and hour range attribute value
# in Facebook's original data.
'range_start':
dt_as_utc.strftime('%Y-%m-%dT%H:%M:%S'),
'entity_id':
entity_id,
'entity_type':
entity_type,
}
def iter_collect_entities_per_page_graph(
job_scope: JobScope) -> Generator[Dict[str, Any], None, None]:
"""
Collects an arbitrary entity for a page using graph API
"""
page_token_manager = PageTokenManager.from_job_scope(job_scope)
with PlatformApiContext(
page_token_manager.get_best_token(
job_scope.ad_account_id)) as fb_ctx:
page_root_fb_entity = fb_ctx.to_fb_model(job_scope.ad_account_id,
Entity.Page)
entity_type = job_scope.report_variant
# page size reduced to avoid error:
# "Please reduce the amount of data you're asking for, then retry your request"
entities = iter_native_entities_per_page_graph(page_root_fb_entity,
entity_type,
page_size=30)
record_id_base_data = job_scope.to_dict()
record_id_base_data.update(entity_type=entity_type, report_variant=None)
with ChunkDumpStore(
job_scope, chunk_size=DEFAULT_CHUNK_SIZE) as store, ChunkDumpStore(
job_scope,
chunk_size=DEFAULT_CHUNK_SIZE,
bucket_type=ColdStoreBucketType.RAW_BUCKET,
custom_namespace=NAMESPACE_RAW,
) as raw_store:
for entity in entities:
entity_data = entity.export_all_data()
entity_data = add_vendor_data(entity_data,
id=generate_universal_id(
entity_id=entity_data.get('id'),
**record_id_base_data))
entity_data['page_id'] = job_scope.ad_account_id
if entity_type == Entity.PagePostPromotable:
# store raw version of response (just to remain consistent)
raw_store(entity_data)
entity_data = _augment_page_post(entity_data)
# Store the individual datum, use job context for the cold
# storage thing to divine whatever it needs from the job context
store(entity_data)
# Signal to the system the new entity
feedback_entity_task.delay(entity_data, entity_type)
yield entity_data
def _from_age_gender_segmented_entity(data: Dict[str, Any],
entity_type: str = None,
**kwargs) -> Dict[str, str]:
"""
Generates Universal record ID from data that is
differentiated only by entity ID
"""
assert entity_type
# ...
# "date_start": "2018-06-02",
# "date_stop": "2018-06-02",
# "age": "18-24", # <----------
# "clicks": "10",
# "cpc": "0.117",
# "cpm": "2.521552",
# "cpp": "2.526998",
# "ctr": "2.155172",
# "gender": "female", # <----------
# "impressions": "464",
# ...
entity_id = data[_entity_type_id_field_map[entity_type]]
# let's be lazy here and assume we always get single day data
# so we'll ignore date_stop for now.
# The rest of data is in kwargs
return {
'id':
generate_universal_id(
fields=universal_id_fields + ['age', 'gender'],
entity_id=entity_id,
entity_type=entity_type,
range_start=data[_date_start],
age=data['age'],
gender=data['gender'],
**kwargs,
),
'range_start':
data[_date_start],
'entity_id':
entity_id,
'entity_type':
entity_type,
}
def _from_platform_segmented_entity(data: Dict[str, Any],
entity_type: str = None,
**kwargs) -> Dict[str, str]:
"""
Generates Universal record ID from data that is
differentiated only by entity ID
"""
assert entity_type
# ...
# "date_start": "2018-06-02",
# "date_stop": "2018-06-02",
# "ctr": "1.88383",
# "frequency": "1.001572",
# "impressions": "637",
# "platform_position": "feed", # <-----------
# "publisher_platform": "facebook", # <-----------
# "reach": "636",
# ...
entity_id = data[_entity_type_id_field_map[entity_type]]
# let's be lazy here and assume we always get single day data
# so we'll ignore date_stop for now.
# The rest of data is in kwargs
return {
'id':
generate_universal_id(
fields=universal_id_fields +
['publisher_platform', 'platform_position'],
entity_id=entity_id,
entity_type=entity_type,
range_start=data[_date_start],
publisher_platform=data['publisher_platform'],
platform_position=data['platform_position'],
**kwargs,
),
'range_start':
data[_date_start],
'entity_id':
entity_id,
'entity_type':
entity_type,
}
def iter_collect_entities_per_adaccount(
job_scope: JobScope) -> Generator[Dict[str, Any], None, None]:
"""
Collects an arbitrary entity for an ad account
"""
token, entity_type, root_fb_entity = _extract_token_entity_type_parent_entity(
job_scope, Entity.AA_SCOPED, Entity.AdAccount, 'ad_account_id')
entities = iter_native_entities_per_adaccount(root_fb_entity, entity_type)
record_id_base_data = job_scope.to_dict()
record_id_base_data.update(entity_type=entity_type, report_variant=None)
token_manager = PlatformTokenManager.from_job_scope(job_scope)
with ChunkDumpStore(job_scope, chunk_size=DEFAULT_CHUNK_SIZE) as store:
for cnt, entity in enumerate(entities):
entity_data = entity.export_all_data()
entity_data = add_vendor_data(
entity_data,
id=generate_universal_id(
# FIXME: add a bug to facebook ads (get_ad_videos doesnt return ad videos but AbstractCrudObject)
# FIXME so it is unable to access entity.Field.id then (only a problem for ad videos)
entity_id=entity_data.get('id'),
**record_id_base_data,
),
)
# Store the individual datum, use job context for the cold
# storage thing to divine whatever it needs from the job context
store(entity_data)
# Signal to the system the new entity
feedback_entity_task.delay(entity_data, entity_type)
yield entity_data
if cnt % 1000 == 0:
# default paging size for entities per parent
# is typically around 200. So, each 200 results
# means about 5 hits to FB
token_manager.report_usage(token, 5)
# Report on the effective task status
token_manager.report_usage(token)
def _from_non_segmented_entity(data: Dict[str, Any],
entity_type: str = None,
**kwargs) -> Dict[str, str]:
"""
Generates Universal record ID from data that is
differentiated only by entity ID
"""
assert entity_type
entity_id = data[_entity_type_id_field_map[entity_type]]
# The rest of data is in kwargs
return {
'id':
generate_universal_id(entity_id=entity_id,
entity_type=entity_type,
**kwargs),
'entity_id':
entity_id,
'entity_type':
entity_type,
}
def test_date_handling(self):
ad_account_id = random.gen_string_id()
report_type = 'blah'
range_start_d = date.today()
range_start_should_be = range_start_d.strftime('%Y-%m-%d')
id_should_be = D.join([
'oprm',
'm',
NS,
ad_account_id,
'', # entity Type
'', # entity ID
report_type,
'', # report variant
range_start_should_be, # Range start
# '', # Range end
])
assert id_should_be == id_tools.generate_universal_id(
ad_account_id=ad_account_id,
report_type=report_type,
range_start=range_start_d)
def test_some_data_trailing_universal(self):
ad_account_id = random.gen_string_id()
report_type = 'blah'
id_should_be = D.join([
'oprm',
'm',
NS,
ad_account_id,
'', # entity Type
'', # entity ID
report_type,
'', # report variant
'', # Range start
'', # Range end
'hocus',
'', # None
'pocus',
])
assert id_should_be == id_tools.generate_universal_id(
ad_account_id=ad_account_id,
report_type=report_type,
trailing_parts=['hocus', None, 'pocus'])
def test_correct_vendor_data_inserted_into_cold_store_payload_campaigns(
self):
entity_types = [
Entity.Campaign,
Entity.AdSet,
Entity.Ad,
Entity.AdCreative,
Entity.AdVideo,
Entity.CustomAudience,
]
fb_model_map = {
Entity.Campaign: FB_CAMPAIGN_MODEL,
Entity.AdSet: FB_ADSET_MODEL,
Entity.Ad: FB_AD_MODEL,
Entity.AdCreative: FB_AD_CREATIVE_MODEL,
Entity.AdVideo: FB_AD_VIDEO_MODEL,
Entity.CustomAudience: FB_CUSTOM_AUDIENCE_MODEL,
}
get_all_method_map = {
Entity.Campaign: 'get_campaigns',
Entity.AdSet: 'get_ad_sets',
Entity.Ad: 'get_ads',
Entity.AdCreative: 'get_ad_creatives',
Entity.AdVideo: 'get_ad_videos',
Entity.CustomAudience: 'get_custom_audiences',
}
for entity_type in entity_types:
fbid = random.gen_string_id()
FB_MODEL = fb_model_map[entity_type]
get_method_name = get_all_method_map[entity_type]
job_scope = JobScope(
sweep_id=self.sweep_id,
ad_account_id=self.ad_account_id,
report_type=ReportType.entity,
report_variant=entity_type,
tokens=['blah'],
)
universal_id_should_be = generate_universal_id(
ad_account_id=self.ad_account_id,
report_type=ReportType.entity,
entity_id=fbid,
entity_type=entity_type)
fb_data = FB_MODEL(fbid=fbid)
if entity_type == Entity.AdVideo:
# Ad videos normally don't have account_id
# but we augment it with it
fb_data['account_id'] = self.ad_account_id
else:
fb_data[FB_MODEL.Field.account_id] = self.ad_account_id
entities_data = [fb_data]
with mock.patch.object(
FB_ADACCOUNT_MODEL,
get_method_name,
return_value=entities_data), mock.patch.object(
ChunkDumpStore, 'store') as store:
list(iter_collect_entities_per_adaccount(job_scope))
assert store.called
store_args, store_keyword_args = store.call_args
assert not store_keyword_args
assert len(
store_args
) == 1, 'Store method should be called with just 1 parameter'
data_actual = store_args[0]
vendor_data_key = '__oprm'
assert (vendor_data_key in data_actual
and type(data_actual[vendor_data_key]) == dict
), 'Special vendor key is present in the returned data'
assert data_actual[vendor_data_key] == {
'id': universal_id_should_be
}, 'Vendor data is set with the right universal id'
def test_correct_vendor_data_inserted_into_cold_store_payload_posts(self):
entity_types = [Entity.PagePostPromotable]
fb_model_map = {Entity.PagePostPromotable: FB_PAGE_POST_MODEL}
for entity_type in entity_types:
with self.subTest(f'Entity type - "{entity_type}"'):
fbid = random.gen_string_id()
FB_MODEL = fb_model_map[entity_type]
job_scope = JobScope(
sweep_id=self.sweep_id,
ad_account_id=self.ad_account_id,
report_type=ReportType.entity,
report_variant=entity_type,
tokens=['user-token'],
)
universal_id_should_be = generate_universal_id(
ad_account_id=self.ad_account_id,
report_type=ReportType.entity,
entity_id=fbid,
entity_type=entity_type,
)
fb_data = FB_MODEL(fbid=fbid)
fb_data['account_id'] = '0'
entities_data = [fb_data]
with mock.patch.object(
PageTokenManager, 'get_best_token', return_value=None
) as get_best_token, mock.patch.object(
FB_PAGE_MODEL, 'get_feed',
return_value=entities_data), mock.patch.object(
FB_PAGE_MODEL,
'get_ads_posts',
return_value=entities_data), mock.patch.object(
ChunkDumpStore,
'store') as store, mock.patch.object(
FB_PAGE_POST_MODEL,
'get',
side_effect=lambda field: field ==
'is_eligible_for_promotion'):
list(iter_collect_entities_per_page_graph(job_scope))
assert get_best_token.called
assert store.called
store_args, store_keyword_args = store.call_args
assert not store_keyword_args
assert len(
store_args
) == 1, 'Store method should be called with just 1 parameter'
data_actual = store_args[0]
vendor_data_key = '__oprm'
assert (vendor_data_key in data_actual
and type(data_actual[vendor_data_key]) == dict
), 'Special vendor key is present in the returned data'
assert data_actual[vendor_data_key] == {
'id': universal_id_should_be
}, 'Vendor data is set with the right universal id'
def test_correct_vendor_data_inserted_into_cold_store_payload_posts(self):
entity_types = [Entity.PagePost, Entity.PageVideo]
fb_model_map = {
Entity.PagePost: FB_PAGE_POST_MODEL,
Entity.PageVideo: FB_AD_VIDEO_MODEL
}
get_all_method_map = {
Entity.PagePost: 'get_posts',
Entity.PageVideo: 'get_videos'
}
for entity_type in entity_types:
with self.subTest(f'Entity type = "{entity_type}"'):
fbid = random.gen_string_id()
FB_MODEL = fb_model_map[entity_type]
get_method_name = get_all_method_map[entity_type]
job_scope = JobScope(
sweep_id=self.sweep_id,
ad_account_id=self.ad_account_id,
report_type=ReportType.entity,
report_variant=entity_type,
tokens=['blah'],
)
universal_id_should_be = generate_universal_id(
ad_account_id=self.ad_account_id,
report_type=ReportType.entity,
entity_id=fbid,
entity_type=entity_type,
)
fb_data = FB_MODEL(fbid=fbid)
fb_data['account_id'] = '0'
entities_data = [fb_data]
with mock.patch.object(
FB_PAGE_MODEL, get_method_name,
return_value=entities_data), mock.patch.object(
ChunkDumpStore, 'store') as store:
list(iter_collect_entities_per_page(job_scope))
assert store.called
store_args, store_keyword_args = store.call_args
assert not store_keyword_args
assert len(
store_args
) == 1, 'Store method should be called with just 1 parameter'
data_actual = store_args[0]
vendor_data_key = '__oprm'
assert (vendor_data_key in data_actual
and type(data_actual[vendor_data_key]) == dict
), 'Special vendor key is present in the returned data'
assert data_actual[vendor_data_key] == {
'id': universal_id_should_be
}, 'Vendor data is set with the right universal id'
def test_universal_works_with_nulls(self):
assert D.join(['oprm', 'm', NS]) == id_tools.generate_universal_id()
