def iter_collect_entities_per_page_post( job_scope: JobScope) -> Generator[Dict[str, Any], None, None]: """ Collects an arbitrary entity for a page post """ entity_type = job_scope.report_variant page_token_manager = PageTokenManager.from_job_scope(job_scope) with PlatformApiContext( page_token_manager.get_best_token( job_scope.ad_account_id)) as fb_ctx: root_fb_entity = fb_ctx.to_fb_model(job_scope.entity_id, Entity.PagePost) entities = iter_native_entities_per_page_post(root_fb_entity, entity_type) record_id_base_data = job_scope.to_dict() record_id_base_data.update(entity_type=entity_type, report_variant=None) del record_id_base_data['entity_id'] with ChunkDumpStore(job_scope, chunk_size=DEFAULT_CHUNK_SIZE) as store: for entity in entities: entity_data = entity.export_all_data() entity_data = add_vendor_data(entity_data, id=generate_universal_id( entity_id=entity_data.get('id'), **record_id_base_data)) entity_data['page_id'] = job_scope.ad_account_id entity_data['page_post_id'] = job_scope.entity_id # Store the individual datum, use job context for the cold # storage thing to divine whatever it needs from the job context store(entity_data) yield entity_data
def _from_universal_segmented_entity(breakdown_field: str, data: Dict[str, Any], entity_type: str = None, **kwargs) -> Dict[str, str]: """ Generates Universal record ID from data that is differentiated only by entity ID and breakdown fields. """ assert entity_type entity_id = data[_entity_type_id_field_map[entity_type]] # let's be lazy here and assume we always get single day data # so we'll ignore date_stop for now. # The rest of data is in kwargs return { 'id': generate_universal_id( fields=universal_id_fields + ['breakdown_field'], entity_id=entity_id, entity_type=entity_type, range_start=data[_date_start], breakdown_field=data[ breakdown_field], # generate_universal_id URL-encodes values. Sleep well. **kwargs, ), 'range_start': data[_date_start], 'entity_id': entity_id, 'entity_type': entity_type, }
def collect_page(job_scope: JobScope, _job_context: JobContext): """ Collect a single facebook page """ if job_scope.report_variant != Entity.Page: raise ValueError( f"Report level {job_scope.report_variant} specified is not: {Entity.Page}" ) token = job_scope.token if not token: raise ValueError( f"Job {job_scope.job_id} cannot proceed. No platform tokens provided." ) # We don't use it for getting a token. Something else that calls us does. # However, we use it to report usages of the token we got. token_manager = PlatformTokenManager.from_job_scope(job_scope) with PlatformApiContext(token) as fb_ctx: page_inst = page.Page(fbid=job_scope.entity_id, api=fb_ctx.api) page_fetched = page_inst.api_get(fields=get_default_fields(Page)) report_job_status_task.delay(ExternalPlatformJobStatus.DataFetched, job_scope) token_manager.report_usage(token, 2) record_id_data = job_scope.to_dict() record_id_data.update(entity_type=Entity.Page, entity_id=job_scope.entity_id, report_variant=None) entity_data = page_fetched.export_all_data() entity_data = add_vendor_data( entity_data, id=generate_universal_id(**record_id_data)) store = NormalStore(job_scope) store.store(entity_data)
def test_datetime_handling_special_zero_hour_handling(self): ad_account_id = random.gen_string_id() report_type = 'blah' range_start_dt = datetime(2000, 1, 31, 0, 0, 0) # even though id-forming logic shortens the string to last # non-zero value, this does not apply to hours. # Whenever value is DateTime type, hours are always # part of string even when hour is zero. range_start_should_be = range_start_dt.strftime('%Y-%m-%dT%H') id_should_be = D.join([ 'oprm', 'm', NS, ad_account_id, '', # entity Type '', # entity ID report_type, '', # report variant range_start_should_be, # Range start # '', # Range end ]) assert id_should_be == id_tools.generate_universal_id( ad_account_id=ad_account_id, report_type=report_type, range_start=range_start_dt)
def test_datetime_handling_and_seconds(self): ad_account_id = random.gen_string_id() report_type = 'blah' range_start_dt = datetime(2000, 1, 31, 3, 0, 47) range_start_should_be = quote_plus( range_start_dt.strftime('%Y-%m-%dT%H:%M:%S')) id_should_be = D.join([ 'oprm', 'm', NS, ad_account_id, '', # entity Type '', # entity ID report_type, '', # report variant range_start_should_be, # Range start # '', # Range end ]) assert id_should_be == id_tools.generate_universal_id( ad_account_id=ad_account_id, report_type=report_type, range_start=range_start_dt)
def collect_pages_from_business(job_scope: JobScope, _job_context: JobContext) -> int: """ Collect all facebook pages that are active """ if job_scope.report_variant != Entity.Page: raise ValueError( f"Report level {job_scope.report_variant} specified is not: {Entity.Page}" ) token = job_scope.token if not token: raise ValueError( f"Job {job_scope.job_id} cannot proceed. No platform tokens provided." ) # We don't use it for getting a token. Something else that calls us does. # However, we use it to report usages of the token we got. token_manager = PlatformTokenManager.from_job_scope(job_scope) with PlatformApiContext(token) as fb_ctx: fb_req = FacebookRequest(node_id="me", method="GET", endpoint="/businesses", api=fb_ctx.api, api_type='EDGE', target_class=Business) businesses = fb_req.execute() report_job_status_task.delay(ExternalPlatformJobStatus.DataFetched, job_scope) token_manager.report_usage(token) entity_type = Entity.Page record_id_base_data = job_scope.to_dict() record_id_base_data.update(entity_type=entity_type, report_variant=None) cnt = 0 for biz in businesses: client_pages = list( biz.get_client_pages(fields=get_default_fields(Page))) owned_pages = list( biz.get_owned_pages(fields=get_default_fields(Page))) pages_list = client_pages + owned_pages for page_inst in pages_list: entity_data = page_inst.export_all_data() record_id_base_data.update(entity_id=entity_data.get('id')) entity_data = add_vendor_data( entity_data, id=generate_universal_id(**record_id_base_data)) store = NormalStore(job_scope) store.store(entity_data) cnt += 1 report_job_status_task.delay(ExternalPlatformJobStatus.Done, job_scope) return cnt
def collect_adaccount(job_scope: JobScope) -> Dict[str, Any]: """ Collects ad account data for a AA specific JobScope definition. :param JobScope job_scope: The JobScope as we get it from the task itself """ if job_scope.report_variant != Entity.AdAccount: raise ValueError( f"Report level {job_scope.report_variant} specified is not: {Entity.AdAccount}" ) token = job_scope.token if not token: raise ValueError( f"Job {job_scope.job_id} cannot proceed. No platform tokens provided." ) assert ( job_scope.ad_account_id == job_scope.entity_id ), f'This is an ad account entity job, account_id should be equal to entity_id' # Used to report token usage by this job token_manager = PlatformTokenManager.from_job_scope(job_scope) with PlatformApiContext(token) as fb_ctx: ad_account = fb_ctx.to_fb_model(job_scope.ad_account_id, Entity.AdAccount) fields = get_default_fields(ad_account.__class__) ad_account_with_selected_fields = ad_account.api_get( fields=fields) # Read just the fields we need ad_account_data_dict = ad_account_with_selected_fields.export_all_data( ) # Export the object to a dict token_manager.report_usage(token) job_scope_base = { # Duplicate the job_scope data to avoid mutating it **job_scope.to_dict(), 'entity_type': Entity.AdAccount, 'report_variant': None, } augmented_ad_account_data = add_vendor_data( # Augment the data returned from the remote API with our vendor data ad_account_data_dict, id=generate_universal_id(**job_scope_base), ) feedback_entity_task.delay(ad_account_data_dict, job_scope.report_variant) store = NormalStore(job_scope) store.store(augmented_ad_account_data) # TODO: feedback account? this probably wouldn't make sense at the moment # because ad accounts are discovered from console and their lifecycle is controlled from there. return ad_account_data_dict
def test_runs_correctly(self): account_id = random.gen_string_id() job_scope = JobScope( ad_account_id=self.ad_account_id, entity_id=self.ad_account_id, tokens=['A_REAL_TOKEN'], report_time=datetime.utcnow(), report_type='entity', report_variant=Entity.AdAccount, sweep_id='1', ) universal_id_should_be = generate_universal_id( ad_account_id=self.ad_account_id, report_type=ReportType.entity, entity_id=self.ad_account_id, entity_type=Entity.AdAccount, ) account_data = AdAccount(fbid=account_id) # Did not find a better way how to set this data on the inner AbstractCrudObject. timezone = 'Europe/Prague' account_data._data['timezone_name'] = timezone account_data._data['account_id'] = account_id with mock.patch.object(FB_ADACCOUNT_MODEL, 'api_get', return_value=account_data), mock.patch.object( NormalStore, 'store') as store: collect_adaccount(job_scope) assert store.called_with( account_data), 'Data should be stored with the cold store module' assert store.called store_args, store_keyword_args = store.call_args assert not store_keyword_args assert len( store_args ) == 1, 'Store method should be called with just 1 parameter' data_actual = store_args[0] vendor_data_key = '__oprm' ad_account_dynamo = AdAccountEntity.get(DEFAULT_SCOPE, account_id) assert ad_account_dynamo.timezone == timezone assert ad_account_dynamo.ad_account_id == account_id assert (vendor_data_key in data_actual and type(data_actual[vendor_data_key]) == dict), 'Special vendor key is present in the returned data' assert data_actual[vendor_data_key] == { 'id': universal_id_should_be }, 'Vendor data is set with the right universal id'
def iter_collect_entities_per_page( job_scope: JobScope) -> Generator[Dict[str, Any], None, None]: """ Collects an arbitrary entity for a page """ token, entity_type, root_fb_entity = _extract_token_entity_type_parent_entity( job_scope, [Entity.PagePost, Entity.PageVideo], Entity.Page, 'ad_account_id') entities = iter_native_entities_per_page(root_fb_entity, entity_type) record_id_base_data = job_scope.to_dict() record_id_base_data.update(entity_type=entity_type, report_variant=None) token_manager = PlatformTokenManager.from_job_scope(job_scope) with ChunkDumpStore( job_scope, chunk_size=DEFAULT_CHUNK_SIZE) as store, ChunkDumpStore( job_scope, chunk_size=DEFAULT_CHUNK_SIZE, bucket_type=ColdStoreBucketType.RAW_BUCKET, custom_namespace=NAMESPACE_RAW, ) as raw_store: cnt = 0 for entity in entities: entity_data = entity.export_all_data() entity_data = add_vendor_data(entity_data, id=generate_universal_id( entity_id=entity_data.get('id'), **record_id_base_data)) entity_data['page_id'] = job_scope.ad_account_id if entity_type == Entity.PagePost: # store raw version of response (just to remain consistent) raw_store(entity_data) entity_data = _augment_page_post(entity_data) # Store the individual datum, use job context for the cold # storage thing to divine whatever it needs from the job context store(entity_data) # Signal to the system the new entity feedback_entity_task.delay(entity_data, entity_type) yield entity_data cnt += 1 if cnt % 1000 == 0: # default paging size for entities per parent # is typically around 200. So, each 200 results # means about 5 hits to FB token_manager.report_usage(token, 5) token_manager.report_usage(token)
def _from_dma_segmented_entity(data: Dict[str, Any], entity_type: str = None, **kwargs) -> Dict[str, str]: """ Generates Universal record ID from data that is differentiated only by entity ID """ assert entity_type # ... # "account_id": "2034428216844013", # "ad_id": "23842698250300224", # "adset_id": "23842698250720224", # "campaign_id": "23842698250110224", # "clicks": "0", # "cpc": "0", # "cpm": "5", # "ctr": "0", # "date_start": "2017-12-31", # "date_stop": "2017-12-31", # "dma": "Macon", <--------------------- id # "impressions": "2", # "reach": "2", # "spend": "0.01" # ... entity_id = data[_entity_type_id_field_map[entity_type]] # let's be lazy here and assume we always get single day data # so we'll ignore date_stop for now. # The rest of data is in kwargs return { 'id': generate_universal_id( fields=universal_id_fields + ['dma'], entity_id=entity_id, entity_type=entity_type, range_start=data[_date_start], dma=data[ 'dma'], # generate_universal_id URL-encodes values. Sleep well. **kwargs, ), 'range_start': data[_date_start], 'entity_id': entity_id, 'entity_type': entity_type, }
def _from_hour_segmented_entity(timezone_name: str, data: Dict[str, Any], entity_type: str = None, **kwargs) -> Dict[str, str]: """ Generates Universal record ID from data that is differentiated only by entity ID """ assert timezone_name assert entity_type # ... # "ctr": "0", # "date_start": "2017-12-31", # "date_stop": "2017-12-31", # "frequency": "0", # "hourly_stats_aggregated_by_advertiser_time_zone": "00:00:00 - 00:59:59", # "impressions": "371", # ... entity_id = data[_entity_type_id_field_map[entity_type]] # let's be lazy here and assume we always get single day data # so we'll ignore date_stop for now. date_str = data[_date_start] hour_str = data[_hourly_stats_aggregated_by_advertiser_time_zone].split( '-', 1)[0].strip() dt = datetime.strptime(date_str + 'T' + hour_str, '%Y-%m-%dT%H:%M:%S') dt_as_utc = dt_to_other_timezone(dt, 'UTC', timezone_name) # The rest of data is in kwargs return { 'id': generate_universal_id(entity_id=entity_id, entity_type=entity_type, range_start=dt_as_utc, **kwargs), # Note that we communicate UTC-based range_start # which will be different in date and time from # data['date_start'] value and hour range attribute value # in Facebook's original data. 'range_start': dt_as_utc.strftime('%Y-%m-%dT%H:%M:%S'), 'entity_id': entity_id, 'entity_type': entity_type, }
def iter_collect_entities_per_page_graph( job_scope: JobScope) -> Generator[Dict[str, Any], None, None]: """ Collects an arbitrary entity for a page using graph API """ page_token_manager = PageTokenManager.from_job_scope(job_scope) with PlatformApiContext( page_token_manager.get_best_token( job_scope.ad_account_id)) as fb_ctx: page_root_fb_entity = fb_ctx.to_fb_model(job_scope.ad_account_id, Entity.Page) entity_type = job_scope.report_variant # page size reduced to avoid error: # "Please reduce the amount of data you're asking for, then retry your request" entities = iter_native_entities_per_page_graph(page_root_fb_entity, entity_type, page_size=30) record_id_base_data = job_scope.to_dict() record_id_base_data.update(entity_type=entity_type, report_variant=None) with ChunkDumpStore( job_scope, chunk_size=DEFAULT_CHUNK_SIZE) as store, ChunkDumpStore( job_scope, chunk_size=DEFAULT_CHUNK_SIZE, bucket_type=ColdStoreBucketType.RAW_BUCKET, custom_namespace=NAMESPACE_RAW, ) as raw_store: for entity in entities: entity_data = entity.export_all_data() entity_data = add_vendor_data(entity_data, id=generate_universal_id( entity_id=entity_data.get('id'), **record_id_base_data)) entity_data['page_id'] = job_scope.ad_account_id if entity_type == Entity.PagePostPromotable: # store raw version of response (just to remain consistent) raw_store(entity_data) entity_data = _augment_page_post(entity_data) # Store the individual datum, use job context for the cold # storage thing to divine whatever it needs from the job context store(entity_data) # Signal to the system the new entity feedback_entity_task.delay(entity_data, entity_type) yield entity_data
def _from_age_gender_segmented_entity(data: Dict[str, Any], entity_type: str = None, **kwargs) -> Dict[str, str]: """ Generates Universal record ID from data that is differentiated only by entity ID """ assert entity_type # ... # "date_start": "2018-06-02", # "date_stop": "2018-06-02", # "age": "18-24", # <---------- # "clicks": "10", # "cpc": "0.117", # "cpm": "2.521552", # "cpp": "2.526998", # "ctr": "2.155172", # "gender": "female", # <---------- # "impressions": "464", # ... entity_id = data[_entity_type_id_field_map[entity_type]] # let's be lazy here and assume we always get single day data # so we'll ignore date_stop for now. # The rest of data is in kwargs return { 'id': generate_universal_id( fields=universal_id_fields + ['age', 'gender'], entity_id=entity_id, entity_type=entity_type, range_start=data[_date_start], age=data['age'], gender=data['gender'], **kwargs, ), 'range_start': data[_date_start], 'entity_id': entity_id, 'entity_type': entity_type, }
def _from_platform_segmented_entity(data: Dict[str, Any], entity_type: str = None, **kwargs) -> Dict[str, str]: """ Generates Universal record ID from data that is differentiated only by entity ID """ assert entity_type # ... # "date_start": "2018-06-02", # "date_stop": "2018-06-02", # "ctr": "1.88383", # "frequency": "1.001572", # "impressions": "637", # "platform_position": "feed", # <----------- # "publisher_platform": "facebook", # <----------- # "reach": "636", # ... entity_id = data[_entity_type_id_field_map[entity_type]] # let's be lazy here and assume we always get single day data # so we'll ignore date_stop for now. # The rest of data is in kwargs return { 'id': generate_universal_id( fields=universal_id_fields + ['publisher_platform', 'platform_position'], entity_id=entity_id, entity_type=entity_type, range_start=data[_date_start], publisher_platform=data['publisher_platform'], platform_position=data['platform_position'], **kwargs, ), 'range_start': data[_date_start], 'entity_id': entity_id, 'entity_type': entity_type, }
def iter_collect_entities_per_adaccount( job_scope: JobScope) -> Generator[Dict[str, Any], None, None]: """ Collects an arbitrary entity for an ad account """ token, entity_type, root_fb_entity = _extract_token_entity_type_parent_entity( job_scope, Entity.AA_SCOPED, Entity.AdAccount, 'ad_account_id') entities = iter_native_entities_per_adaccount(root_fb_entity, entity_type) record_id_base_data = job_scope.to_dict() record_id_base_data.update(entity_type=entity_type, report_variant=None) token_manager = PlatformTokenManager.from_job_scope(job_scope) with ChunkDumpStore(job_scope, chunk_size=DEFAULT_CHUNK_SIZE) as store: for cnt, entity in enumerate(entities): entity_data = entity.export_all_data() entity_data = add_vendor_data( entity_data, id=generate_universal_id( # FIXME: add a bug to facebook ads (get_ad_videos doesnt return ad videos but AbstractCrudObject) # FIXME so it is unable to access entity.Field.id then (only a problem for ad videos) entity_id=entity_data.get('id'), **record_id_base_data, ), ) # Store the individual datum, use job context for the cold # storage thing to divine whatever it needs from the job context store(entity_data) # Signal to the system the new entity feedback_entity_task.delay(entity_data, entity_type) yield entity_data if cnt % 1000 == 0: # default paging size for entities per parent # is typically around 200. So, each 200 results # means about 5 hits to FB token_manager.report_usage(token, 5) # Report on the effective task status token_manager.report_usage(token)
def _from_non_segmented_entity(data: Dict[str, Any], entity_type: str = None, **kwargs) -> Dict[str, str]: """ Generates Universal record ID from data that is differentiated only by entity ID """ assert entity_type entity_id = data[_entity_type_id_field_map[entity_type]] # The rest of data is in kwargs return { 'id': generate_universal_id(entity_id=entity_id, entity_type=entity_type, **kwargs), 'entity_id': entity_id, 'entity_type': entity_type, }
def test_date_handling(self): ad_account_id = random.gen_string_id() report_type = 'blah' range_start_d = date.today() range_start_should_be = range_start_d.strftime('%Y-%m-%d') id_should_be = D.join([ 'oprm', 'm', NS, ad_account_id, '', # entity Type '', # entity ID report_type, '', # report variant range_start_should_be, # Range start # '', # Range end ]) assert id_should_be == id_tools.generate_universal_id( ad_account_id=ad_account_id, report_type=report_type, range_start=range_start_d)
def test_some_data_trailing_universal(self): ad_account_id = random.gen_string_id() report_type = 'blah' id_should_be = D.join([ 'oprm', 'm', NS, ad_account_id, '', # entity Type '', # entity ID report_type, '', # report variant '', # Range start '', # Range end 'hocus', '', # None 'pocus', ]) assert id_should_be == id_tools.generate_universal_id( ad_account_id=ad_account_id, report_type=report_type, trailing_parts=['hocus', None, 'pocus'])
def test_correct_vendor_data_inserted_into_cold_store_payload_campaigns( self): entity_types = [ Entity.Campaign, Entity.AdSet, Entity.Ad, Entity.AdCreative, Entity.AdVideo, Entity.CustomAudience, ] fb_model_map = { Entity.Campaign: FB_CAMPAIGN_MODEL, Entity.AdSet: FB_ADSET_MODEL, Entity.Ad: FB_AD_MODEL, Entity.AdCreative: FB_AD_CREATIVE_MODEL, Entity.AdVideo: FB_AD_VIDEO_MODEL, Entity.CustomAudience: FB_CUSTOM_AUDIENCE_MODEL, } get_all_method_map = { Entity.Campaign: 'get_campaigns', Entity.AdSet: 'get_ad_sets', Entity.Ad: 'get_ads', Entity.AdCreative: 'get_ad_creatives', Entity.AdVideo: 'get_ad_videos', Entity.CustomAudience: 'get_custom_audiences', } for entity_type in entity_types: fbid = random.gen_string_id() FB_MODEL = fb_model_map[entity_type] get_method_name = get_all_method_map[entity_type] job_scope = JobScope( sweep_id=self.sweep_id, ad_account_id=self.ad_account_id, report_type=ReportType.entity, report_variant=entity_type, tokens=['blah'], ) universal_id_should_be = generate_universal_id( ad_account_id=self.ad_account_id, report_type=ReportType.entity, entity_id=fbid, entity_type=entity_type) fb_data = FB_MODEL(fbid=fbid) if entity_type == Entity.AdVideo: # Ad videos normally don't have account_id # but we augment it with it fb_data['account_id'] = self.ad_account_id else: fb_data[FB_MODEL.Field.account_id] = self.ad_account_id entities_data = [fb_data] with mock.patch.object( FB_ADACCOUNT_MODEL, get_method_name, return_value=entities_data), mock.patch.object( ChunkDumpStore, 'store') as store: list(iter_collect_entities_per_adaccount(job_scope)) assert store.called store_args, store_keyword_args = store.call_args assert not store_keyword_args assert len( store_args ) == 1, 'Store method should be called with just 1 parameter' data_actual = store_args[0] vendor_data_key = '__oprm' assert (vendor_data_key in data_actual and type(data_actual[vendor_data_key]) == dict ), 'Special vendor key is present in the returned data' assert data_actual[vendor_data_key] == { 'id': universal_id_should_be }, 'Vendor data is set with the right universal id'
def test_correct_vendor_data_inserted_into_cold_store_payload_posts(self): entity_types = [Entity.PagePostPromotable] fb_model_map = {Entity.PagePostPromotable: FB_PAGE_POST_MODEL} for entity_type in entity_types: with self.subTest(f'Entity type - "{entity_type}"'): fbid = random.gen_string_id() FB_MODEL = fb_model_map[entity_type] job_scope = JobScope( sweep_id=self.sweep_id, ad_account_id=self.ad_account_id, report_type=ReportType.entity, report_variant=entity_type, tokens=['user-token'], ) universal_id_should_be = generate_universal_id( ad_account_id=self.ad_account_id, report_type=ReportType.entity, entity_id=fbid, entity_type=entity_type, ) fb_data = FB_MODEL(fbid=fbid) fb_data['account_id'] = '0' entities_data = [fb_data] with mock.patch.object( PageTokenManager, 'get_best_token', return_value=None ) as get_best_token, mock.patch.object( FB_PAGE_MODEL, 'get_feed', return_value=entities_data), mock.patch.object( FB_PAGE_MODEL, 'get_ads_posts', return_value=entities_data), mock.patch.object( ChunkDumpStore, 'store') as store, mock.patch.object( FB_PAGE_POST_MODEL, 'get', side_effect=lambda field: field == 'is_eligible_for_promotion'): list(iter_collect_entities_per_page_graph(job_scope)) assert get_best_token.called assert store.called store_args, store_keyword_args = store.call_args assert not store_keyword_args assert len( store_args ) == 1, 'Store method should be called with just 1 parameter' data_actual = store_args[0] vendor_data_key = '__oprm' assert (vendor_data_key in data_actual and type(data_actual[vendor_data_key]) == dict ), 'Special vendor key is present in the returned data' assert data_actual[vendor_data_key] == { 'id': universal_id_should_be }, 'Vendor data is set with the right universal id'
def test_correct_vendor_data_inserted_into_cold_store_payload_posts(self): entity_types = [Entity.PagePost, Entity.PageVideo] fb_model_map = { Entity.PagePost: FB_PAGE_POST_MODEL, Entity.PageVideo: FB_AD_VIDEO_MODEL } get_all_method_map = { Entity.PagePost: 'get_posts', Entity.PageVideo: 'get_videos' } for entity_type in entity_types: with self.subTest(f'Entity type = "{entity_type}"'): fbid = random.gen_string_id() FB_MODEL = fb_model_map[entity_type] get_method_name = get_all_method_map[entity_type] job_scope = JobScope( sweep_id=self.sweep_id, ad_account_id=self.ad_account_id, report_type=ReportType.entity, report_variant=entity_type, tokens=['blah'], ) universal_id_should_be = generate_universal_id( ad_account_id=self.ad_account_id, report_type=ReportType.entity, entity_id=fbid, entity_type=entity_type, ) fb_data = FB_MODEL(fbid=fbid) fb_data['account_id'] = '0' entities_data = [fb_data] with mock.patch.object( FB_PAGE_MODEL, get_method_name, return_value=entities_data), mock.patch.object( ChunkDumpStore, 'store') as store: list(iter_collect_entities_per_page(job_scope)) assert store.called store_args, store_keyword_args = store.call_args assert not store_keyword_args assert len( store_args ) == 1, 'Store method should be called with just 1 parameter' data_actual = store_args[0] vendor_data_key = '__oprm' assert (vendor_data_key in data_actual and type(data_actual[vendor_data_key]) == dict ), 'Special vendor key is present in the returned data' assert data_actual[vendor_data_key] == { 'id': universal_id_should_be }, 'Vendor data is set with the right universal id'
def test_universal_works_with_nulls(self): assert D.join(['oprm', 'm', NS]) == id_tools.generate_universal_id()