def test_populate_from_scope_record(self): scope_id = gen_string_id() sweep_id = gen_string_id() console_token = 'console token' platform_token = 'platform token' scope_record = AssetScope() scope_record.scope = scope_id scope_record.scope_api_token = console_token scope_record.set_cache(platform_tokens={platform_token}) PlatformTokenManager.populate_from_scope_entity(scope_record, sweep_id) # now let's make sure we see those tokens: # Scope-centered jobs must result in scope-centered key for token storage job_scope = JobScope(sweep_id=sweep_id, entity_type=Entity.Scope, entity_id=scope_id) assert console_token == PlatformTokenManager.from_job_scope( job_scope).get_best_token() job_scope = JobScope( sweep_id=sweep_id, # uses .namespace default value as 2nd value in redis key. no need to set here. ) assert platform_token == PlatformTokenManager.from_job_scope( job_scope).get_best_token()
def test_key_s3_date_snapped_with_chunk_id(self): """ Check that the key is constructed as we expect """ job_scope = JobScope( ad_account_id=gen_string_id(), report_type=ReportType.day_platform, report_variant=Entity.Ad, range_start=date(2000, 1, 2), ) chunk_marker = 7 dt_should_be = datetime(2000, 1, 2, 0, 0, 0) with mock.patch.object(uuid, 'uuid4', return_value='UUID-HERE'): storage_key = cold_storage.store({'data': 'yeah!'}, job_scope, chunk_marker=7) prefix = xxhash.xxh64(job_scope.ad_account_id).hexdigest()[:6] expected_key = ( f'fb/' + f'{prefix}-{job_scope.ad_account_id}/' + f'{job_scope.report_type}/' + f'{dt_should_be.strftime("%Y")}/' + f'{dt_should_be.strftime("%m")}/' + f'{dt_should_be.strftime("%d")}/' + f'{dt_should_be.strftime("%Y-%m-%dT%H:%M:%SZ")}-' + f'{job_scope.job_id}-' + f'{chunk_marker}-' + f'UUID-HERE' + f'.json' ) assert storage_key == expected_key
def test_key_s3_date_less(self): """ Check that the key is constructed as we expect """ import common.tztools job_scope = JobScope( ad_account_id=gen_string_id(), report_type=ReportType.entity, report_variant=Entity.Campaign ) now_dt = datetime(2000, 1, 2, 3, 4, 5) with mock.patch.object(common.tztools, 'now', return_value=now_dt) as now_mocked, mock.patch.object( uuid, 'uuid4', return_value='UUID-HERE' ): storage_key = cold_storage.store({'data': 'yeah!'}, job_scope) assert now_mocked.called prefix = xxhash.xxh64(job_scope.ad_account_id).hexdigest()[:6] expected_key = ( f'fb/' + f'{prefix}-{job_scope.ad_account_id}/' + f'{job_scope.report_type}/' + f'{now_dt.strftime("%Y")}/' + f'{now_dt.strftime("%m")}/' + f'{now_dt.strftime("%d")}/' + f'{now_dt.strftime("%Y-%m-%dT%H:%M:%SZ")}-' + f'{job_scope.job_id}-' + f'UUID-HERE' + f'.json' ) assert storage_key == expected_key
def test_fails_with_wrong_report_variant(self): job_scope = JobScope(tokens=['blah'], report_time=datetime.utcnow(), report_type='entity', report_variant=None, sweep_id='1') with SweepRunningFlag(job_scope.sweep_id), mock.patch.object( report_job_status_task, 'delay') as status_task, mock.patch( 'common.error_inspector.BugSnagContextData.notify' ) as bugsnag_notify, mock.patch( 'common.error_inspector.API_KEY', 'something'): collect_pages_from_business_task(job_scope, JobContext()) assert bugsnag_notify.called actual_args, actual_kwargs = bugsnag_notify.call_args assert (isinstance(actual_args[0], ValueError) and str( actual_args[0]) == 'Report level None specified is not: P' ), 'Notify bugsnag correctly using correct Exception' assert { 'severity': SEVERITY_ERROR, 'job_scope': job_scope, 'error_type': ErrorTypesReport.UNKNOWN, } == actual_kwargs, 'Notify bugsnag correctly' assert status_task.called parameters, _ = status_task.call_args assert (JobStatus.GenericError, job_scope ) == parameters, 'Must report status correctly on failure'
def test_fails_without_a_token(self): job_scope = JobScope(tokens=[None], report_time=datetime.utcnow(), report_type='entity', report_variant=Entity.Page, sweep_id='1') with SweepRunningFlag(job_scope.sweep_id), mock.patch.object( report_job_status_task, 'delay') as status_task, mock.patch( 'common.error_inspector.BugSnagContextData.notify' ) as bugsnag_notify, mock.patch( 'common.error_inspector.API_KEY', 'something'): collect_pages_from_business_task(job_scope, JobContext()) assert bugsnag_notify.called actual_args, actual_kwargs = bugsnag_notify.call_args assert ( str(actual_args[0]) == 'Job fb||||entity|P cannot proceed. No platform tokens provided.' ), 'Notify bugsnag correctly using correct Exception' assert { 'severity': SEVERITY_ERROR, 'job_scope': job_scope, 'error_type': ErrorTypesReport.UNKNOWN, } == actual_kwargs, 'Notify bugsnag correctly' assert status_task.called status_task_args, _ = status_task.call_args assert ( JobStatus.GenericError, job_scope ) == status_task_args, 'Must report status correctly on failure'
def test_set_inaccessible_entity_task(): mock_model = Mock() mock_factory = Mock(return_value=mock_model) with patch.dict('oozer.set_inaccessible_entity_task.ENTITY_TYPE_MODEL_MAP', {Entity.PagePost: mock_factory}): set_inaccessible_entity_task(JobScope(report_variant=Entity.PagePost)) mock_model.update.assert_called_once_with(actions=[mock_factory.is_accessible.set(False)])
def iter_tasks( self ) -> Generator[Tuple[CeleryTask, JobScope, JobContext, int], None, None]: """Read persisted jobs and pass-through context objects for inspection""" with self.queue.JobsReader() as jobs_iter: for job_id, job_scope_additional_data, score in jobs_iter: job_id_parts = parse_id(job_id) job_scope = JobScope(job_scope_additional_data, job_id_parts, sweep_id=self.sweep_id, score=score) try: celery_task = resolve_job_scope_to_celery_task(job_scope) # TODO: Decide what to do with this. # Was designed for massive hash collection and such, # but cannot have too much data in there because we pickle it and put in on Redis job_context = JobContext() yield celery_task, job_scope, job_context, score logger.info( f"#{self.sweep_id}: Scheduling job_id {job_id} with score {score}." ) except InvalidJobScopeException as e: ErrorInspector.inspect(e, job_scope.ad_account_id, { 'sweep_id': job_scope.sweep_id, 'job_id': job_scope.job_id })
def test_resolve_job_scope_to_celery_task_ad_account( self, mock_lifetime_iter, mock_breakdowns_iter): real_claim = RealityClaim(entity_id='A1', ad_account_id='A1', entity_type=Entity.AdAccount, timezone='America/Los_Angeles') mock_lifetime_iter.return_value = [] mock_breakdowns_iter.return_value = [ RealityClaim( entity_id='AD1', ad_account_id='A1', entity_type=Entity.Ad, range_start=datetime(2019, 1, 20, 12, 0), timezone='America/Los_Angeles', ) ] for job_generator in entity_expectation_generator_map[ Entity.AdAccount]: for exp_claim in job_generator(real_claim): with self.subTest(job_generator=job_generator, exp_claim=exp_claim): job_scope = JobScope(parse_id(exp_claim.job_id)) assert inventory.resolve_job_scope_to_celery_task( job_scope)
def test_task_error_is_logged_into_job_report(self): from oozer.common.report_job_status_task import report_job_status_task class MyException(Exception): pass sync_expectations_job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.sync_expectations, ) with mock.patch.object(report_job_status_task, 'delay') as job_report, mock.patch.object( sync_expectations_task, 'sync_expectations', side_effect=MyException('nope!')): with self.assertRaises(MyException): sync_expectations_task.sync_expectations_task.delay( sync_expectations_job_scope, None) assert job_report.called aa, kk = job_report.call_args assert not kk code, job_scope_actual = aa assert code < 0 # some sort of *Failure* code assert job_scope_actual == sync_expectations_job_scope
def test_task_does_not_blow_up(self): # this is almost same thing as the next test # where we check that call signature is right, # but when call signature changes and our tests don't, # it becomes irrelevant if we have tests - they check for wrong thing # So, here we actually call "store" and in next test # we intercept the call and check payload. # Don't remove me. Not duplicate. expectation_job_id = generate_id( ad_account_id=random.gen_string_id(), report_type=ReportType.day_hour, report_variant=Entity.Ad, range_start='2000-01-01', ) rr = [expectation_job_id] sync_expectations_job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.sync_expectations, ) with mock.patch.object(expecations_store, 'iter_expectations_per_ad_account', return_value=rr): sync_expectations_task.sync_expectations( sync_expectations_job_scope)
def test__job_scope_to_metadata(): scope = JobScope( job_id='job identifier', namespace='fb', ad_account_id='007', report_type='report type', entity_type=Entity.Campaign, report_variant=Entity.Ad, range_start=datetime.fromtimestamp(1), score=10, ) result = _job_scope_to_metadata(scope) result.pop('extracted_at') result.pop('build_id') assert { 'job_id': 'fb|007|C||report+type|A|1970-01-01T00%3A00%3A01', 'ad_account_id': '007', 'report_type': 'report type', 'entity_type': 'A', 'platform_api_version': 'v6.0', 'platform': 'fb', 'score': '10', } == result
def test_lifetime_campaigns(self): job_scope = JobScope( ad_account_id=AD_ACCOUNT, report_type=ReportType.lifetime, report_variant=Entity.Campaign, sweep_id='sweep', tokens=[TOKEN], ) captured_data = [] # type: List[Tuple[Dict, JobScope, int]] def _store(data, job_scope, chunk_marker=0): captured_data.append((data, job_scope, chunk_marker)) with mock.patch.object(cold_storage, 'store', _store): data_iter = Insights.iter_collect_insights(job_scope, None) cnt = 0 for datum in data_iter: cnt += 1 if cnt == 4: break assert cnt == 4 for datum, job_scope_inner, _ in captured_data: assert datum['campaign_id'] == job_scope_inner.entity_id
def test_task_is_called_with_right_data(self): range_start = now() range_start_should_be = range_start.strftime('%Y-%m-%d') expected_job_id = generate_id( ad_account_id=random.gen_string_id(), report_type=ReportType.day_hour, report_variant=Entity.Ad, range_start=range_start, ) rr = [expected_job_id] expected_job_id_parts = parse_id_parts(expected_job_id) sync_expectations_job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.sync_expectations, ) with mock.patch.object(expecations_store, 'iter_expectations_per_ad_account', return_value=rr) as jid_iter, mock.patch.object( cold_storage.ChunkDumpStore, 'store') as store: sync_expectations_task.sync_expectations( sync_expectations_job_scope) assert jid_iter.called aa, kk = jid_iter.call_args assert not kk assert aa == (sync_expectations_job_scope.ad_account_id, sync_expectations_job_scope.sweep_id) assert store.called aa, kk = store.call_args assert not kk assert len(aa) == 1 data = aa[0] assert data == { 'job_id': expected_job_id, # missing "ad_" is intentional. # this matches this attr name as sent by FB # and ysed by us elsewhere in the company 'account_id': expected_job_id_parts.ad_account_id, 'entity_type': expected_job_id_parts.entity_type, 'entity_id': expected_job_id_parts.entity_id, 'report_type': expected_job_id_parts.report_type, 'report_variant': expected_job_id_parts.report_variant, 'range_start': range_start_should_be, # checking manually to ensure it's properly stringified 'range_end': None, 'platform_namespace': JobScope.namespace, # default platform value }
def setUp(self): self.job_scope = JobScope( sweep_id=gen_string_id(), ad_account_id=gen_string_id(), entity_id=gen_string_id(), entity_type=Entity.Campaign, report_type=ReportType.entity, )
def test_runs_correctly(self): account_id = random.gen_string_id() job_scope = JobScope( ad_account_id=self.ad_account_id, entity_id=self.ad_account_id, tokens=['A_REAL_TOKEN'], report_time=datetime.utcnow(), report_type='entity', report_variant=Entity.AdAccount, sweep_id='1', ) universal_id_should_be = generate_universal_id( ad_account_id=self.ad_account_id, report_type=ReportType.entity, entity_id=self.ad_account_id, entity_type=Entity.AdAccount, ) account_data = AdAccount(fbid=account_id) # Did not find a better way how to set this data on the inner AbstractCrudObject. timezone = 'Europe/Prague' account_data._data['timezone_name'] = timezone account_data._data['account_id'] = account_id with mock.patch.object(FB_ADACCOUNT_MODEL, 'api_get', return_value=account_data), mock.patch.object( NormalStore, 'store') as store: collect_adaccount(job_scope) assert store.called_with( account_data), 'Data should be stored with the cold store module' assert store.called store_args, store_keyword_args = store.call_args assert not store_keyword_args assert len( store_args ) == 1, 'Store method should be called with just 1 parameter' data_actual = store_args[0] vendor_data_key = '__oprm' ad_account_dynamo = AdAccountEntity.get(DEFAULT_SCOPE, account_id) assert ad_account_dynamo.timezone == timezone assert ad_account_dynamo.ad_account_id == account_id assert (vendor_data_key in data_actual and type(data_actual[vendor_data_key]) == dict), 'Special vendor key is present in the returned data' assert data_actual[vendor_data_key] == { 'id': universal_id_should_be }, 'Vendor data is set with the right universal id'
def test_resolve_job_scope_to_celery_task_page_post(self): real_claim = RealityClaim(entity_id='PP1', ad_account_id='P1', entity_type=Entity.PagePost) for job_generator in entity_expectation_generator_map[Entity.PagePost]: with self.subTest(job_generator=job_generator): exp_claim = next(job_generator(real_claim)) job_scope = JobScope(parse_id(exp_claim.job_id)) assert inventory.resolve_job_scope_to_celery_task(job_scope)
def store(self, datum): from oozer.common.report_job_status_task import report_job_status_task entity_id = datum.get(self.id_attribute_name) or datum.get('id') assert entity_id, "This code must have an entity ID for building of unique insertion ID" normative_job_scope = JobScope(self.job_scope_base_data, entity_id=entity_id) # and store data under that per-entity, normative JobScope. self._store(datum, normative_job_scope, DEFAULT_CHUNK_NUMBER, self.bucket_type) # since we report for many entities in this code, # must also communicate out the status inside of the for-loop # at the normative level. report_job_status_task.delay(JobStatus.Done, normative_job_scope)
def test_from_job_scope(self): key_gen = '{asset_scope}-{sweep_id}-sorted-token-queue'.format sweep_id = gen_string_id() entity_id = gen_string_id() scope_id = gen_string_id() # Scope-centered jobs must result in scope-centered key for token storage job_scope = JobScope(sweep_id=sweep_id, entity_type=Entity.Scope, entity_id=scope_id) token_manager = PlatformTokenManager.from_job_scope(job_scope) assert token_manager.queue_key == key_gen(asset_scope=scope_id, sweep_id=sweep_id) # non-Scope-centered jobs must result in 'fb'-centered key for token storage job_scope = JobScope(sweep_id=sweep_id) token_manager = PlatformTokenManager.from_job_scope(job_scope) assert token_manager.queue_key == key_gen( asset_scope=JobScope.namespace, sweep_id=sweep_id)
def test_resolve_job_scope_to_celery_task_page(self): real_claim = RealityClaim(entity_id='P1', ad_account_id='P1', entity_type=Entity.Page, timezone='America/Los_Angeles') for job_generator in entity_expectation_generator_map[Entity.Page]: for exp_claim in job_generator(real_claim): with self.subTest(job_generator=job_generator, exp_claim=exp_claim): job_scope = JobScope(parse_id(exp_claim.job_id)) assert inventory.resolve_job_scope_to_celery_task( job_scope)
def test_task_complains_about_bad_report_type(self): sync_expectations_job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.lifetime, # <----------- this is wrong ) with self.assertRaises(AssertionError) as ex_catcher: sync_expectations_task.sync_expectations( sync_expectations_job_scope) assert 'Only sync_expectations report' in str(ex_catcher.exception)
def sync_expectations(job_scope: JobScope): assert (job_scope.report_type == ReportType.sync_expectations ), 'Only sync_expectations report type is processed by this task' if job_scope.ad_account_id: # this is per AA task. No need to iterate over all ad_account_ids_iter = [job_scope.ad_account_id] else: ad_account_ids_iter = expecations_store.iter_expectations_ad_accounts( sweep_id=job_scope.sweep_id) for ad_account_id in ad_account_ids_iter: ad_account_scoped_job_scope = JobScope(job_scope.to_dict(), ad_account_id=ad_account_id, entity_type=Entity.AdAccount, entity_id=ad_account_id) with ChunkDumpStore(ad_account_scoped_job_scope, chunk_size=200) as store: job_ids_iter = expecations_store.iter_expectations_per_ad_account( ad_account_id, ad_account_scoped_job_scope.sweep_id) for job_id in job_ids_iter: job_id_parts = parse_id_parts(job_id) # default is platform namespace and we communicate out only those if job_id_parts.namespace == JobScope.namespace: store({ 'job_id': job_id, # 'status':'expected', 'account_id': job_id_parts.ad_account_id, 'entity_type': job_id_parts.entity_type, 'entity_id': job_id_parts.entity_id, 'report_type': job_id_parts.report_type, 'report_variant': job_id_parts.report_variant, 'range_start': _to_date_string_if_set(job_id_parts.range_start), 'range_end': _to_date_string_if_set(job_id_parts.range_end), 'platform_namespace': job_id_parts.namespace, })
def test_fails_with_wrong_report_variant(self): job_scope = JobScope( ad_account_id=self.ad_account_id, entity_id=self.ad_account_id, tokens=['blah'], report_time=datetime.utcnow(), report_type='entity', report_variant=None, # This actually should be set to AdAccount sweep_id='1', ) with self.assertRaises(ValueError) as ex_trap: collect_adaccount(job_scope) assert 'Report level' in str(ex_trap.exception)
def test_page_import(self): page_id = random.gen_string_id() pages = [dict(ad_account_id=page_id)] job_scope = JobScope( sweep_id=self.sweep_id, entity_type=Entity.Scope, entity_id=self.scope_id, report_type=ReportType.import_accounts, report_variant=Entity.Page, tokens=['token'], ) with mock.patch.object( ConsoleApi, 'get_pages', return_value=pages ) as gp, mock.patch.object( PageEntity, 'upsert' ) as page_upsert, mock.patch.object( report_job_status_task, 'delay' ) as status_task, mock.patch( 'oozer.entities.import_scope_entities_task._have_entity_access', return_value=True) as _have_entity_access_mock: import_pages_task(job_scope, None) assert gp.called assert status_task.called # it was called many times, but we care about the last time only aa, kk = status_task.call_args assert not kk assert aa == (JobStatus.Done, job_scope) assert page_upsert.call_count == 1 page_upsert_args = ( (self.scope_id, page_id), { 'is_active': True, 'updated_by_sweep_id': self.sweep_id, 'is_accessible': True }, ) args1 = page_upsert.call_args_list[0] assert args1 == page_upsert_args
def test_entity_level_data(self): entity_types = [Entity.Campaign, Entity.AdSet, Entity.Ad] # intentionally NOT reusing collect_insights._entity_type_id_field_map map # effectively, here we are testing it too. entity_id_attr_name_map = { Entity.Campaign: 'campaign_id', Entity.AdSet: 'adset_id', Entity.Ad: 'ad_id' } for entity_type in entity_types: with self.subTest(f'Entity type = "{entity_type}"'): input_data = {entity_id_attr_name_map[entity_type]: 'SomeID'} job_scope = JobScope( sweep_id=self.sweep_id, ad_account_id=self.ad_account_id, report_type=ReportType.lifetime, report_variant=entity_type, tokens=['blah'], ) with mock.patch.object(collect_insights.Insights, 'iter_ads_insights', return_value=[input_data ]), mock.patch.object( ChunkDumpStore, 'store') as store: data_iter = collect_insights.Insights.iter_collect_insights( job_scope, None) assert len(list(data_iter)) == 1 aa, kk = store.call_args assert not kk assert aa == ({ entity_id_attr_name_map[entity_type]: 'SomeID', '__oprm': { 'id': f'oprm|m|fb|{self.ad_account_id}|{entity_type}|SomeID|lifetime', 'entity_id': 'SomeID', 'entity_type': entity_type, }, '__transformed': {}, }, )
def test_hourly_ads_per_parent(self): job_scope = JobScope( ad_account_id=AD_ACCOUNT, range_start='2017-12-31', ad_account_timezone_name='Europe/London', report_type=ReportType.day_hour, report_variant=Entity.Ad, sweep_id='sweep', tokens=[TOKEN], ) data_iter = Insights.iter_collect_insights(job_scope, None) cnt = 0 for datum in data_iter: cnt += 1 if cnt == 4: break
def test_fetch_ad_account_pipeline(self): job_scope = JobScope( ad_account_id=AD_ACCOUNT, entity_id=AD_ACCOUNT, tokens=[TOKEN], report_time=datetime.utcnow(), report_type='entity', report_variant=Entity.AdAccount, sweep_id='1', ) aa_data = collect_adaccount_task(job_scope, None) aa_data_keys = aa_data.keys() assert aa_data for required_field in get_default_fields(AdAccount): assert required_field in aa_data_keys, f'{required_field} should be in the response from FB'
def test_pipeline_page_posts(self): job_scope = JobScope( ad_account_id=AD_ACCOUNT, tokens=[TOKEN], report_time=datetime.utcnow(), report_type='entity', report_variant=Entity.PagePost, sweep_id='1', ) data_iter = iter_collect_entities_per_page(job_scope) cnt = 0 for _ in data_iter: cnt += 1 break assert cnt
def test_pipeline_custom_audiences(self): job_scope = JobScope( ad_account_id=AD_ACCOUNT, tokens=[TOKEN], report_time=datetime.utcnow(), report_type='entity', report_variant=Entity.CustomAudience, sweep_id='1', ) data_iter = iter_collect_entities_per_adaccount(job_scope) cnt = 0 for _ in data_iter: cnt += 1 break assert cnt
def test_hourly_ad(self): job_scope = JobScope( # report_variant=Entity.Ad, ad_account_id=AD_ACCOUNT, entity_id='23842698250300224', entity_type=Entity.Ad, range_start='2017-12-31', report_type=ReportType.day_dma, sweep_id='sweep', tokens=[TOKEN], ) data_iter = Insights.iter_collect_insights(job_scope, None) cnt = 0 for datum in data_iter: cnt += 1 if cnt == 4: break
def _report_job_done_to_cold_store(job_scope: JobScope): reporting_job_scope = JobScope( sweep_id=job_scope.sweep_id, ad_account_id=job_scope.ad_account_id, report_type=ReportType.sync_status ) cold_storage.store( { 'job_id': job_scope.job_id, 'account_id': job_scope.ad_account_id, 'entity_type': job_scope.entity_type, 'entity_id': job_scope.entity_id, 'report_type': job_scope.report_type, 'report_variant': job_scope.report_variant, 'range_start': _to_date_string_if_set(job_scope.range_start), 'range_end': _to_date_string_if_set(job_scope.range_end), 'platform_namespace': job_scope.namespace, }, reporting_job_scope, )