def _entity_factory(self, entity_klazz, ad_account_id=None, entity_id=None, **kwargs): """ Manufactures an entity (based on suppplied entity_klazz) that we can use for testing :param class entity_klazz: the :param dict **kwargs: Individual field values valid for given entity :return AbstractCrudObject: The manufactured entity """ ad_account_id = ad_account_id or gen_string_id() entity_id = entity_id or gen_string_id() entity = entity_klazz(entity_id) if entity_klazz == advideo.AdVideo: entity['account_id'] = ad_account_id else: entity[entity.Field.account_id] = ad_account_id entity_fields = filter(lambda v: not v.startswith('__'), dir(entity_klazz.Field)) # Add additional fields, if any for field in filter(lambda f: f in kwargs, entity_fields): entity[field] = kwargs[field] return entity
def test_task_error_is_logged_into_job_report(self): from oozer.common.report_job_status_task import report_job_status_task class MyException(Exception): pass sync_expectations_job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.sync_expectations, ) with mock.patch.object(report_job_status_task, 'delay') as job_report, mock.patch.object( sync_expectations_task, 'sync_expectations', side_effect=MyException('nope!')): with self.assertRaises(MyException): sync_expectations_task.sync_expectations_task.delay( sync_expectations_job_scope, None) assert job_report.called aa, kk = job_report.call_args assert not kk code, job_scope_actual = aa assert code < 0 # some sort of *Failure* code assert job_scope_actual == sync_expectations_job_scope
def test_task_does_not_blow_up(self): # this is almost same thing as the next test # where we check that call signature is right, # but when call signature changes and our tests don't, # it becomes irrelevant if we have tests - they check for wrong thing # So, here we actually call "store" and in next test # we intercept the call and check payload. # Don't remove me. Not duplicate. expectation_job_id = generate_id( ad_account_id=random.gen_string_id(), report_type=ReportType.day_hour, report_variant=Entity.Ad, range_start='2000-01-01', ) rr = [expectation_job_id] sync_expectations_job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.sync_expectations, ) with mock.patch.object(expecations_store, 'iter_expectations_per_ad_account', return_value=rr): sync_expectations_task.sync_expectations( sync_expectations_job_scope)
def test_populate_from_scope_record(self): scope_id = gen_string_id() sweep_id = gen_string_id() console_token = 'console token' platform_token = 'platform token' scope_record = AssetScope() scope_record.scope = scope_id scope_record.scope_api_token = console_token scope_record.set_cache(platform_tokens={platform_token}) PlatformTokenManager.populate_from_scope_entity(scope_record, sweep_id) # now let's make sure we see those tokens: # Scope-centered jobs must result in scope-centered key for token storage job_scope = JobScope(sweep_id=sweep_id, entity_type=Entity.Scope, entity_id=scope_id) assert console_token == PlatformTokenManager.from_job_scope( job_scope).get_best_token() job_scope = JobScope( sweep_id=sweep_id, # uses .namespace default value as 2nd value in redis key. no need to set here. ) assert platform_token == PlatformTokenManager.from_job_scope( job_scope).get_best_token()
def test_default_bol(self): """ Check that the default BOL values are populated on entities without creation date """ FBModel = adcreative.AdCreative entity_type = FB_MODEL_ENTITY_TYPE_MAP[FBModel] DBModel = ENTITY_TYPE_DB_MODEL_MAP[entity_type] aaid = gen_string_id() eid = gen_string_id() entity_data = dict( # returned value here is FB SDK model, hence the dict( above. self._entity_factory(FBModel, ad_account_id=aaid, id=eid)) feedback_entity_task(entity_data, entity_type) record = DBModel.get(aaid, eid) assert record.to_dict() == { 'ad_account_id': aaid, 'entity_id': eid, 'entity_type': entity_type, 'bol': datetime.now(timezone.utc), 'eol': None, 'is_accessible': True, }
def test_bol_translation(self): """ Check that the default BOL values are populated on entities without creation date """ FBModel = customaudience.CustomAudience entity_type = FB_MODEL_ENTITY_TYPE_MAP[FBModel] DBModel = ENTITY_TYPE_DB_MODEL_MAP[entity_type] aaid = gen_string_id() eid = gen_string_id() entity_data = dict( # returned value here is FB SDK model, hence the dict( above. self._entity_factory(FBModel, account_id=aaid, id=eid, time_created=1523049070, time_updated=1533162823)) feedback_entity_task(entity_data, entity_type) record = DBModel.get(aaid, eid) assert record.to_dict() == { 'ad_account_id': aaid, 'entity_id': eid, 'entity_type': entity_type, 'bol': datetime(2018, 4, 6, 21, 11, 10, tzinfo=timezone.utc), 'eol': None, 'is_accessible': True, }
def setUp(self): self.job_scope = JobScope( sweep_id=gen_string_id(), ad_account_id=gen_string_id(), entity_id=gen_string_id(), entity_type=Entity.Campaign, report_type=ReportType.entity, )
def test_task_is_called_with_right_data(self): range_start = now() range_start_should_be = range_start.strftime('%Y-%m-%d') expected_job_id = generate_id( ad_account_id=random.gen_string_id(), report_type=ReportType.day_hour, report_variant=Entity.Ad, range_start=range_start, ) rr = [expected_job_id] expected_job_id_parts = parse_id_parts(expected_job_id) sync_expectations_job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.sync_expectations, ) with mock.patch.object(expecations_store, 'iter_expectations_per_ad_account', return_value=rr) as jid_iter, mock.patch.object( cold_storage.ChunkDumpStore, 'store') as store: sync_expectations_task.sync_expectations( sync_expectations_job_scope) assert jid_iter.called aa, kk = jid_iter.call_args assert not kk assert aa == (sync_expectations_job_scope.ad_account_id, sync_expectations_job_scope.sweep_id) assert store.called aa, kk = store.call_args assert not kk assert len(aa) == 1 data = aa[0] assert data == { 'job_id': expected_job_id, # missing "ad_" is intentional. # this matches this attr name as sent by FB # and ysed by us elsewhere in the company 'account_id': expected_job_id_parts.ad_account_id, 'entity_type': expected_job_id_parts.entity_type, 'entity_id': expected_job_id_parts.entity_id, 'report_type': expected_job_id_parts.report_type, 'report_variant': expected_job_id_parts.report_variant, 'range_start': range_start_should_be, # checking manually to ensure it's properly stringified 'range_end': None, 'platform_namespace': JobScope.namespace, # default platform value }
def test_base_model_to_dict(self): pid = random.gen_string_id() sid = random.gen_string_id() data = self.Model(pid, sid, data='primary data').to_dict() assert data == dict(primary_id=pid, secondary_id=sid, data='primary data', more_data=None)
def test_all_upserted(entity_type, entity_data, expected): aaid = gen_string_id() eid = gen_string_id() entity_data.update(account_id=aaid, id=eid) expected.update(ad_account_id=aaid, entity_id=eid) feedback_entity_task(entity_data, entity_type) record = ENTITY_TYPE_DB_MODEL_MAP[entity_type].get( entity_data['account_id'], entity_data['id']) assert record.to_dict() == expected
def test_task_complains_about_bad_report_type(self): sync_expectations_job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.lifetime, # <----------- this is wrong ) with self.assertRaises(AssertionError) as ex_catcher: sync_expectations_task.sync_expectations( sync_expectations_job_scope) assert 'Only sync_expectations report' in str(ex_catcher.exception)
def test_aa_collection_expectation(): ad_account_id = gen_string_id() reality_claim = RealityClaim(ad_account_id=ad_account_id, entity_id=ad_account_id, entity_type=Entity.AdAccount) def is_adaccount_entity_job(expectation_claim): parsed_id_parts = parse_id_parts(expectation_claim.job_id) return parsed_id_parts.report_type == ReportType.entity and parsed_id_parts.report_variant == Entity.AdAccount adaccount_entity_expectations = list( filter(is_adaccount_entity_job, iter_expectations([reality_claim]))) assert len(adaccount_entity_expectations) == 1 expectation_claim = adaccount_entity_expectations[0] assert expectation_claim.entity_id == reality_claim.ad_account_id assert expectation_claim.entity_type == reality_claim.entity_type assert expectation_claim.job_id == generate_id( ad_account_id=ad_account_id, entity_id=ad_account_id, report_type=ReportType.entity, report_variant=Entity.AdAccount, )
def test_datetime_handling_and_seconds(self): ad_account_id = random.gen_string_id() report_type = 'blah' range_start_dt = datetime(2000, 1, 31, 3, 0, 47) range_start_should_be = quote_plus( range_start_dt.strftime('%Y-%m-%dT%H:%M:%S')) id_should_be = D.join([ 'oprm', 'm', NS, ad_account_id, '', # entity Type '', # entity ID report_type, '', # report variant range_start_should_be, # Range start # '', # Range end ]) assert id_should_be == id_tools.generate_universal_id( ad_account_id=ad_account_id, report_type=report_type, range_start=range_start_dt)
def test_expectations_store(self): all_job_ids_should_be = set() job_id_template = 'fb|{ad_account_id}|{job_variant}'.format sweep_id = random.gen_string_id() ad_account_ids = ['1', '2', '3'] job_variants = [ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l' ] with JobExpectationsWriter(sweep_id=sweep_id) as add_expectation: for ad_account_id in ad_account_ids: for job_variant in job_variants: job_id = job_id_template(ad_account_id=ad_account_id, job_variant=job_variant) all_job_ids_should_be.add(job_id) add_expectation( job_id, ad_account_id, None) # we don't care about entity ID at this time. all_job_ids_actual = list(iter_expectations(sweep_id)) # must test as list to ensure that duplicate entries are not returned # once collapsed into a set() dupes disappear and it's too late to test assert len(all_job_ids_actual) == len( all_job_ids_should_be), "Must have no dupes returned" # now let's test for equality assert set(all_job_ids_actual) == all_job_ids_should_be
def test_datetime_handling_special_zero_hour_handling(self): ad_account_id = random.gen_string_id() report_type = 'blah' range_start_dt = datetime(2000, 1, 31, 0, 0, 0) # even though id-forming logic shortens the string to last # non-zero value, this does not apply to hours. # Whenever value is DateTime type, hours are always # part of string even when hour is zero. range_start_should_be = range_start_dt.strftime('%Y-%m-%dT%H') id_should_be = D.join([ 'oprm', 'm', NS, ad_account_id, '', # entity Type '', # entity ID report_type, '', # report variant range_start_should_be, # Range start # '', # Range end ]) assert id_should_be == id_tools.generate_universal_id( ad_account_id=ad_account_id, report_type=report_type, range_start=range_start_dt)
def test_key_s3_date_less(self): """ Check that the key is constructed as we expect """ import common.tztools job_scope = JobScope( ad_account_id=gen_string_id(), report_type=ReportType.entity, report_variant=Entity.Campaign ) now_dt = datetime(2000, 1, 2, 3, 4, 5) with mock.patch.object(common.tztools, 'now', return_value=now_dt) as now_mocked, mock.patch.object( uuid, 'uuid4', return_value='UUID-HERE' ): storage_key = cold_storage.store({'data': 'yeah!'}, job_scope) assert now_mocked.called prefix = xxhash.xxh64(job_scope.ad_account_id).hexdigest()[:6] expected_key = ( f'fb/' + f'{prefix}-{job_scope.ad_account_id}/' + f'{job_scope.report_type}/' + f'{now_dt.strftime("%Y")}/' + f'{now_dt.strftime("%m")}/' + f'{now_dt.strftime("%d")}/' + f'{now_dt.strftime("%Y-%m-%dT%H:%M:%SZ")}-' + f'{job_scope.job_id}-' + f'UUID-HERE' + f'.json' ) assert storage_key == expected_key
def test_key_s3_date_snapped_with_chunk_id(self): """ Check that the key is constructed as we expect """ job_scope = JobScope( ad_account_id=gen_string_id(), report_type=ReportType.day_platform, report_variant=Entity.Ad, range_start=date(2000, 1, 2), ) chunk_marker = 7 dt_should_be = datetime(2000, 1, 2, 0, 0, 0) with mock.patch.object(uuid, 'uuid4', return_value='UUID-HERE'): storage_key = cold_storage.store({'data': 'yeah!'}, job_scope, chunk_marker=7) prefix = xxhash.xxh64(job_scope.ad_account_id).hexdigest()[:6] expected_key = ( f'fb/' + f'{prefix}-{job_scope.ad_account_id}/' + f'{job_scope.report_type}/' + f'{dt_should_be.strftime("%Y")}/' + f'{dt_should_be.strftime("%m")}/' + f'{dt_should_be.strftime("%d")}/' + f'{dt_should_be.strftime("%Y-%m-%dT%H:%M:%SZ")}-' + f'{job_scope.job_id}-' + f'{chunk_marker}-' + f'UUID-HERE' + f'.json' ) assert storage_key == expected_key
def test_iter_expectations_generates_jobs_from_map(mock_map): expected_claim = Mock() mock_map.get.return_value = [Mock(return_value=[expected_claim])] reality_claim = RealityClaim(entity_id=gen_string_id(), entity_type=Entity.AdAccount) assert [expected_claim] == list(iter_expectations([reality_claim]))
class TestModel(BaseModel): Meta = BaseMeta(random.gen_string_id()) primary_id = attributes.UnicodeAttribute(hash_key=True, attr_name='pid') secondary_id = attributes.UnicodeAttribute(range_key=True, attr_name='sid') data = attributes.UnicodeAttribute(null=True, attr_name='d') more_data = attributes.UnicodeAttribute(null=True, attr_name='d2')
def test_additional_fields(self): # purposefully messing with real attr names to test .to_dict() class TestModel(BaseModel): _additional_fields = {'record_type'} Meta = BaseMeta(random.gen_string_id()) primary_id = attributes.UnicodeAttribute(hash_key=True, attr_name='pid') secondary_id = attributes.UnicodeAttribute(range_key=True, attr_name='sid') data = attributes.UnicodeAttribute(null=True, attr_name='d') record_type = 'SUPER_RECORD' pid = random.gen_string_id() sid = random.gen_string_id() # ARG!!! for some reason instantiation further below # needs to have the table actually exist. Nuts! # TODO: fix this shit and allow object instantiation NOT require a hit to DB TestModel.create_table() record = TestModel(pid, sid, data='value') assert record.to_dict() == dict( primary_id=pid, secondary_id=sid, data='value', record_type='SUPER_RECORD' # <--- note static attribute ) assert record.to_dict(fields=['data', 'record_type']) == dict( # primary_id=pid, # secondary_id=sid, data='value', record_type='SUPER_RECORD', # <--- note static attribute ) with self.assertRaises(AttributeError) as ex: record.to_dict(fields=['data', 'does_not_exist']) error_message = str(ex.exception) assert 'object has no attribute' in error_message assert 'does_not_exist' in error_message
def test_base_model_upsert(self): pid = random.gen_string_id() sid = random.gen_string_id() # no record should exist, but upsert should succeed with self.assertRaises(self.Model.DoesNotExist): self.Model.get(pid, sid) m = self.Model.upsert(pid, sid, data='primary data') assert isinstance(m, self.Model) assert m.to_dict() == dict(primary_id=pid, secondary_id=sid, data='primary data', more_data=None) m = self.Model.get(pid, sid) assert m.to_dict() == dict(primary_id=pid, secondary_id=sid, data='primary data', more_data=None) # Now let's update same record and ensure we don't clobber # data we do NOT communicate in upsert # note that we don't pass in value for `data` attr m = self.Model.upsert(pid, sid, more_data='more data') assert isinstance(m, self.Model) assert m.to_dict() == dict( primary_id=pid, secondary_id=sid, data= 'primary data', # <------- .update call picks up data that was already in DB more_data='more data', ) # and just in case, fresh get m = self.Model.get(pid, sid) assert m.to_dict() == dict(primary_id=pid, secondary_id=sid, data='primary data', more_data='more data')
def test_runs_correctly(self): account_id = random.gen_string_id() job_scope = JobScope( ad_account_id=self.ad_account_id, entity_id=self.ad_account_id, tokens=['A_REAL_TOKEN'], report_time=datetime.utcnow(), report_type='entity', report_variant=Entity.AdAccount, sweep_id='1', ) universal_id_should_be = generate_universal_id( ad_account_id=self.ad_account_id, report_type=ReportType.entity, entity_id=self.ad_account_id, entity_type=Entity.AdAccount, ) account_data = AdAccount(fbid=account_id) # Did not find a better way how to set this data on the inner AbstractCrudObject. timezone = 'Europe/Prague' account_data._data['timezone_name'] = timezone account_data._data['account_id'] = account_id with mock.patch.object(FB_ADACCOUNT_MODEL, 'api_get', return_value=account_data), mock.patch.object( NormalStore, 'store') as store: collect_adaccount(job_scope) assert store.called_with( account_data), 'Data should be stored with the cold store module' assert store.called store_args, store_keyword_args = store.call_args assert not store_keyword_args assert len( store_args ) == 1, 'Store method should be called with just 1 parameter' data_actual = store_args[0] vendor_data_key = '__oprm' ad_account_dynamo = AdAccountEntity.get(DEFAULT_SCOPE, account_id) assert ad_account_dynamo.timezone == timezone assert ad_account_dynamo.ad_account_id == account_id assert (vendor_data_key in data_actual and type(data_actual[vendor_data_key]) == dict), 'Special vendor key is present in the returned data' assert data_actual[vendor_data_key] == { 'id': universal_id_should_be }, 'Vendor data is set with the right universal id'
def test_from_job_scope(self): key_gen = '{asset_scope}-{sweep_id}-sorted-token-queue'.format sweep_id = gen_string_id() entity_id = gen_string_id() scope_id = gen_string_id() # Scope-centered jobs must result in scope-centered key for token storage job_scope = JobScope(sweep_id=sweep_id, entity_type=Entity.Scope, entity_id=scope_id) token_manager = PlatformTokenManager.from_job_scope(job_scope) assert token_manager.queue_key == key_gen(asset_scope=scope_id, sweep_id=sweep_id) # non-Scope-centered jobs must result in 'fb'-centered key for token storage job_scope = JobScope(sweep_id=sweep_id) token_manager = PlatformTokenManager.from_job_scope(job_scope) assert token_manager.queue_key == key_gen( asset_scope=JobScope.namespace, sweep_id=sweep_id)
class TestModel(BaseModel): _fields = {'secondary_id', 'record_type'} Meta = BaseMeta(random.gen_string_id()) primary_id = attributes.UnicodeAttribute(hash_key=True, attr_name='pid') secondary_id = attributes.UnicodeAttribute(range_key=True, attr_name='sid') data = attributes.UnicodeAttribute(null=True, attr_name='d') record_type = 'SUPER_RECORD'
def test_task_success_is_logged_into_job_report(self): from oozer.common.report_job_status_task import report_job_status_task sync_expectations_job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.sync_expectations, ) with mock.patch.object(report_job_status_task, 'delay') as job_report: sync_expectations_task.sync_expectations_task.delay( sync_expectations_job_scope, None) assert job_report.called aa, kk = job_report.call_args assert not kk code, job_scope_actual = aa assert code == JobStatus.Done assert job_scope_actual == sync_expectations_job_scope
def test_right_data_is_communicated_on_done_signal(self): range_start = now() range_start_should_be = range_start.strftime('%Y-%m-%d') job_scope = JobScope( sweep_id=random.gen_string_id(), ad_account_id=random.gen_string_id(), report_type=ReportType.day_hour, report_variant=Entity.Ad, range_start=now(), ) with mock.patch.object(cold_storage, 'store') as store: report_job_status._report_job_done_to_cold_store(job_scope) assert store.called aa, kk = store.call_args assert not kk data, job_scope_reported = aa assert data == { 'job_id': job_scope.job_id, # missing "ad_" is intentional. # this matches this attr name as sent by FB # and ysed by us elsewhere in the company 'account_id': job_scope.ad_account_id, 'entity_type': None, 'entity_id': None, 'report_type': ReportType.day_hour, 'report_variant': Entity.Ad, 'range_start': range_start_should_be, 'range_end': None, 'platform_namespace': JobScope.namespace, # default platform value } assert job_scope_reported.sweep_id == job_scope.sweep_id assert job_scope_reported.ad_account_id == job_scope.ad_account_id assert job_scope_reported.report_type == ReportType.sync_status
def page_set(ctx, scope, id=None, name='Page', is_active=True, data=None): # PlatformToken.upsert(scope, token=TOKEN) # AssetScope.upsert(scope, platform_token_ids={scope}) if data: data = json.loads(data) else: data = {} a = PageEntity.upsert(scope, gen_string_id() if id is None else id, is_active=is_active, **data) print(a.to_dict())
def test_base_model_upsert_is_not_exists(self): pid = random.gen_string_id() sid = random.gen_string_id() # no record should exist, but upsert should succeed with self.assertRaises(self.Model.DoesNotExist): self.Model.get(pid, sid) # record does not exist self.Model.upsert(pid, sid, data=self.Model.data | 'primary data') # if_not_exists change expression m = self.Model.get(pid, sid) assert m.to_dict() == dict(primary_id=pid, secondary_id=sid, data='primary data', more_data=None) # Now record exists and one attr is set, # so attempt to overwrite it will be discarded self.Model.upsert( pid, sid, data=self.Model.data | 'primary data overwrite NOT', # if_not_exists change expression more_data=self.Model.more_data | 'more data', ) m = self.Model.get(pid, sid) assert m.to_dict() == dict( primary_id=pid, secondary_id=sid, data='primary data', more_data='more data' # <-- still original )
def test_page_import(self): page_id = random.gen_string_id() pages = [dict(ad_account_id=page_id)] job_scope = JobScope( sweep_id=self.sweep_id, entity_type=Entity.Scope, entity_id=self.scope_id, report_type=ReportType.import_accounts, report_variant=Entity.Page, tokens=['token'], ) with mock.patch.object( ConsoleApi, 'get_pages', return_value=pages ) as gp, mock.patch.object( PageEntity, 'upsert' ) as page_upsert, mock.patch.object( report_job_status_task, 'delay' ) as status_task, mock.patch( 'oozer.entities.import_scope_entities_task._have_entity_access', return_value=True) as _have_entity_access_mock: import_pages_task(job_scope, None) assert gp.called assert status_task.called # it was called many times, but we care about the last time only aa, kk = status_task.call_args assert not kk assert aa == (JobStatus.Done, job_scope) assert page_upsert.call_count == 1 page_upsert_args = ( (self.scope_id, page_id), { 'is_active': True, 'updated_by_sweep_id': self.sweep_id, 'is_accessible': True }, ) args1 = page_upsert.call_args_list[0] assert args1 == page_upsert_args
def test_some_data_at_end(self): ad_account_id = random.gen_string_id() report_type = 'blah' id_should_be = D.join([ # 'oprm', # 'm', NS, ad_account_id, '', # entity Type '', # entity ID report_type, # '', # report variant # '', # Range start # '', # Range end ]) assert id_should_be == id_tools.generate_id( ad_account_id=ad_account_id, report_type=report_type)