def test_sharding_does_not_break_and_does_not_duplicate(self):
# Comes from understanding that we need to write more than some multiple
# larger than number of internal shards (so currently 10, let's make it
# double that for fun)
jobs_to_generate = SortedJobsQueue._JOBS_READER_BATCH_SIZE * 20
extra_data = {'timezone': 'Some/Thing'}
jobs_to_write = [(
generate_id(
ad_account_id='AAID',
entity_id=str(bogus_score),
report_type=ReportType.entity,
report_variant=Entity.Campaign,
),
bogus_score,
) for bogus_score in range(0, jobs_to_generate)]
with SortedJobsQueue(self.sweep_id).JobsWriter() as add_to_queue:
for job_id, score in jobs_to_write:
# writes tasks to distributed sorting queues
add_to_queue(job_id, score, **extra_data)
with SortedJobsQueue(self.sweep_id).JobsReader() as jobs_iter:
cnt = 0
for job_id, job_scope_data, score in jobs_iter:
assert job_id is not None
assert job_scope_data is not None
assert score is not None
cnt += 1
assert cnt == jobs_to_generate
def test_task_does_not_blow_up(self):
# this is almost same thing as the next test
# where we check that call signature is right,
# but when call signature changes and our tests don't,
# it becomes irrelevant if we have tests - they check for wrong thing
# So, here we actually call "store" and in next test
# we intercept the call and check payload.
# Don't remove me. Not duplicate.
expectation_job_id = generate_id(
ad_account_id=random.gen_string_id(),
report_type=ReportType.day_hour,
report_variant=Entity.Ad,
range_start='2000-01-01',
)
rr = [expectation_job_id]
sync_expectations_job_scope = JobScope(
sweep_id=random.gen_string_id(),
ad_account_id=random.gen_string_id(),
report_type=ReportType.sync_expectations,
)
with mock.patch.object(expecations_store,
'iter_expectations_per_ad_account',
return_value=rr):
sync_expectations_task.sync_expectations(
sync_expectations_job_scope)
def test_aa_collection_expectation():
ad_account_id = gen_string_id()
reality_claim = RealityClaim(ad_account_id=ad_account_id,
entity_id=ad_account_id,
entity_type=Entity.AdAccount)
def is_adaccount_entity_job(expectation_claim):
parsed_id_parts = parse_id_parts(expectation_claim.job_id)
return parsed_id_parts.report_type == ReportType.entity and parsed_id_parts.report_variant == Entity.AdAccount
adaccount_entity_expectations = list(
filter(is_adaccount_entity_job, iter_expectations([reality_claim])))
assert len(adaccount_entity_expectations) == 1
expectation_claim = adaccount_entity_expectations[0]
assert expectation_claim.entity_id == reality_claim.ad_account_id
assert expectation_claim.entity_type == reality_claim.entity_type
assert expectation_claim.job_id == generate_id(
ad_account_id=ad_account_id,
entity_id=ad_account_id,
report_type=ReportType.entity,
report_variant=Entity.AdAccount,
)
def lifetime_metrics_per_entity_under_ad_account(
entity_type: str, reality_claim: RealityClaim
) -> Generator[ExpectationClaim, None, None]:
"""Generate ad-account level expectation claims for lifetime."""
if not reality_claim.timezone:
return
# TODO: Remove once all entities have parent ids
# Divide tasks only if parent levels are defined for all ads
is_dividing_possible = True
root_node = EntityNode(reality_claim.entity_id, reality_claim.entity_type)
for child_claim in iter_reality_per_ad_account_claim(
reality_claim, entity_types=[entity_type]):
is_dividing_possible = is_dividing_possible and child_claim.all_parent_ids_set
new_node = EntityNode(child_claim.entity_id, child_claim.entity_type)
root_node.add_node(new_node, path=child_claim.parent_entity_ids)
logger.warning(
f'[dividing-possible] Ad Account {reality_claim.ad_account_id} Dividing possible: {is_dividing_possible}'
)
yield ExpectationClaim(
reality_claim.entity_id,
reality_claim.entity_type,
ReportType.lifetime,
entity_type,
JobSignature(
generate_id(ad_account_id=reality_claim.ad_account_id,
report_type=ReportType.lifetime,
report_variant=entity_type)),
ad_account_id=reality_claim.ad_account_id,
entity_hierarchy=root_node if is_dividing_possible else None,
timezone=reality_claim.timezone,
)
def test_task_is_called_with_right_data(self):
range_start = now()
range_start_should_be = range_start.strftime('%Y-%m-%d')
expected_job_id = generate_id(
ad_account_id=random.gen_string_id(),
report_type=ReportType.day_hour,
report_variant=Entity.Ad,
range_start=range_start,
)
rr = [expected_job_id]
expected_job_id_parts = parse_id_parts(expected_job_id)
sync_expectations_job_scope = JobScope(
sweep_id=random.gen_string_id(),
ad_account_id=random.gen_string_id(),
report_type=ReportType.sync_expectations,
)
with mock.patch.object(expecations_store,
'iter_expectations_per_ad_account',
return_value=rr) as jid_iter, mock.patch.object(
cold_storage.ChunkDumpStore,
'store') as store:
sync_expectations_task.sync_expectations(
sync_expectations_job_scope)
assert jid_iter.called
aa, kk = jid_iter.call_args
assert not kk
assert aa == (sync_expectations_job_scope.ad_account_id,
sync_expectations_job_scope.sweep_id)
assert store.called
aa, kk = store.call_args
assert not kk
assert len(aa) == 1
data = aa[0]
assert data == {
'job_id': expected_job_id,
# missing "ad_" is intentional.
# this matches this attr name as sent by FB
# and ysed by us elsewhere in the company
'account_id': expected_job_id_parts.ad_account_id,
'entity_type': expected_job_id_parts.entity_type,
'entity_id': expected_job_id_parts.entity_id,
'report_type': expected_job_id_parts.report_type,
'report_variant': expected_job_id_parts.report_variant,
'range_start':
range_start_should_be, # checking manually to ensure it's properly stringified
'range_end': None,
'platform_namespace': JobScope.namespace, # default platform value
}
def job_id(self) -> str:
return generate_id(
ad_account_id=self.ad_account_id,
entity_type=self.entity_type,
entity_id=self.entity_id,
report_type=self.report_type,
report_variant=self.report_variant,
range_start=self.range_start,
range_end=self.range_end,
namespace=self.namespace,
)
def test_storage_works(self):
# note the order of insertion - not in order of the score
scored_jobs = [
(generate_id(ad_account_id='AAID',
report_type=ReportType.entity,
report_variant=Entity.Campaign), 20),
(generate_id(ad_account_id='AAID',
report_type=ReportType.entity,
report_variant=Entity.AdSet), 30),
(generate_id(ad_account_id='AAID',
report_type=ReportType.entity,
report_variant=Entity.Ad), 10),
]
extra_data = {'timezone': 'Europe/London'}
with SortedJobsQueue(self.sweep_id).JobsWriter() as add_to_queue:
for job_id, score in scored_jobs:
# writes tasks to distributed sorting queues
add_to_queue(job_id, score, **extra_data)
jobs_queued_actual = []
with SortedJobsQueue(self.sweep_id).JobsReader() as jobs_iter:
for job_id, job_scope_data, score in jobs_iter:
jobs_queued_actual.append((job_id, job_scope_data, score))
jobs_queued_should_be = [
(
generate_id(ad_account_id='AAID',
report_type=ReportType.entity,
report_variant=Entity.AdSet),
{
'timezone': 'Europe/London'
},
30.0, # <-----
),
(
generate_id(ad_account_id='AAID',
report_type=ReportType.entity,
report_variant=Entity.Campaign),
{
'timezone': 'Europe/London'
},
20.0, # <-----
),
(
generate_id(ad_account_id='AAID',
report_type=ReportType.entity,
report_variant=Entity.Ad),
{
'timezone': 'Europe/London'
},
10.0, # <-----
),
]
assert jobs_queued_actual == jobs_queued_should_be
def page_entity(
reality_claim: RealityClaim
) -> Generator[ExpectationClaim, None, None]:
assert reality_claim.entity_type == Entity.Page, 'Page expectation should be triggered only by page reality claims'
yield ExpectationClaim(
reality_claim.entity_id,
reality_claim.entity_type,
ReportType.entity,
Entity.Page,
JobSignature(
generate_id(
ad_account_id=reality_claim.ad_account_id,
entity_id=reality_claim.entity_id,
report_type=ReportType.entity,
report_variant=Entity.Page,
)),
ad_account_id=reality_claim.ad_account_id,
)
def test_some_data_at_end(self):
ad_account_id = random.gen_string_id()
report_type = 'blah'
id_should_be = D.join([
# 'oprm',
# 'm',
NS,
ad_account_id,
'', # entity Type
'', # entity ID
report_type,
# '', # report variant
# '', # Range start
# '', # Range end
])
assert id_should_be == id_tools.generate_id(
ad_account_id=ad_account_id, report_type=report_type)
def entities_per_page(
entity_type: str, reality_claim: RealityClaim
) -> Generator[ExpectationClaim, None, None]:
"""
Generates "fetch EntityType entities metadata per given Page" job call sig
"""
assert entity_type in Entity.NON_AA_SCOPED
yield ExpectationClaim(
reality_claim.entity_id,
reality_claim.entity_type,
ReportType.entity,
entity_type,
JobSignature(
generate_id(ad_account_id=reality_claim.ad_account_id,
report_type=ReportType.entity,
report_variant=entity_type)),
ad_account_id=reality_claim.ad_account_id,
)
def test_aa_import_expectation_generated():
entity_id = gen_string_id()
reality_claim = RealityClaim(entity_type=Entity.Scope, entity_id=entity_id)
results = list(iter_expectations([reality_claim]))
assert len(results) == 2
expectation_claim = results[0]
assert expectation_claim.entity_id == reality_claim.entity_id
assert expectation_claim.entity_type == reality_claim.entity_type
assert expectation_claim.job_id == generate_id(
namespace=config.application.UNIVERSAL_ID_SYSTEM_NAMESPACE,
entity_type=Entity.Scope,
entity_id=entity_id,
report_type=ReportType.import_accounts,
report_variant=Entity.AdAccount,
)
def lifetime_page_metrics_per_entity(
entity_type: str, reality_claim: RealityClaim
) -> Generator[ExpectationClaim, None, None]:
assert entity_type in Entity.ALL
yield ExpectationClaim(
reality_claim.entity_id,
reality_claim.entity_type,
ReportType.lifetime,
entity_type,
JobSignature(
generate_id(
ad_account_id=reality_claim.ad_account_id,
entity_type=reality_claim.entity_type,
entity_id=reality_claim.entity_id,
report_type=ReportType.lifetime,
report_variant=entity_type,
)),
ad_account_id=reality_claim.ad_account_id,
)
def generate_child_claims(
claim: ExpectationClaim) -> Generator[ExpectationClaim, None, None]:
for child_entity_node in claim.entity_hierarchy.children:
yield ExpectationClaim(
child_entity_node.entity_id,
child_entity_node.entity_type,
claim.report_type,
claim.report_variant,
JobSignature(
generate_id(
ad_account_id=claim.ad_account_id,
range_start=claim.range_start,
report_type=claim.report_type,
report_variant=claim.report_variant,
entity_id=child_entity_node.entity_id,
entity_type=child_entity_node.entity_type,
)),
ad_account_id=claim.ad_account_id,
timezone=claim.timezone,
entity_hierarchy=child_entity_node,
range_start=claim.range_start,
)
def ad_account_entity(
reality_claim: RealityClaim
) -> Generator[ExpectationClaim, None, None]:
assert (
reality_claim.entity_type == Entity.AdAccount
), 'Ad account expectation should be triggered only by ad account reality claims'
yield ExpectationClaim(
reality_claim.entity_id,
reality_claim.entity_type,
ReportType.entity,
Entity.AdAccount,
JobSignature(
generate_id(
ad_account_id=reality_claim.ad_account_id,
entity_id=reality_claim.entity_id,
report_type=ReportType.entity,
report_variant=Entity.AdAccount,
)),
ad_account_id=reality_claim.ad_account_id,
timezone=reality_claim.timezone,
)
def entities_per_ad_account(
entity_type: str, reality_claim: RealityClaim
) -> Generator[ExpectationClaim, None, None]:
"""
Generates "fetch EntityType entities metadata per given AA" job call sig
"""
# Mental Note:
# This job signature generator is designed to be parked
# *under AdAccount job signatures generators inventory*
# In other words, NOT under EntityType inventory (where it would be called
# for each EntityType).
# Unlike metrics report types,
# We don't have an effective "fetch single EntityType entity data per EntityType ID" task (yet).
# So, instead of generating many job signatures per EntityTypes,
# we create only one per-parent-AA, and making that
# into "normative_job_signature" per AA level.
# When we have a need to have atomic per-EntityType entity data collection celery task,
# atomic per-C entity data job signature would go into normative column
# on ExpectationClaim for each and separate EntityType and per-parent-AA
# job signature will go into "effective_job_signatures" list on those claims,
# AND this function must move from AA-level to EntityType-level signature
# generators inventory.
assert entity_type in Entity.ALL
yield ExpectationClaim(
reality_claim.entity_id,
reality_claim.entity_type,
ReportType.entity,
entity_type,
JobSignature(
generate_id(ad_account_id=reality_claim.ad_account_id,
report_type=ReportType.entity,
report_variant=entity_type)),
ad_account_id=reality_claim.ad_account_id,
timezone=reality_claim.timezone,
)
def pages_per_scope(
reality_claim: RealityClaim
) -> Generator[ExpectationClaim, None, None]:
"""
Generates "fetch Pages active entity metadata per given scope" job ID
To be used by Scope-level RealityClaim / ExpectationClaim.
"""
yield ExpectationClaim(
reality_claim.entity_id,
reality_claim.entity_type,
ReportType.import_pages,
Entity.Page,
JobSignature(
generate_id(
namespace=config.application.UNIVERSAL_ID_SYSTEM_NAMESPACE,
# Note absence of value for Page
# This is "all Pages per scope X" job.
entity_id=reality_claim.entity_id,
entity_type=reality_claim.entity_type,
report_type=ReportType.import_pages,
report_variant=Entity.Page,
)),
)
def test_some_data_trailing(self):
ad_account_id = random.gen_string_id()
report_type = 'blah'
id_should_be = D.join([
# 'oprm',
# 'm',
NS,
ad_account_id,
'', # entity Type
'', # entity ID
report_type,
'', # report variant
'', # Range start
'', # Range end
'hocus',
'', # None
'pocus',
])
assert id_should_be == id_tools.generate_id(
ad_account_id=ad_account_id,
report_type=report_type,
trailing_parts=['hocus', None, 'pocus'])
def test_persister_saves_job_scope_auxiliary_data_to_data_flower(self):
# There is a need to save some context data that does not fit on JobIS
# Persister should store that on the Data Flower.
sweep_id = random.gen_string_id()
entity_id = random.gen_string_id()
ad_account_id = random.gen_string_id()
job_id = generate_id(ad_account_id=ad_account_id,
report_type=ReportType.lifetime,
report_variant=Entity.Campaign)
prioritized_iter = [
PrioritizationClaim(
entity_id,
Entity.Campaign,
ReportType.lifetime,
JobSignature(job_id),
100,
ad_account_id=ad_account_id,
timezone='Europe/London',
# TODO: contemplate moving auxiliary data formation to
# place where JobSignatures are generated and use that
# data for Data Flower (as it was originally intended
# but not implemented because saving each job's data
# individually to Data Flower was too slow)
)
]
persisted = persister.iter_persist_prioritized(sweep_id,
prioritized_iter)
cnt = 0
for item in persisted:
cnt += 1
# just need to spin the generator
# so it does all the saving it needs to do per item
assert cnt == 1
# Now, finally, the testing:
jobs_queued_actual = []
with SortedJobsQueue(sweep_id).JobsReader() as jobs_iter:
for job_id, job_scope_data, score in jobs_iter:
jobs_queued_actual.append((job_id, job_scope_data, score))
jobs_queued_should_be = [(
job_id,
# Contents of this dict is what we are testing here
dict(
# comes from Persister code
# manually peeled off *Claim and injected into Data Flower
ad_account_timezone_name='Europe/London'),
100,
)]
assert jobs_queued_actual == jobs_queued_should_be
# And, another way of looking at it
# looper.iter_tasks preassembles JobScope and should apply aux data to it.
job_scope = None
cnt = 0
for celery_task, job_scope, job_context, score in TaskProducer(
sweep_id).iter_tasks():
cnt += 1
# this just needs to spin once
assert cnt == 1
job_scope_should_be = JobScope(
sweep_id=sweep_id,
namespace='fb',
ad_account_id=ad_account_id,
report_type=ReportType.lifetime,
report_variant=Entity.Campaign,
# \/ This is what we are testing \/
# comes from Persister code
# manually peeled off *Claim and injected into Data Flower
ad_account_timezone_name='Europe/London',
score=100,
)
assert job_scope.to_dict() == job_scope_should_be.to_dict()
def test_it_works_with_nulls(self):
assert NS == id_tools.generate_id()