def test_resolve_job_scope_to_celery_task_ad_account(
self, mock_lifetime_iter, mock_breakdowns_iter):
real_claim = RealityClaim(entity_id='A1',
ad_account_id='A1',
entity_type=Entity.AdAccount,
timezone='America/Los_Angeles')
mock_lifetime_iter.return_value = []
mock_breakdowns_iter.return_value = [
RealityClaim(
entity_id='AD1',
ad_account_id='A1',
entity_type=Entity.Ad,
range_start=datetime(2019, 1, 20, 12, 0),
timezone='America/Los_Angeles',
)
]
for job_generator in entity_expectation_generator_map[
Entity.AdAccount]:
for exp_claim in job_generator(real_claim):
with self.subTest(job_generator=job_generator,
exp_claim=exp_claim):
job_scope = JobScope(parse_id(exp_claim.job_id))
assert inventory.resolve_job_scope_to_celery_task(
job_scope)
def test_day_metrics_per_entity_under_ad_account_not_divisible(
mock_iter_reality_per_ad_account):
reality_claim = RealityClaim(
ad_account_id='ad-account-id',
entity_id='ad-account-id',
entity_type=Entity.AdAccount,
timezone='America/Los_Angeles',
)
mock_iter_reality_per_ad_account.return_value = [
RealityClaim(
entity_type=Entity.Ad,
campaign_id='campaign-1',
adset_id='adset-1',
entity_id='ad-1',
bol=datetime(2019, 1, 1, 12, 0),
eol=datetime(2019, 1, 2, 12, 0),
),
RealityClaim(entity_type=Entity.Ad,
entity_id='ad-2',
bol=datetime(2019, 1, 3, 12, 0),
eol=datetime(2019, 1, 3, 12, 0)),
]
result = list(
day_metrics_per_ads_under_ad_account([ReportType.day], reality_claim))
assert result == [
ExpectationClaim(
'ad-account-id',
Entity.AdAccount,
ReportType.day,
Entity.Ad,
JobSignature('fb|ad-account-id|||day|A|2019-01-01'),
ad_account_id='ad-account-id',
timezone='America/Los_Angeles',
range_start=date(2019, 1, 1),
),
ExpectationClaim(
'ad-account-id',
Entity.AdAccount,
ReportType.day,
Entity.Ad,
JobSignature('fb|ad-account-id|||day|A|2019-01-02'),
ad_account_id='ad-account-id',
timezone='America/Los_Angeles',
range_start=date(2019, 1, 2),
),
ExpectationClaim(
'ad-account-id',
Entity.AdAccount,
ReportType.day,
Entity.Ad,
JobSignature('fb|ad-account-id|||day|A|2019-01-03'),
ad_account_id='ad-account-id',
timezone='America/Los_Angeles',
range_start=date(2019, 1, 3),
),
]
def test_lifetime_metrics_per_entity_under_ad_account_is_divisible(
mock_iter_reality_per_ad_account):
reality_claim = RealityClaim(
ad_account_id='ad-account-id',
entity_id='ad-account-id',
entity_type=Entity.AdAccount,
timezone='America/Los_Angeles',
)
mock_iter_reality_per_ad_account.return_value = [
RealityClaim(
entity_type=Entity.Ad,
campaign_id='campaign-1',
adset_id='adset-1',
entity_id='ad-1',
bol=datetime(2019, 1, 1, 12, 0),
eol=datetime(2019, 1, 1, 12, 0),
)
]
result = list(
lifetime_metrics_per_entity_under_ad_account(Entity.Ad, reality_claim))
assert result == [
ExpectationClaim(
'ad-account-id',
Entity.AdAccount,
ReportType.lifetime,
Entity.Ad,
JobSignature('fb|ad-account-id|||lifetime|A'),
ad_account_id='ad-account-id',
timezone='America/Los_Angeles',
entity_hierarchy=EntityNode(
'ad-account-id',
Entity.AdAccount,
children=[
EntityNode(
'campaign-1',
Entity.Campaign,
children=[
EntityNode(
'adset-1',
Entity.AdSet,
children=[EntityNode('ad-1', Entity.Ad)])
],
)
],
),
)
]
def test_aa_collection_expectation():
ad_account_id = gen_string_id()
reality_claim = RealityClaim(ad_account_id=ad_account_id,
entity_id=ad_account_id,
entity_type=Entity.AdAccount)
def is_adaccount_entity_job(expectation_claim):
parsed_id_parts = parse_id_parts(expectation_claim.job_id)
return parsed_id_parts.report_type == ReportType.entity and parsed_id_parts.report_variant == Entity.AdAccount
adaccount_entity_expectations = list(
filter(is_adaccount_entity_job, iter_expectations([reality_claim])))
assert len(adaccount_entity_expectations) == 1
expectation_claim = adaccount_entity_expectations[0]
assert expectation_claim.entity_id == reality_claim.ad_account_id
assert expectation_claim.entity_type == reality_claim.entity_type
assert expectation_claim.job_id == generate_id(
ad_account_id=ad_account_id,
entity_id=ad_account_id,
report_type=ReportType.entity,
report_variant=Entity.AdAccount,
)
def iter_reality_base() -> Generator[RealityClaim, None, None]:
"""
A generator yielding instances of RealityClaim object, filled
with data about some entity (one of Scope, AdAccount)
These claims represent our knowledge about the what exists.
Some consuming code will match these claims of existence to tasks
we are expected to perform for these objects.
:return: Generator yielding RealityClaim objects pertaining to various levels of entities
"""
# Scopes are top-level objects in our system
# they are like binders into which we put some top-level platform assets
# and their associated stuff (like tokens etc)
for scope_record in iter_scopes():
# Each of the scopes has their own refresh tasks
# These are triggered by our claim of existence of the scope
# For example, reaching out to console API
# to refresh the list of AdAccount records
# This scope exists claim:
yield RealityClaim(entity_id=scope_record.scope,
entity_type=Entity.Scope)
# For all the ad accounts we already know are attached to the scope
# we need to kick of their refresh tasks, thus,
# yielding out existence claims pertaining to AdAccounts per scope
for ad_account in iter_active_ad_accounts_per_scope(
scope_record.scope):
# This reality claim is base for expectation to have
# its children lists refreshed.
yield RealityClaim(
ad_account_id=ad_account.ad_account_id,
entity_id=ad_account.ad_account_id,
entity_type=Entity.AdAccount,
timezone=ad_account.timezone,
)
for page in iter_active_pages_per_scope(scope_record.scope):
yield RealityClaim(ad_account_id=page.page_id,
entity_id=page.page_id,
entity_type=Entity.Page)
def test_iter_expectations_generates_jobs_from_map(mock_map):
expected_claim = Mock()
mock_map.get.return_value = [Mock(return_value=[expected_claim])]
reality_claim = RealityClaim(entity_id=gen_string_id(),
entity_type=Entity.AdAccount)
assert [expected_claim] == list(iter_expectations([reality_claim]))
def test_resolve_job_scope_to_celery_task_page_post(self):
real_claim = RealityClaim(entity_id='PP1',
ad_account_id='P1',
entity_type=Entity.PagePost)
for job_generator in entity_expectation_generator_map[Entity.PagePost]:
with self.subTest(job_generator=job_generator):
exp_claim = next(job_generator(real_claim))
job_scope = JobScope(parse_id(exp_claim.job_id))
assert inventory.resolve_job_scope_to_celery_task(job_scope)
def test_resolve_job_scope_to_celery_task_page(self):
real_claim = RealityClaim(entity_id='P1',
ad_account_id='P1',
entity_type=Entity.Page,
timezone='America/Los_Angeles')
for job_generator in entity_expectation_generator_map[Entity.Page]:
for exp_claim in job_generator(real_claim):
with self.subTest(job_generator=job_generator,
exp_claim=exp_claim):
job_scope = JobScope(parse_id(exp_claim.job_id))
assert inventory.resolve_job_scope_to_celery_task(
job_scope)
def iter_reality_per_page_claim(
page_claim: RealityClaim,
entity_types: List[str] = None) -> Generator[RealityClaim, None, None]:
"""
A generator yielding instances of RealityClaim object, filled
with data about some entity (one of Campaign, AdSet, Ad)
These claims represent our knowledge about the what exists.
Some consuming code will match these claims of existence to tasks
we are expected to perform for these objects.
:param page_claim: A RealityClaim instance representing existence of Page
:param entity_types: If truthy, limits the reality iterator to those types of entities only.
:return: Generator yielding RealityClaim objects pertaining to various levels of entities
"""
for entity_data in iter_entities_per_page_id(
page_claim.entity_id, page_entity_types=entity_types):
yield RealityClaim(entity_data)
def test_aa_import_expectation_generated():
entity_id = gen_string_id()
reality_claim = RealityClaim(entity_type=Entity.Scope, entity_id=entity_id)
results = list(iter_expectations([reality_claim]))
assert len(results) == 2
expectation_claim = results[0]
assert expectation_claim.entity_id == reality_claim.entity_id
assert expectation_claim.entity_type == reality_claim.entity_type
assert expectation_claim.job_id == generate_id(
namespace=config.application.UNIVERSAL_ID_SYSTEM_NAMESPACE,
entity_type=Entity.Scope,
entity_id=entity_id,
report_type=ReportType.import_accounts,
report_variant=Entity.AdAccount,
)
def iter_reality_per_ad_account_claim(
ad_account_claim: RealityClaim,
entity_types: List[str] = None) -> Generator[RealityClaim, None, None]:
"""
A generator yielding instances of RealityClaim object, filled
with data about some entity (one of Campaign, AdSet, Ad)
These claims represent our knowledge about the what exists.
Some consuming code will match these claims of existence to tasks
we are expected to perform for these objects.
:param ad_account_claim: A RealityClaim instance representing existence of AdAccount
:param entity_types: If truethy, limits the reality iterator to those types of entities only.
:return: Generator yielding RealityClaim objects pertaining to various levels of entities
"""
# Naturally, we may know about some of the AdAccount's children
# existing already and might need their supporting data refreshed too.
for entity_data in iter_entities_per_ad_account_id(
ad_account_claim.ad_account_id, entity_types=entity_types):
yield RealityClaim(entity_data, timezone=ad_account_claim.timezone)