class AssetScope(BaseModel, MemoizeMixin):
"""
Stores metadata specific to "Scope" that acts as grouping element
for a collection of attached top-level assets like Ad Accounts
This is approximately mapped to a Facebook User (token) or some
cohesive source of Ad Accounts.
Initially used for tracking / managing the per-sweep sync of Ad Account IDs from
Console into our internal store for later iteration over that collection.
"""
Meta = BaseMeta(dynamodb_config.AD_ACCOUNT_SCOPE_TABLE)
# scope is an ephemeral scoping element
# Imagine "operam business manager system user" being one of the scope's values.
# This is here mostly just to simplify iterating
# through AAs per given known source.
# (even if originally we will have only one scope in entire system)
# At first, scope will be pegged one-to-one to
# one token to be used for all AAs,
# (Later this relationship may need to be inverted to
# allow multiple tokens per AA)
scope = attributes.UnicodeAttribute(hash_key=True, attr_name='scope')
# TODO: move storage of scope's API tokens, credentials to this model
# scope_api_credentials = Blah()
# while we have only one scope - Console - we can fake it
# by sending same exact token for all (ahem.. one and only) models
# This obviously needs to rethought at some point.
# (Part of the rethinking is that this record has to match a
# particular partner API implementation that matches this scope
# It's crazy to expect that AUTH will be exactly same and that it will
# be just by token.)
scope_api_token = operam_console_api_config.TOKEN
platform_token_ids = attributes.UnicodeSetAttribute(default=set,
attr_name='tids')
# Memoized in instance to avoid hitting DB all the time we are called.
@property
@memoized_property
def platform_tokens(self):
"""
Returns a set of actual FB tokens that token_ids attribute point to by ids
:return:
"""
return {
record.token
for record in PlatformToken.scan(
PlatformToken.token_id.is_in(*self.platform_token_ids))
}
@property
def platform_token(self):
try:
return list(self.platform_tokens)[0]
except IndexError:
return None
class TestModel(BaseModel):
Meta = BaseMeta(random.gen_string_id())
primary_id = attributes.UnicodeAttribute(hash_key=True,
attr_name='pid')
secondary_id = attributes.UnicodeAttribute(range_key=True,
attr_name='sid')
data = attributes.UnicodeAttribute(null=True, attr_name='d')
more_data = attributes.UnicodeAttribute(null=True, attr_name='d2')
class TestModel(BaseModel):
_fields = {'secondary_id', 'record_type'}
Meta = BaseMeta(random.gen_string_id())
primary_id = attributes.UnicodeAttribute(hash_key=True,
attr_name='pid')
secondary_id = attributes.UnicodeAttribute(range_key=True,
attr_name='sid')
data = attributes.UnicodeAttribute(null=True, attr_name='d')
record_type = 'SUPER_RECORD'
class PlatformToken(BaseModel):
"""
At this time we have no corporate API for
(a) management of platform user token assets and linking them to FB entities
(b) proxying requests to platform over some corporate platform proxy API
(which would remove the need for passing actual tokens around here)
When these structural parts are put in,
remove this table and migrate code to rely on other sources of token data
"""
Meta = BaseMeta(dynamodb_config.TOKEN_TABLE)
token_id = attributes.UnicodeAttribute(hash_key=True, attr_name='tid')
token = attributes.UnicodeAttribute(attr_name='t')
class JobReport(BaseModel, MemoizeMixin):
# this table is split from JobReportEntityExpectation
# in order to allow blind massive upserts of JobReportEntityExpectation
# and JobReportEntityOutcome records without caring about
# present values.
# Reading these tables combined is very inefficient
# (O(kn) where k is number of pages in JobReportEntityExpectation
# and n number of entities in the system)
# compared to reading from what unified table could have been,
# but we care much more about fast idempotent writes in batches
# more than fast occasional reads.
Meta = BaseMeta(dynamodb_config.JOB_REPORT_TABLE)
# value of job_id here could be super weird.
# It's actually the value of JobReportEntityExpectation.job_id
# which there never has entity_id, entity_type filled in
# but here that exact same ID is used as template and
# entity_id, entity_type are filled in.
# In this job_id you may see both, entity_type=Campaign and
# report_variant=Campaign - not something you would see
# in a wild. This is done just for one reason - create
# an ID that is compound of JobReportEntityExpectation.job_id
# and entity_id for use in this table only.
job_id = attributes.UnicodeAttribute(hash_key=True, attr_name='jid')
last_progress_dt = attributes.UTCDateTimeAttribute(null=True, attr_name='pdt')
last_progress_stage_id = attributes.NumberAttribute(null=True, attr_name='pstid')
last_progress_sweep_id = attributes.UnicodeAttribute(null=True, attr_name='psid')
last_success_dt = attributes.UTCDateTimeAttribute(null=True, attr_name='sdt')
last_success_sweep_id = attributes.UnicodeAttribute(null=True, attr_name='ssid')
last_failure_dt = attributes.UTCDateTimeAttribute(null=True, attr_name='fdt')
last_failure_stage_id = attributes.NumberAttribute(null=True, attr_name='fstid')
last_failure_sweep_id = attributes.UnicodeAttribute(null=True, attr_name='fsid')
last_failure_error = attributes.UnicodeAttribute(null=True, attr_name='fmessage')
last_failure_bucket = attributes.NumberAttribute(null=True, attr_name='fb')
last_total_running_time = attributes.NumberAttribute(null=True, attr_name='trt')
last_total_datapoint_count = attributes.NumberAttribute(null=True, attr_name='tdc')
last_partial_running_time = attributes.NumberAttribute(null=True, attr_name='prt')
last_partial_datapoint_count = attributes.NumberAttribute(null=True, attr_name='pdc')
fails_in_row = attributes.NumberAttribute(attr_name='fir')
class AdAccountEntity(ConsoleEntityMixin, BaseModel):
"""
Represents a single facebook ad account entity
"""
Meta = BaseMeta(dynamodb_config.AD_ACCOUNT_ENTITY_TABLE)
_additional_fields = {'entity_type'}
# scope is an ephemeral scoping element
# Imagine "operam business manager system user" being one of the scope's values.
# This is here mostly just to simplify iterating
# through AAs per given known source.
# (even if originally we will have only one scope in entire system)
# At first, scope will be pegged one-to-one to
# one token to be used for all AAs,
# (Later this relationship may need to be inverted to
# allow multiple tokens per AA)
scope = attributes.UnicodeAttribute(hash_key=True, attr_name='scope')
ad_account_id = attributes.UnicodeAttribute(range_key=True,
attr_name='aaid')
# copied indicator of activity from Console DB per each sync
# (alternative to deletion. To be discussed later if deletion is better)
is_active = attributes.BooleanAttribute(default=False, attr_name='a')
is_accessible = attributes.BooleanAttribute(default=True,
attr_name='is_accessible')
# Provides an option to manually disable accounts from syncing, even if they are imported as active from console.
manually_disabled = attributes.BooleanAttribute(default=False,
attr_name='man_dis')
# utilized by logic that prunes out Ad Accounts
# that are switched to "inactive" on Console
# Expectation is that after a long-running update job
# there is a task at the end that goes back and marks
# all AA records with non-last-sweep_id as "inactive"
# See https://operam.atlassian.net/browse/PROD-1825 for context
updated_by_sweep_id = attributes.UnicodeAttribute(null=True, attr_name='u')
# Each AdAccount on FB side can be set to a particular timezone
# A lot of reporting on FB is pegged to a "day" that is interpreted
# in that AdAccount's timezone (not UTC).
timezone = attributes.UnicodeAttribute(attr_name='tz')
# manually controlled through Dynamo UI. Here we just read it
# does not have to be set for majority of records.
score_multiplier = attributes.NumberAttribute(null=True,
attr_name='score_skew')
refresh_if_older_than = attributes.UTCDateTimeAttribute(
null=True, attr_name='refresh_to')
entity_type = Entity.AdAccount
@property
@memoized_property
def scope_model(self):
from common.store.scope import AssetScope
return AssetScope.get(self.scope)
def to_fb_sdk_ad_account(self, api=None):
"""
Returns an instance of Facebook Ads SDK AdAccount model
with ID matching this DB model's ID
:param api: FB Ads SDK Api instance with token baked in.
"""
from facebook_business.api import FacebookAdsApi, FacebookSession
from facebook_business.adobjects.adaccount import AdAccount
if not api:
# This is very expensive call. Takes 2 DB hits to get token value
# Try to pass api value at all times in prod code to avoid using this.
# Allowing to omit the API value to simplify use of this API in
# testing in console.
api = FacebookAdsApi(
FacebookSession(access_token=self.scope_model.platform_token))
return AdAccount(fbid=f'act_{self.ad_account_id}', api=api)
@classmethod
def upsert_entity_from_console(cls, job_scope: JobScope, entity: Any,
is_accessible: bool):
cls.upsert(
job_scope.entity_id, # scope ID
entity['ad_account_id'],
is_active=entity.get('active', True),
updated_by_sweep_id=job_scope.sweep_id,
is_accessible=is_accessible,
)