def collect_page(job_scope: JobScope, _job_context: JobContext):
"""
Collect a single facebook page
"""
if job_scope.report_variant != Entity.Page:
raise ValueError(
f"Report level {job_scope.report_variant} specified is not: {Entity.Page}"
)
token = job_scope.token
if not token:
raise ValueError(
f"Job {job_scope.job_id} cannot proceed. No platform tokens provided."
)
# We don't use it for getting a token. Something else that calls us does.
# However, we use it to report usages of the token we got.
token_manager = PlatformTokenManager.from_job_scope(job_scope)
with PlatformApiContext(token) as fb_ctx:
page_inst = page.Page(fbid=job_scope.entity_id, api=fb_ctx.api)
page_fetched = page_inst.api_get(fields=get_default_fields(Page))
report_job_status_task.delay(ExternalPlatformJobStatus.DataFetched,
job_scope)
token_manager.report_usage(token, 2)
record_id_data = job_scope.to_dict()
record_id_data.update(entity_type=Entity.Page,
entity_id=job_scope.entity_id,
report_variant=None)
entity_data = page_fetched.export_all_data()
entity_data = add_vendor_data(
entity_data, id=generate_universal_id(**record_id_data))
store = NormalStore(job_scope)
store.store(entity_data)
def _report_failure(job_scope: JobScope, start_time: float, exc: Exception,
**kwargs: Any):
"""Report task stats when task fails."""
end_time = time.time()
job_scope.running_time = math.ceil(end_time - start_time)
job_scope.datapoint_count = kwargs.get('partial_datapoint_count')
ErrorInspector.inspect(exc, job_scope.ad_account_id,
{'job_scope': job_scope})
if isinstance(exc, FacebookRequestError):
failure_status, failure_bucket = FacebookApiErrorInspector(
exc).get_status_and_bucket()
else:
failure_status, failure_bucket = ExternalPlatformJobStatus.GenericError, FailureBucket.Other
# No entity type means we don't know what table to target
if failure_bucket == FailureBucket.InaccessibleObject and job_scope.entity_type is not None:
set_inaccessible_entity_task.delay(job_scope)
report_job_status_task.delay(failure_status, job_scope)
PlatformTokenManager.from_job_scope(
job_scope).report_usage_per_failure_bucket(job_scope.token,
failure_bucket)
SweepStatusTracker(job_scope.sweep_id).report_status(failure_bucket)
_send_measurement_task_runtime(job_scope, failure_bucket)
def collect_pages_from_business(job_scope: JobScope,
_job_context: JobContext) -> int:
"""
Collect all facebook pages that are active
"""
if job_scope.report_variant != Entity.Page:
raise ValueError(
f"Report level {job_scope.report_variant} specified is not: {Entity.Page}"
)
token = job_scope.token
if not token:
raise ValueError(
f"Job {job_scope.job_id} cannot proceed. No platform tokens provided."
)
# We don't use it for getting a token. Something else that calls us does.
# However, we use it to report usages of the token we got.
token_manager = PlatformTokenManager.from_job_scope(job_scope)
with PlatformApiContext(token) as fb_ctx:
fb_req = FacebookRequest(node_id="me",
method="GET",
endpoint="/businesses",
api=fb_ctx.api,
api_type='EDGE',
target_class=Business)
businesses = fb_req.execute()
report_job_status_task.delay(ExternalPlatformJobStatus.DataFetched,
job_scope)
token_manager.report_usage(token)
entity_type = Entity.Page
record_id_base_data = job_scope.to_dict()
record_id_base_data.update(entity_type=entity_type, report_variant=None)
cnt = 0
for biz in businesses:
client_pages = list(
biz.get_client_pages(fields=get_default_fields(Page)))
owned_pages = list(
biz.get_owned_pages(fields=get_default_fields(Page)))
pages_list = client_pages + owned_pages
for page_inst in pages_list:
entity_data = page_inst.export_all_data()
record_id_base_data.update(entity_id=entity_data.get('id'))
entity_data = add_vendor_data(
entity_data, id=generate_universal_id(**record_id_base_data))
store = NormalStore(job_scope)
store.store(entity_data)
cnt += 1
report_job_status_task.delay(ExternalPlatformJobStatus.Done, job_scope)
return cnt
def _report_success(job_scope: JobScope, start_time: float, ret_value: Any):
"""Report task stats when successful."""
end_time = time.time()
job_scope.running_time = math.ceil(end_time - start_time)
if isinstance(ret_value, int):
job_scope.datapoint_count = ret_value
report_job_status_task.delay(ExternalPlatformJobStatus.Done, job_scope)
SweepStatusTracker(job_scope.sweep_id).report_status(FailureBucket.Success)
_send_measurement_task_runtime(job_scope, FailureBucket.Success)
def store(self, datum):
from oozer.common.report_job_status_task import report_job_status_task
entity_id = datum.get(self.id_attribute_name) or datum.get('id')
assert entity_id, "This code must have an entity ID for building of unique insertion ID"
normative_job_scope = JobScope(self.job_scope_base_data, entity_id=entity_id)
# and store data under that per-entity, normative JobScope.
self._store(datum, normative_job_scope, DEFAULT_CHUNK_NUMBER, self.bucket_type)
# since we report for many entities in this code,
# must also communicate out the status inside of the for-loop
# at the normative level.
report_job_status_task.delay(JobStatus.Done, normative_job_scope)
def sync_expectations_task(job_scope: JobScope, _):
from oozer.common.report_job_status_task import report_job_status_task
try:
sync_expectations(job_scope)
except Exception as ex:
BugSnagContextData.notify(ex, job_scope=job_scope)
# This is a generic failure, which does not help us at all, so, we just
# report it and bail
report_job_status_task.delay(JobStatus.GenericError, job_scope)
raise
report_job_status_task.delay(JobStatus.Done, job_scope)