def collect_organic_insights_task(job_scope: JobScope, _: JobContext):
logger.info(f'{job_scope} started')
if not job_scope.tokens:
good_token = PlatformTokenManager.from_job_scope(
job_scope).get_best_token()
if good_token is not None:
job_scope.tokens = [good_token]
data_iter = InsightsOrganic.iter_collect_insights(job_scope)
cnt = 0
try:
for _ in data_iter:
cnt += 1
if cnt % 100 == 0:
logger.info(f'{job_scope} processed {cnt} data points so far')
except Exception as e:
# re-raising causes loss of original stack trace. printing it.
ErrorInspector.inspect(e, job_scope.ad_account_id,
{'job_scope': job_scope})
raise CollectionError(e, cnt)
logger.info(f'{job_scope} complete a total of {cnt} data points')
return cnt
def collect_adaccount_task(job_scope: JobScope, _: JobContext):
logger.info(f'{job_scope} started')
if not job_scope.tokens:
good_token = PlatformTokenManager.from_job_scope(
job_scope).get_best_token()
if good_token is not None:
job_scope.tokens = [good_token]
collect_adaccount(job_scope)
def collect_page_task(job_scope: JobScope, job_context: JobContext) -> int:
logger.info(f'{job_scope} started')
if not job_scope.tokens:
good_token = PlatformTokenManager.from_job_scope(
job_scope).get_best_token()
if good_token is not None:
job_scope.tokens = [good_token]
collect_page(job_scope, job_context)
return 1 # we collect 1 page at a time
def collect_pages_from_business_task(job_scope: JobScope,
job_context: JobContext) -> int:
"""
This task should import pages from FB using Business API. At the moment, it is not used anywhere.
"""
logger.info(f'{job_scope} started')
if not job_scope.tokens:
good_token = PlatformTokenManager.from_job_scope(
job_scope).get_best_token()
if good_token is not None:
job_scope.tokens = [good_token]
return collect_pages_from_business(job_scope, job_context)
def collect_entities_from_iterator(
job_scope: JobScope, entity_iterator: Generator[object, None,
None]) -> int:
logger.info(f'{job_scope} started')
if not job_scope.tokens:
good_token = PlatformTokenManager.from_job_scope(
job_scope).get_best_token()
if good_token is not None:
job_scope.tokens = [good_token]
cnt = 0
try:
for cnt, datum in enumerate(entity_iterator):
if cnt % 100 == 0:
logger.info(f'{job_scope} processed {cnt} data points so far')
except Exception as e:
raise CollectionError(e, cnt)
logger.info(f'{job_scope} complete a total of {cnt} data points')
return cnt
def collect_insights_task(job_scope: JobScope, job_context: JobContext):
logger.info(f'{job_scope} started')
if not job_scope.tokens:
good_token = PlatformTokenManager.from_job_scope(
job_scope).get_best_token()
if good_token is not None:
job_scope.tokens = [good_token]
data_iter = Insights.iter_collect_insights(job_scope, job_context)
cnt = 0
try:
for cnt, datum in enumerate(data_iter):
if cnt % 100 == 0:
logger.info(f'{job_scope} processed {cnt} data points so far')
except Exception as e:
raise CollectionError(e, cnt)
logger.info(f'{job_scope} complete a total of {cnt} data points')
return cnt
