def collect_organic_insights_task(job_scope: JobScope, _: JobContext): logger.info(f'{job_scope} started') if not job_scope.tokens: good_token = PlatformTokenManager.from_job_scope( job_scope).get_best_token() if good_token is not None: job_scope.tokens = [good_token] data_iter = InsightsOrganic.iter_collect_insights(job_scope) cnt = 0 try: for _ in data_iter: cnt += 1 if cnt % 100 == 0: logger.info(f'{job_scope} processed {cnt} data points so far') except Exception as e: # re-raising causes loss of original stack trace. printing it. ErrorInspector.inspect(e, job_scope.ad_account_id, {'job_scope': job_scope}) raise CollectionError(e, cnt) logger.info(f'{job_scope} complete a total of {cnt} data points') return cnt
def collect_adaccount_task(job_scope: JobScope, _: JobContext): logger.info(f'{job_scope} started') if not job_scope.tokens: good_token = PlatformTokenManager.from_job_scope( job_scope).get_best_token() if good_token is not None: job_scope.tokens = [good_token] collect_adaccount(job_scope)
def collect_page_task(job_scope: JobScope, job_context: JobContext) -> int: logger.info(f'{job_scope} started') if not job_scope.tokens: good_token = PlatformTokenManager.from_job_scope( job_scope).get_best_token() if good_token is not None: job_scope.tokens = [good_token] collect_page(job_scope, job_context) return 1 # we collect 1 page at a time
def collect_pages_from_business_task(job_scope: JobScope, job_context: JobContext) -> int: """ This task should import pages from FB using Business API. At the moment, it is not used anywhere. """ logger.info(f'{job_scope} started') if not job_scope.tokens: good_token = PlatformTokenManager.from_job_scope( job_scope).get_best_token() if good_token is not None: job_scope.tokens = [good_token] return collect_pages_from_business(job_scope, job_context)
def collect_entities_from_iterator( job_scope: JobScope, entity_iterator: Generator[object, None, None]) -> int: logger.info(f'{job_scope} started') if not job_scope.tokens: good_token = PlatformTokenManager.from_job_scope( job_scope).get_best_token() if good_token is not None: job_scope.tokens = [good_token] cnt = 0 try: for cnt, datum in enumerate(entity_iterator): if cnt % 100 == 0: logger.info(f'{job_scope} processed {cnt} data points so far') except Exception as e: raise CollectionError(e, cnt) logger.info(f'{job_scope} complete a total of {cnt} data points') return cnt
def collect_insights_task(job_scope: JobScope, job_context: JobContext): logger.info(f'{job_scope} started') if not job_scope.tokens: good_token = PlatformTokenManager.from_job_scope( job_scope).get_best_token() if good_token is not None: job_scope.tokens = [good_token] data_iter = Insights.iter_collect_insights(job_scope, job_context) cnt = 0 try: for cnt, datum in enumerate(data_iter): if cnt % 100 == 0: logger.info(f'{job_scope} processed {cnt} data points so far') except Exception as e: raise CollectionError(e, cnt) logger.info(f'{job_scope} complete a total of {cnt} data points') return cnt