def test_populate_from_scope_record(self):

        scope_id = gen_string_id()
        sweep_id = gen_string_id()

        console_token = 'console token'
        platform_token = 'platform token'

        scope_record = AssetScope()
        scope_record.scope = scope_id
        scope_record.scope_api_token = console_token
        scope_record.set_cache(platform_tokens={platform_token})

        PlatformTokenManager.populate_from_scope_entity(scope_record, sweep_id)

        # now let's make sure we see those tokens:

        # Scope-centered jobs must result in scope-centered key for token storage
        job_scope = JobScope(sweep_id=sweep_id,
                             entity_type=Entity.Scope,
                             entity_id=scope_id)
        assert console_token == PlatformTokenManager.from_job_scope(
            job_scope).get_best_token()

        job_scope = JobScope(
            sweep_id=sweep_id,
            # uses .namespace default value as 2nd value in redis key. no need to set here.
        )
        assert platform_token == PlatformTokenManager.from_job_scope(
            job_scope).get_best_token()
    def test_key_s3_date_snapped_with_chunk_id(self):
        """
        Check that the key is constructed as we expect
        """

        job_scope = JobScope(
            ad_account_id=gen_string_id(),
            report_type=ReportType.day_platform,
            report_variant=Entity.Ad,
            range_start=date(2000, 1, 2),
        )

        chunk_marker = 7

        dt_should_be = datetime(2000, 1, 2, 0, 0, 0)
        with mock.patch.object(uuid, 'uuid4', return_value='UUID-HERE'):
            storage_key = cold_storage.store({'data': 'yeah!'}, job_scope, chunk_marker=7)

        prefix = xxhash.xxh64(job_scope.ad_account_id).hexdigest()[:6]
        expected_key = (
            f'fb/'
            + f'{prefix}-{job_scope.ad_account_id}/'
            + f'{job_scope.report_type}/'
            + f'{dt_should_be.strftime("%Y")}/'
            + f'{dt_should_be.strftime("%m")}/'
            + f'{dt_should_be.strftime("%d")}/'
            + f'{dt_should_be.strftime("%Y-%m-%dT%H:%M:%SZ")}-'
            + f'{job_scope.job_id}-'
            + f'{chunk_marker}-'
            + f'UUID-HERE'
            + f'.json'
        )

        assert storage_key == expected_key
    def test_key_s3_date_less(self):
        """
        Check that the key is constructed as we expect
        """
        import common.tztools

        job_scope = JobScope(
            ad_account_id=gen_string_id(), report_type=ReportType.entity, report_variant=Entity.Campaign
        )

        now_dt = datetime(2000, 1, 2, 3, 4, 5)
        with mock.patch.object(common.tztools, 'now', return_value=now_dt) as now_mocked, mock.patch.object(
            uuid, 'uuid4', return_value='UUID-HERE'
        ):

            storage_key = cold_storage.store({'data': 'yeah!'}, job_scope)

        assert now_mocked.called

        prefix = xxhash.xxh64(job_scope.ad_account_id).hexdigest()[:6]

        expected_key = (
            f'fb/'
            + f'{prefix}-{job_scope.ad_account_id}/'
            + f'{job_scope.report_type}/'
            + f'{now_dt.strftime("%Y")}/'
            + f'{now_dt.strftime("%m")}/'
            + f'{now_dt.strftime("%d")}/'
            + f'{now_dt.strftime("%Y-%m-%dT%H:%M:%SZ")}-'
            + f'{job_scope.job_id}-'
            + f'UUID-HERE'
            + f'.json'
        )

        assert storage_key == expected_key
    def test_fails_with_wrong_report_variant(self):
        job_scope = JobScope(tokens=['blah'],
                             report_time=datetime.utcnow(),
                             report_type='entity',
                             report_variant=None,
                             sweep_id='1')

        with SweepRunningFlag(job_scope.sweep_id), mock.patch.object(
                report_job_status_task, 'delay') as status_task, mock.patch(
                    'common.error_inspector.BugSnagContextData.notify'
                ) as bugsnag_notify, mock.patch(
                    'common.error_inspector.API_KEY', 'something'):
            collect_pages_from_business_task(job_scope, JobContext())

            assert bugsnag_notify.called
            actual_args, actual_kwargs = bugsnag_notify.call_args
            assert (isinstance(actual_args[0], ValueError) and str(
                actual_args[0]) == 'Report level None specified is not: P'
                    ), 'Notify bugsnag correctly using correct Exception'
            assert {
                'severity': SEVERITY_ERROR,
                'job_scope': job_scope,
                'error_type': ErrorTypesReport.UNKNOWN,
            } == actual_kwargs, 'Notify bugsnag correctly'
            assert status_task.called
            parameters, _ = status_task.call_args
            assert (JobStatus.GenericError, job_scope
                    ) == parameters, 'Must report status correctly on failure'
    def test_fails_without_a_token(self):
        job_scope = JobScope(tokens=[None],
                             report_time=datetime.utcnow(),
                             report_type='entity',
                             report_variant=Entity.Page,
                             sweep_id='1')

        with SweepRunningFlag(job_scope.sweep_id), mock.patch.object(
                report_job_status_task, 'delay') as status_task, mock.patch(
                    'common.error_inspector.BugSnagContextData.notify'
                ) as bugsnag_notify, mock.patch(
                    'common.error_inspector.API_KEY', 'something'):
            collect_pages_from_business_task(job_scope, JobContext())

            assert bugsnag_notify.called
            actual_args, actual_kwargs = bugsnag_notify.call_args
            assert (
                str(actual_args[0]) ==
                'Job fb||||entity|P cannot proceed. No platform tokens provided.'
            ), 'Notify bugsnag correctly using correct Exception'
            assert {
                'severity': SEVERITY_ERROR,
                'job_scope': job_scope,
                'error_type': ErrorTypesReport.UNKNOWN,
            } == actual_kwargs, 'Notify bugsnag correctly'
            assert status_task.called

            status_task_args, _ = status_task.call_args
            assert (
                JobStatus.GenericError, job_scope
            ) == status_task_args, 'Must report status correctly on failure'
示例#6
0
def test_set_inaccessible_entity_task():
    mock_model = Mock()
    mock_factory = Mock(return_value=mock_model)
    with patch.dict('oozer.set_inaccessible_entity_task.ENTITY_TYPE_MODEL_MAP', {Entity.PagePost: mock_factory}):
        set_inaccessible_entity_task(JobScope(report_variant=Entity.PagePost))

        mock_model.update.assert_called_once_with(actions=[mock_factory.is_accessible.set(False)])
示例#7
0
    def iter_tasks(
        self
    ) -> Generator[Tuple[CeleryTask, JobScope, JobContext, int], None, None]:
        """Read persisted jobs and pass-through context objects for inspection"""
        with self.queue.JobsReader() as jobs_iter:
            for job_id, job_scope_additional_data, score in jobs_iter:

                job_id_parts = parse_id(job_id)
                job_scope = JobScope(job_scope_additional_data,
                                     job_id_parts,
                                     sweep_id=self.sweep_id,
                                     score=score)

                try:
                    celery_task = resolve_job_scope_to_celery_task(job_scope)
                    # TODO: Decide what to do with this.
                    # Was designed for massive hash collection and such,
                    # but cannot have too much data in there because we pickle it and put in on Redis
                    job_context = JobContext()
                    yield celery_task, job_scope, job_context, score
                    logger.info(
                        f"#{self.sweep_id}: Scheduling job_id {job_id} with score {score}."
                    )
                except InvalidJobScopeException as e:
                    ErrorInspector.inspect(e, job_scope.ad_account_id, {
                        'sweep_id': job_scope.sweep_id,
                        'job_id': job_scope.job_id
                    })
    def test_resolve_job_scope_to_celery_task_ad_account(
            self, mock_lifetime_iter, mock_breakdowns_iter):
        real_claim = RealityClaim(entity_id='A1',
                                  ad_account_id='A1',
                                  entity_type=Entity.AdAccount,
                                  timezone='America/Los_Angeles')
        mock_lifetime_iter.return_value = []
        mock_breakdowns_iter.return_value = [
            RealityClaim(
                entity_id='AD1',
                ad_account_id='A1',
                entity_type=Entity.Ad,
                range_start=datetime(2019, 1, 20, 12, 0),
                timezone='America/Los_Angeles',
            )
        ]
        for job_generator in entity_expectation_generator_map[
                Entity.AdAccount]:
            for exp_claim in job_generator(real_claim):
                with self.subTest(job_generator=job_generator,
                                  exp_claim=exp_claim):
                    job_scope = JobScope(parse_id(exp_claim.job_id))

                    assert inventory.resolve_job_scope_to_celery_task(
                        job_scope)
示例#9
0
    def test_task_error_is_logged_into_job_report(self):
        from oozer.common.report_job_status_task import report_job_status_task

        class MyException(Exception):
            pass

        sync_expectations_job_scope = JobScope(
            sweep_id=random.gen_string_id(),
            ad_account_id=random.gen_string_id(),
            report_type=ReportType.sync_expectations,
        )

        with mock.patch.object(report_job_status_task,
                               'delay') as job_report, mock.patch.object(
                                   sync_expectations_task,
                                   'sync_expectations',
                                   side_effect=MyException('nope!')):

            with self.assertRaises(MyException):
                sync_expectations_task.sync_expectations_task.delay(
                    sync_expectations_job_scope, None)

        assert job_report.called

        aa, kk = job_report.call_args

        assert not kk
        code, job_scope_actual = aa
        assert code < 0  # some sort of *Failure* code
        assert job_scope_actual == sync_expectations_job_scope
示例#10
0
    def test_task_does_not_blow_up(self):
        # this is almost same thing as the next test
        # where we check that call signature is right,
        # but when call signature changes and our tests don't,
        # it becomes irrelevant if we have tests - they check for wrong thing
        # So, here we actually call "store" and in next test
        # we intercept the call and check payload.
        # Don't remove me. Not duplicate.

        expectation_job_id = generate_id(
            ad_account_id=random.gen_string_id(),
            report_type=ReportType.day_hour,
            report_variant=Entity.Ad,
            range_start='2000-01-01',
        )
        rr = [expectation_job_id]

        sync_expectations_job_scope = JobScope(
            sweep_id=random.gen_string_id(),
            ad_account_id=random.gen_string_id(),
            report_type=ReportType.sync_expectations,
        )

        with mock.patch.object(expecations_store,
                               'iter_expectations_per_ad_account',
                               return_value=rr):
            sync_expectations_task.sync_expectations(
                sync_expectations_job_scope)
def test__job_scope_to_metadata():
    scope = JobScope(
        job_id='job identifier',
        namespace='fb',
        ad_account_id='007',
        report_type='report type',
        entity_type=Entity.Campaign,
        report_variant=Entity.Ad,
        range_start=datetime.fromtimestamp(1),
        score=10,
    )

    result = _job_scope_to_metadata(scope)

    result.pop('extracted_at')
    result.pop('build_id')

    assert {
        'job_id': 'fb|007|C||report+type|A|1970-01-01T00%3A00%3A01',
        'ad_account_id': '007',
        'report_type': 'report type',
        'entity_type': 'A',
        'platform_api_version': 'v6.0',
        'platform': 'fb',
        'score': '10',
    } == result
    def test_lifetime_campaigns(self):

        job_scope = JobScope(
            ad_account_id=AD_ACCOUNT,
            report_type=ReportType.lifetime,
            report_variant=Entity.Campaign,
            sweep_id='sweep',
            tokens=[TOKEN],
        )

        captured_data = []  # type: List[Tuple[Dict, JobScope, int]]

        def _store(data, job_scope, chunk_marker=0):
            captured_data.append((data, job_scope, chunk_marker))

        with mock.patch.object(cold_storage, 'store', _store):
            data_iter = Insights.iter_collect_insights(job_scope, None)
            cnt = 0
            for datum in data_iter:
                cnt += 1
                if cnt == 4:
                    break

        assert cnt == 4

        for datum, job_scope_inner, _ in captured_data:
            assert datum['campaign_id'] == job_scope_inner.entity_id
示例#13
0
    def test_task_is_called_with_right_data(self):

        range_start = now()
        range_start_should_be = range_start.strftime('%Y-%m-%d')

        expected_job_id = generate_id(
            ad_account_id=random.gen_string_id(),
            report_type=ReportType.day_hour,
            report_variant=Entity.Ad,
            range_start=range_start,
        )
        rr = [expected_job_id]
        expected_job_id_parts = parse_id_parts(expected_job_id)

        sync_expectations_job_scope = JobScope(
            sweep_id=random.gen_string_id(),
            ad_account_id=random.gen_string_id(),
            report_type=ReportType.sync_expectations,
        )

        with mock.patch.object(expecations_store,
                               'iter_expectations_per_ad_account',
                               return_value=rr) as jid_iter, mock.patch.object(
                                   cold_storage.ChunkDumpStore,
                                   'store') as store:

            sync_expectations_task.sync_expectations(
                sync_expectations_job_scope)

        assert jid_iter.called
        aa, kk = jid_iter.call_args
        assert not kk
        assert aa == (sync_expectations_job_scope.ad_account_id,
                      sync_expectations_job_scope.sweep_id)

        assert store.called
        aa, kk = store.call_args
        assert not kk
        assert len(aa) == 1

        data = aa[0]

        assert data == {
            'job_id': expected_job_id,
            # missing "ad_" is intentional.
            # this matches this attr name as sent by FB
            # and ysed by us elsewhere in the company
            'account_id': expected_job_id_parts.ad_account_id,
            'entity_type': expected_job_id_parts.entity_type,
            'entity_id': expected_job_id_parts.entity_id,
            'report_type': expected_job_id_parts.report_type,
            'report_variant': expected_job_id_parts.report_variant,
            'range_start':
            range_start_should_be,  # checking manually to ensure it's properly stringified
            'range_end': None,
            'platform_namespace': JobScope.namespace,  # default platform value
        }
    def setUp(self):

        self.job_scope = JobScope(
            sweep_id=gen_string_id(),
            ad_account_id=gen_string_id(),
            entity_id=gen_string_id(),
            entity_type=Entity.Campaign,
            report_type=ReportType.entity,
        )
示例#15
0
    def test_runs_correctly(self):
        account_id = random.gen_string_id()
        job_scope = JobScope(
            ad_account_id=self.ad_account_id,
            entity_id=self.ad_account_id,
            tokens=['A_REAL_TOKEN'],
            report_time=datetime.utcnow(),
            report_type='entity',
            report_variant=Entity.AdAccount,
            sweep_id='1',
        )

        universal_id_should_be = generate_universal_id(
            ad_account_id=self.ad_account_id,
            report_type=ReportType.entity,
            entity_id=self.ad_account_id,
            entity_type=Entity.AdAccount,
        )

        account_data = AdAccount(fbid=account_id)
        # Did not find a better way how to set this data on the inner AbstractCrudObject.
        timezone = 'Europe/Prague'
        account_data._data['timezone_name'] = timezone
        account_data._data['account_id'] = account_id

        with mock.patch.object(FB_ADACCOUNT_MODEL,
                               'api_get',
                               return_value=account_data), mock.patch.object(
                                   NormalStore, 'store') as store:
            collect_adaccount(job_scope)

        assert store.called_with(
            account_data), 'Data should be stored with the cold store module'

        assert store.called
        store_args, store_keyword_args = store.call_args
        assert not store_keyword_args
        assert len(
            store_args
        ) == 1, 'Store method should be called with just 1 parameter'

        data_actual = store_args[0]

        vendor_data_key = '__oprm'

        ad_account_dynamo = AdAccountEntity.get(DEFAULT_SCOPE, account_id)
        assert ad_account_dynamo.timezone == timezone
        assert ad_account_dynamo.ad_account_id == account_id

        assert (vendor_data_key in data_actual
                and type(data_actual[vendor_data_key])
                == dict), 'Special vendor key is present in the returned data'
        assert data_actual[vendor_data_key] == {
            'id': universal_id_should_be
        }, 'Vendor data is set with the right universal id'
    def test_resolve_job_scope_to_celery_task_page_post(self):
        real_claim = RealityClaim(entity_id='PP1',
                                  ad_account_id='P1',
                                  entity_type=Entity.PagePost)
        for job_generator in entity_expectation_generator_map[Entity.PagePost]:
            with self.subTest(job_generator=job_generator):
                exp_claim = next(job_generator(real_claim))

                job_scope = JobScope(parse_id(exp_claim.job_id))

                assert inventory.resolve_job_scope_to_celery_task(job_scope)
    def store(self, datum):
        from oozer.common.report_job_status_task import report_job_status_task

        entity_id = datum.get(self.id_attribute_name) or datum.get('id')
        assert entity_id, "This code must have an entity ID for building of unique insertion ID"
        normative_job_scope = JobScope(self.job_scope_base_data, entity_id=entity_id)
        # and store data under that per-entity, normative JobScope.
        self._store(datum, normative_job_scope, DEFAULT_CHUNK_NUMBER, self.bucket_type)
        # since we report for many entities in this code,
        # must also communicate out the status inside of the for-loop
        # at the normative level.
        report_job_status_task.delay(JobStatus.Done, normative_job_scope)
    def test_from_job_scope(self):

        key_gen = '{asset_scope}-{sweep_id}-sorted-token-queue'.format

        sweep_id = gen_string_id()
        entity_id = gen_string_id()
        scope_id = gen_string_id()

        # Scope-centered jobs must result in scope-centered key for token storage
        job_scope = JobScope(sweep_id=sweep_id,
                             entity_type=Entity.Scope,
                             entity_id=scope_id)
        token_manager = PlatformTokenManager.from_job_scope(job_scope)
        assert token_manager.queue_key == key_gen(asset_scope=scope_id,
                                                  sweep_id=sweep_id)

        # non-Scope-centered jobs must result in 'fb'-centered key for token storage
        job_scope = JobScope(sweep_id=sweep_id)
        token_manager = PlatformTokenManager.from_job_scope(job_scope)
        assert token_manager.queue_key == key_gen(
            asset_scope=JobScope.namespace, sweep_id=sweep_id)
    def test_resolve_job_scope_to_celery_task_page(self):
        real_claim = RealityClaim(entity_id='P1',
                                  ad_account_id='P1',
                                  entity_type=Entity.Page,
                                  timezone='America/Los_Angeles')
        for job_generator in entity_expectation_generator_map[Entity.Page]:
            for exp_claim in job_generator(real_claim):
                with self.subTest(job_generator=job_generator,
                                  exp_claim=exp_claim):
                    job_scope = JobScope(parse_id(exp_claim.job_id))

                    assert inventory.resolve_job_scope_to_celery_task(
                        job_scope)
示例#20
0
    def test_task_complains_about_bad_report_type(self):

        sync_expectations_job_scope = JobScope(
            sweep_id=random.gen_string_id(),
            ad_account_id=random.gen_string_id(),
            report_type=ReportType.lifetime,  # <----------- this is wrong
        )

        with self.assertRaises(AssertionError) as ex_catcher:
            sync_expectations_task.sync_expectations(
                sync_expectations_job_scope)

        assert 'Only sync_expectations report' in str(ex_catcher.exception)
def sync_expectations(job_scope: JobScope):
    assert (job_scope.report_type == ReportType.sync_expectations
            ), 'Only sync_expectations report type is processed by this task'

    if job_scope.ad_account_id:
        # this is per AA task. No need to iterate over all
        ad_account_ids_iter = [job_scope.ad_account_id]
    else:
        ad_account_ids_iter = expecations_store.iter_expectations_ad_accounts(
            sweep_id=job_scope.sweep_id)

    for ad_account_id in ad_account_ids_iter:

        ad_account_scoped_job_scope = JobScope(job_scope.to_dict(),
                                               ad_account_id=ad_account_id,
                                               entity_type=Entity.AdAccount,
                                               entity_id=ad_account_id)

        with ChunkDumpStore(ad_account_scoped_job_scope,
                            chunk_size=200) as store:

            job_ids_iter = expecations_store.iter_expectations_per_ad_account(
                ad_account_id, ad_account_scoped_job_scope.sweep_id)

            for job_id in job_ids_iter:
                job_id_parts = parse_id_parts(job_id)

                # default is platform namespace and we communicate out only those
                if job_id_parts.namespace == JobScope.namespace:
                    store({
                        'job_id':
                        job_id,
                        # 'status':'expected',
                        'account_id':
                        job_id_parts.ad_account_id,
                        'entity_type':
                        job_id_parts.entity_type,
                        'entity_id':
                        job_id_parts.entity_id,
                        'report_type':
                        job_id_parts.report_type,
                        'report_variant':
                        job_id_parts.report_variant,
                        'range_start':
                        _to_date_string_if_set(job_id_parts.range_start),
                        'range_end':
                        _to_date_string_if_set(job_id_parts.range_end),
                        'platform_namespace':
                        job_id_parts.namespace,
                    })
示例#22
0
    def test_fails_with_wrong_report_variant(self):
        job_scope = JobScope(
            ad_account_id=self.ad_account_id,
            entity_id=self.ad_account_id,
            tokens=['blah'],
            report_time=datetime.utcnow(),
            report_type='entity',
            report_variant=None,  # This actually should be set to AdAccount
            sweep_id='1',
        )

        with self.assertRaises(ValueError) as ex_trap:
            collect_adaccount(job_scope)

        assert 'Report level' in str(ex_trap.exception)
示例#23
0
    def test_page_import(self):

        page_id = random.gen_string_id()

        pages = [dict(ad_account_id=page_id)]

        job_scope = JobScope(
            sweep_id=self.sweep_id,
            entity_type=Entity.Scope,
            entity_id=self.scope_id,
            report_type=ReportType.import_accounts,
            report_variant=Entity.Page,
            tokens=['token'],
        )

        with mock.patch.object(
                ConsoleApi, 'get_pages', return_value=pages
        ) as gp, mock.patch.object(
                PageEntity, 'upsert'
        ) as page_upsert, mock.patch.object(
                report_job_status_task, 'delay'
        ) as status_task, mock.patch(
                'oozer.entities.import_scope_entities_task._have_entity_access',
                return_value=True) as _have_entity_access_mock:
            import_pages_task(job_scope, None)

        assert gp.called

        assert status_task.called
        # it was called many times, but we care about the last time only
        aa, kk = status_task.call_args
        assert not kk
        assert aa == (JobStatus.Done, job_scope)

        assert page_upsert.call_count == 1

        page_upsert_args = (
            (self.scope_id, page_id),
            {
                'is_active': True,
                'updated_by_sweep_id': self.sweep_id,
                'is_accessible': True
            },
        )

        args1 = page_upsert.call_args_list[0]

        assert args1 == page_upsert_args
    def test_entity_level_data(self):

        entity_types = [Entity.Campaign, Entity.AdSet, Entity.Ad]

        # intentionally NOT reusing collect_insights._entity_type_id_field_map map
        # effectively, here we are testing it too.
        entity_id_attr_name_map = {
            Entity.Campaign: 'campaign_id',
            Entity.AdSet: 'adset_id',
            Entity.Ad: 'ad_id'
        }

        for entity_type in entity_types:
            with self.subTest(f'Entity type = "{entity_type}"'):
                input_data = {entity_id_attr_name_map[entity_type]: 'SomeID'}

                job_scope = JobScope(
                    sweep_id=self.sweep_id,
                    ad_account_id=self.ad_account_id,
                    report_type=ReportType.lifetime,
                    report_variant=entity_type,
                    tokens=['blah'],
                )

                with mock.patch.object(collect_insights.Insights,
                                       'iter_ads_insights',
                                       return_value=[input_data
                                                     ]), mock.patch.object(
                                                         ChunkDumpStore,
                                                         'store') as store:

                    data_iter = collect_insights.Insights.iter_collect_insights(
                        job_scope, None)
                    assert len(list(data_iter)) == 1

                aa, kk = store.call_args
                assert not kk
                assert aa == ({
                    entity_id_attr_name_map[entity_type]: 'SomeID',
                    '__oprm': {
                        'id':
                        f'oprm|m|fb|{self.ad_account_id}|{entity_type}|SomeID|lifetime',
                        'entity_id': 'SomeID',
                        'entity_type': entity_type,
                    },
                    '__transformed': {},
                }, )
    def test_hourly_ads_per_parent(self):

        job_scope = JobScope(
            ad_account_id=AD_ACCOUNT,
            range_start='2017-12-31',
            ad_account_timezone_name='Europe/London',
            report_type=ReportType.day_hour,
            report_variant=Entity.Ad,
            sweep_id='sweep',
            tokens=[TOKEN],
        )

        data_iter = Insights.iter_collect_insights(job_scope, None)
        cnt = 0
        for datum in data_iter:
            cnt += 1
            if cnt == 4:
                break
    def test_fetch_ad_account_pipeline(self):

        job_scope = JobScope(
            ad_account_id=AD_ACCOUNT,
            entity_id=AD_ACCOUNT,
            tokens=[TOKEN],
            report_time=datetime.utcnow(),
            report_type='entity',
            report_variant=Entity.AdAccount,
            sweep_id='1',
        )

        aa_data = collect_adaccount_task(job_scope, None)
        aa_data_keys = aa_data.keys()

        assert aa_data

        for required_field in get_default_fields(AdAccount):
            assert required_field in aa_data_keys, f'{required_field} should be in the response from FB'
    def test_pipeline_page_posts(self):

        job_scope = JobScope(
            ad_account_id=AD_ACCOUNT,
            tokens=[TOKEN],
            report_time=datetime.utcnow(),
            report_type='entity',
            report_variant=Entity.PagePost,
            sweep_id='1',
        )

        data_iter = iter_collect_entities_per_page(job_scope)

        cnt = 0
        for _ in data_iter:
            cnt += 1
            break

        assert cnt
    def test_pipeline_custom_audiences(self):

        job_scope = JobScope(
            ad_account_id=AD_ACCOUNT,
            tokens=[TOKEN],
            report_time=datetime.utcnow(),
            report_type='entity',
            report_variant=Entity.CustomAudience,
            sweep_id='1',
        )

        data_iter = iter_collect_entities_per_adaccount(job_scope)

        cnt = 0
        for _ in data_iter:
            cnt += 1
            break

        assert cnt
    def test_hourly_ad(self):

        job_scope = JobScope(
            # report_variant=Entity.Ad,
            ad_account_id=AD_ACCOUNT,
            entity_id='23842698250300224',
            entity_type=Entity.Ad,
            range_start='2017-12-31',
            report_type=ReportType.day_dma,
            sweep_id='sweep',
            tokens=[TOKEN],
        )

        data_iter = Insights.iter_collect_insights(job_scope, None)
        cnt = 0
        for datum in data_iter:
            cnt += 1
            if cnt == 4:
                break
def _report_job_done_to_cold_store(job_scope: JobScope):
    reporting_job_scope = JobScope(
        sweep_id=job_scope.sweep_id, ad_account_id=job_scope.ad_account_id, report_type=ReportType.sync_status
    )

    cold_storage.store(
        {
            'job_id': job_scope.job_id,
            'account_id': job_scope.ad_account_id,
            'entity_type': job_scope.entity_type,
            'entity_id': job_scope.entity_id,
            'report_type': job_scope.report_type,
            'report_variant': job_scope.report_variant,
            'range_start': _to_date_string_if_set(job_scope.range_start),
            'range_end': _to_date_string_if_set(job_scope.range_end),
            'platform_namespace': job_scope.namespace,
        },
        reporting_job_scope,
    )