def test_read_records_all(self, mocker, api): """1. yield all from mock 2. if read slice 2, 3 state not changed if read slice 2, 3, 1 state changed to 3 """ job = mocker.Mock(spec=InsightAsyncJob) job.get_result.return_value = [ mocker.Mock(), mocker.Mock(), mocker.Mock() ] job.interval = pendulum.Period(pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1)) stream = AdsInsights( api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28, ) records = list( stream.read_records( sync_mode=SyncMode.incremental, stream_slice={"insight_job": job}, )) assert len(records) == 3
def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, recent_start_date): """Stream will use start_date when close to now and start_date close to now""" start_date = recent_start_date end_date = pendulum.now() cursor_value = end_date - duration(days=1) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list( stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) assert slices == [{ "insight_job": 1 }, { "insight_job": 2 }, { "insight_job": 3 }] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) assert len(generated_jobs) == (end_date - start_date).days + 1 assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date( ) + duration(days=1)
def test_stream_slices_with_state(self, api, async_manager_mock, start_date): """Stream will use cursor_value from state when there is state""" end_date = start_date + duration(days=10) cursor_value = start_date + duration(days=5) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list( stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) assert slices == [{ "insight_job": 1 }, { "insight_job": 2 }, { "insight_job": 3 }] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) assert len(generated_jobs) == (end_date - cursor_value).days assert generated_jobs[0].interval.start == cursor_value.date( ) + duration(days=1) assert generated_jobs[1].interval.start == cursor_value.date( ) + duration(days=2)
def test_stream_slices_no_state(self, api, async_manager_mock, start_date): """Stream will use start_date when there is not state""" end_date = start_date + duration(weeks=2) stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list( stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental)) assert slices == [{ "insight_job": 1 }, { "insight_job": 2 }, { "insight_job": 3 }] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) assert len(generated_jobs) == (end_date - start_date).days + 1 assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date( ) + duration(days=1)
def test_get_json_schema(self, api): stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) schema = stream.get_json_schema() assert "device_platform" not in schema["properties"] assert "country" not in schema["properties"] assert not (set(stream.fields) - set( schema["properties"].keys())), "all fields present in schema"
def _update_insights_streams(self, insights: List[InsightConfig], default_args, streams) -> List[Type[Stream]]: """Update method, if insights have values returns streams replacing the default insights streams else returns streams """ if not insights: return streams insights_custom_streams = list() for insight in insights: args = dict( api=default_args["api"], name=f"Custom{insight.name}", fields=list(set(insight.fields)), breakdowns=list(set(insight.breakdowns)), action_breakdowns=list(set(insight.action_breakdowns)), time_increment=insight.time_increment, start_date=insight.start_date or default_args["start_date"], end_date=insight.end_date or default_args["end_date"], insights_lookback_window=insight.insights_lookback_window or default_args["insights_lookback_window"], ) insight_stream = AdsInsights(**args) insights_custom_streams.append(insight_stream) return streams + insights_custom_streams
def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: """Discovery method, returns available streams :param config: A Mapping of the user input configuration as defined in the connector spec. """ config: ConnectorConfig = ConnectorConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK api = API(account_id=config.account_id, access_token=config.access_token) insights_args = dict( api=api, start_date=config.start_date, buffer_days=config.insights_lookback_window, days_per_job=config.insights_days_per_job, ) return [ Campaigns(api=api, start_date=config.start_date, include_deleted=config.include_deleted), AdSets(api=api, start_date=config.start_date, include_deleted=config.include_deleted), Ads(api=api, start_date=config.start_date, include_deleted=config.include_deleted), AdCreatives(api=api), AdsInsights(**insights_args), AdsInsightsAgeAndGender(**insights_args), AdsInsightsCountry(**insights_args), AdsInsightsRegion(**insights_args), AdsInsightsDma(**insights_args), AdsInsightsPlatformAndDevice(**insights_args), ]
def test_state(self, api, state): """State setter/getter should work with all combinations""" stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) assert stream.state == {} stream.state = state actual_state = stream.state actual_state["slices"] = sorted(actual_state.get("slices", [])) state["slices"] = sorted(state.get("slices", [])) state["time_increment"] = 1 assert actual_state == state
def test_init(self, api): stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) assert not stream.breakdowns assert stream.action_breakdowns == AdsInsights.ALL_ACTION_BREAKDOWNS assert stream.name == "ads_insights" assert stream.primary_key == ["date_start", "account_id", "ad_id"]
def test_fields_custom(self, api): stream = AdsInsights( api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], insights_lookback_window=28, ) assert stream.fields == ["account_id", "account_currency"]
def test_fields(self, api): stream = AdsInsights( api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28, ) fields = stream.fields assert "account_id" in fields assert "account_currency" in fields assert "actions" in fields
def test_init_override(self, api): stream = AdsInsights( api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), name="CustomName", breakdowns=["test1", "test2"], action_breakdowns=["field1", "field2"], insights_lookback_window=28, ) assert stream.breakdowns == ["test1", "test2"] assert stream.action_breakdowns == ["field1", "field2"] assert stream.name == "custom_name" assert stream.primary_key == [ "date_start", "account_id", "ad_id", "test1", "test2" ]
def _update_insights_streams(self, insights, args, streams) -> List[Type[Stream]]: """Update method, if insights have values returns streams replacing the default insights streams else returns streams """ if not insights: return streams insights_custom_streams = list() for insight in insights: args["name"] = f"Custom{insight.name}" args["fields"] = list(set(insight.fields)) args["breakdowns"] = list(set(insight.breakdowns)) args["action_breakdowns"] = list(set(insight.action_breakdowns)) insight_stream = AdsInsights(**args) insights_custom_streams.append(insight_stream) return streams + insights_custom_streams
def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: """Discovery method, returns available streams :param config: A Mapping of the user input configuration as defined in the connector spec. :return: list of the stream instances """ config: ConnectorConfig = ConnectorConfig.parse_obj(config) api = API(account_id=config.account_id, access_token=config.access_token) insights_args = dict( api=api, start_date=config.start_date, end_date=config.end_date, insights_lookback_window=config.insights_lookback_window) streams = [ AdAccount(api=api), AdSets( api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, max_batch_size=config.max_batch_size, ), Ads( api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, max_batch_size=config.max_batch_size, ), AdCreatives( api=api, fetch_thumbnail_images=config.fetch_thumbnail_images, page_size=config.page_size, max_batch_size=config.max_batch_size, ), AdsInsights(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), AdsInsightsAgeAndGender(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), AdsInsightsCountry(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), AdsInsightsRegion(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), AdsInsightsDma(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), AdsInsightsPlatformAndDevice(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), AdsInsightsActionType(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), Campaigns( api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, max_batch_size=config.max_batch_size, ), Images( api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, max_batch_size=config.max_batch_size, ), Videos( api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, max_batch_size=config.max_batch_size, ), Activities( api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, max_batch_size=config.max_batch_size, ), ] return self._update_insights_streams(insights=config.custom_insights, default_args=insights_args, streams=streams)