def test_read_records_all(self, mocker, api):
"""1. yield all from mock
2. if read slice 2, 3 state not changed
if read slice 2, 3, 1 state changed to 3
"""
job = mocker.Mock(spec=InsightAsyncJob)
job.get_result.return_value = [
mocker.Mock(), mocker.Mock(),
mocker.Mock()
]
job.interval = pendulum.Period(pendulum.date(2010, 1, 1),
pendulum.date(2010, 1, 1))
stream = AdsInsights(
api=api,
start_date=datetime(2010, 1, 1),
end_date=datetime(2011, 1, 1),
insights_lookback_window=28,
)
records = list(
stream.read_records(
sync_mode=SyncMode.incremental,
stream_slice={"insight_job": job},
))
assert len(records) == 3
def test_stream_slices_with_state_close_to_now(self, api,
async_manager_mock,
recent_start_date):
"""Stream will use start_date when close to now and start_date close to now"""
start_date = recent_start_date
end_date = pendulum.now()
cursor_value = end_date - duration(days=1)
state = {AdsInsights.cursor_field: cursor_value.date().isoformat()}
stream = AdsInsights(api=api,
start_date=start_date,
end_date=end_date,
insights_lookback_window=28)
async_manager_mock.completed_jobs.return_value = [1, 2, 3]
slices = list(
stream.stream_slices(stream_state=state,
sync_mode=SyncMode.incremental))
assert slices == [{
"insight_job": 1
}, {
"insight_job": 2
}, {
"insight_job": 3
}]
async_manager_mock.assert_called_once()
args, kwargs = async_manager_mock.call_args
generated_jobs = list(kwargs["jobs"])
assert len(generated_jobs) == (end_date - start_date).days + 1
assert generated_jobs[0].interval.start == start_date.date()
assert generated_jobs[1].interval.start == start_date.date(
) + duration(days=1)
def test_stream_slices_with_state(self, api, async_manager_mock,
start_date):
"""Stream will use cursor_value from state when there is state"""
end_date = start_date + duration(days=10)
cursor_value = start_date + duration(days=5)
state = {AdsInsights.cursor_field: cursor_value.date().isoformat()}
stream = AdsInsights(api=api,
start_date=start_date,
end_date=end_date,
insights_lookback_window=28)
async_manager_mock.completed_jobs.return_value = [1, 2, 3]
slices = list(
stream.stream_slices(stream_state=state,
sync_mode=SyncMode.incremental))
assert slices == [{
"insight_job": 1
}, {
"insight_job": 2
}, {
"insight_job": 3
}]
async_manager_mock.assert_called_once()
args, kwargs = async_manager_mock.call_args
generated_jobs = list(kwargs["jobs"])
assert len(generated_jobs) == (end_date - cursor_value).days
assert generated_jobs[0].interval.start == cursor_value.date(
) + duration(days=1)
assert generated_jobs[1].interval.start == cursor_value.date(
) + duration(days=2)
def test_stream_slices_no_state(self, api, async_manager_mock, start_date):
"""Stream will use start_date when there is not state"""
end_date = start_date + duration(weeks=2)
stream = AdsInsights(api=api,
start_date=start_date,
end_date=end_date,
insights_lookback_window=28)
async_manager_mock.completed_jobs.return_value = [1, 2, 3]
slices = list(
stream.stream_slices(stream_state=None,
sync_mode=SyncMode.incremental))
assert slices == [{
"insight_job": 1
}, {
"insight_job": 2
}, {
"insight_job": 3
}]
async_manager_mock.assert_called_once()
args, kwargs = async_manager_mock.call_args
generated_jobs = list(kwargs["jobs"])
assert len(generated_jobs) == (end_date - start_date).days + 1
assert generated_jobs[0].interval.start == start_date.date()
assert generated_jobs[1].interval.start == start_date.date(
) + duration(days=1)
def test_get_json_schema(self, api):
stream = AdsInsights(api=api,
start_date=datetime(2010, 1, 1),
end_date=datetime(2011, 1, 1),
insights_lookback_window=28)
schema = stream.get_json_schema()
assert "device_platform" not in schema["properties"]
assert "country" not in schema["properties"]
assert not (set(stream.fields) - set(
schema["properties"].keys())), "all fields present in schema"
def _update_insights_streams(self, insights: List[InsightConfig],
default_args, streams) -> List[Type[Stream]]:
"""Update method, if insights have values returns streams replacing the
default insights streams else returns streams
"""
if not insights:
return streams
insights_custom_streams = list()
for insight in insights:
args = dict(
api=default_args["api"],
name=f"Custom{insight.name}",
fields=list(set(insight.fields)),
breakdowns=list(set(insight.breakdowns)),
action_breakdowns=list(set(insight.action_breakdowns)),
time_increment=insight.time_increment,
start_date=insight.start_date or default_args["start_date"],
end_date=insight.end_date or default_args["end_date"],
insights_lookback_window=insight.insights_lookback_window
or default_args["insights_lookback_window"],
)
insight_stream = AdsInsights(**args)
insights_custom_streams.append(insight_stream)
return streams + insights_custom_streams
def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
"""Discovery method, returns available streams
:param config: A Mapping of the user input configuration as defined in the connector spec.
"""
config: ConnectorConfig = ConnectorConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK
api = API(account_id=config.account_id, access_token=config.access_token)
insights_args = dict(
api=api,
start_date=config.start_date,
buffer_days=config.insights_lookback_window,
days_per_job=config.insights_days_per_job,
)
return [
Campaigns(api=api, start_date=config.start_date, include_deleted=config.include_deleted),
AdSets(api=api, start_date=config.start_date, include_deleted=config.include_deleted),
Ads(api=api, start_date=config.start_date, include_deleted=config.include_deleted),
AdCreatives(api=api),
AdsInsights(**insights_args),
AdsInsightsAgeAndGender(**insights_args),
AdsInsightsCountry(**insights_args),
AdsInsightsRegion(**insights_args),
AdsInsightsDma(**insights_args),
AdsInsightsPlatformAndDevice(**insights_args),
]
def test_state(self, api, state):
"""State setter/getter should work with all combinations"""
stream = AdsInsights(api=api,
start_date=datetime(2010, 1, 1),
end_date=datetime(2011, 1, 1),
insights_lookback_window=28)
assert stream.state == {}
stream.state = state
actual_state = stream.state
actual_state["slices"] = sorted(actual_state.get("slices", []))
state["slices"] = sorted(state.get("slices", []))
state["time_increment"] = 1
assert actual_state == state
def test_init(self, api):
stream = AdsInsights(api=api,
start_date=datetime(2010, 1, 1),
end_date=datetime(2011, 1, 1),
insights_lookback_window=28)
assert not stream.breakdowns
assert stream.action_breakdowns == AdsInsights.ALL_ACTION_BREAKDOWNS
assert stream.name == "ads_insights"
assert stream.primary_key == ["date_start", "account_id", "ad_id"]
def test_fields_custom(self, api):
stream = AdsInsights(
api=api,
start_date=datetime(2010, 1, 1),
end_date=datetime(2011, 1, 1),
fields=["account_id", "account_currency"],
insights_lookback_window=28,
)
assert stream.fields == ["account_id", "account_currency"]
def test_fields(self, api):
stream = AdsInsights(
api=api,
start_date=datetime(2010, 1, 1),
end_date=datetime(2011, 1, 1),
insights_lookback_window=28,
)
fields = stream.fields
assert "account_id" in fields
assert "account_currency" in fields
assert "actions" in fields
def test_init_override(self, api):
stream = AdsInsights(
api=api,
start_date=datetime(2010, 1, 1),
end_date=datetime(2011, 1, 1),
name="CustomName",
breakdowns=["test1", "test2"],
action_breakdowns=["field1", "field2"],
insights_lookback_window=28,
)
assert stream.breakdowns == ["test1", "test2"]
assert stream.action_breakdowns == ["field1", "field2"]
assert stream.name == "custom_name"
assert stream.primary_key == [
"date_start", "account_id", "ad_id", "test1", "test2"
]
def _update_insights_streams(self, insights, args, streams) -> List[Type[Stream]]:
"""Update method, if insights have values returns streams replacing the
default insights streams else returns streams
"""
if not insights:
return streams
insights_custom_streams = list()
for insight in insights:
args["name"] = f"Custom{insight.name}"
args["fields"] = list(set(insight.fields))
args["breakdowns"] = list(set(insight.breakdowns))
args["action_breakdowns"] = list(set(insight.action_breakdowns))
insight_stream = AdsInsights(**args)
insights_custom_streams.append(insight_stream)
return streams + insights_custom_streams
def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
"""Discovery method, returns available streams
:param config: A Mapping of the user input configuration as defined in the connector spec.
:return: list of the stream instances
"""
config: ConnectorConfig = ConnectorConfig.parse_obj(config)
api = API(account_id=config.account_id,
access_token=config.access_token)
insights_args = dict(
api=api,
start_date=config.start_date,
end_date=config.end_date,
insights_lookback_window=config.insights_lookback_window)
streams = [
AdAccount(api=api),
AdSets(
api=api,
start_date=config.start_date,
end_date=config.end_date,
include_deleted=config.include_deleted,
page_size=config.page_size,
max_batch_size=config.max_batch_size,
),
Ads(
api=api,
start_date=config.start_date,
end_date=config.end_date,
include_deleted=config.include_deleted,
page_size=config.page_size,
max_batch_size=config.max_batch_size,
),
AdCreatives(
api=api,
fetch_thumbnail_images=config.fetch_thumbnail_images,
page_size=config.page_size,
max_batch_size=config.max_batch_size,
),
AdsInsights(page_size=config.page_size,
max_batch_size=config.max_batch_size,
**insights_args),
AdsInsightsAgeAndGender(page_size=config.page_size,
max_batch_size=config.max_batch_size,
**insights_args),
AdsInsightsCountry(page_size=config.page_size,
max_batch_size=config.max_batch_size,
**insights_args),
AdsInsightsRegion(page_size=config.page_size,
max_batch_size=config.max_batch_size,
**insights_args),
AdsInsightsDma(page_size=config.page_size,
max_batch_size=config.max_batch_size,
**insights_args),
AdsInsightsPlatformAndDevice(page_size=config.page_size,
max_batch_size=config.max_batch_size,
**insights_args),
AdsInsightsActionType(page_size=config.page_size,
max_batch_size=config.max_batch_size,
**insights_args),
Campaigns(
api=api,
start_date=config.start_date,
end_date=config.end_date,
include_deleted=config.include_deleted,
page_size=config.page_size,
max_batch_size=config.max_batch_size,
),
Images(
api=api,
start_date=config.start_date,
end_date=config.end_date,
include_deleted=config.include_deleted,
page_size=config.page_size,
max_batch_size=config.max_batch_size,
),
Videos(
api=api,
start_date=config.start_date,
end_date=config.end_date,
include_deleted=config.include_deleted,
page_size=config.page_size,
max_batch_size=config.max_batch_size,
),
Activities(
api=api,
start_date=config.start_date,
end_date=config.end_date,
include_deleted=config.include_deleted,
page_size=config.page_size,
max_batch_size=config.max_batch_size,
),
]
return self._update_insights_streams(insights=config.custom_insights,
default_args=insights_args,
streams=streams)
