def test_check_connection_backoff_on_limit_reached(requests_mock, config):
"""Error once, check that we retry and not fail"""
responses = [
{
"json": {
"error": "limit reached"
},
"status_code": 429,
"headers": {
"Retry-After": "0"
}
},
{
"json": [],
"status_code": 200
},
]
requests_mock.register_uri("GET", "/properties/v2/contact/properties",
responses)
source = SourceHubspot()
alive, error = source.check_connection(logger=logger, config=config)
assert alive
assert not error
def test_sync_with_latest_state(self, config,
configured_catalog_with_incremental):
"""Sync first time, save the state and sync second time with saved state from previous sync"""
streams = {
stream.stream.name: stream
for stream in configured_catalog_with_incremental.streams
}
records1, states1 = read_stream(SourceHubspot(), config,
configured_catalog_with_incremental)
assert states1, "should have at least one state emitted"
assert records1, "should have at least few records emitted"
records2, states2 = read_stream(SourceHubspot(), config,
configured_catalog_with_incremental,
states1[-1].data)
assert states1[-1] == states2[-1], "final states should be the same"
for stream_name, state in states2[-1].data.items():
cursor_field = streams[stream_name].cursor_field[0]
old_records1 = records_older(
records1[stream_name],
than=records2[stream_name][0].data[cursor_field],
cursor_field=cursor_field)
old_records2 = records_older(
records2[stream_name],
than=records2[stream_name][0].data[cursor_field],
cursor_field=cursor_field)
assert list(
old_records1), "should have older records from the first read"
assert not list(
old_records2
), "should not have older records from the second read"
def test_it_should_not_read_quotes_stream_if_it_does_not_exist_in_client(
oauth_config, configured_catalog):
"""
If 'quotes' stream is not in the client, it should skip it.
"""
source = SourceHubspot()
all_records = list(
source.read(logger,
config=oauth_config,
catalog=configured_catalog,
state=None))
records = [record for record in all_records if record.type == Type.RECORD]
assert not records
def test_check_connection_backoff_on_server_error(requests_mock, config):
"""Error once, check that we retry and not fail"""
responses = [
{
"json": {
"error": "something bad"
},
"status_code": 500
},
{
"json": [],
"status_code": 200
},
]
requests_mock.register_uri("GET", "/properties/v2/contact/properties",
responses)
source = SourceHubspot()
alive, error = source.check_connection(logger=logger, config=config)
assert alive
assert not error
def read_stream(
source: SourceHubspot, config: Mapping, catalog: ConfiguredAirbyteCatalog, state: MutableMapping = None
) -> Tuple[Mapping, List]:
records = {}
states = []
for message in source.read(AirbyteLogger(), config, catalog, state):
if message.type == Type.RECORD:
records.setdefault(message.record.stream, [])
records[message.record.stream].append(message.record)
elif message.type == Type.STATE:
states.append(message.state)
return records, states
def test_check_connection_ok(requests_mock, config):
responses = [
{
"json": [],
"status_code": 200
},
]
requests_mock.register_uri("GET", "/properties/v2/contact/properties",
responses)
ok, error_msg = SourceHubspot().check_connection(logger, config=config)
assert ok
assert not error_msg
def test_check_credential_title_exception(config):
config["credentials"].pop("credentials_title")
with pytest.raises(Exception):
SourceHubspot().check_connection(logger, config=config)
def test_streams(config):
streams = SourceHubspot().streams(config)
assert len(streams) == 27
def test_check_connection_exception(config):
ok, error_msg = SourceHubspot().check_connection(logger, config=config)
assert not ok
assert error_msg
def test_check_connection_invalid_config(config):
config.pop("start_date")
with pytest.raises(TypeError):
SourceHubspot().check_connection(logger, config=config)
def test_check_connection_empty_config(config):
config = {}
with pytest.raises(KeyError):
SourceHubspot().check_connection(logger, config=config)
def common_params_fixture(config):
source = SourceHubspot()
common_params = source.get_common_params(config=config)
return common_params