def test_check_connection_backoff_on_limit_reached(requests_mock, config): """Error once, check that we retry and not fail""" responses = [ { "json": { "error": "limit reached" }, "status_code": 429, "headers": { "Retry-After": "0" } }, { "json": [], "status_code": 200 }, ] requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) source = SourceHubspot() alive, error = source.check_connection(logger=logger, config=config) assert alive assert not error
def test_sync_with_latest_state(self, config, configured_catalog_with_incremental): """Sync first time, save the state and sync second time with saved state from previous sync""" streams = { stream.stream.name: stream for stream in configured_catalog_with_incremental.streams } records1, states1 = read_stream(SourceHubspot(), config, configured_catalog_with_incremental) assert states1, "should have at least one state emitted" assert records1, "should have at least few records emitted" records2, states2 = read_stream(SourceHubspot(), config, configured_catalog_with_incremental, states1[-1].data) assert states1[-1] == states2[-1], "final states should be the same" for stream_name, state in states2[-1].data.items(): cursor_field = streams[stream_name].cursor_field[0] old_records1 = records_older( records1[stream_name], than=records2[stream_name][0].data[cursor_field], cursor_field=cursor_field) old_records2 = records_older( records2[stream_name], than=records2[stream_name][0].data[cursor_field], cursor_field=cursor_field) assert list( old_records1), "should have older records from the first read" assert not list( old_records2 ), "should not have older records from the second read"
def test_it_should_not_read_quotes_stream_if_it_does_not_exist_in_client( oauth_config, configured_catalog): """ If 'quotes' stream is not in the client, it should skip it. """ source = SourceHubspot() all_records = list( source.read(logger, config=oauth_config, catalog=configured_catalog, state=None)) records = [record for record in all_records if record.type == Type.RECORD] assert not records
def test_check_connection_backoff_on_server_error(requests_mock, config): """Error once, check that we retry and not fail""" responses = [ { "json": { "error": "something bad" }, "status_code": 500 }, { "json": [], "status_code": 200 }, ] requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) source = SourceHubspot() alive, error = source.check_connection(logger=logger, config=config) assert alive assert not error
def read_stream( source: SourceHubspot, config: Mapping, catalog: ConfiguredAirbyteCatalog, state: MutableMapping = None ) -> Tuple[Mapping, List]: records = {} states = [] for message in source.read(AirbyteLogger(), config, catalog, state): if message.type == Type.RECORD: records.setdefault(message.record.stream, []) records[message.record.stream].append(message.record) elif message.type == Type.STATE: states.append(message.state) return records, states
def test_check_connection_ok(requests_mock, config): responses = [ { "json": [], "status_code": 200 }, ] requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) ok, error_msg = SourceHubspot().check_connection(logger, config=config) assert ok assert not error_msg
def test_check_credential_title_exception(config): config["credentials"].pop("credentials_title") with pytest.raises(Exception): SourceHubspot().check_connection(logger, config=config)
def test_streams(config): streams = SourceHubspot().streams(config) assert len(streams) == 27
def test_check_connection_exception(config): ok, error_msg = SourceHubspot().check_connection(logger, config=config) assert not ok assert error_msg
def test_check_connection_invalid_config(config): config.pop("start_date") with pytest.raises(TypeError): SourceHubspot().check_connection(logger, config=config)
def test_check_connection_empty_config(config): config = {} with pytest.raises(KeyError): SourceHubspot().check_connection(logger, config=config)
def common_params_fixture(config): source = SourceHubspot() common_params = source.get_common_params(config=config) return common_params