def check_connection(self, logger: AirbyteLogger,
config: Mapping[str, Any]) -> Tuple[bool, Any]:
try:
params = {"access_key": config["access_key"]}
base = config.get("base")
if base is not None:
params["base"] = base
resp = requests.get(
f"{ExchangeRates.url_base}{config['start_date']}",
params=params)
status = resp.status_code
logger.info(f"Ping response code: {status}")
if status == 200:
return True, None
# When API requests is sent but the requested data is not available or the API call fails
# for some reason, a JSON error is returned.
# https://exchangeratesapi.io/documentation/#errors
error = resp.json().get("error")
code = error.get("code")
message = error.get("message") or error.get("info")
# If code is base_currency_access_restricted, error is caused by switching base currency while using free
# plan
if code == "base_currency_access_restricted":
message = f"{message} (this plan doesn't support selecting the base currency)"
return False, message
except Exception as e:
return False, e
def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]:
try:
logger.info("Checking the config")
GoogleAds(credentials=config["credentials"], customer_id=config["customer_id"])
return True, None
except Exception as error:
return False, f"Unable to connect to Google Ads API with the provided credentials - {repr(error)}"
def set_sub_primary_key(self):
if isinstance(self.primary_key, list):
for index, value in enumerate(self.primary_key):
setattr(self, f"sub_primary_key_{index + 1}", value)
else:
logger = AirbyteLogger()
logger.error(
"Failed during setting sub primary keys. Primary key should be list."
)
def read(
self,
logger: AirbyteLogger,
config: Mapping[str, Any],
catalog: ConfiguredAirbyteCatalog,
state: Optional[MutableMapping[str, Any]] = None,
) -> Iterator[AirbyteMessage]:
"""
Overwritten to dynamically receive only those streams that are necessary for reading for significant speed gains
(Salesforce has a strict API limit on requests).
"""
connector_state = copy.deepcopy(state or {})
config, internal_config = split_config(config)
# get the streams once in case the connector needs to make any queries to generate them
logger.info("Starting generating streams")
stream_instances = {
s.name: s
for s in self.streams(config, catalog=catalog, state=state)
}
logger.info(f"Starting syncing {self.name}")
self._stream_to_instance_map = stream_instances
for configured_stream in catalog.streams:
stream_instance = stream_instances.get(
configured_stream.stream.name)
if not stream_instance:
raise KeyError(
f"The requested stream {configured_stream.stream.name} was not found in the source. Available streams: {stream_instances.keys()}"
)
try:
yield from self._read_stream(
logger=logger,
stream_instance=stream_instance,
configured_stream=configured_stream,
connector_state=connector_state,
internal_config=internal_config,
)
except exceptions.HTTPError as error:
error_data = error.response.json()[0]
error_code = error_data.get("errorCode")
if error.response.status_code == codes.FORBIDDEN and error_code == "REQUEST_LIMIT_EXCEEDED":
logger.warn(
f"API Call limit is exceeded. Error message: '{error_data.get('message')}'"
)
break # if got 403 rate limit response, finish the sync with success.
raise error
except Exception as e:
logger.exception(
f"Encountered an exception while reading stream {self.name}"
)
raise e
logger.info(f"Finished syncing {self.name}")
def read(
self, logger: AirbyteLogger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: MutableMapping[str, Any] = None
) -> Iterator[AirbyteMessage]:
"""
Overwritten to dynamically receive only those streams that are necessary for reading for significant speed gains
(Salesforce has a strict API limit on requests).
"""
connector_state = copy.deepcopy(state or {})
config, internal_config = split_config(config)
# get the streams once in case the connector needs to make any queries to generate them
logger.info("Starting generating streams")
stream_instances = {s.name: s for s in self.streams(config, catalog=catalog)}
logger.info(f"Starting syncing {self.name}")
self._stream_to_instance_map = stream_instances
for configured_stream in catalog.streams:
stream_instance = stream_instances.get(configured_stream.stream.name)
if not stream_instance:
raise KeyError(
f"The requested stream {configured_stream.stream.name} was not found in the source. Available streams: {stream_instances.keys()}"
)
try:
yield from self._read_stream(
logger=logger,
stream_instance=stream_instance,
configured_stream=configured_stream,
connector_state=connector_state,
internal_config=internal_config,
)
except Exception as e:
logger.exception(f"Encountered an exception while reading stream {self.name}")
raise e
logger.info(f"Finished syncing {self.name}")
def create_firebolt_wirter(connection: Connection, config: json,
logger: AirbyteLogger) -> FireboltWriter:
if config["loading_method"]["method"] == "S3":
logger.info("Using the S3 writing strategy")
writer = FireboltS3Writer(
connection,
config["loading_method"]["s3_bucket"],
config["loading_method"]["aws_key_id"],
config["loading_method"]["aws_key_secret"],
config["loading_method"]["s3_region"],
)
else:
logger.info("Using the SQL writing strategy")
writer = FireboltSQLWriter(connection)
return writer
def check_connection(
self, logger: AirbyteLogger,
config: Mapping[str, Any]) -> Tuple[bool, Optional[str]]:
try:
_ = self._get_sf_object(config)
except exceptions.HTTPError as error:
error_data = error.response.json()[0]
error_code = error_data.get("errorCode")
if error.response.status_code == codes.FORBIDDEN and error_code == "REQUEST_LIMIT_EXCEEDED":
logger.warn(
f"API Call limit is exceeded. Error message: '{error_data.get('message')}'"
)
return False, "API Call limit is exceeded"
return True, None
def run_load_dataframes(config, expected_columns=10, expected_rows=42):
df_list = SourceFile.load_dataframes(config=config, logger=AirbyteLogger(), skip_data=False)
assert len(df_list) == 1 # Properly load 1 DataFrame
df = df_list[0]
assert len(df.columns) == expected_columns # DataFrame should have 10 columns
assert len(df.index) == expected_rows # DataFrame should have 42 rows of data
return df
def wrapper_balance_rate_limit(*args, **kwargs):
sleep_time = 0
free_load = float("inf")
# find the Response inside args list
for arg in args:
response = arg if type(
arg) is requests.models.Response else None
# Get the rate_limits from response
rate_limits = ([
(response.headers.get(rate_remaining_limit_header),
response.headers.get(rate_max_limit_header))
for rate_remaining_limit_header, rate_max_limit_header in
rate_limits_headers
] if response else None)
# define current load from rate_limits
if rate_limits:
for current_rate, max_rate_limit in rate_limits:
free_load = min(
free_load,
int(current_rate) / int(max_rate_limit))
# define sleep time based on load conditions
if free_load <= threshold:
sleep_time = sleep_on_high_load
# sleep based on load conditions
sleep(sleep_time)
AirbyteLogger().info(
f"Sleep {sleep_time} seconds based on load conditions.")
return func(*args, **kwargs)
class GoogleSheetsClient:
logger = AirbyteLogger()
def __init__(self, config: Dict):
self.config = config
self.retries = 100 # max number of backoff retries
def authorize(self) -> pygsheets_client:
input_creds = self.config.get("credentials")
auth_creds = client_account.Credentials.from_authorized_user_info(info=input_creds)
client = pygsheets.authorize(custom_credentials=auth_creds)
# Increase max number of retries if Rate Limit is reached. Error: <HttpError 429>
client.drive.retries = self.retries # for google drive api
client.sheet.retries = self.retries # for google sheets api
# check if token is expired and refresh it
if client.oauth.expired:
self.logger.info("Auth session is expired. Refreshing...")
client.oauth.refresh(Request())
if not client.oauth.expired:
self.logger.info("Successfully refreshed auth session")
else:
self.logger.fatal("The token is expired and could not be refreshed, please check the credentials are still valid!")
return client
def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
try:
connection = create_connection(config=config)
except Exception as e:
logger.error(f"Failed to create connection. Error: {e}")
return AirbyteConnectionStatus(status=Status.FAILED, message=f"Could not create connection: {repr(e)}")
try:
channel = connection.channel()
if channel.is_open:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
return AirbyteConnectionStatus(status=Status.FAILED, message="Could not open channel")
except Exception as e:
logger.error(f"Failed to open RabbitMQ channel. Error: {e}")
return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}")
finally:
connection.close()
def check(self, logger: AirbyteLogger,
config: Mapping[str, Any]) -> AirbyteConnectionStatus:
"""
Tests if the input configuration can be used to successfully connect to the destination with the needed permissions
e.g: if a provided API token or password can be used to connect and write to the destination.
:param logger: Logging object to display debug/info/error to the logs
(logs will not be accessible via airbyte UI if they are not passed to this logger)
:param config: Json object containing the configuration of this destination, content of this json is as specified in
the properties of the spec.json file
:return: AirbyteConnectionStatus indicating a Success or Failure
"""
connector_config = ConnectorConfig(**config)
try:
aws_handler = AwsHandler(connector_config, self)
except (ClientError, AttributeError) as e:
logger.error(
f"""Could not create session on {connector_config.aws_account_id} Exception: {repr(e)}"""
)
message = f"""Could not authenticate using {connector_config.credentials_type} on Account {connector_config.aws_account_id} Exception: {repr(e)}"""
return AirbyteConnectionStatus(status=Status.FAILED,
message=message)
try:
aws_handler.head_bucket()
except ClientError as e:
message = f"""Could not find bucket {connector_config.bucket_name} in aws://{connector_config.aws_account_id}:{connector_config.region} Exception: {repr(e)}"""
return AirbyteConnectionStatus(status=Status.FAILED,
message=message)
with LakeformationTransaction(aws_handler) as tx:
table_location = "s3://" + connector_config.bucket_name + "/" + connector_config.bucket_prefix + "/" + "airbyte_test/"
table = aws_handler.get_table(
txid=tx.txid,
database_name=connector_config.lakeformation_database_name,
table_name="airbyte_test",
location=table_location,
)
if table is None:
message = f"Could not create a table in database {connector_config.lakeformation_database_name}"
return AirbyteConnectionStatus(status=Status.FAILED,
message=message)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
class NotImplementedAuth(Exception):
"""Not implemented Auth option error"""
logger = AirbyteLogger()
def __init__(self, auth_method: str = None):
self.message = f"Not implemented Auth method = {auth_method}"
super().__init__(self.logger.error(self.message))
def test_local_storage_spec():
"""Checks spec properties"""
source = SourceFileSecure()
spec = source.spec(logger=AirbyteLogger())
for provider in spec.connectionSpecification["properties"]["provider"][
"oneOf"]:
assert provider["properties"]["storage"][
"const"] != LOCAL_STORAGE_NAME, "This connector shouldn't work with local files."
def discover(self, logger: AirbyteLogger,
config: Mapping) -> AirbyteCatalog:
"""
Returns an AirbyteCatalog representing the available streams and fields in this integration. For example, given valid credentials to a
Remote CSV File, returns an Airbyte catalog where each csv file is a stream, and each column is a field.
"""
client = self._get_client(config)
name = client.stream_name
logger.info(
f"Discovering schema of {name} at {client.reader.full_url}...")
try:
streams = list(client.streams)
except Exception as err:
reason = f"Failed to discover schemas of {name} at {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}"
logger.error(reason)
raise err
return AirbyteCatalog(streams=streams)
class AbstractTestParser(ABC):
""" Prefix this class with Abstract so the tests don't run here but only in the children """
logger = AirbyteLogger()
@property
@abstractmethod
def test_files(self) -> List[Mapping[str, Any]]:
"""return a list of test_file dicts in structure:
[
{"AbstractFileParser": CsvParser(format, master_schema), "filepath": "...", "num_records": 5, "inferred_schema": {...}, line_checks:{}, fails: []},
{"AbstractFileParser": CsvParser(format, master_schema), "filepath": "...", "num_records": 16, "inferred_schema": {...}, line_checks:{}, fails: []}
]
note: line_checks index is 1-based to align with row numbers
"""
def _get_readmode(self, test_name, test_file):
self.logger.info(
f"testing {test_name}() with {test_file.get('test_alias', test_file['filepath'].split('/')[-1])} ..."
)
return "rb" if test_file["AbstractFileParser"].is_binary else "r"
def test_get_inferred_schema(self):
for test_file in self.test_files:
with smart_open(
test_file["filepath"],
self._get_readmode("get_inferred_schema", test_file)) as f:
if "test_get_inferred_schema" in test_file["fails"]:
with pytest.raises(Exception) as e_info:
test_file["AbstractFileParser"].get_inferred_schema(f)
self.logger.debug(str(e_info))
else:
assert test_file["AbstractFileParser"].get_inferred_schema(
f) == test_file["inferred_schema"]
def test_stream_records(self):
for test_file in self.test_files:
with smart_open(test_file["filepath"],
self._get_readmode("stream_records",
test_file)) as f:
if "test_stream_records" in test_file["fails"]:
with pytest.raises(Exception) as e_info:
[
print(r) for r in
test_file["AbstractFileParser"].stream_records(f)
]
self.logger.debug(str(e_info))
else:
records = [
r for r in
test_file["AbstractFileParser"].stream_records(f)
]
assert len(records) == test_file["num_records"]
for index, expected_record in test_file[
"line_checks"].items():
assert records[index - 1] == expected_record
def _read_records(conf, catalog, state=None) -> Tuple[List[AirbyteMessage], List[AirbyteMessage]]:
records = []
states = []
for message in SourceFacebookMarketing().read(AirbyteLogger(), conf, catalog, state=state):
if message.type == Type.RECORD:
records.append(message)
elif message.type == Type.STATE:
states.append(message)
return records, states
def check(self, logger: AirbyteLogger,
config: json) -> AirbyteConnectionStatus:
try:
access_token = config["access_token"]
spreadsheet_id = config["spreadsheet_id"]
smartsheet_client = smartsheet.Smartsheet(access_token)
smartsheet_client.errors_as_exceptions(True)
smartsheet_client.Sheets.get_sheet(spreadsheet_id)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
if isinstance(e, smartsheet.exceptions.ApiError):
err = e.error.result
code = 404 if err.code == 1006 else err.code
reason = f"{err.name}: {code} - {err.message} | Check your spreadsheet ID."
else:
reason = str(e)
logger.error(reason)
return AirbyteConnectionStatus(status=Status.FAILED)
def check_connection(self, logger: AirbyteLogger,
config: Mapping[str, Any]) -> Tuple[bool, any]:
try:
logger.info("Checking the config")
google_api = GoogleAds(credentials=self.get_credentials(config),
customer_id=config["customer_id"])
account_stream = Accounts(api=google_api)
list(account_stream.read_records(sync_mode=SyncMode.full_refresh))
# Check custom query request validity by sending metric request with non-existant time window
for q in config.get("custom_queries", []):
q = q.get("query")
if CustomQuery.cursor_field in q:
raise Exception(
f"Custom query should not contain {CustomQuery.cursor_field}"
)
req_q = CustomQuery.insert_segments_date_expr(
q, "1980-01-01", "1980-01-01")
google_api.send_request(req_q)
return True, None
except GoogleAdsException as error:
return False, f"Unable to connect to Google Ads API with the provided credentials - {repr(error.failure)}"
def check_connection(self, logger: AirbyteLogger, config: Mapping[str, any]) -> Tuple[bool, any]:
"""
:param config: the user-input config object conforming to the connector's spec.json
:param logger: logger object
:return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
"""
url_template = "https://{company}.hellobaton.com/api/"
try:
params = {
"api_key": config["api_key"],
}
base_url = url_template.format(company=config["company"])
# This is just going to return a mapping of available endpoints
response = requests.get(base_url, params=params)
status_code = response.status_code
logger.info(f"Status code: {status_code}")
if status_code == 200:
return True, None
except Exception as e:
return False, e
def check_connection(self, logger: AirbyteLogger,
config: Mapping[str, Any]) -> Tuple[bool, any]:
try:
logger.info("Checking the config")
google_api = GoogleAds(credentials=self.get_credentials(config))
accounts = self.get_account_info(google_api, config)
customers = Customer.from_accounts(accounts)
# Check custom query request validity by sending metric request with non-existant time window
for customer in customers:
for query in config.get("custom_queries", []):
query = query.get("query")
if customer.is_manager_account and self.is_metrics_in_custom_query(
query):
logger.warning(
f"Metrics are not available for manager account {customer.id}. "
f"Please remove metrics fields in your custom query: {query}."
)
if CustomQuery.cursor_field in query:
return False, f"Custom query should not contain {CustomQuery.cursor_field}"
req_q = CustomQuery.insert_segments_date_expr(
query, "1980-01-01", "1980-01-01")
response = google_api.send_request(req_q,
customer_id=customer.id)
# iterate over the response otherwise exceptions will not be raised!
for _ in response:
pass
return True, None
except GoogleAdsException as exception:
error_messages = ", ".join(
[error.message for error in exception.failure.errors])
logger.error(traceback.format_exc())
return False, f"Unable to connect to Google Ads API with the provided configuration - {error_messages}"
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
authenticator = TokenAuthenticator(config["api_token"])
default_start_date = pendulum.parse(config["start_date"])
threads_lookback_window = pendulum.Duration(
days=config["lookback_window"])
streams = [
Channels(authenticator=authenticator),
ChannelMembers(authenticator=authenticator),
ChannelMessages(authenticator=authenticator,
default_start_date=default_start_date),
Threads(authenticator=authenticator,
default_start_date=default_start_date,
lookback_window=threads_lookback_window),
Users(authenticator=authenticator),
]
# To sync data from channels, the bot backed by this token needs to join all those channels. This operation is idempotent.
if config["join_channels"]:
logger = AirbyteLogger()
logger.info("joining Slack channels")
join_channels_stream = JoinChannelsStream(
authenticator=authenticator)
for stream_slice in join_channels_stream.stream_slices():
for message in join_channels_stream.read_records(
sync_mode=SyncMode.full_refresh,
stream_slice=stream_slice):
logger.info(message["message"])
return streams
def read(
self, logger: AirbyteLogger, config: Mapping,
catalog: ConfiguredAirbyteCatalog,
state_path: Mapping[str,
any]) -> Generator[AirbyteMessage, None, None]:
"""Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, catalog, and state."""
client = self._get_client(config)
fields = self.selected_fields(catalog)
name = client.stream_name
logger.info(f"Reading {name} ({client.reader.full_url})...")
try:
for row in client.read(fields=fields):
record = AirbyteRecordMessage(
stream=name,
data=row,
emitted_at=int(datetime.now().timestamp()) * 1000)
yield AirbyteMessage(type=Type.RECORD, record=record)
except Exception as err:
reason = f"Failed to read data of {name} at {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}"
logger.error(reason)
raise err
def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog:
access_token = config["access_token"]
spreadsheet_id = config["spreadsheet_id"]
streams = []
smartsheet_client = smartsheet.Smartsheet(access_token)
try:
sheet = smartsheet_client.Sheets.get_sheet(spreadsheet_id)
sheet = json.loads(str(sheet)) # make it subscriptable
sheet_json_schema = get_json_schema(sheet)
logger.info(
f"Running discovery on sheet: {sheet['name']} with {spreadsheet_id}"
)
stream = AirbyteStream(name=sheet["name"],
json_schema=sheet_json_schema)
streams.append(stream)
except Exception as e:
raise Exception(f"Could not run discovery: {str(e)}")
return AirbyteCatalog(streams=streams)
def test_check_connection_should_fail_when_api_call_fails(mocker):
# We patch the object inside source.py because that's the calling context
# https://docs.python.org/3/library/unittest.mock.html#where-to-patch
mocker.patch("source_google_ads.source.GoogleAds",
MockErroringGoogleAdsClient)
source = SourceGoogleAds()
check_successful, message = source.check_connection(
AirbyteLogger(),
{
"credentials": {
"developer_token": "fake_developer_token",
"client_id": "fake_client_id",
"client_secret": "fake_client_secret",
"refresh_token": "fake_refresh_token",
},
"customer_id":
"fake_customer_id",
"start_date":
"2022-01-01",
"conversion_window_days":
14,
"custom_queries": [
{
"query":
"SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date FROM campaign",
"primary_key": None,
"cursor_field": "campaign.start_date",
"table_name": "happytable",
},
{
"query":
"SELECT segments.ad_destination_type, segments.ad_network_type, segments.day_of_week, customer.auto_tagging_enabled, customer.id, metrics.conversions, campaign.start_date FROM campaign",
"primary_key": "customer.id",
"cursor_field": None,
"table_name": "unhappytable",
},
{
"query":
"SELECT ad_group.targeting_setting.target_restrictions FROM ad_group",
"primary_key": "customer.id",
"cursor_field": None,
"table_name": "ad_group_custom",
},
],
},
)
assert not check_successful
assert message.startswith(
"Unable to connect to Google Ads API with the provided configuration")
def test_invalid_custom_query_handled(mocked_gads_api, config):
# limit to one custom query, otherwise need to mock more side effects
config["custom_queries"] = [next(iter(config["custom_queries"]))]
mocked_gads_api(
response=[{
"customer.id": "8765"
}],
failure_msg=
"Unrecognized field in the query: 'ad_group_ad.ad.video_ad.media_file'",
error_type="request_error",
)
source = SourceGoogleAds()
status_ok, error = source.check_connection(AirbyteLogger(), config)
assert not status_ok
assert error == (
"Unable to connect to Google Ads API with the provided configuration - Unrecognized field in the query: "
"'ad_group_ad.ad.video_ad.media_file'")
def test_check_connection_should_pass_when_config_valid(mocker):
mocker.patch("source_google_ads.source.GoogleAds", MockGoogleAdsClient)
source = SourceGoogleAds()
check_successful, message = source.check_connection(
AirbyteLogger(),
{
"credentials": {
"developer_token": "fake_developer_token",
"client_id": "fake_client_id",
"client_secret": "fake_client_secret",
"refresh_token": "fake_refresh_token",
},
"customer_id":
"fake_customer_id",
"start_date":
"2022-01-01",
"conversion_window_days":
14,
"custom_queries": [
{
"query":
"SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date FROM campaign",
"primary_key": None,
"cursor_field": "campaign.start_date",
"table_name": "happytable",
},
{
"query":
"SELECT segments.ad_destination_type, segments.ad_network_type, segments.day_of_week, customer.auto_tagging_enabled, customer.id, metrics.conversions, campaign.start_date FROM campaign",
"primary_key": "customer.id",
"cursor_field": None,
"table_name": "unhappytable",
},
{
"query":
"SELECT ad_group.targeting_setting.target_restrictions FROM ad_group",
"primary_key": "customer.id",
"cursor_field": None,
"table_name": "ad_group_custom",
},
],
},
)
assert check_successful
assert message is None
class AbstractTestParser(ABC):
"""Prefix this class with Abstract so the tests don't run here but only in the children"""
logger = AirbyteLogger()
def _get_readmode(self, test_name, test_file):
self.logger.info(
f"testing {test_name}() with {test_file.get('test_alias', test_file['filepath'].split('/')[-1])} ..."
)
return "rb" if test_file["AbstractFileParser"].is_binary else "r"
def test_get_inferred_schema(self, test_file):
with smart_open(test_file["filepath"],
self._get_readmode("get_inferred_schema",
test_file)) as f:
if "test_get_inferred_schema" in test_file["fails"]:
with pytest.raises(Exception) as e_info:
test_file["AbstractFileParser"].get_inferred_schema(f)
self.logger.debug(str(e_info))
else:
inferred_schema = test_file[
"AbstractFileParser"].get_inferred_schema(f)
expected_schema = test_file["inferred_schema"]
assert inferred_schema == expected_schema
def test_stream_records(self, test_file):
with smart_open(test_file["filepath"],
self._get_readmode("stream_records", test_file)) as f:
if "test_stream_records" in test_file["fails"]:
with pytest.raises(Exception) as e_info:
[
print(r) for r in
test_file["AbstractFileParser"].stream_records(f)
]
self.logger.debug(str(e_info))
else:
records = [
r
for r in test_file["AbstractFileParser"].stream_records(f)
]
assert len(records) == test_file["num_records"]
for index, expected_record in test_file["line_checks"].items():
assert records[index - 1] == expected_record
def read(
self,
logger: AirbyteLogger,
config: Mapping[str, Any],
catalog: ConfiguredAirbyteCatalog,
state: MutableMapping[str, Any] = None,
) -> Iterable[AirbyteMessage]:
logger.info(I_AM_A_SECRET_VALUE)
logger.info(I_AM_A_SECRET_VALUE + " plus Some non secret Value in the same log record" + NOT_A_SECRET_VALUE)
logger.info(NOT_A_SECRET_VALUE)
yield AirbyteMessage(
record=AirbyteRecordMessage(stream="stream", data={"data": "stuff"}, emitted_at=1),
type=Type.RECORD,
)
def request_params(self,
stream_state: Mapping[str, Any] = None,
next_page_token: Mapping[str, Any] = None,
**kwargs):
params = super().request_params(stream_state=stream_state,
next_page_token=next_page_token,
**kwargs)
AirbyteLogger().log("INFO", f"using params: {params}")
# If there is a next page token then we should only send pagination-related parameters.
if not next_page_token:
params["orderby"] = self.order_field
params["order"] = "asc"
if stream_state:
start_date = stream_state.get(self.cursor_field)
start_date = pendulum.parse(start_date).replace(tzinfo=None)
start_date = start_date.subtract(
days=self.conversion_window_days)
params["after"] = start_date
return params
