def check_config(self, logger: AirbyteLogger, config_path: str,
config: json) -> AirbyteConnectionStatus:
# singer catalog that attempts to pull a stream ("accounts") that should always exists, though it may be empty.
try:
customer_ids = config["customer_ids"].split(",")
for customer_id in customer_ids:
oauth2_client = oauth2.GoogleRefreshTokenClient(
config["oauth_client_id"], config["oauth_client_secret"],
config["refresh_token"])
sdk_client = adwords.AdWordsClient(
config["developer_token"],
oauth2_client,
user_agent=config["user_agent"],
client_customer_id=customer_id)
selector = {
"fields": [
"Name", "CanManageClients", "CustomerId",
"TestAccount", "DateTimeZone", "CurrencyCode"
],
}
accounts = self._get_accounts(logger, sdk_client, selector)
if not accounts:
err = f"No accounts associated with customer id {customer_id}"
error_msg = f"Unable to connect with the provided credentials. Error: {err}"
return AirbyteConnectionStatus(status=Status.FAILED,
message=error_msg)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
return AirbyteConnectionStatus(status=Status.FAILED,
message=f"{str(e)}")
def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus:
client = self._client(config)
alive, error = client.health_check()
if not alive:
return AirbyteConnectionStatus(status=Status.FAILED, message=f"{error}")
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def check(self, logger, config_path) -> AirbyteConnectionStatus:
try:
code = requests.get("https://api.ratesapi.io/").status_code
logger.info(f"Ping response code: {code}")
return AirbyteConnectionStatus(status=Status.SUCCEEDED if (code == 200) else Status.FAILED)
except Exception as e:
return AirbyteConnectionStatus(status=Status.FAILED, message=f"{str(e)}")
def check(self, logger: AirbyteLogger, config_container: ConfigContainer) -> AirbyteConnectionStatus:
client = self._client(config_container)
alive, error = client.health_check()
if not alive:
return AirbyteConnectionStatus(status=Status.FAILED, message=f"{error.title}: {error.detail}")
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def check(self, logger: AirbyteLogger,
config: json) -> AirbyteConnectionStatus:
"""
Tests if the input configuration can be used to successfully connect to the integration
e.g: if a provided Stripe API token can be used to connect to the Stripe API.
:param logger: Logging object to display debug/info/error to the logs
(logs will not be accessible via airbyte UI if they are not passed to this logger)
:param config: Json object containing the configuration of this source, content of this json is as specified in
the properties of the spec.json file
:return: AirbyteConnectionStatus indicating a Success or Failure
"""
try:
# Not Implemented
# client = get_client(logger, config)
self._client = Client(config)
if self._client.login():
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
else:
return AirbyteConnectionStatus(status=Status.FAILED,
message='Invalid credentials')
except Exception as e:
return AirbyteConnectionStatus(
status=Status.FAILED,
message=f"An exception occurred: {str(e)}")
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
client = Client(config["account_sid"], config["auth_token"])
client.api.accounts.list()
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except TwilioException as error:
logger.error(str(error))
return AirbyteConnectionStatus(status=Status.FAILED, message=str(error))
def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
"""Check connection"""
client = self._get_client(config)
alive, error = client.health_check()
if not alive:
return AirbyteConnectionStatus(status=Status.FAILED, message=str(error))
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def check(self, logger: AirbyteLogger,
config_container) -> AirbyteConnectionStatus:
r = self._make_request(config_container.rendered_config)
if r.status_code == 200:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
else:
return AirbyteConnectionStatus(status=Status.FAILED,
message=r.text)
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
client = WebClient(token=config["token"])
client.conversations_list()
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except SlackApiError as e:
logger.error(f"Got an error: {e.args[0]}")
return AirbyteConnectionStatus(status=Status.FAILED, message=str(e.args[0]))
def check(self, logger: AirbyteLogger,
config: json) -> AirbyteConnectionStatus:
# Check involves verifying that the specified spreadsheet is reachable with our credentials.
client = GoogleSheetsClient(json.loads(config["credentials_json"]))
spreadsheet_id = config["spreadsheet_id"]
try:
# Attempt to get first row of sheet
client.get(spreadsheetId=spreadsheet_id,
includeGridData=False,
ranges="1:1")
except errors.HttpError as err:
reason = str(err)
# Give a clearer message if it's a common error like 404.
if err.resp.status == status_codes.NOT_FOUND:
reason = "Requested spreadsheet was not found."
logger.error(f"Formatted error: {reason}")
return AirbyteConnectionStatus(
status=Status.FAILED,
message=
f"Unable to connect with the provided credentials to spreadsheet. Error: {reason}"
)
# Check for duplicate headers
spreadsheet_metadata = Spreadsheet.parse_obj(
client.get(spreadsheetId=spreadsheet_id, includeGridData=False))
sheet_names = [
sheet.properties.title for sheet in spreadsheet_metadata.sheets
]
duplicate_headers_in_sheet = {}
for sheet_name in sheet_names:
try:
header_row_data = Helpers.get_first_row(
client, spreadsheet_id, sheet_name)
_, duplicate_headers = Helpers.get_valid_headers_and_duplicates(
header_row_data)
if duplicate_headers:
duplicate_headers_in_sheet[sheet_name] = duplicate_headers
except Exception as err:
logger.error(str(err))
return AirbyteConnectionStatus(
status=Status.FAILED,
message=
f"Unable to read the schema of sheet {sheet_name}. Error: {str(err)}"
)
if duplicate_headers_in_sheet:
duplicate_headers_error_message = ", ".join([
f"[sheet:{sheet_name}, headers:{duplicate_sheet_headers}]"
for sheet_name, duplicate_sheet_headers in
duplicate_headers_in_sheet.items()
])
return AirbyteConnectionStatus(
status=Status.FAILED,
message=
"The following duplicate headers were found in the following sheets. Please fix them to continue: "
+ duplicate_headers_error_message,
)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def check(self, logger, config_container) -> AirbyteConnectionStatus:
try:
json_config = config_container.rendered_config
client = Client(json_config["account_sid"], json_config["auth_token"])
client.api.accounts.list()
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except TwilioException as error:
logger.error(str(error))
return AirbyteConnectionStatus(status=Status.FAILED, message=str(error))
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
self.try_connect(logger, config)
except self.api_error as err:
logger.error(f"Exception while connecting to {self.tap_name}: {err}")
# this should be in UI
error_msg = f"Unable to connect to {self.tap_name} with the provided credentials. Error: {err}"
return AirbyteConnectionStatus(status=Status.FAILED, message=error_msg)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
r = requests.get("https://api.stripe.com/v1/customers", auth=(config["client_secret"], ""))
if r.status_code == 200:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
else:
return AirbyteConnectionStatus(status=Status.FAILED, message=r.text)
except Exception as e:
return AirbyteConnectionStatus(status=Status.FAILED, message=f"{str(e)}")
def check(self, logger: AirbyteLogger,
config: Mapping[str, Any]) -> AirbyteConnectionStatus:
"""Check connection"""
check_succeeded, error = self.check_connection(logger, config)
if not check_succeeded:
return AirbyteConnectionStatus(status=Status.FAILED,
message=str(error))
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
"""Implements the Check Connection operation from the Airbyte Specification. See https://docs.airbyte.io/architecture/airbyte-specification."""
try:
check_succeeded, error = self.check_connection(logger, config)
if not check_succeeded:
return AirbyteConnectionStatus(status=Status.FAILED, message=str(error))
except Exception as e:
return AirbyteConnectionStatus(status=Status.FAILED, message=str(e))
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
self.discover(logger, config_path)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
logger.error("Exception while connecting to the Marketo API")
logger.error(str(e))
return AirbyteConnectionStatus(
status=Status.FAILED, message="Unable to connect to the Marketo API with the provided credentials. "
)
def check_config(self, logger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
r = requests.get("https://api.github.com/repos/airbytehq/airbyte/commits", auth=(config["access_token"], ""))
if r.status_code == 200:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
else:
return AirbyteConnectionStatus(status=Status.FAILED, message=r.text)
except Exception as e:
logger.error(e)
return AirbyteConnectionStatus(status=Status.FAILED, message=f"{str(e)}")
def check(self, logger, config_container) -> AirbyteConnectionStatus:
try:
json_config = config_container.rendered_config
r = requests.get('https://api.stripe.com/v1/customers', auth=(json_config['client_secret'], ''))
if r.status_code == 200:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
else:
return AirbyteConnectionStatus(status=Status.FAILED, message=r.text)
except Exception as e:
return AirbyteConnectionStatus(status=Status.FAILED, message=f"{str(e)}")
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
error_msg = "Unable to connect with the provided credentials. Error: {}"
try:
r = requests.get("https://api.stripe.com/v1/customers", auth=(config["client_secret"], ""))
if r.status_code == status_codes.OK:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
else:
return AirbyteConnectionStatus(status=Status.FAILED, message=error_msg.format(r.json()["error"]["message"]))
except Exception as e:
return AirbyteConnectionStatus(status=Status.FAILED, message=error_msg.format(e))
def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus:
"""
Tests if the input configuration can be used to successfully connect to the integration
e.g: if a provided Stripe API token can be used to connect to the Stripe API.
:param logger: Logging object to display debug/info/error to the logs
(logs will not be accessible via airbyte UI if they are not passed to this logger)
:param config: Json object containing the configuration of this source, content of this json is as specified in
the properties of the spec.json file
:return: AirbyteConnectionStatus indicating a Success or Failure
"""
try:
# VALIDATE CONFIG AGAINST JSON SCHEMA (spec.json)
validate( instance=config, schema=self.spec(logger).connectionSpecification)
# try to get time (ping) from configured URL
url = config[ConfigPropDataApiUrl]
def assertAliasedParamsNotBothPresent(config,stream,paramName1,paramName2):
if paramName1 in config[stream] and paramName2 in config[stream]:
raise AssertionError(f"{stream}: cannot specify both aliased parameters '{paramName1}' and '{paramName2}'. choose one.")
def assertOneRainDateFormat(config,stream,paramName):
try:
if paramName in config[stream]:
return datetime.strptime(config[stream][paramName],'%Y-%m-%d %H:%M:%S')
except ValueError as e:
raise ValueError(stream,paramName,str(e))
# ADDITIONAL GetSiteMetadata STREAM CONFIG CHECKS
assertAliasedParamsNotBothPresent(config,StreamGetSiteMetaData,"or_site_id","site_id")
# ADDITIONAL GetSensorMetaData STREAM CONFIG CHECKS
assertAliasedParamsNotBothPresent(config,StreamGetSensorMetaData,"or_sensor_id","sensor_id")
assertAliasedParamsNotBothPresent(config,StreamGetSensorMetaData,"or_site_id","site_id")
# ADDITIONAL GetSensorData STREAM CONFIG CHECKS
assertAliasedParamsNotBothPresent(config,StreamGetSensorData,"or_site_id","site_id")
assertAliasedParamsNotBothPresent(config,StreamGetSensorData,"or_sensor_id","sensor_id")
assertOneRainDateFormat(config,StreamGetSensorData,"data_start")
assertOneRainDateFormat(config,StreamGetSensorData,"data_end")
# PING CONFIGURED ONERAIN URL WITH GetTime REQUEST TO MAKE SURE IT'S A VALID ENDPOINT
get_time_url = f'{url}?method=GetTime'
# use GetTime method to validate well formed url and that it responds to this
# basic time get request
r = requests.get(get_time_url,timeout=HttpResponseTimeout)
assert r.status_code == 200
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}")
def check_config(self, logger: AirbyteLogger, config_path: str,
config: json) -> AirbyteConnectionStatus:
error_msg = "Unable to connect with the provided credentials. Error: {}"
try:
api_key = config.get("hapikey")
if api_key:
logger.info("checking with api key")
r = requests.get(
f"https://api.hubapi.com/contacts/v1/lists/all/contacts/all?hapikey={api_key}"
)
else:
logger.info("checking with oauth")
# borrowing from tap-hubspot
# https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/__init__.py#L208-L229
payload = {
"grant_type": "refresh_token",
"redirect_uri": config["redirect_uri"],
"refresh_token": config["refresh_token"],
"client_id": config["client_id"],
"client_secret": config["client_secret"],
}
resp = requests.post("https://api.hubapi.com/oauth/v1/token",
data=payload)
if resp.status_code == status_codes.FORBIDDEN:
return AirbyteConnectionStatus(status=Status.FAILED,
message=error_msg.format(
resp.text))
try:
resp.raise_for_status()
except Exception as e:
logger.error(str(e))
return AirbyteConnectionStatus(status=Status.FAILED,
message=error_msg.format(e))
auth = resp.json()
headers = {
"Authorization": "Bearer {}".format(auth["access_token"])
}
r = requests.get(
"https://api.hubapi.com/contacts/v1/lists/all/contacts/all",
headers=headers)
if r.status_code == status_codes.OK:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
else:
return AirbyteConnectionStatus(status=Status.FAILED,
message=error_msg.format(
r.text))
except Exception as e:
logger.error(str(e))
return AirbyteConnectionStatus(status=Status.FAILED,
message=error_msg.format(e))
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
"""
Tests if the input configuration can be used to successfully connect to the integration
e.g: if a provided Stripe API token can be used to connect to the Stripe API.
:param logger: Logging object to display debug/info/error to the logs
(logs will not be accessible via airbyte UI if they are not passed to this logger)
:param config_path: Path to the file containing the configuration json config
:param config: Json object containing the configuration of this source, content of this json is as specified in
the properties of the spec.json file
:return: AirbyteConnectionStatus indicating a Success or Failure
"""
try:
# If an app on the appstore does not support subscriptions or sales, it cannot pull the relevant reports.
# However, the way the Appstore API expresses this is not via clear error messages. Instead it expresses it by throwing an unrelated
# error, in this case "invalid vendor ID". There is no way to distinguish if this error is due to invalid credentials or due to
# the account not supporting this kind of report. So to "check connection" we see if any of the reports can be pulled and if so
# return success. If no reports can be pulled we display the exception messages generated for all reports and return failure.
api_fields_to_test = {
"subscription_event_report": {
"reportType": "SUBSCRIPTION_EVENT",
"frequency": "DAILY",
"reportSubType": "SUMMARY",
"version": "1_2",
},
"subscriber_report": {"reportType": "SUBSCRIBER", "frequency": "DAILY", "reportSubType": "DETAILED", "version": "1_2"},
"subscription_report": {"reportType": "SUBSCRIPTION", "frequency": "DAILY", "reportSubType": "SUMMARY", "version": "1_2"},
"sales_report": {"reportType": "SALES", "frequency": "DAILY", "reportSubType": "SUMMARY", "version": "1_0"},
}
api = Api(config["key_id"], config["key_file"], config["issuer_id"])
stream_to_error = {}
for stream, params in api_fields_to_test.items():
test_date = date.today() - timedelta(days=2)
report_filters = {"reportDate": test_date.strftime("%Y-%m-%d"), "vendorNumber": f"{config['vendor']}"}
report_filters.update(api_fields_to_test[stream])
try:
rep_tsv = api.download_sales_and_trends_reports(filters=report_filters)
if isinstance(rep_tsv, dict):
raise Exception(f"An exception occurred: Received a JSON response instead of" f" the report: {str(rep_tsv)}")
except Exception as e:
logger.warn(f"Unable to download {stream}: {e}")
stream_to_error[stream] = e
# All streams have failed
if len(stream_to_error.keys()) == api_fields_to_test.keys():
message = "\n".join([f"Unable to access {stream} due to error: {e}" for stream, e in stream_to_error])
return AirbyteConnectionStatus(status=Status.FAILED, message=message)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
logger.warn(e)
return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}")
def check(self, logger, config_container) -> AirbyteConnectionStatus:
try:
# singer catalog that attempts to pull a stream ("accounts") that should always exists, though it may be empty.
singer_check_catalog_path = os.path.abspath(os.path.dirname(__file__)) + "/singer_check_catalog.json"
if self.read(logger, config_container, singer_check_catalog_path, None) is not None:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
else:
return AirbyteConnectionStatus(status=Status.FAILED)
except Exception as e:
return AirbyteConnectionStatus(status=Status.FAILED, message=f"{str(e)}")
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
self.discover(logger, config_path)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception:
logger.error("Exception while connecting to the Zendesk Support API")
return AirbyteConnectionStatus(
status=Status.FAILED,
message="Unable to connect to the Zendesk Support API with the provided credentials. Please make sure the "
"input credentials and environment are correct. ",
)
def check_config(self, logger: AirbyteLogger, config_path: str,
config: json) -> AirbyteConnectionStatus:
"""
Tests if the input configuration can be used to successfully connect to the integration
e.g: if a provided Stripe API token can be used to connect to the Stripe API.
:param logger: Logging object to display debug/info/error to the logs
(logs will not be accessible via airbyte UI if they are not passed to this logger)
:param config_path: Path to the file containing the configuration json config
:param config: Json object containing the configuration of this source, content of this json is as specified in
the properties of the spec.json file
:return: AirbyteConnectionStatus indicating a Success or Failure
"""
try:
# create request fields for testing
api_fields_to_test = {
"subscription_report": {
"reportType": "SUBSCRIPTION",
"frequency": "DAILY",
"reportSubType": "SUMMARY",
"version": "1_2"
}
}
test_date = date.today() - timedelta(days=2)
report_filters = {
"reportDate": test_date.strftime("%Y-%m-%d"),
"vendorNumber": "{}".format(config["vendor"])
}
report_filters.update(api_fields_to_test["subscription_report"])
# fetch data from appstore api
api = Api(config["key_id"], config["key_file"],
config["issuer_id"])
rep_tsv = api.download_sales_and_trends_reports(
filters=report_filters)
if isinstance(rep_tsv, dict):
return AirbyteConnectionStatus(
status=Status.FAILED,
message=
f"An exception occurred: Received a JSON response instead of"
f" the report: {str(rep_tsv)}",
)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
logger.warn(e)
return AirbyteConnectionStatus(
status=Status.FAILED,
message=f"An exception occurred: {str(e)}")
def check(self, logger,
config_container: ConfigContainer) -> AirbyteConnectionStatus:
config = config_container.rendered_config
try:
params = "dbname='{dbname}' user='******' host='{host}' password='******' port='{port}'".format(
**config)
psycopg2.connect(params)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
logger.error(
f"Exception while connecting to postgres database: {e}")
return AirbyteConnectionStatus(status=Status.FAILED,
message=str(e))
def check(self, logger, config_container) -> AirbyteConnectionStatus:
try:
# this config is the one specific to GAClient, which does not match the root Singer config
client_secrets = json.loads(config_container.raw_config["credentials_json"])
additional_fields = {"end_date": "2050-10-01T00:00:00Z", "client_secrets": client_secrets}
augmented_config = dict(additional_fields, **config_container.rendered_config)
client = GAClient(augmented_config)
client.fetch_metadata()
except Exception as e:
logger.error(f"Failed check with exception: {e}")
return AirbyteConnectionStatus(status=Status.FAILED)
else:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def check_config(self, logger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
repositories = config["repository"].split(" ")
for repository in repositories:
url = f"https://api.github.com/repos/{repository}/commits"
response = requests.get(url, auth=(config["access_token"], ""))
if response.status_code != requests.codes.ok:
return AirbyteConnectionStatus(status=Status.FAILED, message=f"{repository} {response.text}")
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as err:
logger.error(err)
error_msg = f"Unable to connect with the provided credentials. Error: {err}"
return AirbyteConnectionStatus(status=Status.FAILED, message=error_msg)
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
session = shopify.Session(f"{config['shop']}.myshopify.com", "2020-10", config["api_key"])
shopify.ShopifyResource.activate_session(session)
# try to read the name of the shop, which should be available with any level of permissions
shopify.GraphQL().execute("{ shop { name id } }")
shopify.ShopifyResource.clear_session()
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
logger.error(f"Exception connecting to Shopify: ${e}")
return AirbyteConnectionStatus(
status=Status.FAILED, message="Unable to connect to the Shopify API with the provided credentials."
)
def check(self, logger: AirbyteLogger,
config_container: ConfigContainer) -> AirbyteConnectionStatus:
try:
self.discover(logger, config_container)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
# TODO parse the exception message for a human readable error
logger.error("Exception while connecting to the FB Marketing API")
logger.error(str(e))
return AirbyteConnectionStatus(
status=Status.FAILED,
message=
"Unable to connect to the FB Marketing API with the provided credentials. "
)
