def check_connection(self, logger: AirbyteLogger,
config: Mapping[str, Any]) -> Tuple[bool, any]:
try:
logger.info("Checking the config")
google_api = GoogleAds(credentials=self.get_credentials(config))
accounts = self.get_account_info(google_api, config)
customers = Customer.from_accounts(accounts)
# Check custom query request validity by sending metric request with non-existant time window
for customer in customers:
for query in config.get("custom_queries", []):
query = query.get("query")
if customer.is_manager_account and self.is_metrics_in_custom_query(
query):
logger.warning(
f"Metrics are not available for manager account {customer.id}. "
f"Please remove metrics fields in your custom query: {query}."
)
if CustomQuery.cursor_field in query:
return False, f"Custom query should not contain {CustomQuery.cursor_field}"
req_q = CustomQuery.insert_segments_date_expr(
query, "1980-01-01", "1980-01-01")
response = google_api.send_request(req_q,
customer_id=customer.id)
# iterate over the response otherwise exceptions will not be raised!
for _ in response:
pass
return True, None
except GoogleAdsException as exception:
error_messages = ", ".join(
[error.message for error in exception.failure.errors])
logger.error(traceback.format_exc())
return False, f"Unable to connect to Google Ads API with the provided configuration - {error_messages}"
def set_sub_primary_key(self):
if isinstance(self.primary_key, list):
for index, value in enumerate(self.primary_key):
setattr(self, f"sub_primary_key_{index + 1}", value)
else:
logger = AirbyteLogger()
logger.error(
"Failed during setting sub primary keys. Primary key should be list."
)
def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
try:
connection = create_connection(config=config)
except Exception as e:
logger.error(f"Failed to create connection. Error: {e}")
return AirbyteConnectionStatus(status=Status.FAILED, message=f"Could not create connection: {repr(e)}")
try:
channel = connection.channel()
if channel.is_open:
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
return AirbyteConnectionStatus(status=Status.FAILED, message="Could not open channel")
except Exception as e:
logger.error(f"Failed to open RabbitMQ channel. Error: {e}")
return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}")
finally:
connection.close()
def check(self, logger: AirbyteLogger,
config: Mapping[str, Any]) -> AirbyteConnectionStatus:
"""
Tests if the input configuration can be used to successfully connect to the destination with the needed permissions
e.g: if a provided API token or password can be used to connect and write to the destination.
:param logger: Logging object to display debug/info/error to the logs
(logs will not be accessible via airbyte UI if they are not passed to this logger)
:param config: Json object containing the configuration of this destination, content of this json is as specified in
the properties of the spec.json file
:return: AirbyteConnectionStatus indicating a Success or Failure
"""
connector_config = ConnectorConfig(**config)
try:
aws_handler = AwsHandler(connector_config, self)
except (ClientError, AttributeError) as e:
logger.error(
f"""Could not create session on {connector_config.aws_account_id} Exception: {repr(e)}"""
)
message = f"""Could not authenticate using {connector_config.credentials_type} on Account {connector_config.aws_account_id} Exception: {repr(e)}"""
return AirbyteConnectionStatus(status=Status.FAILED,
message=message)
try:
aws_handler.head_bucket()
except ClientError as e:
message = f"""Could not find bucket {connector_config.bucket_name} in aws://{connector_config.aws_account_id}:{connector_config.region} Exception: {repr(e)}"""
return AirbyteConnectionStatus(status=Status.FAILED,
message=message)
with LakeformationTransaction(aws_handler) as tx:
table_location = "s3://" + connector_config.bucket_name + "/" + connector_config.bucket_prefix + "/" + "airbyte_test/"
table = aws_handler.get_table(
txid=tx.txid,
database_name=connector_config.lakeformation_database_name,
table_name="airbyte_test",
location=table_location,
)
if table is None:
message = f"Could not create a table in database {connector_config.lakeformation_database_name}"
return AirbyteConnectionStatus(status=Status.FAILED,
message=message)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def discover(self, logger: AirbyteLogger,
config: Mapping) -> AirbyteCatalog:
"""
Returns an AirbyteCatalog representing the available streams and fields in this integration. For example, given valid credentials to a
Remote CSV File, returns an Airbyte catalog where each csv file is a stream, and each column is a field.
"""
client = self._get_client(config)
name = client.stream_name
logger.info(
f"Discovering schema of {name} at {client.reader.full_url}...")
try:
streams = list(client.streams)
except Exception as err:
reason = f"Failed to discover schemas of {name} at {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}"
logger.error(reason)
raise err
return AirbyteCatalog(streams=streams)
def check(self, logger: AirbyteLogger,
config: json) -> AirbyteConnectionStatus:
try:
access_token = config["access_token"]
spreadsheet_id = config["spreadsheet_id"]
smartsheet_client = smartsheet.Smartsheet(access_token)
smartsheet_client.errors_as_exceptions(True)
smartsheet_client.Sheets.get_sheet(spreadsheet_id)
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
if isinstance(e, smartsheet.exceptions.ApiError):
err = e.error.result
code = 404 if err.code == 1006 else err.code
reason = f"{err.name}: {code} - {err.message} | Check your spreadsheet ID."
else:
reason = str(e)
logger.error(reason)
return AirbyteConnectionStatus(status=Status.FAILED)
def read(
self, logger: AirbyteLogger, config: Mapping,
catalog: ConfiguredAirbyteCatalog,
state_path: Mapping[str,
any]) -> Generator[AirbyteMessage, None, None]:
"""Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, catalog, and state."""
client = self._get_client(config)
fields = self.selected_fields(catalog)
name = client.stream_name
logger.info(f"Reading {name} ({client.reader.full_url})...")
try:
for row in client.read(fields=fields):
record = AirbyteRecordMessage(
stream=name,
data=row,
emitted_at=int(datetime.now().timestamp()) * 1000)
yield AirbyteMessage(type=Type.RECORD, record=record)
except Exception as err:
reason = f"Failed to read data of {name} at {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}"
logger.error(reason)
raise err
def read(self, logger: AirbyteLogger, config: json,
catalog: ConfiguredAirbyteCatalog,
state: Dict[str, any]) -> Generator[AirbyteMessage, None, None]:
access_token = config["access_token"]
spreadsheet_id = config["spreadsheet_id"]
smartsheet_client = smartsheet.Smartsheet(access_token)
for configured_stream in catalog.streams:
stream = configured_stream.stream
properties = stream.json_schema["properties"]
if isinstance(properties, list):
columns = tuple(key for dct in properties
for key in dct.keys())
elif isinstance(properties, dict):
columns = tuple(i for i in properties.keys())
else:
logger.error(
"Could not read properties from the JSONschema in this stream"
)
name = stream.name
try:
sheet = smartsheet_client.Sheets.get_sheet(spreadsheet_id)
sheet = json.loads(str(sheet)) # make it subscriptable
logger.info(f"Starting syncing spreadsheet {sheet['name']}")
logger.info(f"Row count: {sheet['totalRowCount']}")
for row in sheet["rows"]:
values = tuple(i["value"] if "value" in i else ""
for i in row["cells"])
try:
data = dict(zip(columns, values))
yield AirbyteMessage(
type=Type.RECORD,
record=AirbyteRecordMessage(
stream=name,
data=data,
emitted_at=int(datetime.now().timestamp()) *
1000),
)
except Exception as e:
logger.error(
f"Unable to encode row into an AirbyteMessage with the following error: {e}"
)
except Exception as e:
logger.error(f"Could not read smartsheet: {name}")
raise e
logger.info(f"Finished syncing spreadsheet with ID: {spreadsheet_id}")
