def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: try: logger.info("Checking the config") google_api = GoogleAds(credentials=self.get_credentials(config)) accounts = self.get_account_info(google_api, config) customers = Customer.from_accounts(accounts) # Check custom query request validity by sending metric request with non-existant time window for customer in customers: for query in config.get("custom_queries", []): query = query.get("query") if customer.is_manager_account and self.is_metrics_in_custom_query( query): logger.warning( f"Metrics are not available for manager account {customer.id}. " f"Please remove metrics fields in your custom query: {query}." ) if CustomQuery.cursor_field in query: return False, f"Custom query should not contain {CustomQuery.cursor_field}" req_q = CustomQuery.insert_segments_date_expr( query, "1980-01-01", "1980-01-01") response = google_api.send_request(req_q, customer_id=customer.id) # iterate over the response otherwise exceptions will not be raised! for _ in response: pass return True, None except GoogleAdsException as exception: error_messages = ", ".join( [error.message for error in exception.failure.errors]) logger.error(traceback.format_exc()) return False, f"Unable to connect to Google Ads API with the provided configuration - {error_messages}"
def set_sub_primary_key(self): if isinstance(self.primary_key, list): for index, value in enumerate(self.primary_key): setattr(self, f"sub_primary_key_{index + 1}", value) else: logger = AirbyteLogger() logger.error( "Failed during setting sub primary keys. Primary key should be list." )
def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: try: connection = create_connection(config=config) except Exception as e: logger.error(f"Failed to create connection. Error: {e}") return AirbyteConnectionStatus(status=Status.FAILED, message=f"Could not create connection: {repr(e)}") try: channel = connection.channel() if channel.is_open: return AirbyteConnectionStatus(status=Status.SUCCEEDED) return AirbyteConnectionStatus(status=Status.FAILED, message="Could not open channel") except Exception as e: logger.error(f"Failed to open RabbitMQ channel. Error: {e}") return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") finally: connection.close()
def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: """ Tests if the input configuration can be used to successfully connect to the destination with the needed permissions e.g: if a provided API token or password can be used to connect and write to the destination. :param logger: Logging object to display debug/info/error to the logs (logs will not be accessible via airbyte UI if they are not passed to this logger) :param config: Json object containing the configuration of this destination, content of this json is as specified in the properties of the spec.json file :return: AirbyteConnectionStatus indicating a Success or Failure """ connector_config = ConnectorConfig(**config) try: aws_handler = AwsHandler(connector_config, self) except (ClientError, AttributeError) as e: logger.error( f"""Could not create session on {connector_config.aws_account_id} Exception: {repr(e)}""" ) message = f"""Could not authenticate using {connector_config.credentials_type} on Account {connector_config.aws_account_id} Exception: {repr(e)}""" return AirbyteConnectionStatus(status=Status.FAILED, message=message) try: aws_handler.head_bucket() except ClientError as e: message = f"""Could not find bucket {connector_config.bucket_name} in aws://{connector_config.aws_account_id}:{connector_config.region} Exception: {repr(e)}""" return AirbyteConnectionStatus(status=Status.FAILED, message=message) with LakeformationTransaction(aws_handler) as tx: table_location = "s3://" + connector_config.bucket_name + "/" + connector_config.bucket_prefix + "/" + "airbyte_test/" table = aws_handler.get_table( txid=tx.txid, database_name=connector_config.lakeformation_database_name, table_name="airbyte_test", location=table_location, ) if table is None: message = f"Could not create a table in database {connector_config.lakeformation_database_name}" return AirbyteConnectionStatus(status=Status.FAILED, message=message) return AirbyteConnectionStatus(status=Status.SUCCEEDED)
def discover(self, logger: AirbyteLogger, config: Mapping) -> AirbyteCatalog: """ Returns an AirbyteCatalog representing the available streams and fields in this integration. For example, given valid credentials to a Remote CSV File, returns an Airbyte catalog where each csv file is a stream, and each column is a field. """ client = self._get_client(config) name = client.stream_name logger.info( f"Discovering schema of {name} at {client.reader.full_url}...") try: streams = list(client.streams) except Exception as err: reason = f"Failed to discover schemas of {name} at {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}" logger.error(reason) raise err return AirbyteCatalog(streams=streams)
def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus: try: access_token = config["access_token"] spreadsheet_id = config["spreadsheet_id"] smartsheet_client = smartsheet.Smartsheet(access_token) smartsheet_client.errors_as_exceptions(True) smartsheet_client.Sheets.get_sheet(spreadsheet_id) return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: if isinstance(e, smartsheet.exceptions.ApiError): err = e.error.result code = 404 if err.code == 1006 else err.code reason = f"{err.name}: {code} - {err.message} | Check your spreadsheet ID." else: reason = str(e) logger.error(reason) return AirbyteConnectionStatus(status=Status.FAILED)
def read( self, logger: AirbyteLogger, config: Mapping, catalog: ConfiguredAirbyteCatalog, state_path: Mapping[str, any]) -> Generator[AirbyteMessage, None, None]: """Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, catalog, and state.""" client = self._get_client(config) fields = self.selected_fields(catalog) name = client.stream_name logger.info(f"Reading {name} ({client.reader.full_url})...") try: for row in client.read(fields=fields): record = AirbyteRecordMessage( stream=name, data=row, emitted_at=int(datetime.now().timestamp()) * 1000) yield AirbyteMessage(type=Type.RECORD, record=record) except Exception as err: reason = f"Failed to read data of {name} at {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}" logger.error(reason) raise err
def read(self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any]) -> Generator[AirbyteMessage, None, None]: access_token = config["access_token"] spreadsheet_id = config["spreadsheet_id"] smartsheet_client = smartsheet.Smartsheet(access_token) for configured_stream in catalog.streams: stream = configured_stream.stream properties = stream.json_schema["properties"] if isinstance(properties, list): columns = tuple(key for dct in properties for key in dct.keys()) elif isinstance(properties, dict): columns = tuple(i for i in properties.keys()) else: logger.error( "Could not read properties from the JSONschema in this stream" ) name = stream.name try: sheet = smartsheet_client.Sheets.get_sheet(spreadsheet_id) sheet = json.loads(str(sheet)) # make it subscriptable logger.info(f"Starting syncing spreadsheet {sheet['name']}") logger.info(f"Row count: {sheet['totalRowCount']}") for row in sheet["rows"]: values = tuple(i["value"] if "value" in i else "" for i in row["cells"]) try: data = dict(zip(columns, values)) yield AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( stream=name, data=data, emitted_at=int(datetime.now().timestamp()) * 1000), ) except Exception as e: logger.error( f"Unable to encode row into an AirbyteMessage with the following error: {e}" ) except Exception as e: logger.error(f"Could not read smartsheet: {name}") raise e logger.info(f"Finished syncing spreadsheet with ID: {spreadsheet_id}")