def _get_authenticator(config: dict) -> Union[TokenAuthenticator, AsanaOauth2Authenticator]: if "access_token" in config: # Before Oauth we had Person Access Token stored under "access_token" # config field, this code here is for backward compatibility return TokenAuthenticator(token=config["access_token"]) creds = config.get("credentials") if "personal_access_token" in creds: return TokenAuthenticator(token=creds["personal_access_token"]) else: return AsanaOauth2Authenticator( token_refresh_endpoint="https://app.asana.com/-/oauth_token", client_secret=creds["client_secret"], client_id=creds["client_id"], refresh_token=creds["refresh_token"], )
def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(config["apikey"]) streams = [ Lists(authenticator=authenticator), Campaigns(authenticator=authenticator), Contacts(authenticator=authenticator), StatsAutomations(authenticator=authenticator), Segments(authenticator=authenticator), SingleSends(authenticator=authenticator), Templates(authenticator=authenticator), GlobalSuppressions(authenticator=authenticator, start_time=config["start_time"]), SuppressionGroups(authenticator=authenticator), SuppressionGroupMembers(authenticator=authenticator), Blocks(authenticator=authenticator, start_time=config["start_time"]), Bounces(authenticator=authenticator, start_time=config["start_time"]), InvalidEmails(authenticator=authenticator, start_time=config["start_time"]), SpamReports(authenticator=authenticator, start_time=config["start_time"]), ] return streams
def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["private_token"]) auth_params = dict(authenticator=auth, api_url=config["api_url"]) groups, projects = self._generate_main_streams(config) pipelines = Pipelines(parent_stream=projects, start_date=config["start_date"], **auth_params) merge_requests = MergeRequests(parent_stream=projects, start_date=config["start_date"], **auth_params) epics = Epics(parent_stream=groups, **auth_params) streams = [ groups, projects, Branches(parent_stream=projects, repository_part=True, **auth_params), Commits(parent_stream=projects, repository_part=True, start_date=config["start_date"], **auth_params), epics, EpicIssues(parent_stream=epics, **auth_params), GroupIssueBoards(parent_stream=groups, **auth_params), Issues(parent_stream=projects, start_date=config["start_date"], **auth_params), Jobs(parent_stream=pipelines, **auth_params), ProjectMilestones(parent_stream=projects, **auth_params), GroupMilestones(parent_stream=groups, **auth_params), ProjectMembers(parent_stream=projects, **auth_params), GroupMembers(parent_stream=groups, **auth_params), ProjectLabels(parent_stream=projects, **auth_params), GroupLabels(parent_stream=groups, **auth_params), merge_requests, MergeRequestCommits(parent_stream=merge_requests, **auth_params), Releases(parent_stream=projects, **auth_params), Tags(parent_stream=projects, repository_part=True, **auth_params), pipelines, PipelinesExtended(parent_stream=pipelines, **auth_params), Users(parent_stream=projects, **auth_params), ] return streams
def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: try: auth = TokenAuthenticator(token=self._convert_auth_to_token(config["api_key"], config["secret_key"]), auth_method="Basic") list(Cohorts(authenticator=auth).read_records(SyncMode.full_refresh)) return True, None except Exception as error: return False, f"Unable to connect to Amplitude API with the provided credentials - {repr(error)}"
def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(token=config["access_token"], auth_method="token") full_refresh_args = { "authenticator": authenticator, "repository": config["repository"] } incremental_args = { **full_refresh_args, "start_date": config["start_date"] } return [ Assignees(**full_refresh_args), Collaborators(**full_refresh_args), Comments(**incremental_args), CommitComments(**incremental_args), Commits(**incremental_args), Events(**incremental_args), IssueEvents(**incremental_args), IssueLabels(**full_refresh_args), IssueMilestones(**incremental_args), Issues(**incremental_args), Projects(**incremental_args), PullRequests(**incremental_args), Releases(**incremental_args), Reviews(**full_refresh_args), Stargazers(**incremental_args), Teams(**full_refresh_args), ]
def generate_streams(cls, config: Mapping[str, Any], stream_names: List[str], sf_object: Salesforce) -> List[Stream]: """ "Generates a list of stream by their names. It can be used for different tests too""" authenticator = TokenAuthenticator(sf_object.access_token) streams_kwargs = {} if config["api_type"] == "REST": full_refresh, incremental = SalesforceStream, IncrementalSalesforceStream else: full_refresh, incremental = BulkSalesforceStream, BulkIncrementalSalesforceStream streams_kwargs["wait_timeout"] = config.get("wait_timeout") streams = [] for stream_name in stream_names: json_schema = sf_object.generate_schema(stream_name) pk, replication_key = sf_object.get_pk_and_replication_key( json_schema) streams_kwargs.update( dict(sf_api=sf_object, pk=pk, stream_name=stream_name, schema=json_schema, authenticator=authenticator)) if replication_key and stream_name not in UNSUPPORTED_FILTERING_STREAMS: streams.append( incremental(**streams_kwargs, replication_key=replication_key, start_date=config.get("start_date"))) else: streams.append(full_refresh(**streams_kwargs)) return streams
def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(config["api_token"]) default_start_date = pendulum.parse(config["start_date"]) threads_lookback_window = pendulum.Duration( days=config["lookback_window"]) streams = [ Channels(authenticator=authenticator), ChannelMembers(authenticator=authenticator), ChannelMessages(authenticator=authenticator, default_start_date=default_start_date), Threads(authenticator=authenticator, default_start_date=default_start_date, lookback_window=threads_lookback_window), Users(authenticator=authenticator), ] # To sync data from channels, the bot backed by this token needs to join all those channels. This operation is idempotent. if config["join_channels"]: logger = AirbyteLogger() logger.info("joining Slack channels") join_channels_stream = JoinChannelsStream( authenticator=authenticator) for stream_slice in join_channels_stream.stream_slices(): for message in join_channels_stream.read_records( sync_mode=SyncMode.full_refresh, stream_slice=stream_slice): logger.info(message["message"]) return streams
def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(config["client_secret"]) start_date = pendulum.parse(config["start_date"]).int_timestamp args = { "authenticator": authenticator, "account_id": config["account_id"], "start_date": start_date } incremental_args = { **args, "lookback_window_days": config.get("lookback_window_days") } return [ BalanceTransactions(**incremental_args), BankAccounts(**args), Charges(**incremental_args), CheckoutSessions(**args), CheckoutSessionsLineItems(**args), Coupons(**incremental_args), CustomerBalanceTransactions(**args), Customers(**incremental_args), Disputes(**incremental_args), Events(**incremental_args), InvoiceItems(**incremental_args), InvoiceLineItems(**args), Invoices(**incremental_args), PaymentIntents(**incremental_args), Payouts(**incremental_args), Plans(**incremental_args), Products(**incremental_args), PromotionCodes(**incremental_args), Refunds(**incremental_args), SubscriptionItems(**args), Subscriptions(**incremental_args), Transfers(**incremental_args), ]
def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(config["client_secret"]) args = { "authenticator": authenticator, "account_id": config["account_id"] } return [ BankAccounts(**args), BalanceTransactions(**args), Charges(**args), Coupons(**args), Customers(**args), CustomerBalanceTransactions(**args), Disputes(**args), Events(**args), InvoiceItems(**args), InvoiceLineItems(**args), Invoices(**args), Plans(**args), Payouts(**args), Products(**args), Subscriptions(**args), SubscriptionItems(**args), Refunds(**args), Transfers(**args), ]
def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["api_key"]) args = { "authenticator": auth, "is_sandbox": config["is_sandbox"], "api_version": self.api_version, "start_date": config["start_date"], "include_deleted_objects": config["include_deleted_objects"], } return [ Items(**args), Categories(**args), Discounts(**args), Taxes(**args), Locations(**args), TeamMembers(**args), TeamMemberWages(**args), Refunds(**args), Payments(**args), Customers(**args), ModifierList(**args), Shifts(**args), Orders(**args), ]
def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ event/sessions stream is dynamic. Probably, it contains a list of CURRENT sessions. In Next day session may expire and wont be available via this endpoint. So we need a dynamic load data before tests. This stream was requested to be removed due to this reason. """ authenticator = TokenAuthenticator(token=config["api_key"]) base_url = config.get("base_url", DEFAULT_BASE_URL) return [ Annotations(authenticator=authenticator, start_date=config["start_date"], base_url=base_url), Cohorts(authenticator=authenticator, base_url=base_url), Events(authenticator=authenticator, start_date=config["start_date"], base_url=base_url), EventsSessions(authenticator=authenticator, base_url=base_url), FeatureFlags(authenticator=authenticator, base_url=base_url), Insights(authenticator=authenticator, base_url=base_url), InsightsPath(authenticator=authenticator, base_url=base_url), InsightsSessions(authenticator=authenticator, base_url=base_url), Persons(authenticator=authenticator, base_url=base_url), Trends(authenticator=authenticator, base_url=base_url), ]
def streams(self, config: Mapping[str, Any]) -> List[Stream]: talkdesk_auth = TalkdeskAuth(config) token_request = talkdesk_auth.request_bearer_token() talkdesk_auth_token = token_request.get("access_token", None) authenticator = TokenAuthenticator(token=talkdesk_auth_token) start_date = config.get("start_date", None) timezone = config.get("timezone", None) streams_ = [ Calls(start_date=start_date, timezone=timezone, authenticator=authenticator), UserStatus(start_date=start_date, timezone=timezone, authenticator=authenticator), StudioFlowExecution(start_date=start_date, timezone=timezone, authenticator=authenticator), Contacts(start_date=start_date, timezone=timezone, authenticator=authenticator), RingAttempts(start_date=start_date, timezone=timezone, authenticator=authenticator), ] return streams_
def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["api_token"]) return [ Items(authenticator=auth), Boards(authenticator=auth), Teams(authenticator=auth), Updates(authenticator=auth), Users(authenticator=auth), ]
def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ TODO: Replace the streams below with your own streams. :param config: A Mapping of the user input configuration as defined in the connector spec. """ # TODO remove the authenticator if not required. auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support return [Customers(authenticator=auth), Employees(authenticator=auth)]
def check_connection(self, logger, config) -> Tuple[bool, any]: try: authenticator = TokenAuthenticator(config["apikey"]) scopes_gen = Scopes(authenticator=authenticator).read_records( sync_mode=SyncMode.full_refresh) next(scopes_gen) return True, None except Exception as error: return False, f"Unable to connect to Sendgrid API with the provided credentials - {error}"
def check_connection(self, logger, config) -> Tuple[bool, any]: auth = TokenAuthenticator(token=f'token={config["api_key"]}', auth_method="Token").get_auth_header() url = f'https://{config["domain_name"]}/crm/sales/api/contacts/filters' try: session = requests.get(url, headers=auth) session.raise_for_status() return True, None except requests.exceptions.RequestException as e: return False, e
def get_auth( config: Mapping[str, Any] ) -> Union[Base64HttpAuthenticator, TokenAuthenticator]: credentials = config.get("credentials", {}) token = config.get("api_token") or credentials.get("api_token") email = config.get("email") or credentials.get("email") if email and token: return Base64HttpAuthenticator(auth=(f"{email}/token", token)) return TokenAuthenticator(token=credentials["access_token"])
def test_token_authenticator(): """ Should match passed in token, no matter how many times token is retrieved. """ token = TokenAuthenticator("test-token") header = token.get_auth_header() assert {"Authorization": "Bearer test-token"} == header header = token.get_auth_header() assert {"Authorization": "Bearer test-token"} == header
def get_stream(): config = { "start_date": "2021-01-01T00:00:00", "access_token": "something" } authenticator = TokenAuthenticator(token=config["access_token"]) start_date = pendulum.parse(config["start_date"]) stream = Surveys(authenticator=authenticator, start_date=start_date) return stream
def check_connection(self, logger, config) -> Tuple[bool, any]: url = "https://api.monday.com/v2" params = {"query": "{boards(limit:1){id name}}"} auth = TokenAuthenticator(config["api_token"]).get_auth_header() try: response = requests.post(url, params=params, headers=auth) response.raise_for_status() return True, None except requests.exceptions.RequestException as e: return False, e
def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(token=config["access_token"]) start_date = pendulum.parse(config["start_date"]) args = {"authenticator": authenticator, "start_date": start_date} return [ Surveys(**args), SurveyPages(**args), SurveyQuestions(**args), SurveyResponses(**args) ]
def check_connection(self, logger, config) -> Tuple[bool, Any]: try: projects_stream = Projects( authenticator=TokenAuthenticator(token=config["auth_token"]), hostname=config.get("hostname"), ) next(projects_stream.read_records(sync_mode=SyncMode.full_refresh)) return True, None except Exception as e: return False, e
def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["token"]) return [ Forms(authenticator=auth, **config), Responses(authenticator=auth, **config), Webhooks(authenticator=auth, **config), Workspaces(authenticator=auth, **config), Images(authenticator=auth, **config), Themes(authenticator=auth, **config), ]
def check_connection(self, logger, config) -> Tuple[bool, any]: authenticator = TokenAuthenticator(token=config["access_token"]) try: url = f"{IntercomStream.url_base}/tags" auth_headers = {"Accept": "application/json", **authenticator.get_auth_header()} session = requests.get(url, headers=auth_headers) session.raise_for_status() return True, None except requests.exceptions.RequestException as e: return False, e
def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["api_token"]) return [ ConversationExport( authenticator=auth, start_date=datetime.strptime(config["start_date"], "%Y-%m-%d"), batch_size=int(config["batch_size"]), logger=AirbyteLogger(), ) ]
def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: _ = pendulum.parse(config["start_date"], strict=True) authenticator = TokenAuthenticator(token=config["api_key"]) stream = Cohorts(authenticator=authenticator) records = stream.read_records(sync_mode=SyncMode.full_refresh) _ = next(records) return True, None except Exception as e: return False, repr(e)
def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: workspaces_stream = Workspaces(authenticator=TokenAuthenticator( token=config["access_token"])) next( workspaces_stream.read_records( sync_mode=SyncMode.full_refresh)) return True, None except Exception as e: return False, e
def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(token=config["api_key"]) base_url = config.get("base_url", DEFAULT_BASE_URL) return [ Annotations(authenticator=authenticator, start_date=config["start_date"], base_url=base_url), Cohorts(authenticator=authenticator, base_url=base_url), Events(authenticator=authenticator, start_date=config["start_date"], base_url=base_url), FeatureFlags(authenticator=authenticator, base_url=base_url), Persons(authenticator=authenticator, base_url=base_url), ]
def check_connection(self, logger, config) -> Tuple[bool, any]: try: api_key = config["api_key"] auth_method = f"api_key={api_key}" auth = TokenAuthenticator(token="", auth_method=auth_method) stream = Companies(auth) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) return True, None except Exception as e: return False, repr(e)
def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: authenticator = TokenAuthenticator(token=config["access_token"]) start_date = pendulum.parse(config["start_date"]) stream = Surveys(authenticator=authenticator, start_date=start_date) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) return True, None except Exception as e: return False, repr(e)