def __init__( self, domain: str, api_key: str, requests_per_minute: int = None, verify: bool = True, proxies: MutableMapping[str, Any] = None, ): """Basic HTTP interface to read from endpoints""" self._api_prefix = f"https://{domain.rstrip('/')}/api/v2/" self._session = requests.Session() self._session.auth = (api_key, "unused_with_api_key") self._session.verify = verify self._session.proxies = proxies self._session.headers = { "Content-Type": "application/json", "User-Agent": "Airbyte", } self._call_credit = CallCredit( balance=requests_per_minute) if requests_per_minute else None if domain.find("freshdesk.com") < 0: raise AttributeError( "Freshdesk v2 API works only via Freshdesk domains and not via custom CNAMEs" )
def __init__( self, domain: str, api_key: str, requests_per_minute: int = None, verify: bool = True, proxies: MutableMapping[str, Any] = None, start_date: str = None, ): """Basic HTTP interface to read from endpoints""" self._api_prefix = f"https://{domain.rstrip('/')}/api/v2/" self._session = requests.Session() self._session.auth = (api_key, "unused_with_api_key") self._session.verify = verify self._session.proxies = proxies self._session.headers = { "Content-Type": "application/json", "User-Agent": "Airbyte", } self._call_credit = CallCredit(balance=requests_per_minute) if requests_per_minute else None # By default, only tickets that have been created within the past 30 days will be returned. # Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations. # So updated_since parameter will be always used in tickets streams. And start_date will be used too # with default value 30 days look back. self._start_date = pendulum.parse(start_date) if start_date else pendulum.now() - pendulum.duration(days=30) if domain.find("freshdesk.com") < 0: raise AttributeError("Freshdesk v2 API works only via Freshdesk domains and not via custom CNAMEs")
def test_consume_one(): """Multiple consumptions of 1 cred will reach limit""" credit = CallCredit(balance=3, reload_period=1) ts_1 = time.time() for i in range(4): credit.consume(1) ts_2 = time.time() assert 1 <= ts_2 - ts_1 < 2
def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): super().__init__(authenticator=authenticator) requests_per_minute = config.get("requests_per_minute") self.domain = config["domain"] self._call_credit = CallCredit( balance=requests_per_minute) if requests_per_minute else None # By default, only tickets that have been created within the past 30 days will be returned. # Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations. # So updated_since parameter will be always used in tickets streams. And start_date will be used too # with default value 30 days look back. self.start_date = config.get("start_date") or ( pendulum.now() - pendulum.duration(days=30)).strftime("%Y-%m-%dT%H:%M:%SZ")
def test_consume_many(): """Consumptions of N creds will reach limit and decrease balance""" credit = CallCredit(balance=3, reload_period=1) ts_1 = time.time() credit.consume(1) credit.consume(3) ts_2 = time.time() # the balance decreased already, so single cred will be enough to reach limit credit.consume(1) ts_3 = time.time() assert 1 <= ts_2 - ts_1 < 2 assert 1 <= ts_3 - ts_2 < 2
class API: def __init__( self, domain: str, api_key: str, requests_per_minute: int = None, verify: bool = True, proxies: MutableMapping[str, Any] = None, start_date: str = None, ): """Basic HTTP interface to read from endpoints""" self._api_prefix = f"https://{domain.rstrip('/')}/api/v2/" self._session = requests.Session() self._session.auth = (api_key, "unused_with_api_key") self._session.verify = verify self._session.proxies = proxies self._session.headers = { "Content-Type": "application/json", "User-Agent": "Airbyte", } self._call_credit = CallCredit(balance=requests_per_minute) if requests_per_minute else None # By default, only tickets that have been created within the past 30 days will be returned. # Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations. # So updated_since parameter will be always used in tickets streams. And start_date will be used too # with default value 30 days look back. self._start_date = pendulum.parse(start_date) if start_date else pendulum.now() - pendulum.duration(days=30) if domain.find("freshdesk.com") < 0: raise AttributeError("Freshdesk v2 API works only via Freshdesk domains and not via custom CNAMEs") @staticmethod def _parse_and_handle_errors(response): try: body = response.json() except ValueError: body = {} error_message = "Freshdesk Request Failed" if "errors" in body: error_message = f"{body.get('description')}: {body['errors']}" # API docs don't mention this clearly, but in the case of bad credentials the returned JSON will have a # "message" field at the top level elif "message" in body: error_message = f"{body.get('code')}: {body['message']}" if response.status_code == 400: raise FreshdeskBadRequest(error_message or "Wrong input, check your data", response=response) elif response.status_code == 401: raise FreshdeskUnauthorized(error_message or "Invalid credentials", response=response) elif response.status_code == 403: raise FreshdeskAccessDenied(error_message or "You don't have enough permissions", response=response) elif response.status_code == 404: raise FreshdeskNotFound(error_message or "Resource not found", response=response) elif response.status_code == 429: retry_after = response.headers.get("Retry-After") raise FreshdeskRateLimited( f"429 Rate Limit Exceeded: API rate-limit has been reached until {retry_after} seconds." " See http://freshdesk.com/api#ratelimit", response=response, ) elif 500 <= response.status_code < 600: raise FreshdeskServerError(f"{response.status_code}: Server Error", response=response) # Catch any other errors try: response.raise_for_status() except HTTPError as err: raise FreshdeskError(f"{err}: {body}", response=response) from err return body @retry_connection_handler(max_tries=5, factor=5) @retry_after_handler(max_tries=3) def get(self, url: str, params: Mapping = None): """Wrapper around request.get() to use the API prefix. Returns a JSON response.""" params = params or {} response = self._session.get(self._api_prefix + url, params=params) return self._parse_and_handle_errors(response) def consume_credit(self, credit): """Consume call credit, if there is no credit left within current window will sleep til next period""" if self._call_credit: self._call_credit.consume(credit)
class FreshdeskStream(HttpStream, ABC): """Basic stream API that allows to iterate over entities""" call_credit = 1 # see https://developers.freshdesk.com/api/#embedding result_return_limit = 100 primary_key = "id" link_regex = re.compile(r'<(.*?)>;\s*rel="next"') def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): super().__init__(authenticator=authenticator) requests_per_minute = config.get("requests_per_minute") self.domain = config["domain"] self._call_credit = CallCredit( balance=requests_per_minute) if requests_per_minute else None # By default, only tickets that have been created within the past 30 days will be returned. # Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations. # So updated_since parameter will be always used in tickets streams. And start_date will be used too # with default value 30 days look back. self.start_date = config.get("start_date") or ( pendulum.now() - pendulum.duration(days=30)).strftime("%Y-%m-%dT%H:%M:%SZ") @property def url_base(self) -> str: return parse.urljoin(f"https://{self.domain.rstrip('/')}", "/api/v2") def backoff_time(self, response: requests.Response) -> Optional[float]: if response.status_code == requests.codes.too_many_requests: return float(response.headers.get("Retry-After", 0)) def next_page_token( self, response: requests.Response) -> Optional[Mapping[str, Any]]: link_header = response.headers.get("Link") if not link_header: return {} match = self.link_regex.search(link_header) next_url = match.group(1) params = parse.parse_qs(parse.urlparse(next_url).query) return self.parse_link_params(link_query_params=params) def parse_link_params( self, link_query_params: Mapping[str, List[str]]) -> Mapping[str, Any]: return { "per_page": link_query_params["per_page"][0], "page": link_query_params["page"][0] } def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None) -> MutableMapping[str, Any]: params = {"per_page": self.result_return_limit} if next_page_token and "page" in next_page_token: params["page"] = next_page_token["page"] return params def _consume_credit(self, credit): """Consume call credit, if there is no credit left within current window will sleep til next period""" if self._call_credit: self._call_credit.consume(credit) def read_records( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Mapping[str, Any]]: self._consume_credit(self.call_credit) yield from super().read_records(sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state) def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: data = response.json() return data if data else []