def __init__(
self,
domain: str,
api_key: str,
requests_per_minute: int = None,
verify: bool = True,
proxies: MutableMapping[str, Any] = None,
):
"""Basic HTTP interface to read from endpoints"""
self._api_prefix = f"https://{domain.rstrip('/')}/api/v2/"
self._session = requests.Session()
self._session.auth = (api_key, "unused_with_api_key")
self._session.verify = verify
self._session.proxies = proxies
self._session.headers = {
"Content-Type": "application/json",
"User-Agent": "Airbyte",
}
self._call_credit = CallCredit(
balance=requests_per_minute) if requests_per_minute else None
if domain.find("freshdesk.com") < 0:
raise AttributeError(
"Freshdesk v2 API works only via Freshdesk domains and not via custom CNAMEs"
)
def __init__(
self,
domain: str,
api_key: str,
requests_per_minute: int = None,
verify: bool = True,
proxies: MutableMapping[str, Any] = None,
start_date: str = None,
):
"""Basic HTTP interface to read from endpoints"""
self._api_prefix = f"https://{domain.rstrip('/')}/api/v2/"
self._session = requests.Session()
self._session.auth = (api_key, "unused_with_api_key")
self._session.verify = verify
self._session.proxies = proxies
self._session.headers = {
"Content-Type": "application/json",
"User-Agent": "Airbyte",
}
self._call_credit = CallCredit(balance=requests_per_minute) if requests_per_minute else None
# By default, only tickets that have been created within the past 30 days will be returned.
# Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations.
# So updated_since parameter will be always used in tickets streams. And start_date will be used too
# with default value 30 days look back.
self._start_date = pendulum.parse(start_date) if start_date else pendulum.now() - pendulum.duration(days=30)
if domain.find("freshdesk.com") < 0:
raise AttributeError("Freshdesk v2 API works only via Freshdesk domains and not via custom CNAMEs")
def test_consume_one():
"""Multiple consumptions of 1 cred will reach limit"""
credit = CallCredit(balance=3, reload_period=1)
ts_1 = time.time()
for i in range(4):
credit.consume(1)
ts_2 = time.time()
assert 1 <= ts_2 - ts_1 < 2
def __init__(self, authenticator: AuthBase, config: Mapping[str, Any],
*args, **kwargs):
super().__init__(authenticator=authenticator)
requests_per_minute = config.get("requests_per_minute")
self.domain = config["domain"]
self._call_credit = CallCredit(
balance=requests_per_minute) if requests_per_minute else None
# By default, only tickets that have been created within the past 30 days will be returned.
# Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations.
# So updated_since parameter will be always used in tickets streams. And start_date will be used too
# with default value 30 days look back.
self.start_date = config.get("start_date") or (
pendulum.now() -
pendulum.duration(days=30)).strftime("%Y-%m-%dT%H:%M:%SZ")
def test_consume_many():
"""Consumptions of N creds will reach limit and decrease balance"""
credit = CallCredit(balance=3, reload_period=1)
ts_1 = time.time()
credit.consume(1)
credit.consume(3)
ts_2 = time.time()
# the balance decreased already, so single cred will be enough to reach limit
credit.consume(1)
ts_3 = time.time()
assert 1 <= ts_2 - ts_1 < 2
assert 1 <= ts_3 - ts_2 < 2
class API:
def __init__(
self,
domain: str,
api_key: str,
requests_per_minute: int = None,
verify: bool = True,
proxies: MutableMapping[str, Any] = None,
start_date: str = None,
):
"""Basic HTTP interface to read from endpoints"""
self._api_prefix = f"https://{domain.rstrip('/')}/api/v2/"
self._session = requests.Session()
self._session.auth = (api_key, "unused_with_api_key")
self._session.verify = verify
self._session.proxies = proxies
self._session.headers = {
"Content-Type": "application/json",
"User-Agent": "Airbyte",
}
self._call_credit = CallCredit(balance=requests_per_minute) if requests_per_minute else None
# By default, only tickets that have been created within the past 30 days will be returned.
# Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations.
# So updated_since parameter will be always used in tickets streams. And start_date will be used too
# with default value 30 days look back.
self._start_date = pendulum.parse(start_date) if start_date else pendulum.now() - pendulum.duration(days=30)
if domain.find("freshdesk.com") < 0:
raise AttributeError("Freshdesk v2 API works only via Freshdesk domains and not via custom CNAMEs")
@staticmethod
def _parse_and_handle_errors(response):
try:
body = response.json()
except ValueError:
body = {}
error_message = "Freshdesk Request Failed"
if "errors" in body:
error_message = f"{body.get('description')}: {body['errors']}"
# API docs don't mention this clearly, but in the case of bad credentials the returned JSON will have a
# "message" field at the top level
elif "message" in body:
error_message = f"{body.get('code')}: {body['message']}"
if response.status_code == 400:
raise FreshdeskBadRequest(error_message or "Wrong input, check your data", response=response)
elif response.status_code == 401:
raise FreshdeskUnauthorized(error_message or "Invalid credentials", response=response)
elif response.status_code == 403:
raise FreshdeskAccessDenied(error_message or "You don't have enough permissions", response=response)
elif response.status_code == 404:
raise FreshdeskNotFound(error_message or "Resource not found", response=response)
elif response.status_code == 429:
retry_after = response.headers.get("Retry-After")
raise FreshdeskRateLimited(
f"429 Rate Limit Exceeded: API rate-limit has been reached until {retry_after} seconds."
" See http://freshdesk.com/api#ratelimit",
response=response,
)
elif 500 <= response.status_code < 600:
raise FreshdeskServerError(f"{response.status_code}: Server Error", response=response)
# Catch any other errors
try:
response.raise_for_status()
except HTTPError as err:
raise FreshdeskError(f"{err}: {body}", response=response) from err
return body
@retry_connection_handler(max_tries=5, factor=5)
@retry_after_handler(max_tries=3)
def get(self, url: str, params: Mapping = None):
"""Wrapper around request.get() to use the API prefix. Returns a JSON response."""
params = params or {}
response = self._session.get(self._api_prefix + url, params=params)
return self._parse_and_handle_errors(response)
def consume_credit(self, credit):
"""Consume call credit, if there is no credit left within current window will sleep til next period"""
if self._call_credit:
self._call_credit.consume(credit)
class FreshdeskStream(HttpStream, ABC):
"""Basic stream API that allows to iterate over entities"""
call_credit = 1 # see https://developers.freshdesk.com/api/#embedding
result_return_limit = 100
primary_key = "id"
link_regex = re.compile(r'<(.*?)>;\s*rel="next"')
def __init__(self, authenticator: AuthBase, config: Mapping[str, Any],
*args, **kwargs):
super().__init__(authenticator=authenticator)
requests_per_minute = config.get("requests_per_minute")
self.domain = config["domain"]
self._call_credit = CallCredit(
balance=requests_per_minute) if requests_per_minute else None
# By default, only tickets that have been created within the past 30 days will be returned.
# Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations.
# So updated_since parameter will be always used in tickets streams. And start_date will be used too
# with default value 30 days look back.
self.start_date = config.get("start_date") or (
pendulum.now() -
pendulum.duration(days=30)).strftime("%Y-%m-%dT%H:%M:%SZ")
@property
def url_base(self) -> str:
return parse.urljoin(f"https://{self.domain.rstrip('/')}", "/api/v2")
def backoff_time(self, response: requests.Response) -> Optional[float]:
if response.status_code == requests.codes.too_many_requests:
return float(response.headers.get("Retry-After", 0))
def next_page_token(
self, response: requests.Response) -> Optional[Mapping[str, Any]]:
link_header = response.headers.get("Link")
if not link_header:
return {}
match = self.link_regex.search(link_header)
next_url = match.group(1)
params = parse.parse_qs(parse.urlparse(next_url).query)
return self.parse_link_params(link_query_params=params)
def parse_link_params(
self, link_query_params: Mapping[str,
List[str]]) -> Mapping[str, Any]:
return {
"per_page": link_query_params["per_page"][0],
"page": link_query_params["page"][0]
}
def request_params(
self,
stream_state: Mapping[str, Any],
stream_slice: Mapping[str, any] = None,
next_page_token: Mapping[str,
Any] = None) -> MutableMapping[str, Any]:
params = {"per_page": self.result_return_limit}
if next_page_token and "page" in next_page_token:
params["page"] = next_page_token["page"]
return params
def _consume_credit(self, credit):
"""Consume call credit, if there is no credit left within current window will sleep til next period"""
if self._call_credit:
self._call_credit.consume(credit)
def read_records(
self,
sync_mode: SyncMode,
cursor_field: List[str] = None,
stream_slice: Mapping[str, Any] = None,
stream_state: Mapping[str, Any] = None,
) -> Iterable[Mapping[str, Any]]:
self._consume_credit(self.call_credit)
yield from super().read_records(sync_mode=sync_mode,
cursor_field=cursor_field,
stream_slice=stream_slice,
stream_state=stream_state)
def parse_response(self, response: requests.Response,
**kwargs) -> Iterable[Mapping]:
data = response.json()
return data if data else []