def test_transform_with_jq():
assert transform_with_jq(data=[1, 2, 3], jq_filter='.[]+1') == [2, 3, 4]
assert transform_with_jq([[1, 2, 3]], '.[]') == [1, 2, 3]
assert transform_with_jq([{
'col1': [1, 2],
'col2': [3, 4]
}], '.') == [{
'col1': [1, 2],
'col2': [3, 4]
}]
def _retrieve_data(self, data_source: RokDataSource) -> pd.DataFrame:
endpoint = f'{self.host}/graphql?DatabaseName={data_source.database}'
auth_query = """
query Auth($database: String!, $user: String!, $password: String!)
{authenticate(database: $database, user: $user, password: $password)}"""
auth_vars = {
'database': data_source.database,
'user': self.username,
'password': self.password,
}
auth_res = requests.post(endpoint,
json={
'query': auth_query,
'variables': auth_vars
}).json()
if 'errors' in auth_res:
raise ValueError(str(auth_res['errors']))
payload = {
'query': data_source.query,
'variables': data_source.parameters
}
headers = {'Token': auth_res['data']['authenticate']}
res = requests.post(endpoint, json=payload, headers=headers).json()
if 'errors' in res:
raise ValueError(str(res['errors']))
return pd.DataFrame(transform_with_jq(res, data_source.filter))
def do_request(self, query, session):
"""
Get some json data with an HTTP request and run a jq filter on it.
Args:
query (dict): specific infos about the request (url, http headers, etc.)
session (requests.Session):
Returns:
data (list): The response from the API in the form of a list of dict
"""
jq_filter = query['filter']
available_params = [
'url', 'method', 'params', 'data', 'json', 'headers', 'proxies'
]
query = {k: v for k, v in query.items() if k in available_params}
query['url'] = '/'.join(
[self.baseroute.rstrip('/'), query['url'].lstrip('/')])
if self.cert:
# `cert` is a list of PosixPath. `request` needs a list of strings for certificates
query['cert'] = [str(c) for c in self.cert]
res = session.request(**query)
try:
data = res.json()
except ValueError:
HttpAPIConnector.logger.error(f'Could not decode {res.content!r}')
raise
try:
return transform_with_jq(data, jq_filter)
except ValueError:
HttpAPIConnector.logger.error(
f'Could not transform {data} using {jq_filter}')
raise
async def _get_data(self, endpoint, jq_filter):
"""
Basic data retrieval function
- builds the full url
- retrieves built headers
- calls a fetch and returns filtered data or an error
"""
full_url = f'{self.baseroute}/{endpoint}'
api_key = self.authentication.api_key
api_secret: str = self.authentication.api_secret.get_secret_value()
username = self.authentication.username
timestamp = int(datetime.datetime.now().timestamp())
headers = build_headers(api_key, api_secret, username, str(timestamp))
async with ClientSession(headers=headers) as session:
data = await fetch(full_url, session)
try:
return transform_with_jq(data['content'], jq_filter)
except ValueError:
# This follows the HTTP connector's behavior for a similar situation
LOGGER.error('Could not transform the data using %s as filter',
jq_filter)
raise
except ScriptRuntimeError:
LOGGER.error('Could not transform the data using %s as filter',
jq_filter)
raise
def _retrieve_data(self, data_source: RokDataSource) -> pd.DataFrame:
# Endpoint depends on the authentication mode
endpoint = f'{self.host}/graphql'
date_viewid_parameters = {
'start_date': data_source.start_date,
'end_date': data_source.end_date,
'viewId': data_source.viewId,
}
if data_source.parameters:
parameters = {**data_source.parameters, **date_viewid_parameters}
else:
parameters = date_viewid_parameters
data_source.query = nosql_apply_parameters_to_query(
data_source.query, parameters)
if self.authenticated_with_token:
if not data_source.live_data:
raise InvalidAuthenticationMethodError(
"""Request with ROK token is not possible while not
in live data mode. Change the connector configuration to live data"""
)
if not self.secret:
raise NoROKSecretAvailableError('secrets not defined')
res = self.retrieve_data_with_jwt(data_source, endpoint)
else:
endpoint = f'{endpoint}?DatabaseName={data_source.database}'
# First retrieve the authentication token
rok_token = self.retrieve_token_with_password(
data_source.database, endpoint)
# Then retrieve the data
payload = {'query': data_source.query}
res = requests.post(endpoint,
json=payload,
headers={
'Token': rok_token
}).json()
if 'errors' in res:
raise ValueError(str(res['errors']))
return pd.DataFrame(transform_with_jq(res, data_source.filter))
def use_jq_to_parse(data, jq_filter):
try:
return transform_with_jq(data, jq_filter)
except ValueError:
WorkdayConnector.logger.error(f'Could not transform {data} using {jq_filter}')
raise