def _notify(result,
error=None,
notify_url: str = None,
notify_args: dict = None) -> bool:
result_param_convention = '[=]'
error_param_convention = '[!]'
if notify_url:
if isinstance(notify_args, MutableMapping):
result_param_name = notify_args.pop(result_param_convention, None)
error_param_name = notify_args.pop(error_param_convention, None)
else:
result_param_name = error_param_name = None
if result_param_name:
notify_args[result_param_name] = result
if error_param_name:
notify_args[error_param_name] = error
rest(notify_url, notify_args)
return True
else:
return False
def run_query(connection_string:str, command_text:str,
result_model:str='DictOfList', column_mapping:Mapping={},
mdx_retries:int=1, delay_retry:float=10.0,
pass_result_to_url:str=None, more_args:Mapping=None,
notify_url:str=None, notify_args:MutableMapping=None):
result = {}
retries = 0
max_retries = int(mdx_retries) if isinstance(mdx_retries, (int, str)) else 0
delay = float(delay_retry) if isinstance(delay_retry, (int, float, str)) else 10.0
if delay < 1.0:
delay = 1.0
while True:
try:
with AdomdClient(connection_string) as client:
result = client.execute(command_text, result_model, column_mapping)
break
except Exception as err:
if retries < max_retries:
retries += 1
sleep(delay)
else:
if _notify(result, err, notify_url, notify_args): # Send a notification with result data and/or error information
return result
else:
raise
try:
if pass_result_to_url:
if more_args:
if isinstance(result, MutableMapping):
result.update(more_args)
elif isinstance(result, list):
for row in result:
if isinstance(row, MutableMapping):
row.update(more_args)
result = rest(pass_result_to_url, result) # Chain above result to DbWebApi for storage or further processing
except Exception as err:
if not _notify(result, err, notify_url, notify_args): # Send a notification with result data and/or error information
raise
else:
_notify(result, None, notify_url, notify_args) # Send a notification with result data
return result
def invoke_powerbi_rest(access_token: str,
http_method: str,
rest_path: str,
request_payload: Mapping = None,
organization: str = 'myorg',
api_version: str = 'v1.0',
**kwargs):
"""Executes a REST call to the Power BI service, with the specified URL and body.
:param access_token: The authentication access token for the Power BI REST call.
:param http_method: Method for the request: ``GET``, ``POST``, ``DELETE``, ``PUT``, ``PATCH``, ``OPTIONS``.
:param rest_path: Relative or absolute URL of the Power BI entity you want to access. For example, if you want to access https://api.powerbi.com/v1.0/myorg/groups, then specify 'groups', or pass in the entire URL.
:param request_payload: Body of the request. This is optional unless the request method is POST, PUT, or PATCH.
:param organization: (optional) Organization name or tenant GUID to include in the URL. Default is 'myorg'.
:param api_version: (optional) Version of the API to include in the URL. Default is 'v1.0'. Ignored if ``rest_path`` is an absolute URL.
:param kwargs: (optional) Please refer to https://requests.readthedocs.io for other optional arguments.
:return: A JSON decoded object.
"""
def full_url(relative_url: str, organization: str,
api_version: str) -> str:
if not organization:
organization = 'myorg'
if not api_version:
api_version = 'v1.0'
return urljoin(
f"https://api.powerbi.com/{api_version}/{organization}/",
relative_url)
explicit_headers = kwargs.get('headers', {})
if isinstance(explicit_headers, str):
explicit_headers = json_decode(explicit_headers)
if not isinstance(explicit_headers, Mapping):
explicit_headers = {}
headers = {
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
'Authorization': 'Bearer ' + access_token,
'Accept': 'application/json'
}
headers.update(explicit_headers)
kwargs['headers'] = headers
return rest(full_url(rest_path, organization, api_version),
request_payload,
http_method,
error_extractor=lambda x: x['error']['message'],
**kwargs)
def invoke_task_sp(sp_url: str, sp_args: dict, sp_timeout: float) -> dict:
def check_dbwebapi(result: dict) -> bool:
if not isinstance(result, Mapping):
return False
if 'ResultSets' in result and 'OutputParameters' in result and 'ReturnValue' in result:
return True
else:
return False
result = rest(sp_url, sp_args, timeout=sp_timeout)
if result and not check_dbwebapi(result):
raise TypeError(f"{repr(sp_url)} is not a dbwebapi call")
return result
def _invoke_sp(sp_url: str, sp_args: dict, sp_timeout: float) -> dict:
def check_dbwebapi(result: dict) -> bool:
if not isinstance(result, Mapping):
return False
if 'ResultSets' in result and 'OutputParameters' in result and 'ReturnValue' in result:
return True
else:
return False
result = rest(sp_url, sp_args, timeout=sp_timeout)
if result and not check_dbwebapi(result):
raise TypeError(f"{repr(sp_url)} is not a dbwebapi call")
result_sets = result['ResultSets']
count_sets = len(result_sets)
if count_sets < 2 or len(result_sets[0]) != count_sets - 1:
raise ValueError(
f"the first result set must be used to indicate the corresponding Power BI table name and optional push sequence number for all subsequent result sets"
)
return result
def start(task_sp_url: str,
sp_args: dict,
mdx_conn_str: str,
timeout: float = 1800,
mdx_column: str = 'MDX_QUERY',
column_map_column: str = 'COLUMN_MAPPING',
callback_sp_column: str = 'CALLBACK_SP',
callback_args_column: str = 'CALLBACK_ARGS',
db_type='oracle',
post_sp_outparam: str = 'OUT_POST_SP',
post_sp_args_outparam: str = 'OUT_POST_SP_ARGS',
notify_url: str = None,
notify_args: dict = None):
def invoke_task_sp(sp_url: str, sp_args: dict, sp_timeout: float) -> dict:
def check_dbwebapi(result: dict) -> bool:
if not isinstance(result, Mapping):
return False
if 'ResultSets' in result and 'OutputParameters' in result and 'ReturnValue' in result:
return True
else:
return False
result = rest(sp_url, sp_args, timeout=sp_timeout)
if result and not check_dbwebapi(result):
raise TypeError(f"{repr(sp_url)} is not a dbwebapi call")
return result
def get_tasks(sp_result: dict) -> dict:
out_params = CaseInsensitiveDict(sp_result['OutputParameters'])
post_sp = out_params.pop(post_sp_outparam, None)
post_sp_args = json.loads(out_params.pop(post_sp_args_outparam, '{}'))
if post_sp:
post_url = urljoin(task_sp_url, '../' + post_sp)
post_sp_args.update(out_params)
else:
post_url = None
post_sp_args = None
if db_type and isinstance(db_type,
str) and db_type[:3].lower() == 'ora':
result_model = 'DictOfList'
else:
result_model = 'SqlTvp'
serial_tasks = []
for rs in sp_result['ResultSets']:
parallel_tasks = []
for row in rs:
task = CaseInsensitiveDict(row)
mdx_query = task.get(mdx_column)
if not mdx_query:
if parallel_tasks:
continue # skip a row if mdx_column is missing from a subsequent row
else:
break # skip the whole resultset if mdx_column is missing from the first row
callback_sp = task.get(callback_sp_column)
if callback_sp:
column_map = json.loads(task.get(column_map_column))
callback_url = urljoin(task_sp_url, '../' + callback_sp)
callback_args = json.loads(
task.get(callback_args_column, '{}'))
if out_params:
callback_args.update(out_params)
parallel_tasks.append({
"(://)": _url_mdx_reader,
"(...)": {
"connection_string": mdx_conn_str,
"command_text": mdx_query,
"result_model": result_model,
"column_mapping": column_map,
"pass_result_to_url": callback_url,
"more_args": callback_args
},
"(:!!)": timeout
})
if parallel_tasks:
serial_tasks.append({"[###]": parallel_tasks})
if serial_tasks:
if len(serial_tasks) == 1:
svc_grp = serial_tasks[0]
else:
svc_grp = {"[+++]": serial_tasks}
else:
svc_grp = None
return (svc_grp, post_url, post_sp_args)
try:
result = None
while True:
result = invoke_task_sp(task_sp_url, sp_args, timeout)
svc_grp, post_url, post_sp_args = get_tasks(result)
if svc_grp:
result = rest(_url_svc_grp, {"rest": svc_grp})
if post_url:
task_sp_url, sp_args = post_url, post_sp_args
else:
break
else:
break
except Exception as err:
if not _notify(
result, err, notify_url, notify_args
): # Send a notification with result data and/or error information
raise
else:
_notify(result, None, notify_url,
notify_args) # Send a notification with result data
return result
def _task_func(url:str, data:dict=None, timeout:float=None, headers:dict=None):
return rest(url, data, timeout=timeout, headers=headers)