def _notify(result, error=None, notify_url: str = None, notify_args: dict = None) -> bool: result_param_convention = '[=]' error_param_convention = '[!]' if notify_url: if isinstance(notify_args, MutableMapping): result_param_name = notify_args.pop(result_param_convention, None) error_param_name = notify_args.pop(error_param_convention, None) else: result_param_name = error_param_name = None if result_param_name: notify_args[result_param_name] = result if error_param_name: notify_args[error_param_name] = error rest(notify_url, notify_args) return True else: return False
def run_query(connection_string:str, command_text:str, result_model:str='DictOfList', column_mapping:Mapping={}, mdx_retries:int=1, delay_retry:float=10.0, pass_result_to_url:str=None, more_args:Mapping=None, notify_url:str=None, notify_args:MutableMapping=None): result = {} retries = 0 max_retries = int(mdx_retries) if isinstance(mdx_retries, (int, str)) else 0 delay = float(delay_retry) if isinstance(delay_retry, (int, float, str)) else 10.0 if delay < 1.0: delay = 1.0 while True: try: with AdomdClient(connection_string) as client: result = client.execute(command_text, result_model, column_mapping) break except Exception as err: if retries < max_retries: retries += 1 sleep(delay) else: if _notify(result, err, notify_url, notify_args): # Send a notification with result data and/or error information return result else: raise try: if pass_result_to_url: if more_args: if isinstance(result, MutableMapping): result.update(more_args) elif isinstance(result, list): for row in result: if isinstance(row, MutableMapping): row.update(more_args) result = rest(pass_result_to_url, result) # Chain above result to DbWebApi for storage or further processing except Exception as err: if not _notify(result, err, notify_url, notify_args): # Send a notification with result data and/or error information raise else: _notify(result, None, notify_url, notify_args) # Send a notification with result data return result
def invoke_powerbi_rest(access_token: str, http_method: str, rest_path: str, request_payload: Mapping = None, organization: str = 'myorg', api_version: str = 'v1.0', **kwargs): """Executes a REST call to the Power BI service, with the specified URL and body. :param access_token: The authentication access token for the Power BI REST call. :param http_method: Method for the request: ``GET``, ``POST``, ``DELETE``, ``PUT``, ``PATCH``, ``OPTIONS``. :param rest_path: Relative or absolute URL of the Power BI entity you want to access. For example, if you want to access https://api.powerbi.com/v1.0/myorg/groups, then specify 'groups', or pass in the entire URL. :param request_payload: Body of the request. This is optional unless the request method is POST, PUT, or PATCH. :param organization: (optional) Organization name or tenant GUID to include in the URL. Default is 'myorg'. :param api_version: (optional) Version of the API to include in the URL. Default is 'v1.0'. Ignored if ``rest_path`` is an absolute URL. :param kwargs: (optional) Please refer to https://requests.readthedocs.io for other optional arguments. :return: A JSON decoded object. """ def full_url(relative_url: str, organization: str, api_version: str) -> str: if not organization: organization = 'myorg' if not api_version: api_version = 'v1.0' return urljoin( f"https://api.powerbi.com/{api_version}/{organization}/", relative_url) explicit_headers = kwargs.get('headers', {}) if isinstance(explicit_headers, str): explicit_headers = json_decode(explicit_headers) if not isinstance(explicit_headers, Mapping): explicit_headers = {} headers = { 'Pragma': 'no-cache', 'Cache-Control': 'no-cache', 'Authorization': 'Bearer ' + access_token, 'Accept': 'application/json' } headers.update(explicit_headers) kwargs['headers'] = headers return rest(full_url(rest_path, organization, api_version), request_payload, http_method, error_extractor=lambda x: x['error']['message'], **kwargs)
def invoke_task_sp(sp_url: str, sp_args: dict, sp_timeout: float) -> dict: def check_dbwebapi(result: dict) -> bool: if not isinstance(result, Mapping): return False if 'ResultSets' in result and 'OutputParameters' in result and 'ReturnValue' in result: return True else: return False result = rest(sp_url, sp_args, timeout=sp_timeout) if result and not check_dbwebapi(result): raise TypeError(f"{repr(sp_url)} is not a dbwebapi call") return result
def _invoke_sp(sp_url: str, sp_args: dict, sp_timeout: float) -> dict: def check_dbwebapi(result: dict) -> bool: if not isinstance(result, Mapping): return False if 'ResultSets' in result and 'OutputParameters' in result and 'ReturnValue' in result: return True else: return False result = rest(sp_url, sp_args, timeout=sp_timeout) if result and not check_dbwebapi(result): raise TypeError(f"{repr(sp_url)} is not a dbwebapi call") result_sets = result['ResultSets'] count_sets = len(result_sets) if count_sets < 2 or len(result_sets[0]) != count_sets - 1: raise ValueError( f"the first result set must be used to indicate the corresponding Power BI table name and optional push sequence number for all subsequent result sets" ) return result
def start(task_sp_url: str, sp_args: dict, mdx_conn_str: str, timeout: float = 1800, mdx_column: str = 'MDX_QUERY', column_map_column: str = 'COLUMN_MAPPING', callback_sp_column: str = 'CALLBACK_SP', callback_args_column: str = 'CALLBACK_ARGS', db_type='oracle', post_sp_outparam: str = 'OUT_POST_SP', post_sp_args_outparam: str = 'OUT_POST_SP_ARGS', notify_url: str = None, notify_args: dict = None): def invoke_task_sp(sp_url: str, sp_args: dict, sp_timeout: float) -> dict: def check_dbwebapi(result: dict) -> bool: if not isinstance(result, Mapping): return False if 'ResultSets' in result and 'OutputParameters' in result and 'ReturnValue' in result: return True else: return False result = rest(sp_url, sp_args, timeout=sp_timeout) if result and not check_dbwebapi(result): raise TypeError(f"{repr(sp_url)} is not a dbwebapi call") return result def get_tasks(sp_result: dict) -> dict: out_params = CaseInsensitiveDict(sp_result['OutputParameters']) post_sp = out_params.pop(post_sp_outparam, None) post_sp_args = json.loads(out_params.pop(post_sp_args_outparam, '{}')) if post_sp: post_url = urljoin(task_sp_url, '../' + post_sp) post_sp_args.update(out_params) else: post_url = None post_sp_args = None if db_type and isinstance(db_type, str) and db_type[:3].lower() == 'ora': result_model = 'DictOfList' else: result_model = 'SqlTvp' serial_tasks = [] for rs in sp_result['ResultSets']: parallel_tasks = [] for row in rs: task = CaseInsensitiveDict(row) mdx_query = task.get(mdx_column) if not mdx_query: if parallel_tasks: continue # skip a row if mdx_column is missing from a subsequent row else: break # skip the whole resultset if mdx_column is missing from the first row callback_sp = task.get(callback_sp_column) if callback_sp: column_map = json.loads(task.get(column_map_column)) callback_url = urljoin(task_sp_url, '../' + callback_sp) callback_args = json.loads( task.get(callback_args_column, '{}')) if out_params: callback_args.update(out_params) parallel_tasks.append({ "(://)": _url_mdx_reader, "(...)": { "connection_string": mdx_conn_str, "command_text": mdx_query, "result_model": result_model, "column_mapping": column_map, "pass_result_to_url": callback_url, "more_args": callback_args }, "(:!!)": timeout }) if parallel_tasks: serial_tasks.append({"[###]": parallel_tasks}) if serial_tasks: if len(serial_tasks) == 1: svc_grp = serial_tasks[0] else: svc_grp = {"[+++]": serial_tasks} else: svc_grp = None return (svc_grp, post_url, post_sp_args) try: result = None while True: result = invoke_task_sp(task_sp_url, sp_args, timeout) svc_grp, post_url, post_sp_args = get_tasks(result) if svc_grp: result = rest(_url_svc_grp, {"rest": svc_grp}) if post_url: task_sp_url, sp_args = post_url, post_sp_args else: break else: break except Exception as err: if not _notify( result, err, notify_url, notify_args ): # Send a notification with result data and/or error information raise else: _notify(result, None, notify_url, notify_args) # Send a notification with result data return result
def _task_func(url:str, data:dict=None, timeout:float=None, headers:dict=None): return rest(url, data, timeout=timeout, headers=headers)