def set_method(self): method = Utilities.dict_lookup(self.settings, "endpoints", self.actual_endpoint_name, "method") if method: self.method = method return method method = Utilities.dict_lookup(self.settings, "common_endpoint_settings", "all", "method") if method: self.method = method return method self.method = "get" return method
def setup_oauth_header(self, global_vars): self._config.set_method() request_params = self._config.get_endpoint_setting('parameters') include_oauth_header = self._config.get_endpoint_setting('include_oauth_header') if not include_oauth_header: return if not Utilities.dict_lookup(self._config.settings, 'authentication', 'oauth_header_parameters'): return oauth_nonce = hashlib.sha1(str(random())).hexdigest() oauth_timestamp = str(int(time.time())) oauth_params = self._get_oauth_params(oauth_nonce, oauth_timestamp) oauth_signature = self._get_oauth_signature(oauth_params, request_params) oauth_header_dict = {} for key, value in oauth_params.items(): value = urllib.quote(value, safe='') oauth_header_dict[key] = '{}="{}"'.format(key, value) oauth_header_dict['oauth_signature'] = 'oauth_signature="{}"'.format(urllib.quote(oauth_signature, safe='')) oauth_header_ordered = [t[1] for t in sorted(oauth_header_dict.items())] # print("*** SDKless:Authentication::setup_oauth_header:signature_params:") # print(oauth_header_ordered) global_vars['merge']['OAUTH-HEADER-PARAMS'] = ', '.join(oauth_header_ordered) # print("*** SDKless:Authentication::setup_oauth_header:OAUTH-HEADER-PARAMS:") # print(global_vars['merge']['OAUTH-HEADER-PARAMS']) self._config.apply_global_vars(global_vars);
def set_actual_endpoint_name(self): # if passed endpoint_name is mapped in custom config, set to maps_to name maps_to = Utilities.dict_lookup(self.settings_custom, "endpoints", self.custom_endpoint_name, "maps_to") if maps_to: self.actual_endpoint_name = maps_to else: self.actual_endpoint_name = self.custom_endpoint_name
def _get_setting(self, settings, endpoint_name, setting): # endpoint setting can be in endpoint, common_endpoint_settings[self.method], or common_endpoint_settings['all'] (in order of precedence) # check in order of precedence and immediately return if non-object # if have object(s), merge them in reverse order since the latter ones overwrite results = [] setting_value = Utilities.dict_lookup(settings, "endpoints", endpoint_name, setting) if setting_value is not None: if isinstance(setting_value, dict): results.append(setting_value) else: return setting_value setting_value = Utilities.dict_lookup(settings, "common_endpoint_settings", self.method, setting) if setting_value is not None: if isinstance(setting_value, dict): results.append(setting_value) else: return setting_value setting_value = Utilities.dict_lookup(settings, "common_endpoint_settings", "all", setting) if setting_value is not None: if isinstance(setting_value, dict): results.append(setting_value) else: return setting_value if not results: return None ret = {} results.reverse() for s in results: for key, value in s.items(): ret[key] = value self._clean_endpoint_setting(ret) return ret
def _map_endpoint_parameter(self, endpoint, param): endpoint_param = Utilities.dict_lookup(self.settings_custom, "endpoints", endpoint, "parameter_maps", param) if endpoint_param: return endpoint_param global_param = self._map_global_parameter(param) if global_param: return global_param return param
def prepare_auth_step(self, step, incoming_params, global_vars): if incoming_params: if step['type'] == 'endpoint': if not Utilities.dict_lookup(self._config.settings, 'endpoints', step['endpoint']): return # merge incoming params with global_vars and re-setup config if Utilities.dict_lookup(self._config.settings, 'endpoints', step['endpoint'], 'merge_maps'): for incoming_key, merge_key in self._config.settings['endpoints'][step['endpoint']]['merge_maps'].items(): if incoming_params.get(incoming_key): global_vars['merge'][merge_key] = incoming_params[incoming_key] self._config.apply_global_vars(global_vars) param_location = self._config.settings['endpoints'][step['endpoint']] else: param_location = step if param_location.get('parameter_maps'): for key, value in incoming_params.items(): param_key = key; if param_location['parameter_maps'].get(key): param_key = param_location['parameter_maps'][key] if not param_location.get('parameters'): param_location['parameters'] = {} # if params are specified in config, only set those if param_location['parameters'].get(param_key): param_location['parameters'][param_key] = value else: # if parameters is a dict, merge in the incoming values; otherwise set parameters to None for next step if param_location.get('parameters') and isinstance(param_location.get('parameters'), dict): # if params are specified in config, only set those for key, value in param_location['parameters'].items(): if incoming_params.get(key): param_location['parameters'][key] = incoming_params[key] else: param_location['parameters'] = None
def apply_custom_global_vars(self): global_merge = Utilities.dict_lookup(self.settings_custom, "global", "merge") if global_merge: settings = json.dumps(self.settings) for key, value in global_merge.items(): merge_key = "{}{}{}".format(self._merge_open, key, self._merge_close) settings = settings.replace(merge_key, value) try: settings = json.loads(settings) except: raise Exception("invalid JSON caused by custom config global merge") self.settings = settings global_set = Utilities.dict_lookup(self.settings_custom, "global", "set") if global_set: for key, value in global_set.items(): self.settings[key] = value
def _get_data(self, data): output_config = self._config.get_endpoint_setting('output') location = Utilities.dict_lookup(output_config, 'data', 'location') if not location: return data if not isinstance(location, list): raise Exception('endpoint output location must be a list') # drill down to desired data for location_key in location: data = data.get(location_key) if data == None: raise Exception("specified key not found in response: {}".format(location_key)) return data
def apply_custom_endpoint_params(self): custom_endpoint = Utilities.dict_lookup(self.settings_custom, "endpoints", self.custom_endpoint_name) if custom_endpoint: if custom_endpoint.get("parameters"): Utilities.dict_nested_add(self.settings, {}, "endpoints", self.actual_endpoint_name, "parameters") for key, value in custom_endpoint["parameters"].items(): self.settings["endpoints"][self.actual_endpoint_name]["parameters"][key] = value if custom_endpoint.get("output"): Utilities.dict_nested_add(self.settings, {}, "endpoints", self.actual_endpoint_name, "output") for key, value in custom_endpoint["output"].items(): self.settings["endpoints"][self.actual_endpoint_name]["output"][key] = value if custom_endpoint.get("limit") is not None: Utilities.dict_nested_add( self.settings, custom_endpoint["limit"], "endpoints", self.actual_endpoint_name, "limit" ) if custom_endpoint.get("paging") is not None: Utilities.dict_nested_add( self.settings, custom_endpoint["paging"], "endpoints", self.actual_endpoint_name, "paging" )
def apply_global_vars(self, global_vars): if global_vars.get("merge"): settings = json.dumps(self.settings) for key, value in global_vars["merge"].items(): if key == "OAUTH-HEADER-PARAMS": value = Utilities.add_slashes(value) settings = settings.replace("{}{}{}".format(self._merge_open, key, self._merge_close), value) try: settings = json.loads(settings) except: raise Exception("invalid JSON caused by global merge vars") self.settings = settings if global_vars.get("set"): self._apply_global_set_vars(global_vars["set"])
def _process_prerequisites(self): if Utilities.dict_lookup(self.config.settings,'endpoint_prerequisites'): if not isinstance(self.config.settings['endpoint_prerequisites'], list): raise Exception('endpoint_prerequisites must be an array') for prerequisite in self.config.settings['endpoint_prerequisites']: if not prerequisite.get('repeat') and self._prerequisites_complete: continue if prerequisite.get('protocol'): if prerequisite['protocol'] == 'cookie': cookie_file = "{}/sdkless_{}_cookie".format(tempfile.gettempdir(), self.api_name) if self.local_vars.get('cookie_id'): cookie_file = "{}_{}".format(cookie_file, self.local_vars['cookie_id']) self.config.settings['common_endpoint_settings']['all']['request_options']['COOKIEFILE'] = cookie_file self.config.settings['common_endpoint_settings']['all']['request_options']['COOKIEJAR'] = cookie_file if prerequisite.get('endpoint'): if not self.config.settings['endpoints'].get(prerequisite['endpoint']): raise Exception('specified prerequisite endpoint does not exist in config') self.config.custom_endpoint_name = prerequisite['endpoint'] self.config.set_actual_endpoint_name() self.config.apply_custom_endpoint_params() response = self.request.get_response(self._time_limit) if response and prerequisite.get('merge_maps'): for response_key, merge_key in prerequisite['merge_maps'].items(): if response.get(response_key): self.global_vars['merge'][merge_key] = response[response_key] self.config.apply_global_vars(self.global_vars) self.prerequisites_complete = True
def go(self, endpoint_name, endpoint_vars = {}, local_vars = {}): if isinstance(local_vars, dict): self.local_vars = local_vars # must set endpoint name before checking for bypass_prerequisites self.config.custom_endpoint_name = endpoint_name self.config.set_actual_endpoint_name() self.config.apply_custom_endpoint_params() if not Utilities.dict_lookup(self.config.settings, 'endpoints', self.config.custom_endpoint_name, 'bypass_prerequisites'): self._process_prerequisites() # must set endpoint name after processing prerequisites to setup requested endpoint self.config.custom_endpoint_name = endpoint_name self.config.set_actual_endpoint_name() self.config.apply_custom_endpoint_params() if self.config.actual_endpoint_name not in self.config.settings['endpoints']: raise Exception('specified endpoint does not exist in config: {}'.format(self.config['actual_endpoint_name'])) if isinstance(endpoint_vars, dict): self.endpoint_vars = endpoint_vars else: self.endpoint_vars = {} array_set = endpoint_vars.get('array_set') if array_set: self.config.apply_endpoint_array_set_vars(array_set) self.config.apply_endpoint_vars(endpoint_vars) self.config.set_method() self._auth.setup_oauth_header(self.global_vars) self._time_limit = self.local_vars.get('time_limit') or self.config.get_custom_endpoint_setting('time_limit') output = self.request.get_response(self._time_limit) output_config = self.config.get_endpoint_setting('output') # filter output if output_config and 'filter' in output_config and Utilities.is_structure(output): unfiltered_output = list(output) output = [] if not Utilities.is_structure(output_config['filter']): raise Exception('onfig endpoint output filter must be a structure') for filter in output_config['filter']: match_found = False if 'search_key' not in filter or 'search_value' not in filter: raise Exception('search_key and search_value are required for output filtering') for item in unfiltered_output: item_value = item.get(filter['search_key']) if item_value == None: continue if item_value == filter['search_value']: match_found = True if filter['return_key']: return item.get(filter['return_key']) output.append(item) if filter.get('return_type'): if filter['return_type'] == 'boolean': return match_found elif filter['return_type'] == '!boolean': return not match_found self.config.reset_to_unmerged() return output
def _map_global_parameter(self, param): param = Utilities.dict_lookup(self.settings_custom, "global", "parameter_maps", param) return param
def apply_endpoint_array_set_vars(self, array_set): # create static method variables if not hasattr(Configuration.apply_endpoint_array_set_vars.__func__, "new_endpoint_setting"): Configuration.apply_endpoint_array_set_vars.__func__.new_endpoint_setting = [] if not hasattr(Configuration.apply_endpoint_array_set_vars.__func__, "endpoint_setting_keys"): Configuration.apply_endpoint_array_set_vars.__func__.endpoint_setting_keys = [] array_set_templates = Utilities.dict_lookup( self.settings_custom, "endpoints", self.custom_endpoint_name, "array_set_templates" ) if not array_set_templates: return for key, value in array_set.items(): new_key = self._map_endpoint_parameter(self.custom_endpoint_name, key) Configuration.apply_endpoint_array_set_vars.__func__.endpoint_setting_keys.append(new_key) # when list is found, generate required format from template and populate # list must contain dict children if isinstance(value, list): template = array_set_templates.get(new_key) if not template: raise Exception("array_set_templates key {} does not exist in custom config".format(new_key)) if ( not isinstance(template, str) and not isinstance(template, unicode) and not isinstance(template, dict) ): raise Exception("array_set template must be a string or a dict") for entry in value: if not isinstance(entry, dict): raise Exception("array_set must contain dictionaries to apply to template") if isinstance(template, str) or isinstance(template, unicode): new_template = entry.get(template) if new_template is None: raise Exception("array_set template key not found in array set") else: new_template = copy.deepcopy(template) for entry_key, entry_value in entry.items(): self._update_template_value(new_template, entry_key, entry_value) Configuration.apply_endpoint_array_set_vars.__func__.new_endpoint_setting.append(new_template) self.set_endpoint_setting( Configuration.apply_endpoint_array_set_vars.endpoint_setting_keys, Configuration.apply_endpoint_array_set_vars.new_endpoint_setting, ) Configuration.apply_endpoint_array_set_vars.__func__.endpoint_setting_keys = [] Configuration.apply_endpoint_array_set_vars.__func__.new_endpoint_setting = [] elif isinstance(value, dict): self.apply_endpoint_array_set_vars(value) else: raise Exception("array set must contain dict and/or list")
def set_endpoint_setting(self, keys, value): new_keys = ["endpoints", self.actual_endpoint_name] + keys Utilities.dict_nested_update(self.settings, value, *new_keys)
def populate(self, config, response, output): self._config = config response_count = 0 output_config = config.get_endpoint_setting('output') paging = config.get_endpoint_setting('paging') ret = {} if output == None: output = [] if not output_config: if paging: if isinstance(response, list): return len(response), output + response else: output.append(response) return 1, output else: return 1, response data = self._get_data(response) if not data: return 0, output output_format = Utilities.dict_lookup(output_config, 'data', 'format') if output_format == 'iterable': # like an array of contact records if not Utilities.is_structure(data): raise Exception('output config specifies structure data format but response is not a structure'); # put in specified output format, if applicable if Utilities.dict_lookup(output_config, 'data', 'items', 'locations'): if isinstance(data, dict): for data_key, data_value in data.items(): key_filter = Utilities.dict_lookup(output_config, 'data', 'key_filter') if key_filter: if (key_filter == 'numeric') and not isinstance(data_key, int): continue # if output_config.data.items is specified, we are expecting the data structure to contain child structures if not Utilities.is_structure(data_value): raise Exception('output config specifies data items but response children are not structures') output_item = self._get_item(data_value); output.append(output_item) response_count += 1 elif isinstance(data, list): for data_value in data: # if output_config.data.items is specified, we are expecting the data structure to contain child structures if not Utilities.is_structure(data_value): raise Exception('output config specifies data items but response children are not structures') output_item = self._get_item(data_value); output.append(output_item) response_count += 1 else: output = output + data response_count = len(data) else: # non-iterable (like scalar value or single contact record) if isinstance(data, dict) and Utilities.dict_lookup(output_config, 'data', 'items', 'locations'): return_output = self._get_item(data) else: return_output = data # leave data as is if paging: output.append(return_output) else: output = return_output response_count = 1 return response_count, output