def set_method(self):
method = Utilities.dict_lookup(self.settings, "endpoints", self.actual_endpoint_name, "method")
if method:
self.method = method
return method
method = Utilities.dict_lookup(self.settings, "common_endpoint_settings", "all", "method")
if method:
self.method = method
return method
self.method = "get"
return method
def setup_oauth_header(self, global_vars):
self._config.set_method()
request_params = self._config.get_endpoint_setting('parameters')
include_oauth_header = self._config.get_endpoint_setting('include_oauth_header')
if not include_oauth_header:
return
if not Utilities.dict_lookup(self._config.settings, 'authentication', 'oauth_header_parameters'):
return
oauth_nonce = hashlib.sha1(str(random())).hexdigest()
oauth_timestamp = str(int(time.time()))
oauth_params = self._get_oauth_params(oauth_nonce, oauth_timestamp)
oauth_signature = self._get_oauth_signature(oauth_params, request_params)
oauth_header_dict = {}
for key, value in oauth_params.items():
value = urllib.quote(value, safe='')
oauth_header_dict[key] = '{}="{}"'.format(key, value)
oauth_header_dict['oauth_signature'] = 'oauth_signature="{}"'.format(urllib.quote(oauth_signature, safe=''))
oauth_header_ordered = [t[1] for t in sorted(oauth_header_dict.items())]
# print("*** SDKless:Authentication::setup_oauth_header:signature_params:")
# print(oauth_header_ordered)
global_vars['merge']['OAUTH-HEADER-PARAMS'] = ', '.join(oauth_header_ordered)
# print("*** SDKless:Authentication::setup_oauth_header:OAUTH-HEADER-PARAMS:")
# print(global_vars['merge']['OAUTH-HEADER-PARAMS'])
self._config.apply_global_vars(global_vars);
def set_actual_endpoint_name(self):
# if passed endpoint_name is mapped in custom config, set to maps_to name
maps_to = Utilities.dict_lookup(self.settings_custom, "endpoints", self.custom_endpoint_name, "maps_to")
if maps_to:
self.actual_endpoint_name = maps_to
else:
self.actual_endpoint_name = self.custom_endpoint_name
def _get_setting(self, settings, endpoint_name, setting):
# endpoint setting can be in endpoint, common_endpoint_settings[self.method], or common_endpoint_settings['all'] (in order of precedence)
# check in order of precedence and immediately return if non-object
# if have object(s), merge them in reverse order since the latter ones overwrite
results = []
setting_value = Utilities.dict_lookup(settings, "endpoints", endpoint_name, setting)
if setting_value is not None:
if isinstance(setting_value, dict):
results.append(setting_value)
else:
return setting_value
setting_value = Utilities.dict_lookup(settings, "common_endpoint_settings", self.method, setting)
if setting_value is not None:
if isinstance(setting_value, dict):
results.append(setting_value)
else:
return setting_value
setting_value = Utilities.dict_lookup(settings, "common_endpoint_settings", "all", setting)
if setting_value is not None:
if isinstance(setting_value, dict):
results.append(setting_value)
else:
return setting_value
if not results:
return None
ret = {}
results.reverse()
for s in results:
for key, value in s.items():
ret[key] = value
self._clean_endpoint_setting(ret)
return ret
def _map_endpoint_parameter(self, endpoint, param):
endpoint_param = Utilities.dict_lookup(self.settings_custom, "endpoints", endpoint, "parameter_maps", param)
if endpoint_param:
return endpoint_param
global_param = self._map_global_parameter(param)
if global_param:
return global_param
return param
def prepare_auth_step(self, step, incoming_params, global_vars):
if incoming_params:
if step['type'] == 'endpoint':
if not Utilities.dict_lookup(self._config.settings, 'endpoints', step['endpoint']):
return
# merge incoming params with global_vars and re-setup config
if Utilities.dict_lookup(self._config.settings, 'endpoints', step['endpoint'], 'merge_maps'):
for incoming_key, merge_key in self._config.settings['endpoints'][step['endpoint']]['merge_maps'].items():
if incoming_params.get(incoming_key):
global_vars['merge'][merge_key] = incoming_params[incoming_key]
self._config.apply_global_vars(global_vars)
param_location = self._config.settings['endpoints'][step['endpoint']]
else:
param_location = step
if param_location.get('parameter_maps'):
for key, value in incoming_params.items():
param_key = key;
if param_location['parameter_maps'].get(key):
param_key = param_location['parameter_maps'][key]
if not param_location.get('parameters'):
param_location['parameters'] = {}
# if params are specified in config, only set those
if param_location['parameters'].get(param_key):
param_location['parameters'][param_key] = value
else:
# if parameters is a dict, merge in the incoming values; otherwise set parameters to None for next step
if param_location.get('parameters') and isinstance(param_location.get('parameters'), dict):
# if params are specified in config, only set those
for key, value in param_location['parameters'].items():
if incoming_params.get(key):
param_location['parameters'][key] = incoming_params[key]
else:
param_location['parameters'] = None
def apply_custom_global_vars(self):
global_merge = Utilities.dict_lookup(self.settings_custom, "global", "merge")
if global_merge:
settings = json.dumps(self.settings)
for key, value in global_merge.items():
merge_key = "{}{}{}".format(self._merge_open, key, self._merge_close)
settings = settings.replace(merge_key, value)
try:
settings = json.loads(settings)
except:
raise Exception("invalid JSON caused by custom config global merge")
self.settings = settings
global_set = Utilities.dict_lookup(self.settings_custom, "global", "set")
if global_set:
for key, value in global_set.items():
self.settings[key] = value
def _get_data(self, data):
output_config = self._config.get_endpoint_setting('output')
location = Utilities.dict_lookup(output_config, 'data', 'location')
if not location:
return data
if not isinstance(location, list):
raise Exception('endpoint output location must be a list')
# drill down to desired data
for location_key in location:
data = data.get(location_key)
if data == None:
raise Exception("specified key not found in response: {}".format(location_key))
return data
def apply_custom_endpoint_params(self):
custom_endpoint = Utilities.dict_lookup(self.settings_custom, "endpoints", self.custom_endpoint_name)
if custom_endpoint:
if custom_endpoint.get("parameters"):
Utilities.dict_nested_add(self.settings, {}, "endpoints", self.actual_endpoint_name, "parameters")
for key, value in custom_endpoint["parameters"].items():
self.settings["endpoints"][self.actual_endpoint_name]["parameters"][key] = value
if custom_endpoint.get("output"):
Utilities.dict_nested_add(self.settings, {}, "endpoints", self.actual_endpoint_name, "output")
for key, value in custom_endpoint["output"].items():
self.settings["endpoints"][self.actual_endpoint_name]["output"][key] = value
if custom_endpoint.get("limit") is not None:
Utilities.dict_nested_add(
self.settings, custom_endpoint["limit"], "endpoints", self.actual_endpoint_name, "limit"
)
if custom_endpoint.get("paging") is not None:
Utilities.dict_nested_add(
self.settings, custom_endpoint["paging"], "endpoints", self.actual_endpoint_name, "paging"
)
def apply_global_vars(self, global_vars):
if global_vars.get("merge"):
settings = json.dumps(self.settings)
for key, value in global_vars["merge"].items():
if key == "OAUTH-HEADER-PARAMS":
value = Utilities.add_slashes(value)
settings = settings.replace("{}{}{}".format(self._merge_open, key, self._merge_close), value)
try:
settings = json.loads(settings)
except:
raise Exception("invalid JSON caused by global merge vars")
self.settings = settings
if global_vars.get("set"):
self._apply_global_set_vars(global_vars["set"])
def _process_prerequisites(self):
if Utilities.dict_lookup(self.config.settings,'endpoint_prerequisites'):
if not isinstance(self.config.settings['endpoint_prerequisites'], list):
raise Exception('endpoint_prerequisites must be an array')
for prerequisite in self.config.settings['endpoint_prerequisites']:
if not prerequisite.get('repeat') and self._prerequisites_complete:
continue
if prerequisite.get('protocol'):
if prerequisite['protocol'] == 'cookie':
cookie_file = "{}/sdkless_{}_cookie".format(tempfile.gettempdir(), self.api_name)
if self.local_vars.get('cookie_id'):
cookie_file = "{}_{}".format(cookie_file, self.local_vars['cookie_id'])
self.config.settings['common_endpoint_settings']['all']['request_options']['COOKIEFILE'] = cookie_file
self.config.settings['common_endpoint_settings']['all']['request_options']['COOKIEJAR'] = cookie_file
if prerequisite.get('endpoint'):
if not self.config.settings['endpoints'].get(prerequisite['endpoint']):
raise Exception('specified prerequisite endpoint does not exist in config')
self.config.custom_endpoint_name = prerequisite['endpoint']
self.config.set_actual_endpoint_name()
self.config.apply_custom_endpoint_params()
response = self.request.get_response(self._time_limit)
if response and prerequisite.get('merge_maps'):
for response_key, merge_key in prerequisite['merge_maps'].items():
if response.get(response_key):
self.global_vars['merge'][merge_key] = response[response_key]
self.config.apply_global_vars(self.global_vars)
self.prerequisites_complete = True
def go(self, endpoint_name, endpoint_vars = {}, local_vars = {}):
if isinstance(local_vars, dict):
self.local_vars = local_vars
# must set endpoint name before checking for bypass_prerequisites
self.config.custom_endpoint_name = endpoint_name
self.config.set_actual_endpoint_name()
self.config.apply_custom_endpoint_params()
if not Utilities.dict_lookup(self.config.settings, 'endpoints', self.config.custom_endpoint_name, 'bypass_prerequisites'):
self._process_prerequisites()
# must set endpoint name after processing prerequisites to setup requested endpoint
self.config.custom_endpoint_name = endpoint_name
self.config.set_actual_endpoint_name()
self.config.apply_custom_endpoint_params()
if self.config.actual_endpoint_name not in self.config.settings['endpoints']:
raise Exception('specified endpoint does not exist in config: {}'.format(self.config['actual_endpoint_name']))
if isinstance(endpoint_vars, dict):
self.endpoint_vars = endpoint_vars
else:
self.endpoint_vars = {}
array_set = endpoint_vars.get('array_set')
if array_set:
self.config.apply_endpoint_array_set_vars(array_set)
self.config.apply_endpoint_vars(endpoint_vars)
self.config.set_method()
self._auth.setup_oauth_header(self.global_vars)
self._time_limit = self.local_vars.get('time_limit') or self.config.get_custom_endpoint_setting('time_limit')
output = self.request.get_response(self._time_limit)
output_config = self.config.get_endpoint_setting('output')
# filter output
if output_config and 'filter' in output_config and Utilities.is_structure(output):
unfiltered_output = list(output)
output = []
if not Utilities.is_structure(output_config['filter']):
raise Exception('onfig endpoint output filter must be a structure')
for filter in output_config['filter']:
match_found = False
if 'search_key' not in filter or 'search_value' not in filter:
raise Exception('search_key and search_value are required for output filtering')
for item in unfiltered_output:
item_value = item.get(filter['search_key'])
if item_value == None:
continue
if item_value == filter['search_value']:
match_found = True
if filter['return_key']:
return item.get(filter['return_key'])
output.append(item)
if filter.get('return_type'):
if filter['return_type'] == 'boolean':
return match_found
elif filter['return_type'] == '!boolean':
return not match_found
self.config.reset_to_unmerged()
return output
def _map_global_parameter(self, param):
param = Utilities.dict_lookup(self.settings_custom, "global", "parameter_maps", param)
return param
def apply_endpoint_array_set_vars(self, array_set):
# create static method variables
if not hasattr(Configuration.apply_endpoint_array_set_vars.__func__, "new_endpoint_setting"):
Configuration.apply_endpoint_array_set_vars.__func__.new_endpoint_setting = []
if not hasattr(Configuration.apply_endpoint_array_set_vars.__func__, "endpoint_setting_keys"):
Configuration.apply_endpoint_array_set_vars.__func__.endpoint_setting_keys = []
array_set_templates = Utilities.dict_lookup(
self.settings_custom, "endpoints", self.custom_endpoint_name, "array_set_templates"
)
if not array_set_templates:
return
for key, value in array_set.items():
new_key = self._map_endpoint_parameter(self.custom_endpoint_name, key)
Configuration.apply_endpoint_array_set_vars.__func__.endpoint_setting_keys.append(new_key)
# when list is found, generate required format from template and populate
# list must contain dict children
if isinstance(value, list):
template = array_set_templates.get(new_key)
if not template:
raise Exception("array_set_templates key {} does not exist in custom config".format(new_key))
if (
not isinstance(template, str)
and not isinstance(template, unicode)
and not isinstance(template, dict)
):
raise Exception("array_set template must be a string or a dict")
for entry in value:
if not isinstance(entry, dict):
raise Exception("array_set must contain dictionaries to apply to template")
if isinstance(template, str) or isinstance(template, unicode):
new_template = entry.get(template)
if new_template is None:
raise Exception("array_set template key not found in array set")
else:
new_template = copy.deepcopy(template)
for entry_key, entry_value in entry.items():
self._update_template_value(new_template, entry_key, entry_value)
Configuration.apply_endpoint_array_set_vars.__func__.new_endpoint_setting.append(new_template)
self.set_endpoint_setting(
Configuration.apply_endpoint_array_set_vars.endpoint_setting_keys,
Configuration.apply_endpoint_array_set_vars.new_endpoint_setting,
)
Configuration.apply_endpoint_array_set_vars.__func__.endpoint_setting_keys = []
Configuration.apply_endpoint_array_set_vars.__func__.new_endpoint_setting = []
elif isinstance(value, dict):
self.apply_endpoint_array_set_vars(value)
else:
raise Exception("array set must contain dict and/or list")
def set_endpoint_setting(self, keys, value):
new_keys = ["endpoints", self.actual_endpoint_name] + keys
Utilities.dict_nested_update(self.settings, value, *new_keys)
def populate(self, config, response, output):
self._config = config
response_count = 0
output_config = config.get_endpoint_setting('output')
paging = config.get_endpoint_setting('paging')
ret = {}
if output == None:
output = []
if not output_config:
if paging:
if isinstance(response, list):
return len(response), output + response
else:
output.append(response)
return 1, output
else:
return 1, response
data = self._get_data(response)
if not data:
return 0, output
output_format = Utilities.dict_lookup(output_config, 'data', 'format')
if output_format == 'iterable': # like an array of contact records
if not Utilities.is_structure(data):
raise Exception('output config specifies structure data format but response is not a structure');
# put in specified output format, if applicable
if Utilities.dict_lookup(output_config, 'data', 'items', 'locations'):
if isinstance(data, dict):
for data_key, data_value in data.items():
key_filter = Utilities.dict_lookup(output_config, 'data', 'key_filter')
if key_filter:
if (key_filter == 'numeric') and not isinstance(data_key, int):
continue
# if output_config.data.items is specified, we are expecting the data structure to contain child structures
if not Utilities.is_structure(data_value):
raise Exception('output config specifies data items but response children are not structures')
output_item = self._get_item(data_value);
output.append(output_item)
response_count += 1
elif isinstance(data, list):
for data_value in data:
# if output_config.data.items is specified, we are expecting the data structure to contain child structures
if not Utilities.is_structure(data_value):
raise Exception('output config specifies data items but response children are not structures')
output_item = self._get_item(data_value);
output.append(output_item)
response_count += 1
else:
output = output + data
response_count = len(data)
else: # non-iterable (like scalar value or single contact record)
if isinstance(data, dict) and Utilities.dict_lookup(output_config, 'data', 'items', 'locations'):
return_output = self._get_item(data)
else:
return_output = data # leave data as is
if paging:
output.append(return_output)
else:
output = return_output
response_count = 1
return response_count, output
