def __call__(self, parser, namespace, value, option_string=None):
config = read_config_file()
format = namespace.format if namespace.format else config.get('DEFAULT','format')
oauth = config.getboolean('DEFAULT','oauth')
# Checking ig OAuth params are defined
if oauth :
oauth = YOAuth(None, None, from_file=config.get('auth','from_file'))
attr = {
'community': True,
'format': format,
#'jsonCompact': namespace.jsonCompact if namespace.jsonCompact else config.getboolean(format, 'jsonCompact'),
'debug': namespace.debug if namespace.debug else config.getboolean(format, 'debug'),
'oauth': oauth
}
yql = MYQL(**attr)
yql.diagnostics = namespace.diagnostics if namespace.diagnostics else config.getboolean(format, 'diagnostics')
for v in value:
response = yql.rawQuery(v)
if not response.status_code == 200:
print(response.content)
sys.exit(1)
if format == 'json':
print(pretty_json(response.content))
else:
print(pretty_xml(response.content))
sys.exit(0)
def get_best_excerpt(self, config_name, split_name):
try:
best_excerpt = ""
MIN_LENGTH = 100
MAX_LENGTH = 1000
for i, e in enumerate(
self.dataset_per_config[config_name][split_name]):
if i > 100:
break
excerpt = pretty_json(e)
if len(excerpt) > len(best_excerpt):
if len(excerpt) < MAX_LENGTH or len(best_excerpt) == 0:
best_excerpt = excerpt
else:
if len(best_excerpt) > MAX_LENGTH:
if len(excerpt) > MIN_LENGTH:
best_excerpt = excerpt
return best_excerpt
except:
self.warn(f"Could not find excerpt for {config_name}/{split_name}")
return ""
def _output_aggregated_report(self, out):
out.write(
pretty_json(self._make_aggregated_report()).replace(
'"', "'").replace("\\'", '"') + "\n")
def __parse_command(self, command, service, scenario_root):
json_pattern = None
result_name = None
print_result = False
print_body = False
export_result = None
eval_expr = None
pre_eval_expr = None
check_code = 200
check_message = None
repeat = None
description = None
order = None
hooks = None
body_to_print = False
# load the check_result json file if provided
if 'check_result' in command:
check_json_val = command.pop('check_result')
if isinstance(check_json_val, dict):
json_pattern = check_json_val
elif isinstance(check_json_val, str) or isinstance(check_json_val, unicode):
check_json_file = self.get_filepath(self.scenario_root, check_json_val)
with open(check_json_file, 'r') as check:
json_pattern = json.load(check)
if 'save_result' in command:
result_name = command.pop('save_result')
if 'check_code' in command:
check_code = command.pop('check_code')
if 'check_message' in command:
check_message = command.pop('check_message')
if 'print_result' in command:
print_result = command.pop('print_result')
if 'print_body' in command:
print_body = command.pop('print_body')
if 'export_result' in command:
export_result = command.pop('export_result')
if 'eval_expr' in command:
eval_expr = command.pop('eval_expr')
if 'pre_eval_expr' in command:
pre_eval_expr = command.pop('pre_eval_expr')
if 'repeat' in command:
repeat = command.pop('repeat')
if 'description' in command:
description = unicode(command.pop('description'))
if 'check_order' in command:
order = command.pop('check_order')
if 'hooks' in command:
hooks = command.pop('hooks')
if len(command.keys()) != 1:
raise ValueError("You must provide one and only one endpoint per command, see the manual.\n{}".format(
"\n".join(['- {}'.format(k) for k in command])))
# build the endpoint request
key = command.keys()[0]
if hooks and "setup" in hooks:
self.run_hook(hooks.get("setup"), "setup")
repeat_bool = True
times = 0
while repeat_bool:
# put the () for the endpoints
endpoint = 'service.' + key.replace('.', '().')
endpoint += '('
endpoint_args = []
if isinstance(command[key], dict):
for arg in command[key]:
val = command[key][arg]
# for body, read the json file
if arg == 'body':
# if we do not receive a json object, load it from a file
if not isinstance(val, dict):
match = self.expression_matcher.match(val)
if match:
# raises a ValueError, to be catched upper in the stack
val = self.__parse_expression(match.group(1))
else:
body_file = self.get_filepath(self.scenario_root, val)
if body_file:
with open(body_file, 'r') as body:
val = json.load(body)
# parse expressions in the body
val = self._parse_body(val)
# avoid to use `self.eval_expr` in the yml by using `expr` directly
expr = lambda e: self.eval_expr('{{' + e + '}}')
saved_results = self.output_results
saved_results['body'] = val
ns = {'saved_results': saved_results,
'body': val,
'expr': lambda e: self.eval_expr('{{' + e + '}}', container=saved_results)}
if pre_eval_expr:
if isinstance(pre_eval_expr, str) or isinstance(pre_eval_expr, unicode):
exec pre_eval_expr in ns
elif isinstance(pre_eval_expr, list):
for e in pre_eval_expr:
exec e in ns
val = ns.get('body', val)
if print_body is True:
body_to_print = val
else:
if isinstance(val, basestring):
val = '"' + str(self.eval_expr(val)) + '"'
elif isinstance(val, list):
# if we have a list here, we want to transform it into arg=[val1,val2]
# resolving each element of the list
val = ','.join([str(self.eval_expr(v)) for v in val])
val = "[{}]".format(val)
endpoint_args.append("{} = {}".format(arg, val))
endpoint += ','.join(endpoint_args) + ').execute()'
print "\n{}{}Executing : {}{}".format(ju.bold, ju.yellow, key, ju.end_color)
if description:
print "Description: {}\n".format(description)
if body_to_print:
print ju.info_color
print "Body JSON:"
pretty_json(body_to_print)
print ju.end_color
exec_time = time.time()
# run the endpoint request
status = 200
message = None
result = None
retry = True
nb_retries = 5
while retry and nb_retries > 0:
nb_retries -= 1
try:
ns = {'service': service}
exec "result = {}".format(endpoint) in ns
result = ns['result']
retry = False
except BadStatusLine as e:
print "RETRYING: {}".format(endpoint)
retry = True
time.sleep(1)
except Exception as e:
retry = False
try:
message = json.loads(e.content).get('error').get('message')
except:
pass
if check_code and hasattr(e, 'resp') and e.resp["status"] == str(check_code):
status = check_code
elif not repeat:
if len(e.message) == 0:
msg = e.__str__()
else:
msg = e.message
print traceback.format_exc()
raise RuntimeError("The executed command was: {}\nMessage: {}".
format(endpoint.replace("\.execute()", ""), msg))
else:
if hasattr(e, 'resp'):
status = e.resp["status"]
result = None
print "Done in {}ms".format(int(round((time.time() - exec_time) * 1000)))
if not repeat and status != check_code:
raise RuntimeError("The executed command was: {}\nMessage: {}".format(
endpoint.replace("\.execute()", ""),
"HTTP status code is {} and expected is {}.".format(status, check_code)
))
elif not repeat and int(check_code) - 200 >= 100:
result = None
if check_message and check_message != message:
raise RuntimeError("The executed command was: {}\nMessage: {}".format(
endpoint.replace("\.execute()", ""),
"HTTP error message is {} and expected is {}.".format(message, check_message)
))
if eval_expr:
ns = {'saved_results': self.output_results,
'result': result,
'expr': lambda e, container=self.output_results: self.eval_expr('{{' + e + '}}', container)}
if isinstance(eval_expr, str) or isinstance(eval_expr, unicode):
exec eval_expr in ns
elif isinstance(eval_expr, list):
for e in eval_expr:
exec e in ns
result = ns.get('result', result)
if result_name and result:
self.output_results[result_name] = result
if export_result and result:
with open("{}".format(export_result), 'w') as f:
json.dump(result, f, indent=4, separators=(',', ': '))
if result:
if print_result is True:
print ju.info_color
print "Result JSON:"
pretty_json(result)
print ju.end_color
elif isinstance(print_result, str) or isinstance(print_result, unicode):
# we have an expression!
match = self.expression_matcher.match(print_result)
if match:
val = None
try:
val = self.__parse_expression(match.group(1), container=result)
except Exception as e:
if self.debug:
print traceback.format_exc()
print e
if val:
print ju.info_color
print "Content of {}:".format(match.group(1))
pretty_json(val)
print ju.end_color
elif isinstance(print_result, list):
for expr in print_result:
# we have an expression!
match = self.expression_matcher.match(expr)
if match:
val = None
try:
val = self.__parse_expression(match.group(1), container=result)
except Exception as e:
if self.debug:
print traceback.format_exc()
print e
if val:
print ju.info_color
print "Content of {}:".format(match.group(1))
pretty_json(val)
print ju.end_color
val = None
if repeat:
repeat_bool = self.__parse_repeat(repeat, times, result, status, message)
if repeat_bool:
print "Calling the endpoint again"
else:
print "Done repeating the call"
times += 1
else:
repeat_bool = False
if json_pattern:
json_pattern = self._parse_body(json_pattern)
check_json(result, json_pattern, exit_on_error=self.exit_on_error)
if order:
if isinstance(order, str) or isinstance(order, unicode):
match = self.expression_matcher.match(order)
raise RuntimeError("Expression {} for check_order is incorrect".format(match.group(1)))
elif isinstance(order, list):
values = []
directions = []
paths = []
for criteria in order:
for expr, direction in criteria.iteritems():
# we have a list!
match = self.expression_matcher.match(expr)
if match:
val = self.__parse_expression(match.group(1), container=result)
values.append(val)
directions.append(direction)
paths.append(match.group(1))
check_order_values(values, directions, paths, exit_on_error=self.exit_on_error)
if hooks and "teardown" in hooks:
self.run_hook(hooks.get("teardown"), "teardown")
def _output_aggregated_report(self, out):
out.write(pretty_json(self._make_aggregated_report()).replace('"', "'").replace("\\'", '"') + "\n")
def _output_aggregated_report(self, out, run_stats):
output = OrderedDict([('runStats', run_stats),
('results', self._report.get_reports())])
#out.write(pretty_json(output))
out.write(pretty_json(output).replace('"', "'").replace("\\'", '"') + "\n")
def perform_search(self):
if self.verbose:
print utils.pretty_json(self.body)
return self.es.search(
index=self.index, doc_type=self.doc_type, body=self.body)
exec e in ns
result = ns.get('result', result)
if result_name and result:
self.output_results[result_name] = result
if export_result and result:
with open("{}".format(export_result), 'w') as f:
json.dump(result, f, indent=4, separators=(',', ': '))
if result:
if print_result is True:
print ju.info_color
print "Result JSON:"
pretty_json(result)
print ju.end_color
elif isinstance(print_result, str) or isinstance(print_result, unicode):
# we have an expression!
match = self.expression_matcher.match(print_result)
if match:
val = None
try:
val = self.__parse_expression(match.group(1), container=result)
except Exception, e:
if self.debug:
print traceback.format_exc()
print e
if val:
print ju.info_color