Ejemplo n.º 1
0
    def __call__(self, parser, namespace, value, option_string=None):

        config = read_config_file()
        
        format =  namespace.format if namespace.format else config.get('DEFAULT','format')
        oauth = config.getboolean('DEFAULT','oauth')

        # Checking ig OAuth params are defined
        if oauth :
            oauth = YOAuth(None, None, from_file=config.get('auth','from_file'))

        attr = {
            'community': True,
            'format': format,
            #'jsonCompact': namespace.jsonCompact if namespace.jsonCompact else config.getboolean(format, 'jsonCompact'),
            'debug': namespace.debug if namespace.debug else config.getboolean(format, 'debug'),
            'oauth': oauth
        }

        
        yql = MYQL(**attr)
        yql.diagnostics = namespace.diagnostics if namespace.diagnostics else config.getboolean(format, 'diagnostics')

        for v in value:
            response = yql.rawQuery(v)

            if not response.status_code == 200:
                print(response.content)
                sys.exit(1)

            if format == 'json':
                print(pretty_json(response.content))
            else:
                print(pretty_xml(response.content))
        sys.exit(0)
Ejemplo n.º 2
0
    def get_best_excerpt(self, config_name, split_name):
        try:
            best_excerpt = ""

            MIN_LENGTH = 100
            MAX_LENGTH = 1000
            for i, e in enumerate(
                    self.dataset_per_config[config_name][split_name]):
                if i > 100:
                    break
                excerpt = pretty_json(e)
                if len(excerpt) > len(best_excerpt):
                    if len(excerpt) < MAX_LENGTH or len(best_excerpt) == 0:
                        best_excerpt = excerpt
                else:
                    if len(best_excerpt) > MAX_LENGTH:
                        if len(excerpt) > MIN_LENGTH:
                            best_excerpt = excerpt
            return best_excerpt
        except:
            self.warn(f"Could not find excerpt for {config_name}/{split_name}")
            return ""
Ejemplo n.º 3
0
Archivo: dex.py Proyecto: WalterShe/dex
 def _output_aggregated_report(self, out):
     out.write(
         pretty_json(self._make_aggregated_report()).replace(
             '"', "'").replace("\\'", '"') + "\n")
Ejemplo n.º 4
0
    def __parse_command(self, command, service, scenario_root):
        json_pattern = None
        result_name = None
        print_result = False
        print_body = False
        export_result = None
        eval_expr = None
        pre_eval_expr = None
        check_code = 200
        check_message = None
        repeat = None
        description = None
        order = None
        hooks = None
        body_to_print = False

        # load the check_result json file if provided
        if 'check_result' in command:
            check_json_val = command.pop('check_result')
            if isinstance(check_json_val, dict):
                json_pattern = check_json_val
            elif isinstance(check_json_val, str) or isinstance(check_json_val, unicode):
                check_json_file = self.get_filepath(self.scenario_root, check_json_val)

                with open(check_json_file, 'r') as check:
                    json_pattern = json.load(check)

        if 'save_result' in command:
            result_name = command.pop('save_result')

        if 'check_code' in command:
            check_code = command.pop('check_code')

        if 'check_message' in command:
            check_message = command.pop('check_message')

        if 'print_result' in command:
            print_result = command.pop('print_result')

        if 'print_body' in command:
            print_body = command.pop('print_body')

        if 'export_result' in command:
            export_result = command.pop('export_result')

        if 'eval_expr' in command:
            eval_expr = command.pop('eval_expr')

        if 'pre_eval_expr' in command:
            pre_eval_expr = command.pop('pre_eval_expr')

        if 'repeat' in command:
            repeat = command.pop('repeat')

        if 'description' in command:
            description = unicode(command.pop('description'))

        if 'check_order' in command:
            order = command.pop('check_order')

        if 'hooks' in command:
            hooks = command.pop('hooks')

        if len(command.keys()) != 1:
            raise ValueError("You must provide one and only one endpoint per command, see the manual.\n{}".format(
                "\n".join(['- {}'.format(k) for k in command])))

        # build the endpoint request
        key = command.keys()[0]

        if hooks and "setup" in hooks:
            self.run_hook(hooks.get("setup"), "setup")

        repeat_bool = True
        times = 0

        while repeat_bool:
            # put the () for the endpoints
            endpoint = 'service.' + key.replace('.', '().')
            endpoint += '('

            endpoint_args = []
            if isinstance(command[key], dict):
                for arg in command[key]:
                    val = command[key][arg]

                    # for body, read the json file
                    if arg == 'body':
                        # if we do not receive a json object, load it from a file
                        if not isinstance(val, dict):
                            match = self.expression_matcher.match(val)
                            if match:
                                # raises a ValueError, to be catched upper in the stack
                                val = self.__parse_expression(match.group(1))
                            else:
                                body_file = self.get_filepath(self.scenario_root, val)

                                if body_file:
                                    with open(body_file, 'r') as body:
                                        val = json.load(body)

                        # parse expressions in the body
                        val = self._parse_body(val)

                        # avoid to use `self.eval_expr` in the yml by using `expr` directly
                        expr = lambda e: self.eval_expr('{{' + e + '}}')

                        saved_results = self.output_results
                        saved_results['body'] = val
                        ns = {'saved_results': saved_results,
                              'body': val,
                              'expr': lambda e: self.eval_expr('{{' + e + '}}', container=saved_results)}

                        if pre_eval_expr:
                            if isinstance(pre_eval_expr, str) or isinstance(pre_eval_expr, unicode):
                                exec pre_eval_expr in ns
                            elif isinstance(pre_eval_expr, list):
                                for e in pre_eval_expr:
                                    exec e in ns

                        val = ns.get('body', val)
                        if print_body is True:
                            body_to_print = val
                    else:
                        if isinstance(val, basestring):
                            val = '"' + str(self.eval_expr(val)) + '"'
                        elif isinstance(val, list):
                            # if we have a list here, we want to transform it into arg=[val1,val2]
                            # resolving each element of the list
                            val = ','.join([str(self.eval_expr(v)) for v in val])
                            val = "[{}]".format(val)

                    endpoint_args.append("{} = {}".format(arg, val))

            endpoint += ','.join(endpoint_args) + ').execute()'

            print "\n{}{}Executing : {}{}".format(ju.bold, ju.yellow, key, ju.end_color)
            if description:
                print "Description: {}\n".format(description)

            if body_to_print:
                print ju.info_color
                print "Body JSON:"
                pretty_json(body_to_print)
                print ju.end_color

            exec_time = time.time()
            # run the endpoint request
            status = 200
            message = None
            result = None

            retry = True
            nb_retries = 5
            while retry and nb_retries > 0:
                nb_retries -= 1
                try:
                    ns = {'service': service}
                    exec "result = {}".format(endpoint) in ns
                    result = ns['result']
                    retry = False
                except BadStatusLine as e:
                    print "RETRYING: {}".format(endpoint)
                    retry = True
                    time.sleep(1)
                except Exception as e:
                    retry = False
                    try:
                        message = json.loads(e.content).get('error').get('message')
                    except:
                        pass

                    if check_code and hasattr(e, 'resp') and e.resp["status"] == str(check_code):
                        status = check_code
                    elif not repeat:
                        if len(e.message) == 0:
                            msg = e.__str__()
                        else:
                            msg = e.message
                        print traceback.format_exc()
                        raise RuntimeError("The executed command was: {}\nMessage: {}".
                                           format(endpoint.replace("\.execute()", ""), msg))
                    else:
                        if hasattr(e, 'resp'):
                            status = e.resp["status"]
                        result = None
            print "Done in {}ms".format(int(round((time.time() - exec_time) * 1000)))

            if not repeat and status != check_code:
                raise RuntimeError("The executed command was: {}\nMessage: {}".format(
                    endpoint.replace("\.execute()", ""),
                    "HTTP status code is {} and expected is {}.".format(status, check_code)
                ))
            elif not repeat and int(check_code) - 200 >= 100:
                result = None

            if check_message and check_message != message:
                raise RuntimeError("The executed command was: {}\nMessage: {}".format(
                    endpoint.replace("\.execute()", ""),
                    "HTTP error message is {} and expected is {}.".format(message, check_message)
                ))

            if eval_expr:
                ns = {'saved_results': self.output_results,
                      'result': result,
                      'expr': lambda e, container=self.output_results: self.eval_expr('{{' + e + '}}', container)}

                if isinstance(eval_expr, str) or isinstance(eval_expr, unicode):
                    exec eval_expr in ns
                elif isinstance(eval_expr, list):
                    for e in eval_expr:
                        exec e in ns

                result = ns.get('result', result)

            if result_name and result:
                self.output_results[result_name] = result

            if export_result and result:
                with open("{}".format(export_result), 'w') as f:
                    json.dump(result, f, indent=4, separators=(',', ': '))

            if result:
                if print_result is True:
                    print ju.info_color
                    print "Result JSON:"
                    pretty_json(result)
                    print ju.end_color
                elif isinstance(print_result, str) or isinstance(print_result, unicode):
                    # we have an expression!
                    match = self.expression_matcher.match(print_result)
                    if match:
                        val = None
                        try:
                            val = self.__parse_expression(match.group(1), container=result)
                        except Exception as e:
                            if self.debug:
                                print traceback.format_exc()
                            print e

                        if val:
                            print ju.info_color
                            print "Content of {}:".format(match.group(1))
                            pretty_json(val)
                            print ju.end_color
                elif isinstance(print_result, list):
                    for expr in print_result:
                        # we have an expression!
                        match = self.expression_matcher.match(expr)
                        if match:
                            val = None
                            try:
                                val = self.__parse_expression(match.group(1), container=result)
                            except Exception as e:
                                if self.debug:
                                    print traceback.format_exc()
                                print e

                            if val:
                                print ju.info_color
                                print "Content of {}:".format(match.group(1))
                                pretty_json(val)
                                print ju.end_color
                            val = None

            if repeat:
                repeat_bool = self.__parse_repeat(repeat, times, result, status, message)
                if repeat_bool:
                    print "Calling the endpoint again"
                else:
                    print "Done repeating the call"
                times += 1
            else:
                repeat_bool = False

            if json_pattern:
                json_pattern = self._parse_body(json_pattern)
                check_json(result, json_pattern, exit_on_error=self.exit_on_error)

            if order:
                if isinstance(order, str) or isinstance(order, unicode):
                    match = self.expression_matcher.match(order)
                    raise RuntimeError("Expression {} for check_order is incorrect".format(match.group(1)))

                elif isinstance(order, list):
                    values = []
                    directions = []
                    paths = []
                    for criteria in order:
                        for expr, direction in criteria.iteritems():
                            # we have a list!
                            match = self.expression_matcher.match(expr)
                            if match:
                                val = self.__parse_expression(match.group(1), container=result)
                                values.append(val)
                                directions.append(direction)
                                paths.append(match.group(1))

                    check_order_values(values, directions, paths, exit_on_error=self.exit_on_error)

        if hooks and "teardown" in hooks:
            self.run_hook(hooks.get("teardown"), "teardown")
Ejemplo n.º 5
0
Archivo: dex.py Proyecto: LeslieM/dex
 def _output_aggregated_report(self, out):
     out.write(pretty_json(self._make_aggregated_report()).replace('"', "'").replace("\\'", '"') + "\n")
Ejemplo n.º 6
0
    def _output_aggregated_report(self, out, run_stats):
        output = OrderedDict([('runStats', run_stats),
                              ('results', self._report.get_reports())])

        #out.write(pretty_json(output))
        out.write(pretty_json(output).replace('"', "'").replace("\\'", '"') + "\n")
Ejemplo n.º 7
0
 def perform_search(self):
     if self.verbose:
         print utils.pretty_json(self.body)
     return self.es.search(
         index=self.index, doc_type=self.doc_type, body=self.body)
Ejemplo n.º 8
0
                        exec e in ns

                result = ns.get('result', result)

            if result_name and result:
                self.output_results[result_name] = result

            if export_result and result:
                with open("{}".format(export_result), 'w') as f:
                    json.dump(result, f, indent=4, separators=(',', ': '))

            if result:
                if print_result is True:
                    print ju.info_color
                    print "Result JSON:"
                    pretty_json(result)
                    print ju.end_color
                elif isinstance(print_result, str) or isinstance(print_result, unicode):
                    # we have an expression!
                    match = self.expression_matcher.match(print_result)
                    if match:
                        val = None
                        try:
                            val = self.__parse_expression(match.group(1), container=result)
                        except Exception, e:
                            if self.debug:
                                print traceback.format_exc()
                            print e

                        if val:
                            print ju.info_color