Exemplo n.º 1
0
    def __init__(self, options, ammofile):
        self.core = TankCore()

        self.options = options
        self.ammofile = ammofile

        self.baseconfigs_location = '/etc/yandex-tank'

        self.log_filename = self.options.log
        self.core.add_artifact_file(self.log_filename)
        self.log = logging.getLogger(__name__)

        self.signal_count = 0
        self.scheduled_start = None
Exemplo n.º 2
0
    def __init__(self, options, ammofile):
        self.core = TankCore()

        self.options = options
        self.ammofile = ammofile
        
        self.baseconfigs_location = '/etc/yandex-tank' 

        self.log_filename = self.options.log
        self.core.add_artifact_file(self.log_filename)
        self.log = logging.getLogger(__name__)

        self.signal_count = 0
        self.scheduled_start = None
Exemplo n.º 3
0
class ConsoleTank:
    """
    Worker class that runs tank core accepting cmdline params
    """

    IGNORE_LOCKS = "ignore_locks"

    def __init__(self, options, ammofile):
        self.core = TankCore()

        self.options = options
        self.ammofile = ammofile
        
        self.baseconfigs_location = '/etc/yandex-tank' 

        self.log_filename = self.options.log
        self.core.add_artifact_file(self.log_filename)
        self.log = logging.getLogger(__name__)

        self.signal_count = 0
        self.scheduled_start = None


    def set_baseconfigs_dir(self, directory):
        '''
        Set directory where to read configs set
        '''
        self.baseconfigs_location = directory
        
        
    def init_logging(self):
        '''
        Set up logging, as it is very important for console tool
        '''
        logger = logging.getLogger()
        logger.setLevel(logging.DEBUG)

        # create file handler which logs even debug messages
        if (self.log_filename):
            file_handler = logging.FileHandler(self.log_filename)
            file_handler.setLevel(logging.DEBUG)
            file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(message)s"))
            logger.addHandler(file_handler)
            
        # create console handler with a higher log level
        console_handler = logging.StreamHandler(sys.stdout)
        stderr_hdl = logging.StreamHandler(sys.stderr)
        
        fmt_verbose = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(message)s")
        fmt_regular = logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S")
        
        if self.options.verbose:
            console_handler.setLevel(logging.DEBUG)
            console_handler.setFormatter(fmt_verbose)
            stderr_hdl.setFormatter(fmt_verbose)
        elif self.options.quiet:
            console_handler.setLevel(logging.WARNING)
            console_handler.setFormatter(fmt_regular)
            stderr_hdl.setFormatter(fmt_regular)
        else:
            console_handler.setLevel(logging.INFO)
            console_handler.setFormatter(fmt_regular)
            stderr_hdl.setFormatter(fmt_regular)

        f_err = SingleLevelFilter(logging.ERROR, True)
        f_warn = SingleLevelFilter(logging.WARNING, True)
        f_crit = SingleLevelFilter(logging.CRITICAL, True)
        console_handler.addFilter(f_err)
        console_handler.addFilter(f_warn)
        console_handler.addFilter(f_crit)
        logger.addHandler(console_handler)

        f_info = SingleLevelFilter(logging.INFO, True)
        f_debug = SingleLevelFilter(logging.DEBUG, True)
        stderr_hdl.addFilter(f_info)
        stderr_hdl.addFilter(f_debug)
        logger.addHandler(stderr_hdl)


    def __override_config_from_cmdline(self):
        ''' override config options from command line'''
        if self.options.option: 
            self.core.apply_shorthand_options(self.options.option)            
    

    def get_default_configs(self):
        ''' returns default configs list, from /etc and home dir '''
        configs = []
        try:
            conf_files = os.listdir(self.baseconfigs_location)
            conf_files.sort()
            for filename in conf_files:
                if fnmatch.fnmatch(filename, '*.ini'):
                    configs += [os.path.realpath(self.baseconfigs_location + os.sep + filename)]
        except OSError:
            self.log.warn(self.baseconfigs_location + ' is not acessible to get configs list')

        configs += [os.path.expanduser('~/.yandex-tank')]
        return configs
    
    
    def configure(self):
        '''         Make all console-specific preparations before running Tank        '''
        if self.options.ignore_lock:
            self.log.warn("Lock files ignored. This is highly unrecommended practice!")
        
        while True:        
            try:
                self.core.get_lock(self.options.ignore_lock)
                break
            except Exception, exc:
                if self.options.lock_fail:
                    raise RuntimeError("Lock file present, cannot continue")
                self.log.debug("Failed to get lock: %s", traceback.format_exc(exc))
                self.log.info("Waiting 5s for retry...")
                time.sleep(5)
        
        try:
            configs = []
            
            if not self.options.no_rc:
                configs = self.get_default_configs()
            
            if not self.options.config:
                if os.path.exists(os.path.realpath('load.ini')):
                    self.log.info("No config passed via cmdline, using ./load.ini")
                    configs += [os.path.realpath('load.ini')]
                    self.core.add_artifact_file(os.path.realpath('load.ini'), True)
                elif os.path.exists(os.path.realpath('load.conf')):
                    # just for old 'lunapark' compatibility
                    self.log.warn("Using 'load.conf' is unrecommended, please use 'load.ini' instead")
                    conf_file = os.path.realpath('load.conf')
                    configs += [conf_file]
                    self.core.add_artifact_file(conf_file, True)
            else:
                for config_file in self.options.config:
                    configs.append(config_file)
    
            self.core.load_configs(configs)
    
            if self.ammofile:
                self.log.debug("Ammofile: %s", self.ammofile)
                self.core.set_option("phantom", 'ammofile', self.ammofile[0])
    
            self.__override_config_from_cmdline()
                        
            self.core.load_plugins()
            
            if self.options.scheduled_start:
                try:
                    self.scheduled_start = datetime.datetime.strptime(self.options.scheduled_start, '%Y-%m-%d %H:%M:%S')
                except ValueError:
                    self.scheduled_start = datetime.datetime.strptime(datetime.datetime.now().strftime('%Y-%m-%d ') + self.options.scheduled_start, '%Y-%m-%d %H:%M:%S')

            if self.options.ignore_lock:
                self.core.set_option(self.core.SECTION, self.IGNORE_LOCKS, "1")
                
        except Exception, ex:
            self.log.info("Exception: %s", traceback.format_exc(ex))
            sys.stderr.write(RealConsoleMarkup.RED)
            self.log.error("%s", ex)
            sys.stderr.write(RealConsoleMarkup.RESET)
            sys.stderr.write(RealConsoleMarkup.TOTAL_RESET)
            self.core.release_lock()
            raise ex
Exemplo n.º 4
0
class ConsoleTank:
    """    Worker class that runs tank core accepting cmdline params    """

    IGNORE_LOCKS = "ignore_locks"

    def __init__(self, options, ammofile):
        self.core = TankCore()

        self.options = options
        self.ammofile = ammofile

        self.baseconfigs_location = '/etc/yandex-tank'

        self.log_filename = self.options.log
        self.core.add_artifact_file(self.log_filename)
        self.log = logging.getLogger(__name__)

        self.signal_count = 0
        self.scheduled_start = None

    def set_baseconfigs_dir(self, directory):
        """        Set directory where to read configs set        """
        self.baseconfigs_location = directory

    def init_logging(self):
        """ Set up logging, as it is very important for console tool """
        logger = logging.getLogger('')
        logger.setLevel(logging.DEBUG)

        # create file handler which logs even debug messages
        if self.log_filename:
            file_handler = logging.FileHandler(self.log_filename)
            file_handler.setLevel(logging.DEBUG)
            file_handler.setFormatter(
                logging.Formatter(
                    "%(asctime)s [%(levelname)s] %(name)s %(message)s"))
            logger.addHandler(file_handler)

        # create console handler with a higher log level
        console_handler = logging.StreamHandler(sys.stdout)
        stderr_hdl = logging.StreamHandler(sys.stderr)

        fmt_verbose = logging.Formatter(
            "%(asctime)s [%(levelname)s] %(name)s %(message)s")
        fmt_regular = logging.Formatter(
            "%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S")

        if self.options.verbose:
            console_handler.setLevel(logging.DEBUG)
            console_handler.setFormatter(fmt_verbose)
            stderr_hdl.setFormatter(fmt_verbose)
        elif self.options.quiet:
            console_handler.setLevel(logging.WARNING)
            console_handler.setFormatter(fmt_regular)
            stderr_hdl.setFormatter(fmt_regular)
        else:
            console_handler.setLevel(logging.INFO)
            console_handler.setFormatter(fmt_regular)
            stderr_hdl.setFormatter(fmt_regular)

        f_err = SingleLevelFilter(logging.ERROR, True)
        f_warn = SingleLevelFilter(logging.WARNING, True)
        f_crit = SingleLevelFilter(logging.CRITICAL, True)
        console_handler.addFilter(f_err)
        console_handler.addFilter(f_warn)
        console_handler.addFilter(f_crit)
        logger.addHandler(console_handler)

        f_info = SingleLevelFilter(logging.INFO, True)
        f_debug = SingleLevelFilter(logging.DEBUG, True)
        stderr_hdl.addFilter(f_info)
        stderr_hdl.addFilter(f_debug)
        logger.addHandler(stderr_hdl)

    def __override_config_from_cmdline(self):
        """ override config options from command line"""
        if self.options.option:
            self.core.apply_shorthand_options(self.options.option)

    def get_default_configs(self):
        """ returns default configs list, from /etc and home dir """
        # initialize basic defaults
        configs = [resource_filename(__name__, 'config/00-base.ini')]
        try:
            conf_files = os.listdir(self.baseconfigs_location)
            conf_files.sort()
            for filename in conf_files:
                if fnmatch.fnmatch(filename, '*.ini'):
                    configs += [
                        os.path.realpath(self.baseconfigs_location + os.sep +
                                         filename)
                    ]
        except OSError:
            self.log.warn(self.baseconfigs_location +
                          ' is not acessible to get configs list')

        configs += [os.path.expanduser('~/.yandex-tank')]
        return configs

    def configure(self):
        """ Make all console-specific preparations before running Tank """
        if self.options.ignore_lock:
            self.log.warn(
                "Lock files ignored. This is highly unrecommended practice!")

        if self.options.lock_dir:
            self.core.set_option(self.core.SECTION, "lock_dir",
                                 self.options.lock_dir)

        while True:
            try:
                self.core.get_lock(self.options.ignore_lock)
                break
            except Exception, exc:
                if self.options.lock_fail:
                    raise RuntimeError("Lock file present, cannot continue")
                self.log.info(
                    "Couldn't get lock. Will retry in 5 seconds... (%s)",
                    str(exc))
                time.sleep(5)

        try:
            configs = []

            if not self.options.no_rc:
                configs = self.get_default_configs()

            if not self.options.config:
                if os.path.exists(os.path.realpath('load.ini')):
                    self.log.info(
                        "No config passed via cmdline, using ./load.ini")
                    configs += [os.path.realpath('load.ini')]
                    self.core.add_artifact_file(os.path.realpath('load.ini'),
                                                True)
                elif os.path.exists(os.path.realpath('load.conf')):
                    # just for old 'lunapark' compatibility
                    self.log.warn(
                        "Using 'load.conf' is unrecommended, please use 'load.ini' instead"
                    )
                    conf_file = os.path.realpath('load.conf')
                    configs += [conf_file]
                    self.core.add_artifact_file(conf_file, True)
            else:
                for config_file in self.options.config:
                    configs.append(config_file)

            self.core.load_configs(configs)

            if self.ammofile:
                self.log.debug("Ammofile: %s", self.ammofile)
                self.core.set_option("phantom", 'ammofile', self.ammofile[0])

            self.__override_config_from_cmdline()

            self.core.load_plugins()

            if self.options.scheduled_start:
                try:
                    self.scheduled_start = datetime.datetime.strptime(
                        self.options.scheduled_start, '%Y-%m-%d %H:%M:%S')
                except ValueError:
                    self.scheduled_start = datetime.datetime.strptime(
                        datetime.datetime.now().strftime('%Y-%m-%d ') +
                        self.options.scheduled_start, '%Y-%m-%d %H:%M:%S')

            if self.options.ignore_lock:
                self.core.set_option(self.core.SECTION, self.IGNORE_LOCKS, "1")

        except Exception, ex:
            self.log.info("Exception: %s", traceback.format_exc(ex))
            sys.stderr.write(RealConsoleMarkup.RED)
            self.log.error("%s", ex)
            sys.stderr.write(RealConsoleMarkup.RESET)
            sys.stderr.write(RealConsoleMarkup.TOTAL_RESET)
            self.core.release_lock()
            raise ex
Exemplo n.º 5
0
class ConsoleTank:
    """
    Worker class that runs tank core accepting cmdline params
    """

    MIGRATE_SECTION = 'migrate_old'

    old_options_mapping = {
        'instances': ('phantom', ''),
        'tank_type': ('phantom', ''),
        'gatling_ip': ('phantom', ''),
        'ssl': ('phantom', ''),
        'address': ('phantom', ''),
        'port': ('phantom', ''),
        'writelog': ('phantom', ''),
        'phantom_http_line': ('phantom', ''),
        'phantom_http_field_num': ('phantom', ''),
        'phantom_http_field': ('phantom', ''),
        'phantom_http_entity': ('phantom', ''),

        'load': ('phantom', 'rps_schedule'),
        'instances_schedule': ('phantom', ''),
        'ammofile': ('phantom', ''),
        'loop': ('phantom', ''),
        'autocases': ('phantom', ''),
        'chosen_cases': ('phantom', ''),
        'uri': ('phantom', 'uris'),
        'header': ('phantom', 'headers'),
        
        'time_periods': ('aggregator', ''),
        'detailed_field': ('aggregator', ''),

        'task': ('meta', ''),
        'job_name': ('meta', ''),
        'job_dsc': ('meta', ''),
        'component': ('meta', ''),
        'regress': ('meta', ''),
        'ver': ('meta', ''),
        'inform': ('meta', 'notify'),

        'autostop': ('autostop', ''),
        'monitoring_config': ('monitoring', 'config'),
        'manual_start': ('tank', ''),
        'http_base': ('meta', 'api_address')
    }
    
    
    def __init__(self, options, ammofile):
        self.core = TankCore()

        self.options = options
        self.ammofile = ammofile
        
        self.baseconfigs_location = '/etc/yandex-tank' 

        self.log_filename = self.options.log
        self.core.add_artifact_file(self.log_filename)
        self.log = logging.getLogger(__name__)

        self.signal_count = 0
        self.scheduled_start = None

    def set_baseconfigs_dir(self, directory):
        '''
        Set directory where to read configs set
        '''
        self.baseconfigs_location = directory
        
    def init_logging(self):
        '''
        Set up logging, as it is very important for console tool
        '''
        logger = logging.getLogger()
        logger.setLevel(logging.DEBUG)

        # create file handler which logs even debug messages
        if (self.log_filename):
            file_handler = logging.FileHandler(self.log_filename)
            file_handler.setLevel(logging.DEBUG)
            file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(message)s"))
            logger.addHandler(file_handler)
            
        # create console handler with a higher log level
        console_handler = logging.StreamHandler(sys.stdout)
        
        if self.options.verbose:
            console_handler.setLevel(logging.DEBUG)
            console_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(message)s"))
        elif self.options.quiet:
            console_handler.setLevel(logging.WARNING)
            console_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S"))
        else:
            console_handler.setLevel(logging.INFO)
            console_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S"))
        logger.addHandler(console_handler)


    def __convert_old_multiline_options(self, old_lines):
        ''' supports old-school 'lunapark' tool configs ''' 
        opts = {}
        option = None
        res = ''
        for line in old_lines:
            try:
                if line.strip() and line.strip()[0] == '#':
                    res += line
                    continue
                option = line[:line.index('=')]
                value = line[line.index('=') + 1:]
                if option not in opts.keys():
                    opts[option] = []
                opts[option].append(value.strip())
            except Exception:
                if option:
                    opts[option].append(line.strip())
                else:
                    res += line.strip() + "\n"
    
        for option, values in opts.iteritems():
            res += option + '=' + "\n\t".join(values) + "\n"
    
        return res
    
    
    def __adapt_old_config(self, config):
        ''' supports old-school 'lunapark' tool configs ''' 
        test_parser = ConfigParser.ConfigParser()
        try:
            test_parser.read(config)
            self.log.debug("Config passed ini format test: %s", config)
            return config
        except Exception:
            self.log.warning("Config failed INI format test, consider upgrading it: %s", config)
            file_handle, corrected_file = tempfile.mkstemp(".ini", "corrected_")
            self.log.debug("Creating corrected INI config for it: %s", corrected_file)
            os.write(file_handle, "[" + self.MIGRATE_SECTION + "]\n")
            os.write(file_handle, self.__convert_old_multiline_options(open(config, 'r').readlines()))
            os.close(file_handle)
            return corrected_file

    def __add_adapted_config(self, configs, conf_file):
        ''' supports old-school 'lunapark' tool configs ''' 
        conf_file = self.__adapt_old_config(conf_file)
        configs += [conf_file]
        self.core.add_artifact_file(conf_file, True)


    def __override_config_from_cmdline(self):
        ''' override config options from command line'''
        if self.options.option: 
            self.core.apply_shorthand_options(self.options.option, self.MIGRATE_SECTION)            
    
    def __translate_old_options(self):
        ''' supports old-school 'lunapark' tool configs ''' 
        for old_option, value in self.core.config.get_options(self.MIGRATE_SECTION):
            if old_option in self.old_options_mapping.keys():
                new_sect = self.old_options_mapping[old_option][0]
                new_opt = self.old_options_mapping[old_option][1] if self.old_options_mapping[old_option][1] else old_option
                self.log.debug("Translating old option %s=%s into new: %s.%s", old_option, value, new_sect, new_opt)
                self.core.set_option(new_sect, new_opt, value)
            else:
                self.log.warn("Unknown old option, please add it to translation mapping: %s=%s", old_option, value)

        if self.core.config.config.has_section(self.MIGRATE_SECTION):                
            self.core.config.config.remove_section(self.MIGRATE_SECTION)

                
    def configure(self):
        '''
        Make all console-specific preparations before running Tank
        '''
        if self.options.ignore_lock:
            self.log.warn("Lock files ignored. This is highly unrecommended practice!")        
        
        while True:        
            try:
                self.core.get_lock(self.options.ignore_lock)
                break
            except Exception:
                if self.options.lock_fail:
                    raise RuntimeError("Lock file present, cannot continue")
                self.log.info("Waiting 5s for retry...")
                time.sleep(5)
        
        try:
            configs = []
            
            if not self.options.no_rc:
                try:
                    conf_files = os.listdir(self.baseconfigs_location)
                    conf_files.sort()
                    for filename in conf_files:
                        if fnmatch.fnmatch(filename, '*.ini'):
                            configs += [os.path.realpath(self.baseconfigs_location + os.sep + filename)]
                except OSError:
                    self.log.warn(self.baseconfigs_location + ' is not acessible to get configs list')
        
                configs += [os.path.expanduser('~/.yandex-tank')]
            
            if not self.options.config:
                # just for old 'lunapark' compatibility
                self.log.debug("No config passed via cmdline, using ./load.conf")
                conf_file = self.__adapt_old_config(os.path.realpath('load.conf'))
                configs += [conf_file]
                self.core.add_artifact_file(conf_file, True)
            else:
                for config_file in self.options.config:
                    self.__add_adapted_config(configs, config_file)
    
            self.core.load_configs(configs)
    
            if self.ammofile:
                self.log.debug("Ammofile: %s", self.ammofile)
                self.core.set_option(self.MIGRATE_SECTION, 'ammofile', self.ammofile[0])
    
            self.__translate_old_options()
            self.__override_config_from_cmdline()
                        
            self.core.load_plugins()
            
            if self.options.scheduled_start:
                try:
                    self.scheduled_start = datetime.datetime.strptime(self.options.scheduled_start, '%Y-%m-%d %H:%M:%S')
                except ValueError:
                    self.scheduled_start = datetime.datetime.strptime(datetime.datetime.now().strftime('%Y-%m-%d ') + self.options.scheduled_start, '%Y-%m-%d %H:%M:%S')
        except Exception, ex:
            self.log.info("Exception: %s", traceback.format_exc(ex))
            sys.stdout.write(RealConsoleMarkup.RED)
            self.log.error("%s", ex)
            sys.stdout.write(RealConsoleMarkup.RESET)
            sys.stdout.write(RealConsoleMarkup.TOTAL_RESET)
            self.core.release_lock()
            raise ex
Exemplo n.º 6
0
class ConsoleTank:
    """
    Worker class that runs tank core accepting cmdline params
    """

    IGNORE_LOCKS = "ignore_locks"

    MIGRATE_SECTION = "migrate_old"

    old_options_mapping = {
        "instances": ("phantom", ""),
        "tank_type": ("phantom", ""),
        "gatling_ip": ("phantom", ""),
        "ssl": ("phantom", ""),
        "address": ("phantom", ""),
        "port": ("phantom", ""),
        "writelog": ("phantom", ""),
        "phantom_http_line": ("phantom", ""),
        "phantom_http_field_num": ("phantom", ""),
        "phantom_http_field": ("phantom", ""),
        "phantom_http_entity": ("phantom", ""),
        "load": ("phantom", "rps_schedule"),
        "instances_schedule": ("phantom", ""),
        "ammofile": ("phantom", ""),
        "loop": ("phantom", ""),
        "autocases": ("phantom", ""),
        "chosen_cases": ("phantom", ""),
        "uri": ("phantom", "uris"),
        "header": ("phantom", "headers"),
        "time_periods": ("aggregator", ""),
        "detailed_field": ("aggregator", ""),
        "task": ("meta", ""),
        "job_name": ("meta", ""),
        "job_dsc": ("meta", ""),
        "component": ("meta", ""),
        "regress": ("meta", ""),
        "ver": ("meta", ""),
        "inform": ("meta", "notify"),
        "autostop": ("autostop", ""),
        "monitoring_config": ("monitoring", "config"),
        "manual_start": ("tank", ""),
        "http_base": ("meta", "api_address"),
    }

    def __init__(self, options, ammofile):
        self.core = TankCore()

        self.options = options
        self.ammofile = ammofile

        self.baseconfigs_location = "/etc/yandex-tank"

        self.log_filename = self.options.log
        self.core.add_artifact_file(self.log_filename)
        self.log = logging.getLogger(__name__)

        self.signal_count = 0
        self.scheduled_start = None

    def set_baseconfigs_dir(self, directory):
        """
        Set directory where to read configs set
        """
        self.baseconfigs_location = directory

    def init_logging(self):
        """
        Set up logging, as it is very important for console tool
        """
        logger = logging.getLogger()
        logger.setLevel(logging.DEBUG)

        # create file handler which logs even debug messages
        if self.log_filename:
            file_handler = logging.FileHandler(self.log_filename)
            file_handler.setLevel(logging.DEBUG)
            file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(message)s"))
            logger.addHandler(file_handler)

        # create console handler with a higher log level
        console_handler = logging.StreamHandler(sys.stdout)

        if self.options.verbose:
            console_handler.setLevel(logging.DEBUG)
            console_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(message)s"))
        elif self.options.quiet:
            console_handler.setLevel(logging.WARNING)
            console_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S"))
        else:
            console_handler.setLevel(logging.INFO)
            console_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S"))
        logger.addHandler(console_handler)

    def __convert_old_multiline_options(self, old_lines):
        """ supports old-school 'lunapark' tool configs """
        opts = {}
        option = None
        res = ""
        for line in old_lines:
            try:
                if line.strip() and line.strip()[0] == "#":
                    res += line
                    continue
                option = line[: line.index("=")]
                value = line[line.index("=") + 1 :]
                if option not in opts.keys():
                    opts[option] = []
                opts[option].append(value.strip())
            except Exception:
                if option:
                    opts[option].append(line.strip())
                else:
                    res += line.strip() + "\n"

        for option, values in opts.iteritems():
            res += option + "=" + "\n\t".join(values) + "\n"

        return res

    def __adapt_old_config(self, config):
        """ supports old-school 'lunapark' tool configs """
        test_parser = ConfigParser.ConfigParser()
        try:
            test_parser.read(config)
            self.log.debug("Config passed ini format test: %s", config)
            return config
        except Exception:
            self.log.warning("Config failed INI format test, consider upgrading it: %s", config)
            file_handle, corrected_file = tempfile.mkstemp(".ini", "corrected_")
            self.log.debug("Creating corrected INI config for it: %s", corrected_file)
            os.write(file_handle, "[" + self.MIGRATE_SECTION + "]\n")
            os.write(file_handle, self.__convert_old_multiline_options(open(config, "r").readlines()))
            os.close(file_handle)
            return corrected_file

    def __add_adapted_config(self, configs, conf_file):
        """ supports old-school 'lunapark' tool configs """
        conf_file = self.__adapt_old_config(conf_file)
        configs += [conf_file]
        self.core.add_artifact_file(conf_file, True)

    def __override_config_from_cmdline(self):
        """ override config options from command line"""
        if self.options.option:
            self.core.apply_shorthand_options(self.options.option, self.MIGRATE_SECTION)

    def __translate_old_options(self):
        """ supports old-school 'lunapark' tool configs """
        for old_option, value in self.core.config.get_options(self.MIGRATE_SECTION):
            if old_option in self.old_options_mapping.keys():
                new_sect = self.old_options_mapping[old_option][0]
                new_opt = (
                    self.old_options_mapping[old_option][1] if self.old_options_mapping[old_option][1] else old_option
                )
                self.log.debug("Translating old option %s=%s into new: %s.%s", old_option, value, new_sect, new_opt)
                self.core.set_option(new_sect, new_opt, value)
            else:
                self.log.warn("Unknown old option, please add it to translation mapping: %s=%s", old_option, value)

        if self.core.config.config.has_section(self.MIGRATE_SECTION):
            self.core.config.config.remove_section(self.MIGRATE_SECTION)

    def configure(self):
        """
        Make all console-specific preparations before running Tank
        """
        if self.options.ignore_lock:
            self.log.warn("Lock files ignored. This is highly unrecommended practice!")

        while True:
            try:
                self.core.get_lock(self.options.ignore_lock)
                break
            except Exception, exc:
                if self.options.lock_fail:
                    raise RuntimeError("Lock file present, cannot continue")
                self.log.debug("Failed to get lock: %s", traceback.format_exc(exc))
                self.log.info("Waiting 5s for retry...")
                time.sleep(5)

        try:
            configs = []

            if not self.options.no_rc:
                try:
                    conf_files = os.listdir(self.baseconfigs_location)
                    conf_files.sort()
                    for filename in conf_files:
                        if fnmatch.fnmatch(filename, "*.ini"):
                            configs += [os.path.realpath(self.baseconfigs_location + os.sep + filename)]
                except OSError:
                    self.log.warn(self.baseconfigs_location + " is not acessible to get configs list")

                configs += [os.path.expanduser("~/.yandex-tank")]

            if not self.options.config:
                if os.path.exists(os.path.realpath("load.ini")):
                    self.log.info("No config passed via cmdline, using ./load.ini")
                    configs += [os.path.realpath("load.ini")]
                    self.core.add_artifact_file(os.path.realpath("load.ini"), True)
                elif os.path.exists(os.path.realpath("load.conf")):
                    # just for old 'lunapark' compatibility
                    self.log.warn("Using 'load.conf' is unrecommended, please use 'load.ini' instead")
                    conf_file = self.__adapt_old_config(os.path.realpath("load.conf"))
                    configs += [conf_file]
                    self.core.add_artifact_file(conf_file, True)
            else:
                for config_file in self.options.config:
                    self.__add_adapted_config(configs, config_file)

            self.core.load_configs(configs)

            if self.ammofile:
                self.log.debug("Ammofile: %s", self.ammofile)
                self.core.set_option(self.MIGRATE_SECTION, "ammofile", self.ammofile[0])

            self.__translate_old_options()
            self.__override_config_from_cmdline()

            self.core.load_plugins()

            if self.options.scheduled_start:
                try:
                    self.scheduled_start = datetime.datetime.strptime(self.options.scheduled_start, "%Y-%m-%d %H:%M:%S")
                except ValueError:
                    self.scheduled_start = datetime.datetime.strptime(
                        datetime.datetime.now().strftime("%Y-%m-%d ") + self.options.scheduled_start,
                        "%Y-%m-%d %H:%M:%S",
                    )

            if self.options.ignore_lock:
                self.core.set_option(self.core.SECTION, self.IGNORE_LOCKS, "1")

        except Exception, ex:
            self.log.info("Exception: %s", traceback.format_exc(ex))
            sys.stdout.write(RealConsoleMarkup.RED)
            self.log.error("%s", ex)
            sys.stdout.write(RealConsoleMarkup.RESET)
            sys.stdout.write(RealConsoleMarkup.TOTAL_RESET)
            self.core.release_lock()
            raise ex
Exemplo n.º 7
0
class ConsoleTank:
    """
    Worker class that runs tank core accepting cmdline params
    """

    MIGRATE_SECTION = 'migrate_old'

    old_options_mapping = {
        'instances': ('phantom', ''),
        'tank_type': ('phantom', ''),
        'gatling_ip': ('phantom', ''),
        'ssl': ('phantom', ''),
        'address': ('phantom', ''),
        'port': ('phantom', ''),
        'writelog': ('phantom', ''),
        'phantom_http_line': ('phantom', ''),
        'phantom_http_field_num': ('phantom', ''),
        'phantom_http_field': ('phantom', ''),
        'phantom_http_entity': ('phantom', ''),

        'load': ('phantom', 'rps_schedule'),
        'instances_schedule': ('phantom', ''),
        'ammofile': ('phantom', ''),
        'loop': ('phantom', ''),
        'autocases': ('phantom', ''),
        'chosen_cases': ('phantom', ''),
        'uri': ('phantom', 'uris'),
        'header': ('phantom', 'headers'),
        
        'time_periods': ('aggregator', ''),
        'detailed_field': ('aggregator', ''),

        'task': ('meta', ''),
        'job_name': ('meta', ''),
        'job_dsc': ('meta', ''),
        'component': ('meta', ''),
        'regress': ('meta', ''),
        'ver': ('meta', ''),
        'inform': ('meta', 'notify'),

        'autostop': ('autostop', ''),
        'monitoring_config': ('monitoring', 'config'),
        'manual_start': ('tank', ''),
        'http_base': ('meta', 'api_address')
    }
    
    
    def __init__(self, options, ammofile):
        self.core = TankCore()

        self.options = options
        self.ammofile = ammofile
        
        self.baseconfigs_location = '/etc/yandex-tank' 

        self.log_filename = self.options.log
        self.core.add_artifact_file(self.log_filename)
        self.log = logging.getLogger(__name__)

        self.signal_count = 0
        self.scheduled_start = None

    def set_baseconfigs_dir(self, directory):
        '''
        Set directory where to read configs set
        '''
        self.baseconfigs_location = directory
        
    def init_logging(self):
        '''
        Set up logging, as it is very important for console tool
        '''
        logger = logging.getLogger()
        logger.setLevel(logging.DEBUG)

        # create file handler which logs even debug messages
        if (self.log_filename):
            file_handler = logging.FileHandler(self.log_filename)
            file_handler.setLevel(logging.DEBUG)
            file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(message)s"))
            logger.addHandler(file_handler)
            
        # create console handler with a higher log level
        console_handler = logging.StreamHandler(sys.stdout)
        
        if self.options.verbose:
            console_handler.setLevel(logging.DEBUG)
            console_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(message)s"))
        elif self.options.quiet:
            console_handler.setLevel(logging.WARNING)
            console_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S"))
        else:
            console_handler.setLevel(logging.INFO)
            console_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S"))
        logger.addHandler(console_handler)


    def __convert_old_multiline_options(self, old_lines):
        ''' supports old-school 'lunapark' tool configs ''' 
        opts = {}
        option = None
        res = ''
        for line in old_lines:
            try:
                if line.strip() and line.strip()[0] == '#':
                    res += line
                    continue
                option = line[:line.index('=')]
                value = line[line.index('=') + 1:]
                if option not in opts.keys():
                    opts[option] = []
                opts[option].append(value.strip())
            except Exception:
                if option:
                    opts[option].append(line.strip())
                else:
                    res += line.strip() + "\n"
    
        for option, values in opts.iteritems():
            res += option + '=' + "\n\t".join(values) + "\n"
    
        return res
    
    
    def __adapt_old_config(self, config):
        ''' supports old-school 'lunapark' tool configs ''' 
        test_parser = ConfigParser.ConfigParser()
        try:
            test_parser.read(config)
            self.log.debug("Config passed ini format test: %s", config)
            return config
        except Exception:
            self.log.warning("Config failed INI format test, consider upgrading it: %s", config)
            file_handle, corrected_file = tempfile.mkstemp(".ini", "corrected_")
            self.log.debug("Creating corrected INI config for it: %s", corrected_file)
            os.write(file_handle, "[" + self.MIGRATE_SECTION + "]\n")
            os.write(file_handle, self.__convert_old_multiline_options(open(config, 'r').readlines()))
            os.close(file_handle)
            return corrected_file

    def __add_adapted_config(self, configs, conf_file):
        ''' supports old-school 'lunapark' tool configs ''' 
        conf_file = self.__adapt_old_config(conf_file)
        configs += [conf_file]
        self.core.add_artifact_file(conf_file, True)


    def __override_config_from_cmdline(self):
        ''' override config options from command line'''
        if self.options.option: 
            self.core.apply_shorthand_options(self.options.option, self.MIGRATE_SECTION)            
    
    def __translate_old_options(self):
        ''' supports old-school 'lunapark' tool configs ''' 
        for old_option, value in self.core.config.get_options(self.MIGRATE_SECTION):
            if old_option in self.old_options_mapping.keys():
                new_sect = self.old_options_mapping[old_option][0]
                new_opt = self.old_options_mapping[old_option][1] if self.old_options_mapping[old_option][1] else old_option
                self.log.debug("Translating old option %s=%s into new: %s.%s", old_option, value, new_sect, new_opt)
                self.core.set_option(new_sect, new_opt, value)
            else:
                self.log.warn("Unknown old option, please add it to translation mapping: %s=%s", old_option, value)

        if self.core.config.config.has_section(self.MIGRATE_SECTION):                
            self.core.config.config.remove_section(self.MIGRATE_SECTION)

                
    def configure(self):
        '''
        Make all console-specific preparations before running Tank
        '''
        if self.options.ignore_lock:
            self.log.warn("Lock files ignored. This is highly unrecommended practice!")        
        
        while True:        
            try:
                self.core.get_lock(self.options.ignore_lock)
                break
            except Exception:
                if self.options.lock_fail:
                    raise RuntimeError("Lock file present, cannot continue")
                self.log.info("Waiting 5s for retry...")
                time.sleep(5)
        
        try:
            configs = []
            
            if not self.options.no_rc:
                try:
                    conf_files = os.listdir(self.baseconfigs_location)
                    conf_files.sort()
                    for filename in conf_files:
                        if fnmatch.fnmatch(filename, '*.ini'):
                            configs += [os.path.realpath(self.baseconfigs_location + os.sep + filename)]
                except OSError:
                    self.log.warn(self.baseconfigs_location + ' is not acessible to get configs list')
        
                configs += [os.path.expanduser('~/.yandex-tank')]
            
            if not self.options.config:
                if os.path.exists(os.path.realpath('load.ini')):
                    self.log.info("No config passed via cmdline, using ./load.ini")
                    configs += [os.path.realpath('load.ini')]
                    self.core.add_artifact_file(os.path.realpath('load.ini'), True)
                elif os.path.exists(os.path.realpath('load.conf')):
                    # just for old 'lunapark' compatibility
                    self.log.warn("Using 'load.conf' is unrecommended, please use 'load.ini' instead")
                    conf_file = self.__adapt_old_config(os.path.realpath('load.conf'))
                    configs += [conf_file]
                    self.core.add_artifact_file(conf_file, True)
            else:
                for config_file in self.options.config:
                    self.__add_adapted_config(configs, config_file)
    
            self.core.load_configs(configs)
    
            if self.ammofile:
                self.log.debug("Ammofile: %s", self.ammofile)
                self.core.set_option(self.MIGRATE_SECTION, 'ammofile', self.ammofile[0])
    
            self.__translate_old_options()
            self.__override_config_from_cmdline()
                        
            self.core.load_plugins()
            
            if self.options.scheduled_start:
                try:
                    self.scheduled_start = datetime.datetime.strptime(self.options.scheduled_start, '%Y-%m-%d %H:%M:%S')
                except ValueError:
                    self.scheduled_start = datetime.datetime.strptime(datetime.datetime.now().strftime('%Y-%m-%d ') + self.options.scheduled_start, '%Y-%m-%d %H:%M:%S')
        except Exception, ex:
            self.log.info("Exception: %s", traceback.format_exc(ex))
            sys.stdout.write(RealConsoleMarkup.RED)
            self.log.error("%s", ex)
            sys.stdout.write(RealConsoleMarkup.RESET)
            sys.stdout.write(RealConsoleMarkup.TOTAL_RESET)
            self.core.release_lock()
            raise ex
Exemplo n.º 8
0
 def get_core(self):
     self.core = TankCore()
     self.core.artifacts_base_dir = tempfile.mkdtemp()
     return self.core