def config(self): """ :rtype: str """ if self._config is None: value = self.get_option('config') if value.lower() == "none": self.monitoring = None self.die_on_fail = False self._config = value # handle http/https url or file path else: if value.startswith("<"): config_contents = value elif value.lower() == "auto": self.die_on_fail = False with open(resource.resource_filename(self.default_config), 'rb') as def_config: config_contents = def_config.read() else: with open(resource.resource_filename(value), 'rb') as config: config_contents = config.read() self._config = self._save_config_contents(config_contents) return self._config
def __init__(self, options, ammofile): overwrite_options = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {} self.options = options # self.lock_dir = options.lock_dir if options.lock_dir else '/var/lock' self.baseconfigs_location = '/etc/yandex-tank' self.init_logging() self.log = logging.getLogger(__name__) if ammofile: self.log.debug("Ammofile: %s", ammofile) overwrite_options['phantom'] = { 'use_caching': False, 'ammofile': ammofile } self.core = load_tank_core([resource_manager.resource_filename(cfg) for cfg in options.config], options.option, options.no_rc, [], overwrite_options) raw_cfg_file, raw_cfg_path = tempfile.mkstemp(suffix='_pre-validation-config.yaml') os.close(raw_cfg_file) self.core.config.save_raw(raw_cfg_path) self.core.add_artifact_file(raw_cfg_path) self.core.add_artifact_file(options.log) self.signal_count = 0 self.scheduled_start = None
def __init__(self, options, ammofile): overwrite_options = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {} self.options = options self.baseconfigs_location = '/etc/yandex-tank' self.init_logging() self.log = logging.getLogger(__name__) if ammofile: self.log.debug("Ammofile: %s", ammofile) overwrite_options['phantom'] = { 'use_caching': False, 'ammofile': ammofile } self.core = load_tank_core([resource_manager.resource_filename(cfg) for cfg in options.config], options.option, options.no_rc, [], overwrite_options, options.patches) raw_cfg_file, raw_cfg_path = tempfile.mkstemp(suffix='_pre-validation-config.yaml') os.close(raw_cfg_file) self.core.config.save_raw(raw_cfg_path) self.core.add_artifact_file(raw_cfg_path) self.core.add_artifact_file(options.log) self.core.add_artifact_file(options.error_log) self.core.error_log = options.error_log self.signal_count = 0 self.scheduled_start = None
def configure(self): pandora_path = self.get_option("pandora_cmd").strip() if pandora_path.startswith("http://") or pandora_path.startswith( "https://"): self.pandora_cmd = resource_manager.resource_filename(pandora_path) os.chmod(self.pandora_cmd, 0o755) else: self.pandora_cmd = pandora_path self.buffered_seconds = self.get_option("buffered_seconds") self.affinity = self.get_option("affinity", "") # get config_contents and patch it: expand resources via resource manager # config_content option has more priority over config_file if self.get_option("config_content"): logger.info('Found config_content option configuration') self.config_contents = self.__patch_raw_config_and_dump( self.get_option("config_content")) elif self.get_option("config_file"): logger.info('Found config_file option configuration') with open(self.get_option("config_file"), 'rb') as config: external_file_config_contents = yaml.load(config.read()) self.config_contents = self.__patch_raw_config_and_dump( external_file_config_contents) else: raise RuntimeError( "Neither pandora.config_content, nor pandora.config_file specified" ) logger.error('Config after parsing for patching: %s', self.config_contents) # find report filename and add to artifacts self.sample_log = self.__find_report_filename() with open(self.sample_log, 'w'): pass self.core.add_artifact_file(self.sample_log)
def patch_config(config): """ download remote resources, replace links with local filenames :param dict config: pandora config """ for pool in config['pools']: if 'file' in pool.get('ammo', {}): pool['ammo']['file'] = resource_manager.resource_filename(pool['ammo']['file']) return config
def load_files(self, configs): """ Read configs set into storage """ logger.debug("Reading configs: %s", configs) config_filenames = [resource.resource_filename(config) for config in configs] try: self.config.read(config_filenames) except Exception as ex: logger.error("Can't load configs: %s", ex) raise ex
def patch_config(config): """ download remote resources, replace links with local filenames :param dict config: pandora config """ for pool in config['pools']: if 'file' in pool.get('ammo', {}): pool['ammo']['file'] = resource_manager.resource_filename( pool['ammo']['file']) return config
def patch_ini_config_with_monitoring(ini_config, mon_section_name): """ :type ini_config: ConfigParser """ CONFIG = 'config' telegraf_cfg = ini_config.get(mon_section_name, CONFIG) if not telegraf_cfg.startswith('<') and not telegraf_cfg.lower() == 'auto': with open(resource_manager.resource_filename(telegraf_cfg), 'rb') as telegraf_cfg_file: config_contents = telegraf_cfg_file.read() ini_config.set(mon_section_name, CONFIG, config_contents) return ini_config
def load_ini_cfgs(config_files): config_filenames = [resource_manager.resource_filename(config) for config in config_files] cfg = ConfigParser() cfg.read(config_filenames) dotted_options = [] if cfg.has_section('tank'): for option, value in cfg.items('tank'): if '.' in option: dotted_options += [option + '=' + value] else: cfg.add_section('tank') cfg = apply_shorthand_options(cfg, dotted_options) cfg.set('tank', 'pid', str(os.getpid())) return cfg
def patch_config(self, config): """ download remote resources, replace links with local filenames add result file section :param dict config: pandora config """ for pool in config['pools']: if 'file' in pool.get('ammo', {}).get('source', {}).get('type', ''): self.ammofile = pool['ammo']['source']['path'] pool['ammo']['source']['path'] = resource_manager.resource_filename( self.ammofile ) if not pool.get('result') or 'phout' not in pool.get('result', {}).get('type', ''): logger.warning('Seems like pandora result file not specified... adding defaults') pool['result'] = dict( destination=self.DEFAULT_REPORT_FILE, type='phout', ) return config
def patch_config(self, config): """ download remote resources, replace links with local filenames add result file section :param dict config: pandora config """ # get expvar parameters if config.get("monitoring"): if config["monitoring"].get("expvar"): self.expvar = config["monitoring"]["expvar"].get("enabled") if config["monitoring"]["expvar"].get("port"): self.expvar_port = config["monitoring"]["expvar"].get( "port") else: self.expvar_port = self.DEFAULT_EXPVAR_PORT # or set if expvar not exists else: config["monitoring"] = { "expvar": { "enabled": True, } } self.expvar = True self.expvar_port = self.DEFAULT_EXPVAR_PORT # FIXME this is broken for custom ammo providers due to interface incompatibility # FIXME refactor pandora plx for pool in config['pools']: if pool.get('ammo', {}).get('file', ''): self.ammofile = pool['ammo']['file'] pool['ammo']['file'] = resource_manager.resource_filename( self.ammofile) if not pool.get('result') or 'phout' not in pool.get( 'result', {}).get('type', ''): logger.warning( 'Seems like pandora result file not specified... adding defaults' ) pool['result'] = dict( destination=self.DEFAULT_REPORT_FILE, type='phout', ) return config
def patch_config(self, config): """ download remote resources, replace links with local filenames add result file section :param dict config: pandora config """ # FIXME this is broken for custom ammo providers due to interface incompatibility # FIXME refactor pandora plx for pool in config['pools']: if pool.get('ammo', {}).get('file', ''): self.ammofile = pool['ammo']['file'] pool['ammo']['file'] = resource_manager.resource_filename( self.ammofile ) if not pool.get('result') or 'phout' not in pool.get('result', {}).get('type', ''): logger.warning('Seems like pandora result file not specified... adding defaults') pool['result'] = dict( destination=self.DEFAULT_REPORT_FILE, type='phout', ) return config
def __init__(self, options, ammofile): cli_kwargs = { 'core': { 'lock_dir': options.lock_dir } } if options.lock_dir else {} if options.ignore_lock: cli_kwargs.setdefault('core', {})['ignore_lock'] = options.ignore_lock self.options = options self.baseconfigs_location = '/etc/yandex-tank' self.init_logging() self.log = logging.getLogger(__name__) if ammofile: self.log.debug("Ammofile: %s", ammofile) cli_kwargs['phantom'] = { 'use_caching': False, 'ammofile': ammofile } self.core = load_tank_core([ resource_manager.resource_filename(cfg) for cfg in options.config ], options.option, options.no_rc, [], cli_kwargs, options.patches) raw_cfg_file, raw_cfg_path = tempfile.mkstemp( suffix='_pre-validation-config.yaml') os.close(raw_cfg_file) self.core.config.save_raw(raw_cfg_path) self.core.add_artifact_file(raw_cfg_path) self.core.add_artifact_file(options.log) self.core.add_artifact_file(options.error_log) self.core.error_log = options.error_log self.signal_count = 0 self.scheduled_start = None
def patch_config(self, config): """ download remote resources, replace links with local filenames add result file section :param dict config: pandora config """ # FIXME this is broken for custom ammo providers due to interface incompatibility # FIXME refactor pandora plx for pool in config['pools']: if pool.get('ammo', {}).get('file', ''): self.ammofile = pool['ammo']['file'] pool['ammo']['file'] = resource_manager.resource_filename( self.ammofile) if not pool.get('result') or 'phout' not in pool.get( 'result', {}).get('type', ''): logger.warning( 'Seems like pandora result file not specified... adding defaults' ) pool['result'] = dict( destination=self.DEFAULT_REPORT_FILE, type='phout', ) return config
def main(): parser = OptionParser() parser.add_option( '-c', '--config', action='append', help= "Path to YAML file containing run options, multiple options accepted", default=[]) parser.add_option('-f', '--fail-lock', action='store_true', dest='lock_fail', help="Don't wait for lock to release, fail test instead") parser.add_option( '-i', '--ignore-lock', action='store_true', dest='ignore_lock', help= "Ignore lock files from concurrent instances, has precedence before --lock-fail" ) parser.add_option('-k', '--lock-dir', action='store', dest='lock_dir', type="string", help="Directory for lock file") parser.add_option('-l', '--log', action='store', default="tank.log", help="Tank log file location") parser.add_option('--error_log', action='store', dest='error_log', default="tank_errors.log", help="Tank errors log file location") parser.add_option('-m', '--manual-start', action='store_true', dest='manual_start', help="Wait for Enter key to start the test") parser.add_option( '-n', '--no-rc', action='store_true', dest='no_rc', help="Don't load config files from /etc/yandex-tank and ~/.yandex-tank" ) parser.add_option( '-o', '--option', action='append', help= "Set config option, multiple options accepted, example: -o 'shellexec.start=pwd'" ) parser.add_option('-q', '--quiet', action='store_true', help="Less console output, only errors and warnings") parser.add_option( '-s', '--scheduled-start', action='store', dest='scheduled_start', help= "Start test at specified time, format 'YYYY-MM-DD hh:mm:ss', date part is optional" ) parser.add_option('-v', '--verbose', action='store_true', help="More console output, +debug messages") parser.add_option( '-p', '--patch-cfg', action='append', help= 'Patch config with yaml snippet (similar to -o, but has full compatibility to\ and the exact scheme of yaml format config)', dest='patches') parser.add_option('--version', action='store_true', dest='version') # FIXME: restore completion helper # completion_helper = CompletionHelperOptionParser() # completion_helper.handle_request(parser) options, ammofiles = parser.parse_args() if options.version: print(('YandexTank/{}'.format( pkg_resources.require('yandextank')[0].version))) return ammofile = ammofiles[0] if len(ammofiles) > 0 else None handlers = init_logging(options.error_log, options.verbose, options.quiet) cli_kwargs = { 'core': { 'lock_dir': options.lock_dir } } if options.lock_dir else {} if options.ignore_lock: cli_kwargs.setdefault('core', {})['ignore_lock'] = options.ignore_lock if ammofile: logging.debug("Ammofile: %s", ammofile) cli_kwargs['phantom'] = {'use_caching': False, 'ammofile': ammofile} try: worker = TankWorker( [ resource_manager.resource_filename(cfg) for cfg in options.config ], options.option, options.patches, [cli_kwargs], options.no_rc, ammo_file=ammofile if ammofile else None, log_handlers=handlers, ) except ValidationError as e: logging.error('Config validation error:\n{}'.format(e.errors)) return worker.start() try: while True: worker.join(timeout=2) if not worker.is_alive(): break except KeyboardInterrupt: worker.stop() worker.join() sys.exit(worker.retcode)
def main(): parser = OptionParser() parser.add_option( '-c', '--config', action='append', help="Path to INI file containing run options, multiple options accepted", default=[] ) parser.add_option( '-f', '--fail-lock', action='store_true', dest='lock_fail', help="Don't wait for lock to release, fail test instead") parser.add_option( '-i', '--ignore-lock', action='store_true', dest='ignore_lock', help="Ignore lock files from concurrent instances, has precedence before --lock-fail" ) parser.add_option( '-k', '--lock-dir', action='store', dest='lock_dir', type="string", help="Directory for lock file") parser.add_option( '-l', '--log', action='store', default="tank.log", help="Tank log file location") parser.add_option( '--error_log', action='store', dest='error_log', default="tank_errors.log", help="Tank errors log file location") parser.add_option( '-m', '--manual-start', action='store_true', dest='manual_start', help="Wait for Enter key to start the test") parser.add_option( '-n', '--no-rc', action='store_true', dest='no_rc', help="Don't load config files from /etc/yandex-tank and ~/.yandex-tank") parser.add_option( '-o', '--option', action='append', help="Set config option, multiple options accepted, example: -o 'shellexec.start=pwd'" ) parser.add_option( '-q', '--quiet', action='store_true', help="Less console output, only errors and warnings") parser.add_option( '-s', '--scheduled-start', action='store', dest='scheduled_start', help="Start test at specified time, format 'YYYY-MM-DD hh:mm:ss', date part is optional" ) parser.add_option( '-v', '--verbose', action='store_true', help="More console output, +debug messages") parser.add_option( '-p', '--patch-cfg', action='append', help='Patch config with yaml snippet (similar to -o, but has full compatibility to\ and the exact scheme of yaml format config)', dest='patches' ) parser.add_option( '--version', action='store_true', dest='version' ) # FIXME: restore completion helper # completion_helper = CompletionHelperOptionParser() # completion_helper.handle_request(parser) options, ammofiles = parser.parse_args() if options.version: print('YandexTank/{}'.format(pkg_resources.require('yandextank')[0].version)) return ammofile = ammofiles[0] if len(ammofiles) > 0 else None handlers = init_logging(options.error_log, options.verbose, options.quiet) cli_kwargs = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {} if options.ignore_lock: cli_kwargs.setdefault('core', {})['ignore_lock'] = options.ignore_lock if ammofile: logging.debug("Ammofile: %s", ammofile) cli_kwargs['phantom'] = { 'use_caching': False, 'ammofile': ammofile } try: worker = TankWorker([resource_manager.resource_filename(cfg) for cfg in options.config], options.option, options.patches, [cli_kwargs], options.no_rc, ammo_file=ammofile if ammofile else None, log_handlers=handlers ) except ValidationError as e: logging.error('Config validation error:\n{}'.format(e.errors)) return worker.start() try: while True: worker.join(timeout=2) if not worker.is_alive(): break except KeyboardInterrupt: worker.stop() worker.join()