def config(self):
"""
:rtype: str
"""
if self._config is None:
value = self.get_option('config')
if value.lower() == "none":
self.monitoring = None
self.die_on_fail = False
self._config = value
# handle http/https url or file path
else:
if value.startswith("<"):
config_contents = value
elif value.lower() == "auto":
self.die_on_fail = False
with open(resource.resource_filename(self.default_config),
'rb') as def_config:
config_contents = def_config.read()
else:
with open(resource.resource_filename(value),
'rb') as config:
config_contents = config.read()
self._config = self._save_config_contents(config_contents)
return self._config
def config(self):
"""
:rtype: str
"""
if self._config is None:
value = self.get_option('config')
if value.lower() == "none":
self.monitoring = None
self.die_on_fail = False
self._config = value
# handle http/https url or file path
else:
if value.startswith("<"):
config_contents = value
elif value.lower() == "auto":
self.die_on_fail = False
with open(resource.resource_filename(self.default_config), 'rb') as def_config:
config_contents = def_config.read()
else:
with open(resource.resource_filename(value), 'rb') as config:
config_contents = config.read()
self._config = self._save_config_contents(config_contents)
return self._config
def __init__(self, options, ammofile):
overwrite_options = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {}
self.options = options
# self.lock_dir = options.lock_dir if options.lock_dir else '/var/lock'
self.baseconfigs_location = '/etc/yandex-tank'
self.init_logging()
self.log = logging.getLogger(__name__)
if ammofile:
self.log.debug("Ammofile: %s", ammofile)
overwrite_options['phantom'] = {
'use_caching': False,
'ammofile': ammofile
}
self.core = load_tank_core([resource_manager.resource_filename(cfg) for cfg in options.config],
options.option,
options.no_rc,
[],
overwrite_options)
raw_cfg_file, raw_cfg_path = tempfile.mkstemp(suffix='_pre-validation-config.yaml')
os.close(raw_cfg_file)
self.core.config.save_raw(raw_cfg_path)
self.core.add_artifact_file(raw_cfg_path)
self.core.add_artifact_file(options.log)
self.signal_count = 0
self.scheduled_start = None
def __init__(self, options, ammofile):
overwrite_options = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {}
self.options = options
self.baseconfigs_location = '/etc/yandex-tank'
self.init_logging()
self.log = logging.getLogger(__name__)
if ammofile:
self.log.debug("Ammofile: %s", ammofile)
overwrite_options['phantom'] = {
'use_caching': False,
'ammofile': ammofile
}
self.core = load_tank_core([resource_manager.resource_filename(cfg) for cfg in options.config],
options.option,
options.no_rc,
[],
overwrite_options,
options.patches)
raw_cfg_file, raw_cfg_path = tempfile.mkstemp(suffix='_pre-validation-config.yaml')
os.close(raw_cfg_file)
self.core.config.save_raw(raw_cfg_path)
self.core.add_artifact_file(raw_cfg_path)
self.core.add_artifact_file(options.log)
self.core.add_artifact_file(options.error_log)
self.core.error_log = options.error_log
self.signal_count = 0
self.scheduled_start = None
def configure(self):
pandora_path = self.get_option("pandora_cmd").strip()
if pandora_path.startswith("http://") or pandora_path.startswith(
"https://"):
self.pandora_cmd = resource_manager.resource_filename(pandora_path)
os.chmod(self.pandora_cmd, 0o755)
else:
self.pandora_cmd = pandora_path
self.buffered_seconds = self.get_option("buffered_seconds")
self.affinity = self.get_option("affinity", "")
# get config_contents and patch it: expand resources via resource manager
# config_content option has more priority over config_file
if self.get_option("config_content"):
logger.info('Found config_content option configuration')
self.config_contents = self.__patch_raw_config_and_dump(
self.get_option("config_content"))
elif self.get_option("config_file"):
logger.info('Found config_file option configuration')
with open(self.get_option("config_file"), 'rb') as config:
external_file_config_contents = yaml.load(config.read())
self.config_contents = self.__patch_raw_config_and_dump(
external_file_config_contents)
else:
raise RuntimeError(
"Neither pandora.config_content, nor pandora.config_file specified"
)
logger.error('Config after parsing for patching: %s',
self.config_contents)
# find report filename and add to artifacts
self.sample_log = self.__find_report_filename()
with open(self.sample_log, 'w'):
pass
self.core.add_artifact_file(self.sample_log)
def patch_config(config):
"""
download remote resources, replace links with local filenames
:param dict config: pandora config
"""
for pool in config['pools']:
if 'file' in pool.get('ammo', {}):
pool['ammo']['file'] = resource_manager.resource_filename(pool['ammo']['file'])
return config
def load_files(self, configs):
""" Read configs set into storage """
logger.debug("Reading configs: %s", configs)
config_filenames = [resource.resource_filename(config) for config in configs]
try:
self.config.read(config_filenames)
except Exception as ex:
logger.error("Can't load configs: %s", ex)
raise ex
def load_files(self, configs):
""" Read configs set into storage """
logger.debug("Reading configs: %s", configs)
config_filenames = [resource.resource_filename(config) for config in configs]
try:
self.config.read(config_filenames)
except Exception as ex:
logger.error("Can't load configs: %s", ex)
raise ex
def patch_config(config):
"""
download remote resources, replace links with local filenames
:param dict config: pandora config
"""
for pool in config['pools']:
if 'file' in pool.get('ammo', {}):
pool['ammo']['file'] = resource_manager.resource_filename(
pool['ammo']['file'])
return config
def patch_ini_config_with_monitoring(ini_config, mon_section_name):
"""
:type ini_config: ConfigParser
"""
CONFIG = 'config'
telegraf_cfg = ini_config.get(mon_section_name, CONFIG)
if not telegraf_cfg.startswith('<') and not telegraf_cfg.lower() == 'auto':
with open(resource_manager.resource_filename(telegraf_cfg), 'rb') as telegraf_cfg_file:
config_contents = telegraf_cfg_file.read()
ini_config.set(mon_section_name, CONFIG, config_contents)
return ini_config
def patch_ini_config_with_monitoring(ini_config, mon_section_name):
"""
:type ini_config: ConfigParser
"""
CONFIG = 'config'
telegraf_cfg = ini_config.get(mon_section_name, CONFIG)
if not telegraf_cfg.startswith('<') and not telegraf_cfg.lower() == 'auto':
with open(resource_manager.resource_filename(telegraf_cfg), 'rb') as telegraf_cfg_file:
config_contents = telegraf_cfg_file.read()
ini_config.set(mon_section_name, CONFIG, config_contents)
return ini_config
def load_ini_cfgs(config_files):
config_filenames = [resource_manager.resource_filename(config) for config in config_files]
cfg = ConfigParser()
cfg.read(config_filenames)
dotted_options = []
if cfg.has_section('tank'):
for option, value in cfg.items('tank'):
if '.' in option:
dotted_options += [option + '=' + value]
else:
cfg.add_section('tank')
cfg = apply_shorthand_options(cfg, dotted_options)
cfg.set('tank', 'pid', str(os.getpid()))
return cfg
def load_ini_cfgs(config_files):
config_filenames = [resource_manager.resource_filename(config) for config in config_files]
cfg = ConfigParser()
cfg.read(config_filenames)
dotted_options = []
if cfg.has_section('tank'):
for option, value in cfg.items('tank'):
if '.' in option:
dotted_options += [option + '=' + value]
else:
cfg.add_section('tank')
cfg = apply_shorthand_options(cfg, dotted_options)
cfg.set('tank', 'pid', str(os.getpid()))
return cfg
def patch_config(self, config):
"""
download remote resources, replace links with local filenames
add result file section
:param dict config: pandora config
"""
for pool in config['pools']:
if 'file' in pool.get('ammo', {}).get('source', {}).get('type', ''):
self.ammofile = pool['ammo']['source']['path']
pool['ammo']['source']['path'] = resource_manager.resource_filename(
self.ammofile
)
if not pool.get('result') or 'phout' not in pool.get('result', {}).get('type', ''):
logger.warning('Seems like pandora result file not specified... adding defaults')
pool['result'] = dict(
destination=self.DEFAULT_REPORT_FILE,
type='phout',
)
return config
def patch_config(self, config):
"""
download remote resources, replace links with local filenames
add result file section
:param dict config: pandora config
"""
# get expvar parameters
if config.get("monitoring"):
if config["monitoring"].get("expvar"):
self.expvar = config["monitoring"]["expvar"].get("enabled")
if config["monitoring"]["expvar"].get("port"):
self.expvar_port = config["monitoring"]["expvar"].get(
"port")
else:
self.expvar_port = self.DEFAULT_EXPVAR_PORT
# or set if expvar not exists
else:
config["monitoring"] = {
"expvar": {
"enabled": True,
}
}
self.expvar = True
self.expvar_port = self.DEFAULT_EXPVAR_PORT
# FIXME this is broken for custom ammo providers due to interface incompatibility
# FIXME refactor pandora plx
for pool in config['pools']:
if pool.get('ammo', {}).get('file', ''):
self.ammofile = pool['ammo']['file']
pool['ammo']['file'] = resource_manager.resource_filename(
self.ammofile)
if not pool.get('result') or 'phout' not in pool.get(
'result', {}).get('type', ''):
logger.warning(
'Seems like pandora result file not specified... adding defaults'
)
pool['result'] = dict(
destination=self.DEFAULT_REPORT_FILE,
type='phout',
)
return config
def patch_config(self, config):
"""
download remote resources, replace links with local filenames
add result file section
:param dict config: pandora config
"""
# FIXME this is broken for custom ammo providers due to interface incompatibility
# FIXME refactor pandora plx
for pool in config['pools']:
if pool.get('ammo', {}).get('file', ''):
self.ammofile = pool['ammo']['file']
pool['ammo']['file'] = resource_manager.resource_filename(
self.ammofile
)
if not pool.get('result') or 'phout' not in pool.get('result', {}).get('type', ''):
logger.warning('Seems like pandora result file not specified... adding defaults')
pool['result'] = dict(
destination=self.DEFAULT_REPORT_FILE,
type='phout',
)
return config
def __init__(self, options, ammofile):
cli_kwargs = {
'core': {
'lock_dir': options.lock_dir
}
} if options.lock_dir else {}
if options.ignore_lock:
cli_kwargs.setdefault('core',
{})['ignore_lock'] = options.ignore_lock
self.options = options
self.baseconfigs_location = '/etc/yandex-tank'
self.init_logging()
self.log = logging.getLogger(__name__)
if ammofile:
self.log.debug("Ammofile: %s", ammofile)
cli_kwargs['phantom'] = {
'use_caching': False,
'ammofile': ammofile
}
self.core = load_tank_core([
resource_manager.resource_filename(cfg) for cfg in options.config
], options.option, options.no_rc, [], cli_kwargs, options.patches)
raw_cfg_file, raw_cfg_path = tempfile.mkstemp(
suffix='_pre-validation-config.yaml')
os.close(raw_cfg_file)
self.core.config.save_raw(raw_cfg_path)
self.core.add_artifact_file(raw_cfg_path)
self.core.add_artifact_file(options.log)
self.core.add_artifact_file(options.error_log)
self.core.error_log = options.error_log
self.signal_count = 0
self.scheduled_start = None
def patch_config(self, config):
"""
download remote resources, replace links with local filenames
add result file section
:param dict config: pandora config
"""
# FIXME this is broken for custom ammo providers due to interface incompatibility
# FIXME refactor pandora plx
for pool in config['pools']:
if pool.get('ammo', {}).get('file', ''):
self.ammofile = pool['ammo']['file']
pool['ammo']['file'] = resource_manager.resource_filename(
self.ammofile)
if not pool.get('result') or 'phout' not in pool.get(
'result', {}).get('type', ''):
logger.warning(
'Seems like pandora result file not specified... adding defaults'
)
pool['result'] = dict(
destination=self.DEFAULT_REPORT_FILE,
type='phout',
)
return config
def main():
parser = OptionParser()
parser.add_option(
'-c',
'--config',
action='append',
help=
"Path to YAML file containing run options, multiple options accepted",
default=[])
parser.add_option('-f',
'--fail-lock',
action='store_true',
dest='lock_fail',
help="Don't wait for lock to release, fail test instead")
parser.add_option(
'-i',
'--ignore-lock',
action='store_true',
dest='ignore_lock',
help=
"Ignore lock files from concurrent instances, has precedence before --lock-fail"
)
parser.add_option('-k',
'--lock-dir',
action='store',
dest='lock_dir',
type="string",
help="Directory for lock file")
parser.add_option('-l',
'--log',
action='store',
default="tank.log",
help="Tank log file location")
parser.add_option('--error_log',
action='store',
dest='error_log',
default="tank_errors.log",
help="Tank errors log file location")
parser.add_option('-m',
'--manual-start',
action='store_true',
dest='manual_start',
help="Wait for Enter key to start the test")
parser.add_option(
'-n',
'--no-rc',
action='store_true',
dest='no_rc',
help="Don't load config files from /etc/yandex-tank and ~/.yandex-tank"
)
parser.add_option(
'-o',
'--option',
action='append',
help=
"Set config option, multiple options accepted, example: -o 'shellexec.start=pwd'"
)
parser.add_option('-q',
'--quiet',
action='store_true',
help="Less console output, only errors and warnings")
parser.add_option(
'-s',
'--scheduled-start',
action='store',
dest='scheduled_start',
help=
"Start test at specified time, format 'YYYY-MM-DD hh:mm:ss', date part is optional"
)
parser.add_option('-v',
'--verbose',
action='store_true',
help="More console output, +debug messages")
parser.add_option(
'-p',
'--patch-cfg',
action='append',
help=
'Patch config with yaml snippet (similar to -o, but has full compatibility to\
and the exact scheme of yaml format config)',
dest='patches')
parser.add_option('--version', action='store_true', dest='version')
# FIXME: restore completion helper
# completion_helper = CompletionHelperOptionParser()
# completion_helper.handle_request(parser)
options, ammofiles = parser.parse_args()
if options.version:
print(('YandexTank/{}'.format(
pkg_resources.require('yandextank')[0].version)))
return
ammofile = ammofiles[0] if len(ammofiles) > 0 else None
handlers = init_logging(options.error_log, options.verbose, options.quiet)
cli_kwargs = {
'core': {
'lock_dir': options.lock_dir
}
} if options.lock_dir else {}
if options.ignore_lock:
cli_kwargs.setdefault('core', {})['ignore_lock'] = options.ignore_lock
if ammofile:
logging.debug("Ammofile: %s", ammofile)
cli_kwargs['phantom'] = {'use_caching': False, 'ammofile': ammofile}
try:
worker = TankWorker(
[
resource_manager.resource_filename(cfg)
for cfg in options.config
],
options.option,
options.patches,
[cli_kwargs],
options.no_rc,
ammo_file=ammofile if ammofile else None,
log_handlers=handlers,
)
except ValidationError as e:
logging.error('Config validation error:\n{}'.format(e.errors))
return
worker.start()
try:
while True:
worker.join(timeout=2)
if not worker.is_alive():
break
except KeyboardInterrupt:
worker.stop()
worker.join()
sys.exit(worker.retcode)
def main():
parser = OptionParser()
parser.add_option(
'-c',
'--config',
action='append',
help="Path to INI file containing run options, multiple options accepted",
default=[]
)
parser.add_option(
'-f',
'--fail-lock',
action='store_true',
dest='lock_fail',
help="Don't wait for lock to release, fail test instead")
parser.add_option(
'-i',
'--ignore-lock',
action='store_true',
dest='ignore_lock',
help="Ignore lock files from concurrent instances, has precedence before --lock-fail"
)
parser.add_option(
'-k',
'--lock-dir',
action='store',
dest='lock_dir',
type="string",
help="Directory for lock file")
parser.add_option(
'-l',
'--log',
action='store',
default="tank.log",
help="Tank log file location")
parser.add_option(
'--error_log',
action='store',
dest='error_log',
default="tank_errors.log",
help="Tank errors log file location")
parser.add_option(
'-m',
'--manual-start',
action='store_true',
dest='manual_start',
help="Wait for Enter key to start the test")
parser.add_option(
'-n',
'--no-rc',
action='store_true',
dest='no_rc',
help="Don't load config files from /etc/yandex-tank and ~/.yandex-tank")
parser.add_option(
'-o',
'--option',
action='append',
help="Set config option, multiple options accepted, example: -o 'shellexec.start=pwd'"
)
parser.add_option(
'-q',
'--quiet',
action='store_true',
help="Less console output, only errors and warnings")
parser.add_option(
'-s',
'--scheduled-start',
action='store',
dest='scheduled_start',
help="Start test at specified time, format 'YYYY-MM-DD hh:mm:ss', date part is optional"
)
parser.add_option(
'-v',
'--verbose',
action='store_true',
help="More console output, +debug messages")
parser.add_option(
'-p',
'--patch-cfg',
action='append',
help='Patch config with yaml snippet (similar to -o, but has full compatibility to\
and the exact scheme of yaml format config)',
dest='patches'
)
parser.add_option(
'--version',
action='store_true',
dest='version'
)
# FIXME: restore completion helper
# completion_helper = CompletionHelperOptionParser()
# completion_helper.handle_request(parser)
options, ammofiles = parser.parse_args()
if options.version:
print('YandexTank/{}'.format(pkg_resources.require('yandextank')[0].version))
return
ammofile = ammofiles[0] if len(ammofiles) > 0 else None
handlers = init_logging(options.error_log, options.verbose, options.quiet)
cli_kwargs = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {}
if options.ignore_lock:
cli_kwargs.setdefault('core', {})['ignore_lock'] = options.ignore_lock
if ammofile:
logging.debug("Ammofile: %s", ammofile)
cli_kwargs['phantom'] = {
'use_caching': False,
'ammofile': ammofile
}
try:
worker = TankWorker([resource_manager.resource_filename(cfg) for cfg in options.config],
options.option,
options.patches,
[cli_kwargs],
options.no_rc,
ammo_file=ammofile if ammofile else None,
log_handlers=handlers
)
except ValidationError as e:
logging.error('Config validation error:\n{}'.format(e.errors))
return
worker.start()
try:
while True:
worker.join(timeout=2)
if not worker.is_alive():
break
except KeyboardInterrupt:
worker.stop()
worker.join()
