def __build_launcher(self):
modified_launcher = self.engine.create_artifact('gatling-launcher', EXE_SUFFIX)
origin_launcher = get_full_path(self.settings['path'])
origin_dir = get_full_path(origin_launcher, step_up=2)
with open(origin_launcher) as origin:
origin_lines = origin.readlines()
modified_lines = []
mod_success = False
for line in origin_lines:
if is_windows() and line.startswith('set COMPILATION_CLASSPATH=""'):
mod_success = True
continue
if not is_windows() and line.startswith('COMPILATION_CLASSPATH='):
mod_success = True
line = line.rstrip() + '":${COMPILATION_CLASSPATH}"\n'
modified_lines.append(line)
if not mod_success:
raise ValueError("Can't modify gatling launcher for jar usage, ability isn't supported")
if is_windows():
first_line = 'set "GATLING_HOME=%s"\n' % origin_dir
else:
first_line = 'GATLING_HOME="%s"\n' % origin_dir
modified_lines.insert(1, first_line)
with open(modified_launcher, 'w') as modified:
modified.writelines(modified_lines)
if not is_windows():
os.chmod(modified_launcher, 0o755)
return modified_launcher
def test_case_of_variables(self):
env = {'aaa': 333, 'AAA': 666}
line_tpl = "echo %%%s%%" if is_windows() else "echo $%s"
cmdlines = [line_tpl % "aaa", line_tpl % "AAA"]
results = set()
for cmdline in cmdlines:
self.executor.env.set(env)
process = self.executor.execute(cmdline, shell=True)
stdout, _ = communicate(process)
results.add(stdout.strip())
if is_windows():
self.assertEqual(1, len(results))
else:
self.assertEqual(2, len(results))
def test_external_jar_built_launcher_v2(self):
jars = ['tests/resources/grinder/fake_grinder.jar', 'tests/resources/selenium/junit/another_dummy.jar']
self.obj.execution.merge({
'files': [
jars[0]],
'scenario': {
"script": RESOURCES_DIR + "gatling/BasicSimulation.scala",
"additional-classpath": [jars[1]],
"simulation": "mytest.BasicSimulation"}})
self.obj.prepare()
self.obj.startup()
self.obj.shutdown()
modified_launcher = self.obj.tool.tool_path
with open(modified_launcher) as modified:
modified_lines = modified.readlines()
for var in ("JAVA_CLASSPATH", "COMPILATION_CLASSPATH"):
self.assertNotIn(jars[0], self.obj.env.get(var))
self.assertIn(jars[1], self.obj.env.get(var))
for line in modified_lines:
if not is_windows() and '"$JAVA"' in line:
self.assertTrue(line.startswith('eval'))
self.assertFalse(line.startswith('set COMPILATION_CLASSPATH=""')) # win
if line.startswith('COMPILATION_CLASSPATH='): # linux
self.assertTrue(line.endswith(':"${COMPILATION_CLASSPATH}"\n'))
def test_properties_2levels(self):
self.obj.settings.merge({
"properties": {
"settlevel": u"settval",
"unc": u"Ä",
"override": 1,
},
})
self.obj.execution.merge({
"scenario": {
"properties": {
"scenlevel": "scenval",
"override": 2,
},
"requests": ["http://blazedemo.com/"]}
})
self.obj._execute = lambda *args, **kwargs: None
self.obj.prepare()
self.obj.startup()
if is_windows():
form = '%s'
else:
form = '%r'
self.assertIn("-Dscenlevel=" + form % 'scenval', self.obj.env.get("JAVA_OPTS"))
self.assertIn("-Dsettlevel=" + form % 'settval', self.obj.env.get("JAVA_OPTS"))
self.assertIn("-Doverride=2", self.obj.env.get("JAVA_OPTS"))
def test_chrome_proxy(self):
self.obj.responses = [
ResponseEmul(200, '{"result" : {}}'),
ResponseEmul(200, '{"result" : {"port": "port1", "host": "host1"}}'),
ResponseEmul(200, ''),
ResponseEmul(200, ''), # startup: startRecording
ResponseEmul(200, ''), # shutdown: stopRecording
ResponseEmul(200, '{"result" : "http://jmx_url"}'),
ResponseEmul(200, 'regular jmx contents'),
ResponseEmul(200, '{"result" : "http://smartjmx_url"}'),
ResponseEmul(200, 'smartjmx content')]
self.obj.engine.config.merge({
'modules': {
'recorder': {
'token': '123'}}})
self.obj.settings = self.obj.engine.config.get('modules').get('recorder')
self.sniff_log(self.obj.log)
executor = SeleniumExecutor()
self.obj.engine.provisioning.executors = [executor]
self.obj.prepare()
if is_linux():
self._check_linux()
elif is_windows():
self._check_windows()
else: # MacOS
self._check_mac()
self.obj.shutdown()
self.obj.post_process()
def test_passes_artifacts_dir(self):
cmdline = "echo %TAURUS_ARTIFACTS_DIR%" if is_windows() else "echo $TAURUS_ARTIFACTS_DIR"
self.engine.eval_env()
self.engine.prepare()
process = self.obj._execute(cmdline, shell=True)
stdout, _ = communicate(process)
self.assertEquals(self.engine.artifacts_dir, stdout.strip())
def _generate_id():
if os.getenv("JENKINS_HOME"):
prefix = "jenkins"
elif os.getenv("TRAVIS"):
prefix = "travis"
elif any([key.startswith("bamboo") for key in os.environ.keys()]):
prefix = "bamboo"
elif os.getenv("TEAMCITY_VERSION"):
prefix = "teamcity"
elif os.getenv("DOCKER_HOST"):
prefix = "docker"
elif os.getenv("AWS_"):
prefix = "amazon"
elif os.getenv("GOOGLE_APPLICATION_CREDENTIALS") or os.getenv("CLOUDSDK_CONFIG"):
prefix = "google_cloud"
elif os.getenv("WEBJOBS_NAME"):
prefix = "azure"
elif is_linux():
prefix = 'linux'
elif is_windows():
prefix = 'windows'
else:
prefix = 'macos'
return "%s-%x" % (prefix, uuid.getnode())
def prepare(self):
self.install_required_tools()
scenario = self.get_scenario()
jars = self.get_additional_classpath()
self.log.debug("JAR files list for Gatling: %s", jars)
for element in jars:
self.env.add_path({"JAVA_CLASSPATH": element})
self.env.add_path({"COMPILATION_CLASSPATH": element})
if is_windows() or jars:
self.log.debug("Building Gatling launcher")
self.launcher = self.__build_launcher()
else:
self.log.debug("Will not build Gatling launcher")
self.launcher = self.settings["path"]
self.script = self.get_script_path()
if not self.script:
if "requests" in scenario:
self.get_scenario()['simulation'], self.script = self.__generate_script()
self.__copy_data_sources()
else:
msg = "There must be a script file or requests for its generation "
msg += "to run Gatling tool (%s)" % self.execution.get('scenario')
raise TaurusConfigError(msg)
self.dir_prefix = self.settings.get("dir_prefix", self.dir_prefix)
self.reader = DataLogReader(self.engine.artifacts_dir, self.log, self.dir_prefix)
if isinstance(self.engine.aggregator, ConsolidatingAggregator):
self.engine.aggregator.add_underling(self.reader)
def post_process(self):
super(Proxy2JMX, self).post_process()
if self.engine.stopping_reason and not isinstance(self.engine.stopping_reason, KeyboardInterrupt):
self.log.info("Will not pick converted JMX due to exception: %s", self.engine.stopping_reason)
return
# self.log.info("Downloading JSON...")
# jmx_text = self.proxy.get_json()
# jmx_file = self.engine.create_artifact(self.label, '.recording.json')
# with open(jmx_file, 'w') as _file:
# _file.writelines(jmx_text)
# self.log.info("JSON saved into %s", jmx_file)
self.log.info("Downloading simple JMX...")
jmx_text = self.proxy.get_jmx()
jmx_file = self.engine.create_artifact(self.label, '.simple.jmx')
with open(jmx_file, 'w') as _file:
_file.writelines(jmx_text)
self.log.info("Simple JMX saved into %s", jmx_file)
self.log.info("Waiting for proxy to generate SmartJMX...")
jmx_text = self.proxy.get_smart_jmx()
jmx_file = self.engine.create_artifact(self.label, '.smart.jmx')
with open(jmx_file, 'w') as _file:
_file.writelines(jmx_text)
self.log.info("Smart JMX saved into %s", jmx_file)
if 'HTTPSampler' not in jmx_text:
self.log.warning("There aren't requests recorded by proxy2jmx, check your proxy configuration")
# log of chrome-loader not found under windows
if is_windows() and not os.path.isfile(join(self.engine.artifacts_dir, 'chrome-loader.log')):
msg = "Problems with chrome tuning are encountered, "
msg += "take look at http://http://gettaurus.org/docs/Proxy2JMX/ for help"
self.log.warning(msg)
def main():
"""
This function is used as entrypoint by setuptools
"""
usage = "Usage: bzt [options] [configs] [-aliases]"
dsc = "BlazeMeter Taurus Tool v%s, the configuration-driven test running engine" % bzt.VERSION
parser = OptionParserWithAliases(usage=usage, description=dsc, prog="bzt")
parser.add_option("-l", "--log", action="store", default="bzt.log", help="Log file location")
parser.add_option("-o", "--option", action="append", help="Override option in config")
parser.add_option("-q", "--quiet", action="store_true", help="Only errors and warnings printed to console")
parser.add_option("-v", "--verbose", action="store_true", help="Prints all logging messages to console")
parser.add_option("-n", "--no-system-configs", action="store_true", help="Skip system and user config files")
parsed_options, parsed_configs = parser.parse_args()
executor = CLI(parsed_options)
if not is_windows():
readable = select([sys.stdin], [], [], 0.1)[0]
for stream in readable:
stdin = stream.read()
if stdin:
with NamedTemporaryFile(prefix="stdin_", suffix=".config", delete=False) as fhd:
fhd.write(b(stdin))
parsed_configs.append(fhd.name)
try:
code = executor.perform(parsed_configs)
except BaseException as exc_top:
logging.error("%s: %s", type(exc_top).__name__, exc_top)
logging.debug("Exception: %s", traceback.format_exc())
code = 1
exit(code)
def prepare(self):
self._check_installed()
scenario = self.get_scenario()
jar_files = []
files = self.execution.get('files', [])
for _file in files:
if os.path.isfile(_file) and _file.lower().endswith('.jar'):
jar_files.append(_file)
elif os.path.isdir(_file):
for element in os.listdir(_file):
element = os.path.join(_file, element)
if os.path.isfile(element) and element.lower().endswith('.jar'):
jar_files.append(element)
if jar_files:
separator = os.pathsep
self.jar_list = separator + separator.join(jar_files)
if is_windows() or jar_files:
self.launcher = self.__build_launcher()
else:
self.launcher = self.settings["path"]
if Scenario.SCRIPT in scenario and scenario[Scenario.SCRIPT]:
self.script = self.get_script_path()
elif "requests" in scenario:
self.get_scenario()['simulation'], self.script = self.__generate_script()
else:
raise ValueError("There must be a script file to run Gatling")
self.dir_prefix = 'gatling-%s' % id(self)
self.reader = DataLogReader(self.engine.artifacts_dir, self.log, self.dir_prefix)
if isinstance(self.engine.aggregator, ConsolidatingAggregator):
self.engine.aggregator.add_underling(self.reader)
def install_required_tools(self):
if is_windows():
return
self.mono = self._get_tool(Mono)
self.log.debug("Checking for Mono")
if not self.mono.check_if_installed():
self.mono.install()
def __init__(self, config=None, **kwargs):
settings = config or {}
version = settings.get('version', self.VERSION)
base_dir = get_full_path(SeleniumExecutor.SELENIUM_TOOLS_DIR)
filename = 'chromedriver.exe' if is_windows() else 'chromedriver'
tool_path = os.path.join(base_dir, 'chromedriver', version, filename)
link = settings.get('download-link', self.DOWNLOAD_LINK)
if is_windows():
arch = 'win32' # no 64-bit windows builds, :(
elif is_mac():
arch = 'mac64'
else:
arch = 'linux32' if platform_bitness() == 32 else 'linux64'
link = link.format(version=version, arch=arch)
super(ChromeDriver, self).__init__(tool_path=tool_path, version=version, download_link=link, **kwargs)
def full_run(self, config):
self.configure(config)
dummy = RESOURCES_DIR + 'selenium/ruby/ruby' + ('.bat' if is_windows() else '.sh')
self.obj.prepare()
self.obj.runner.settings.merge({"interpreter": dummy})
self.obj.startup()
while not self.obj.check():
time.sleep(self.obj.engine.check_interval)
self.obj.shutdown()
def check_if_installed(self):
try:
rspec_exec = "rspec.bat" if is_windows() else "rspec"
output = check_output([rspec_exec, '--version'], stderr=STDOUT)
self.log.debug("%s output: %s", self.tool_name, output)
return True
except (CalledProcessError, OSError):
self.log.debug("RSpec check exception: %s", traceback.format_exc())
return False
def test_screen_invalid(self):
obj = ConsoleStatusReporter()
obj.settings["screen"] = "invalid"
if not sys.stdout.isatty():
self.assertEqual(obj._get_screen_type(), "dummy")
elif is_windows():
self.assertEqual(obj._get_screen(), "gui")
else:
self.assertEqual(obj._get_screen_type(), "console")
def test_task_prepare(self):
self.obj.settings['env'] = {"VAR": 1}
if is_windows():
task = "dir .. && cd .."
else:
task = "ls .. && cd .."
self.obj.parameters.merge({"prepare": [task]})
self.obj.prepare()
self.obj.startup()
self.obj.shutdown()
def full_run(self, config, script_dir):
self.configure(config)
self.run_command(["npm.cmd" if is_windows() else "npm", "install"], "npm-install", script_dir)
self.obj.prepare()
self.obj.startup()
while not self.obj.check():
time.sleep(self.obj.engine.check_interval)
self.obj.shutdown()
def startup(self):
cmdline = []
if not is_windows():
if self.mono.tool_path:
cmdline.append(self.mono.tool_path)
cmdline += [self.runner_executable,
"--target", self.script,
"--report-file", self.report_file]
load = self.get_load()
if load.iterations:
cmdline += ['--iterations', str(load.iterations)]
if load.hold:
cmdline += ['--duration', str(int(load.hold))]
if not is_windows():
self.env.add_path({"MONO_PATH": self.runner_dir})
self._start_subprocess(cmdline)
def _get_chromedriver_link(self):
settings = self.settings.get('chromedriver')
link = settings.get('download-link', SeleniumExecutor.CHROMEDRIVER_DOWNLOAD_LINK)
version = settings.get('version', SeleniumExecutor.CHROMEDRIVER_VERSION)
if is_windows():
arch = 'win32' # no 64-bit windows builds, :(
elif is_mac():
arch = 'mac64'
else:
arch = 'linux32' if platform_bitness() == 32 else 'linux64'
return link.format(version=version, arch=arch)
def check_if_installed(self):
rspec_exec = "rspec.bat" if is_windows() else "rspec"
try:
out, err = self.call([rspec_exec, '--version'])
except CALL_PROBLEMS as exc:
self.log.warning("%s check failed: %s", self.tool_name, exc)
return False
if err:
out += err
self.log.debug("%s output: %s", self.tool_name, out)
return True
def _get_chromedriver_link(self):
settings = self.settings.get('chromedriver')
link = settings.get('download-link',
SeleniumExecutor.CHROMEDRIVER_DOWNLOAD_LINK)
version = settings.get('version',
SeleniumExecutor.CHROMEDRIVER_VERSION)
if is_windows():
arch = 'win32'
elif is_mac():
arch = 'mac64'
else:
arch = 'linux32' if platform_bitness() == 32 else 'linux64'
return link.format(version=version, arch=arch)
def test_interpreter(self):
self.configure({
'execution': {
'iterations': 3,
'scenario': {'script': RESOURCES_DIR + 'selenium/ruby/example_spec.rb'},
'executor': 'selenium'
},
})
self.obj.settings.merge(self.obj.engine.config.get("modules").get("selenium"))
dummy = RESOURCES_DIR + 'selenium/ruby/ruby-dummy'
dummy += '.bat' if is_windows() else ''
self.obj.settings.merge({"interpreter": dummy})
self.obj.prepare()
def __build_launcher(self):
modified_launcher = self.engine.create_artifact(
'gatling-launcher', EXE_SUFFIX)
origin_launcher = get_full_path(self.settings['path'])
origin_dir = get_full_path(origin_launcher, step_up=2)
with open(origin_launcher) as origin:
origin_lines = origin.readlines()
modified_lines = []
mod_success = False
for line in origin_lines:
if is_windows() and line.startswith(
'set COMPILATION_CLASSPATH=""'):
mod_success = True
continue
if not is_windows() and line.startswith('COMPILATION_CLASSPATH='):
mod_success = True
line = line.rstrip() + '":${COMPILATION_CLASSPATH}"\n'
modified_lines.append(line)
if not mod_success:
raise ToolError(
"Can't modify gatling launcher for jar usage, ability isn't supported"
)
if is_windows():
first_line = 'set "GATLING_HOME=%s"\n' % origin_dir
else:
first_line = 'GATLING_HOME="%s"\n' % origin_dir
modified_lines.insert(1, first_line)
with open(modified_launcher, 'w') as modified:
modified.writelines(modified_lines)
if not is_windows():
os.chmod(modified_launcher, 0o755)
return modified_launcher
def build_launcher(self, new_name):
def convert_v2():
modified_lines = []
mod_success = False
with open(self.tool_path) as fds:
for line in fds.readlines():
if is_windows() and line.startswith('set COMPILATION_CLASSPATH=""'):
mod_success = True
continue # don't add it to modified_lines - just remove
if not is_windows() and line.startswith('COMPILATION_CLASSPATH='):
mod_success = True
line = line.rstrip() + ':"${COMPILATION_CLASSPATH}"\n' # add from env
modified_lines.append(line)
if not mod_success:
raise ToolError("Can't modify gatling launcher for jar usage, ability isn't supported")
return modified_lines
def convert_v3():
modified_lines = []
mod_success = False
with open(self.tool_path) as fds:
for line in fds.readlines():
if is_windows() and line.startswith('set COMPILER_CLASSPATH='):
mod_success = True
line = line.rstrip() + ';%COMPILATION_CLASSPATH%\n' # add from env
if not is_windows() and line.startswith('COMPILER_CLASSPATH='):
mod_success = True
line = line.rstrip()[:-1] + '${COMPILATION_CLASSPATH}"\n' # add from env
modified_lines.append(line)
if not mod_success:
raise ToolError("Can't modify gatling launcher for jar usage, ability isn't supported")
return modified_lines
if LooseVersion(self.version) < LooseVersion('3'):
converted_lines = convert_v2()
else:
converted_lines = convert_v3()
self.tool_path = new_name
with open(self.tool_path, 'w') as modified:
modified.writelines(converted_lines)
if not is_windows():
os.chmod(self.tool_path, 0o755)
def test_streams(self):
self.sniff_log()
print('test1')
with log_std_streams(logger=self.captured_logger, stdout_level=logging.DEBUG):
print('test2')
with log_std_streams(stdout_level=logging.DEBUG):
print('test3')
with log_std_streams(stdout_level=logging.DEBUG):
sys.stdout.write('test3')
with log_std_streams(logger=self.captured_logger, stdout_level=logging.DEBUG):
cmd = ['echo', '"test5"']
if is_windows():
cmd = ['cmd', '/c'] + cmd
process = Popen(cmd)
process.wait()
missed_file = get_uniq_name('.', 'test6', '')
with log_std_streams(logger=self.captured_logger, stderr_level=logging.WARNING):
if is_windows():
cmd = ['cmd', '/c', 'dir']
else:
cmd = ['ls']
process = Popen(cmd + [missed_file])
process.wait()
debug_buf = self.log_recorder.debug_buff.getvalue()
warn_buf = self.log_recorder.warn_buff.getvalue()
self.assertNotIn('test1', debug_buf)
self.assertIn('test2', debug_buf)
self.assertNotIn('test3', debug_buf)
self.assertIn('test5', debug_buf)
self.assertTrue(len(warn_buf) > 0)
def test_streams(self):
self.sniff_log()
print('test1')
with log_std_streams(logger=self.captured_logger, stdout_level=logging.DEBUG):
print('test2')
with log_std_streams(stdout_level=logging.DEBUG):
print('test3')
with log_std_streams(stdout_level=logging.DEBUG):
sys.stdout.write('test3')
with log_std_streams(logger=self.captured_logger, stdout_level=logging.DEBUG):
cmd = ['echo', '"test5"']
if is_windows():
cmd = ['cmd', '/c'] + cmd
process = Popen(cmd)
process.wait()
missed_file = get_uniq_name('.', 'test6', '')
with log_std_streams(logger=self.captured_logger, stderr_level=logging.WARNING):
if is_windows():
cmd = ['cmd', '/c', 'dir']
else:
cmd = ['ls']
process = Popen(cmd + [missed_file])
process.wait()
debug_buf = self.log_recorder.debug_buff.getvalue()
warn_buf = self.log_recorder.warn_buff.getvalue()
self.assertNotIn('test1', debug_buf)
self.assertIn('test2', debug_buf)
self.assertNotIn('test3', debug_buf)
self.assertIn('test5', debug_buf)
self.assertTrue(len(warn_buf) > 0)
def startup(self):
cmdline = []
if not is_windows():
if self.mono.tool_path:
cmdline.append(self.mono.tool_path)
cmdline += [self.runner_executable,
"--target", self.script,
"--report-file", self.report_file]
load = self.get_load()
if load.iterations:
cmdline += ['--iterations', str(load.iterations)]
if load.hold:
cmdline += ['--duration', str(int(load.hold))]
if load.concurrency:
cmdline += ['--concurrency', str(int(load.concurrency))]
if load.ramp_up:
cmdline += ['--ramp_up', str(int(load.ramp_up))]
if not is_windows():
self.env.add_path({"MONO_PATH": self.runner_dir})
self.process = self._execute(cmdline)
def startup(self):
super(Proxy2JMX, self).startup()
self.log.info('Starting BlazeMeter recorder...')
labels = []
additional_env = {}
if is_linux():
self.log.info('Set proxy for selenium: %s', self.proxy_addr)
additional_env.update({'http_proxy': self.proxy_addr, # set vars anyway for case
'https_proxy': self.proxy_addr, # linux system can't say correct name
'HTTP_PROXY': self.proxy_addr,
'HTTPS_PROXY': self.proxy_addr,
"CHROMIUM_USER_FLAGS": "--proxy-server=%s" % self.proxy_addr, # for Linux chrome
'XDG_CURRENT_DESKTOP': None, # (it might be in Docker, etc.)
'DESKTOP_SESSION': None,
'GNOME_DESKTOP_SESSION_ID': None,
'KDE_FULL_SESSION': None})
elif is_windows():
self.log.info('Prepare chrome loader')
chrome_path = self._get_chrome_path()
if chrome_path:
self._prepare_chrome_loader()
new_path = join(self.engine.artifacts_dir, 'chrome-loader') + os.pathsep + os.getenv('PATH', '')
additional_env.update({
'path_to_chrome': chrome_path,
'additional_chrome_params': '--proxy-server="%s"' % self.proxy_addr,
'chrome_loader_log': self.engine.create_artifact('chrome-loader', '.log'),
'path': new_path
})
else:
self.log.warning('Chrome not found')
else: # probably we are in MacOS
self.log.warning("Your system doesn't support settings of proxy by Taurus way")
for executor in self.engine.provisioning.executors:
if isinstance(executor, AbstractSeleniumExecutor):
if executor.label:
labels.append(executor.label)
executor.env.set(additional_env)
if isinstance(executor, SubprocessedExecutor):
if executor.label:
labels.append(executor.label)
executor.env.set(additional_env)
if len(labels) == 1:
self.label += '_' + labels[0]
self.proxy.start()
def _get_geckodriver_link(self):
settings = self.settings.get('geckodriver')
link = settings.get('download-link', SeleniumExecutor.GECKODRIVER_DOWNLOAD_LINK)
version = settings.get('version', SeleniumExecutor.GECKODRIVER_VERSION)
if is_windows():
arch = 'win64' # no 32-bit windows builds, :(
ext = 'zip'
elif is_mac():
arch = 'macos'
ext = 'tar.gz'
else:
arch = 'linux32' if platform_bitness() == 32 else 'linux64'
ext = 'tar.gz'
return link.format(version=version, arch=arch, ext=ext)
def _expand_download_link(self):
if is_windows():
arch = 'win64'
ext = 'zip'
elif is_mac():
arch = 'macos'
ext = 'tar.gz'
else:
arch = 'linux64'
ext = 'tar.gz'
self.download_link = self.download_link.format(version=self.version,
arch=arch,
ext=ext)
def convert_v2():
modified_lines = []
mod_success = False
with open(self.tool_path) as fds:
for line in fds.readlines():
if is_windows() and line.startswith(
'set COMPILATION_CLASSPATH=""'):
mod_success = True
continue # don't add it to modified_lines - just remove
if not is_windows() and line.startswith(
'COMPILATION_CLASSPATH='):
mod_success = True
line = line.rstrip(
) + ':"${COMPILATION_CLASSPATH}"\n' # add from env
modified_lines.append(line)
if not mod_success:
raise ToolError(
"Can't modify gatling launcher for jar usage, ability isn't supported"
)
return modified_lines
def startup(self):
super(Proxy2JMX, self).startup()
self.log.info('Starting BlazeMeter recorder...')
labels = []
additional_env = {}
if is_linux():
self.log.info('Set proxy for selenium: %s', self.proxy_addr)
additional_env.update({'http_proxy': self.proxy_addr, # set vars anyway for case
'https_proxy': self.proxy_addr, # linux system can't say correct name
'HTTP_PROXY': self.proxy_addr,
'HTTPS_PROXY': self.proxy_addr,
"CHROMIUM_USER_FLAGS": "--proxy-server=%s" % self.proxy_addr, # for Linux chrome
'XDG_CURRENT_DESKTOP': None, # (it might be in Docker, etc.)
'DESKTOP_SESSION': None,
'GNOME_DESKTOP_SESSION_ID': None,
'KDE_FULL_SESSION': None})
elif is_windows():
self.log.info('Prepare chrome loader')
chrome_path = self._get_chrome_path()
if chrome_path:
self._prepare_chrome_loader()
new_path = join(self.engine.artifacts_dir, 'chrome-loader') + os.pathsep + os.getenv('PATH', '')
additional_env.update({
'path_to_chrome': chrome_path,
'additional_chrome_params': '--proxy-server="%s"' % self.proxy_addr,
'chrome_loader_log': self.engine.create_artifact('chrome-loader', '.log'),
'path': new_path
})
else:
self.log.warning('Chrome not found')
else: # probably we are in MacOS
self.log.warning("Your system doesn't support settings of proxy by Taurus way")
for executor in self.engine.provisioning.executors:
if isinstance(executor, SeleniumExecutor):
if executor.label:
labels.append(executor.label)
executor.env.set(additional_env)
if isinstance(executor, SubprocessedExecutor):
if executor.label:
labels.append(executor.label)
executor.env.set(additional_env)
if len(labels) == 1:
self.label += '_' + labels[0]
self.proxy.start()
def _get_geckodriver_link(self):
settings = self.settings.get('geckodriver')
link = settings.get('download-link', SeleniumExecutor.GECKODRIVER_DOWNLOAD_LINK)
version = settings.get('version', SeleniumExecutor.GECKODRIVER_VERSION)
if is_windows():
arch = 'win64' # no 32-bit windows builds, :(
ext = 'zip'
elif is_mac():
arch = 'macos'
ext = 'tar.gz'
else:
arch = 'linux32' if platform_bitness() == 32 else 'linux64'
ext = 'tar.gz'
return link.format(version=version, arch=arch, ext=ext)
def _get_download_link(self):
download_link = "https://chromedriver.storage.googleapis.com/{version}/chromedriver_{arch}.zip"
latest_driver_version = requests.get(
'https://chromedriver.storage.googleapis.com/LATEST_RELEASE').text
if is_windows():
arch = 'win32'
elif is_mac():
arch = 'mac64'
else:
arch = 'linux32' if platform_bitness() == 32 else 'linux64'
self.download_link = download_link.format(
version=latest_driver_version, arch=arch)
def set_virtual_display(self):
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
else:
if self.engine in VirtualDisplay.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = VirtualDisplay.SHARED_VIRTUAL_DISPLAY[self.engine]
else:
width = self.parameters.get("width", 1024)
height = self.parameters.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size, self.virtual_display.new_display_var)
self.virtual_display.start()
VirtualDisplay.SHARED_VIRTUAL_DISPLAY[self.engine] = self.virtual_display
def prepare(self):
self.install_required_tools()
scenario = self.get_scenario()
jar_files = []
files = self.execution.get('files', [])
for candidate in files:
candidate = get_full_path(self.engine.find_file(candidate))
if os.path.isfile(candidate) and candidate.lower().endswith(
'.jar'):
jar_files.append(candidate)
elif os.path.isdir(candidate):
for element in os.listdir(candidate):
element = os.path.join(candidate, element)
if os.path.isfile(element) and element.lower().endswith(
'.jar'):
jar_files.append(element)
self.log.debug("JAR files list for Gatling: %s", jar_files)
for element in jar_files:
self.env.add_path({"JAVA_CLASSPATH": element})
self.env.add_path({"COMPILATION_CLASSPATH": element})
if is_windows() or jar_files:
self.log.debug("Building Gatling launcher")
self.launcher = self.__build_launcher()
else:
self.log.debug("Will not build Gatling launcher")
self.launcher = self.settings["path"]
self.script = self.get_script_path()
if not self.script:
if "requests" in scenario:
self.get_scenario(
)['simulation'], self.script = self.__generate_script()
self.__copy_data_sources()
else:
msg = "There must be a script file or requests for its generation "
msg += "to run Gatling tool (%s)" % self.execution.get(
'scenario')
raise TaurusConfigError(msg)
self.dir_prefix = self.settings.get('dir_prefix', None)
if self.dir_prefix is None:
self.dir_prefix = 'gatling-%s' % id(self)
self.reader = DataLogReader(self.engine.artifacts_dir, self.log,
self.dir_prefix)
if isinstance(self.engine.aggregator, ConsolidatingAggregator):
self.engine.aggregator.add_underling(self.reader)
def __init__(self, config=None, **kwargs):
settings = config or {}
version = settings.get('version', self.VERSION)
base_dir = get_full_path(SeleniumExecutor.SELENIUM_TOOLS_DIR)
filename = 'geckodriver.exe' if is_windows() else 'geckodriver'
tool_path = os.path.join(base_dir, 'geckodriver', version, filename)
link = settings.get('download-link', self.DOWNLOAD_LINK)
if is_windows():
arch = 'win64' # no 32-bit windows builds, :(
ext = 'zip'
elif is_mac():
arch = 'macos'
ext = 'tar.gz'
else:
arch = 'linux32' if platform_bitness() == 32 else 'linux64'
ext = 'tar.gz'
link = link.format(version=version, arch=arch, ext=ext)
super(GeckoDriver, self).__init__(tool_path=tool_path,
version=version,
download_link=link,
**kwargs)
def check_if_installed(self):
candidates = ["npm"]
if is_windows():
candidates.append("npm.cmd")
for candidate in candidates:
try:
self.log.debug("Trying %r", candidate)
output = subprocess.check_output([candidate, '--version'], stderr=subprocess.STDOUT)
self.log.debug("%s output: %s", candidate, output)
self.executable = candidate
return True
except (CalledProcessError, OSError):
self.log.debug("%r is not installed", candidate)
continue
return False
def check_if_installed(self):
candidates = ["npm"]
if is_windows():
candidates.append("npm.cmd")
for candidate in candidates:
try:
self.log.debug("Trying %r", candidate)
output = sync_run([candidate, '--version'])
self.log.debug("%s output: %s", candidate, output)
self.tool_path = candidate
return True
except CALL_PROBLEMS:
self.log.debug("%r is not installed", candidate)
continue
return False
def test_interpreter(self):
self.configure({
'execution': {
'iterations': 3,
'scenario': {'script': RESOURCES_DIR + 'selenium/ruby/example_spec.rb'},
'executor': 'selenium'
},
})
self.obj.settings.merge(self.obj.engine.config.get("modules").get("selenium"))
dummy = RESOURCES_DIR + 'selenium/ruby/ruby' + ('.bat' if is_windows() else '.sh')
self.obj.settings.merge({"interpreter": dummy})
self.obj.settings.merge({"path": dummy})
self.obj.prepare()
def check_if_installed(self):
candidates = ["npm"]
if is_windows():
candidates.append("npm.cmd")
for candidate in candidates:
try:
self.log.debug("Trying %r", candidate)
output = subprocess.check_output([candidate, '--version'], stderr=subprocess.STDOUT)
self.log.debug("%s output: %s", candidate, output)
self.executable = candidate
return True
except (CalledProcessError, OSError):
self.log.debug("%r is not installed", candidate)
continue
return False
def _set_env(self):
props = BetterDict()
props.merge(self.settings.get('properties'))
props.merge(self.get_scenario().get("properties"))
props['gatling.core.outputDirectoryBaseName'] = self.dir_prefix
props['gatling.core.directory.resources'] = self.engine.artifacts_dir
props['gatling.core.directory.results'] = self.engine.artifacts_dir
props.merge(self._get_simulation_props())
props.merge(self._get_load_props())
props.merge(self._get_scenario_props())
for key in sorted(props.keys()):
prop = props[key]
val_tpl = "%s"
if isinstance(prop, string_types):
if not is_windows(
): # extend properties support (contained separators/quotes/etc.) on lin/mac
val_tpl = "%r"
if PY2:
prop = prop.encode(
"utf-8", 'ignore') # to convert from unicode into str
if is_gatling2(self.tool.version) or not key.startswith(
'gatling.'): # send param through java_opts
self.env.add_java_param(
{"JAVA_OPTS": ("-D%s=" + val_tpl) % (key, prop)})
self.env.set({"NO_PAUSE": "TRUE"})
self.env.add_java_param(
{"JAVA_OPTS": self.settings.get("java-opts", None)})
self.log.debug('JAVA_OPTS: "%s"', self.env.get("JAVA_OPTS"))
if not is_gatling2(self.tool.version): # cook prop file
prop_lines = []
for key in props:
if key.startswith("gatling."):
prop_lines.append("%s = %s" % (key, props[key]))
conf_dir = self.engine.create_artifact("conf", "")
os.mkdir(conf_dir)
with open(os.path.join(conf_dir, "gatling.conf"),
'w') as conf_file:
conf_file.write('\n'.join(prop_lines))
self.env.add_path({"GATLING_CONF": conf_dir})
def set_virtual_display(self):
display_conf = self.settings.get("virtual-display")
if display_conf:
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
else:
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine]
else:
width = display_conf.get("width", 1024)
height = display_conf.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size, self.virtual_display.new_display_var)
self.virtual_display.start()
SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine] = self.virtual_display
def prepare(self):
"""
1) Locate script or folder
2) detect script type
3) create runner instance, prepare runner
"""
self.scenario = self.get_scenario()
self._verify_script()
self.kpi_file = self.engine.create_artifact("selenium_tests_report", ".csv")
self.err_jtl = self.engine.create_artifact("selenium_tests_err", ".xml")
script_type = self.detect_script_type(self.scenario.get(Scenario.SCRIPT))
if script_type == ".py":
self.runner = NoseTester
runner_config = self.settings.get("selenium-tools").get("nose")
elif script_type == ".jar" or script_type == ".java":
self.runner = JunitTester
runner_config = self.settings.get("selenium-tools").get("junit")
else:
raise ValueError("Unsupported script type: %s" % script_type)
runner_config["script-type"] = script_type
self.runner_working_dir = self.engine.create_artifact(runner_config.get("working-dir", "classes"), "")
runner_config["working-dir"] = self.runner_working_dir
runner_config.get("artifacts-dir", self.engine.artifacts_dir)
runner_config.get("working-dir", self.runner_working_dir)
runner_config.get("report-file", self.kpi_file)
runner_config.get("err-file", self.err_jtl)
runner_config.get("stdout", self.engine.create_artifact("junit", ".out"))
runner_config.get("stderr", self.engine.create_artifact("junit", ".err"))
self._cp_resource_files(self.runner_working_dir)
self.runner = self.runner(runner_config, self.scenario, self.log)
self.runner.prepare()
self.reader = JTLReader(self.kpi_file, self.log, self.err_jtl)
if isinstance(self.engine.aggregator, ConsolidatingAggregator):
self.engine.aggregator.add_underling(self.reader)
display_conf = self.settings.get("virtual-display")
if display_conf:
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
else:
width = display_conf.get("width", 1024)
height = display_conf.get("height", 768)
self.virtual_display = Display(size=(width, height))
def _start(self):
super(GUIScreen, self)._start()
self.root = tkinter.Tk()
self.root.geometry("%sx%s" % (self.size[0] * 7, self.size[1] * 15))
self.root.bind("<Configure>", self.resize)
if is_windows():
self.root.bind("<Control-MouseWheel>", self.change_font)
else:
self.root.bind("<Control-4>", self.change_font)
self.root.bind("<Control-5>", self.change_font)
self.root.protocol("WM_DELETE_WINDOW", self.closed_window)
self.text = tkinter.Text(self.root, font="TkFixedFont", wrap=tkinter.NONE, state=tkinter.DISABLED,
background="black", foreground="light gray")
self.text.pack(side=tkinter.LEFT, fill=tkinter.BOTH, expand=tkinter.YES)
self.font = tkfont.Font(self.root, self.text.cget("font"))
self.text.config(font=self.font)
self.__prepare_tags()
def __init__(self, config=None, **kwargs):
settings = config or {}
version = settings.get("version", self.VERSION)
self.tool_path = get_full_path(
settings.get("path",
self.LOCAL_PATH.format(version=version) + 'vegeta'))
if not is_windows():
platform = 'darwin' if is_mac() else 'linux'
download_link = settings.get(
"download-link", self.DOWNLOAD_LINK).format(version=version,
platform=platform)
else:
download_link = ''
super(Vegeta, self).__init__(tool_path=self.tool_path,
download_link=download_link,
version=version,
**kwargs)
def set_virtual_display(self):
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
return
if VirtualDisplay.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = VirtualDisplay.SHARED_VIRTUAL_DISPLAY
else:
width = self.parameters.get("width", 1024)
height = self.parameters.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size, self.virtual_display.new_display_var)
self.virtual_display.start()
VirtualDisplay.SHARED_VIRTUAL_DISPLAY = self.virtual_display
self.engine.shared_env.set({'DISPLAY': os.environ['DISPLAY']}) # backward compatibility
def install(self):
_dir = self.get_dir()
if not os.path.exists(_dir):
os.makedirs(_dir)
dist = self._download(use_link=True)
if dist:
if self.download_link.endswith('.zip'):
self.log.info("Unzipping %s to %s", dist, _dir)
unzip(dist, self.get_dir())
else:
self.log.info("Untaring %s to %s", dist, _dir)
untar(dist, _dir)
os.remove(dist)
if not is_windows():
os.chmod(self.tool_path, 0o755)
def execute(self,
args,
cwd=None,
stdout=PIPE,
stderr=PIPE,
stdin=PIPE,
shell=False,
env=None):
self.preprocess_args(args)
if cwd is None:
cwd = self.engine.default_cwd
environ = BetterDict()
environ.merge(dict(os.environ))
if env is not None:
if is_windows():
# as variables in windows are case insensitive we should provide correct merging
cur_env = {name.upper(): environ[name] for name in environ}
old_keys = set(env.keys())
env = {name.upper(): env[name] for name in env}
new_keys = set(env.keys())
if old_keys != new_keys:
msg = 'Some taurus environment variables might be been lost: %s'
self.log.debug(msg, list(old_keys - new_keys))
environ = BetterDict()
environ.merge(cur_env)
environ.merge(env)
environ.merge({"TAURUS_ARTIFACTS_DIR": self.engine.artifacts_dir})
environ = {
key: environ[key]
for key in environ.keys() if environ[key] is not None
}
self.log.debug("Executing shell from %s: %s", cwd, args)
return shell_exec(args,
cwd=cwd,
stdout=stdout,
stderr=stderr,
stdin=stdin,
shell=shell,
env=environ)
def install(self):
dest = self.get_driver_dir()
if not os.path.exists(dest):
os.makedirs(dest)
self.log.info("Will install %s into %s", self.tool_name, dest)
dist = self._download(use_link=True)
try:
self.log.info("Unzipping %s to %s", dist, dest)
unzip(dist, dest)
finally:
os.remove(dist)
if not is_windows():
os.chmod(self.tool_path, 0o755)
if not self.check_if_installed():
raise ToolError("Unable to find %s after installation!" % self.tool_name)
def test_passes_artifacts_dir_with_envs(self):
cmdline = "echo %TAURUS_ARTIFACTS_DIR%" if is_windows() else "echo $TAURUS_ARTIFACTS_DIR"
engine = EngineEmul({
"settings": {
"env": {"BZT_ARTIFACTS_DIR_ENV_TEST": "custom_dir_from_env"},
"artifacts-dir": get_uniq_name(directory=get_full_path(TEST_DIR),
prefix="${BZT_ARTIFACTS_DIR_ENV_TEST}/%Y-%m-%d_%H-%M-%S.%f")
}})
engine.eval_env()
engine.prepare()
executor = self.obj
executor.engine = engine
process = executor._execute(cmdline, shell=True)
stdout, _ = communicate(process)
self.assertEqual(engine.artifacts_dir, stdout.strip())
if "BZT_ARTIFACTS_DIR_ENV_TEST" in os.environ:
os.environ.pop("BZT_ARTIFACTS_DIR_ENV_TEST")
def __close_log(self):
"""
Close log handlers, move log to artifacts dir
:return:
"""
if self.options.log:
if is_windows():
# need to finalize the logger before moving file
for handler in self.log.handlers:
if issubclass(handler.__class__, logging.FileHandler):
self.log.debug("Closing log handler: %s", handler.baseFilename)
handler.close()
self.log.handlers.remove(handler)
if os.path.exists(self.options.log):
self.engine.existing_artifact(self.options.log)
os.remove(self.options.log)
else:
self.engine.existing_artifact(self.options.log, True)
def test_interpreter(self):
self.configure({
'execution': {
'iterations': 3,
'scenario': {
'script':
__dir__() +
'/../../resources/selenium/ruby/example_spec.rb'
},
'executor': 'selenium'
},
})
self.obj.settings.merge(
self.obj.engine.config.get("modules").get("selenium"))
dummy = __dir__() + '/../../resources/selenium/ruby/ruby-dummy'
dummy += '.bat' if is_windows() else ''
self.obj.settings.merge({"interpreter": dummy})
self.obj.prepare()
def post_process(self):
super(Proxy2JMX, self).post_process()
if self.engine.stopping_reason and not isinstance(
self.engine.stopping_reason, KeyboardInterrupt):
self.log.info("Will not pick converted JMX due to exception: %s",
self.engine.stopping_reason)
return
# self.log.info("Downloading JSON...")
# jmx_text = self.proxy.get_json()
# jmx_file = self.engine.create_artifact(self.label, '.recording.json')
# with open(jmx_file, 'w') as _file:
# _file.writelines(jmx_text)
# self.log.info("JSON saved into %s", jmx_file)
self.log.info("Downloading simple JMX...")
jmx_text = self.proxy.get_jmx()
if not self.output_simple:
self.output_simple = self.engine.create_artifact(
self.label, '.simple.jmx')
with open(self.output_simple, 'w') as _file:
_file.writelines(jmx_text)
self.log.info("Simple JMX saved into %s", self.output_simple)
self.log.info("Waiting for proxy to generate SmartJMX...")
jmx_text = self.proxy.get_jmx(smart=True)
if not self.output_smart:
self.output_smart = self.engine.create_artifact(
self.label, '.smart.jmx')
with open(self.output_smart, 'w') as _file:
_file.writelines(jmx_text)
self.log.info("Smart JMX saved into %s", self.output_smart)
if 'HTTPSampler' not in jmx_text:
self.log.warning(
"There aren't requests recorded by proxy2jmx, check your proxy configuration"
)
# log of chrome-loader not found under windows
if is_windows() and not os.path.isfile(
join(self.engine.artifacts_dir, 'chrome-loader.log')):
msg = "Problems with chrome tuning are encountered, "
msg += "take look at http://gettaurus.org/docs/Proxy2JMX/ for help"
self.log.warning(msg)
def __redirect_streams(self):
if self.__streams_redirected:
return
if sys.stdout.isatty():
if not is_windows():
self.__detect_console_logger()
if self.orig_streams:
raise TaurusInternalException("Console: original streams already set")
elif self.logger_handlers and not self.orig_streams:
self.log.debug("Overriding logging streams")
for handler in self.logger_handlers:
self.orig_streams[handler] = handler.stream
handler.stream = self.temp_stream
self.log.debug("Redirected logging streams, %s/%s", self.logger_handlers, self.orig_streams)
self.__streams_redirected = True
else:
self.log.info("Did not mute console logging")
def check_if_installed(self):
candidates = ["npm"]
if is_windows():
candidates.append("npm.cmd")
for candidate in candidates:
self.log.debug("Trying '%r' as NPM Tool...", candidate)
try:
out, err = self.call([candidate, '--version'])
except CALL_PROBLEMS as exc:
self.log.debug("%r is not installed: %s", candidate, exc)
continue
if err:
out += err
self.log.debug("%s output: %s", candidate, out)
self.tool_path = candidate
return True
return False