def http_proxy():
config_dir = os.path.join(os.path.dirname(__file__), "resources", "squid")
lines = process.run_subprocess_with_output(f"docker run --rm --name squid -d "
f"-v {config_dir}/squidpasswords:/etc/squid/squidpasswords "
f"-v {config_dir}/squid.conf:/etc/squid/squid.conf "
f"-p 3128:3128 datadog/squid")
proxy_container_id = lines[0].strip()
proxy = HttpProxy(authenticated_url=f"http://*****:*****@127.0.0.1:3128",
anonymous_url=f"http://127.0.0.1:3128")
yield proxy
process.run_subprocess(f"docker stop {proxy_container_id}")
def run(self, task):
try:
src_dir = self._config.opts("source", "local.src.dir")
except config.ConfigError:
logging.exception("Rally is not configured to build from sources")
raise SystemSetupError(
"Rally is not setup to build from sources. You can either benchmark a binary distribution or "
"install the required software and reconfigure Rally with %s --configure."
% PROGRAM_NAME)
logger.info("Building Elasticsearch from sources in [%s]." % src_dir)
gradle = self._config.opts("build", "gradle.bin")
java_home = self._config.opts("runtime", "java8.home")
log_dir = self._config.opts("system", "log.dir")
logger.info("Executing %s %s..." % (gradle, task))
io.ensure_dir(log_dir)
log_file = "%s/build.log" % log_dir
# we capture all output to a dedicated build log file
if process.run_subprocess(
"export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" %
(java_home, src_dir, gradle, task, log_file)):
msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (
gradle, task)
msg += "=========================================================================================================\n"
with open(log_file, "r") as f:
msg += "\t"
msg += "\t".join(f.readlines()[-20:])
msg += "=========================================================================================================\n"
msg += "The full build log is available at [%s]." % log_file
raise BuildError(msg)
def run(self, task):
try:
src_dir = self._config.opts("source", "local.src.dir")
except config.ConfigError:
logging.exception("Rally is not configured to build from sources")
raise SystemSetupError("Rally is not setup to build from sources. You can either benchmark a binary distribution or "
"install the required software and reconfigure Rally with %s --configure." % PROGRAM_NAME)
logger.info("Building Elasticsearch from sources in [%s]." % src_dir)
gradle = self._config.opts("build", "gradle.bin")
java_home = self._config.opts("runtime", "java8.home")
log_dir = self._config.opts("system", "log.dir")
logger.info("Executing %s %s..." % (gradle, task))
io.ensure_dir(log_dir)
log_file = "%s/build.log" % log_dir
# we capture all output to a dedicated build log file
if process.run_subprocess("export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (java_home, src_dir, gradle, task, log_file)):
msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (gradle, task)
msg += "=========================================================================================================\n"
with open(log_file, "r") as f:
msg += "\t"
msg += "\t".join(f.readlines()[-20:])
msg += "=========================================================================================================\n"
msg += "The full build log is available at [%s]." % log_file
raise BuildError(msg)
def run(self, command, override_src_dir=None):
src_dir = self.src_dir if override_src_dir is None else override_src_dir
logger.info("Building from sources in [%s].", src_dir)
logger.info("Executing %s...", command)
io.ensure_dir(self.log_dir)
log_file = os.path.join(self.log_dir, "build.log")
# we capture all output to a dedicated build log file
build_cmd = "export JAVA_HOME={}; cd {}; {} >> {} 2>&1".format(
self.java_home, src_dir, command, log_file)
logger.info("Running build command [%s]", build_cmd)
if process.run_subprocess(build_cmd):
msg = "Executing '{}' failed. The last 20 lines in the build log file are:\n".format(
command)
msg += "=========================================================================================================\n"
with open(log_file, "r", encoding="utf-8") as f:
msg += "\t"
msg += "\t".join(f.readlines()[-20:])
msg += "=========================================================================================================\n"
msg += "The full build log is available at [{}].".format(log_file)
raise BuildError(msg)
def run(self, task):
from esrally.utils import jvm
logger.info("Building from sources in [%s]." % self.src_dir)
logger.info("Executing %s %s..." % (self.gradle, task))
io.ensure_dir(self.log_dir)
log_file = "%s/build.log" % self.log_dir
# we capture all output to a dedicated build log file
jvm_major_version = jvm.major_version(self.java_home)
if jvm_major_version > 8:
logger.info(
"Detected JVM with major version [%d]. Adjusting JDK module access options for the build."
% jvm_major_version)
gradle_opts = "export GRADLE_OPTS=\"%s\"; " % Builder.JAVA_9_GRADLE_OPTS
else:
gradle_opts = ""
if process.run_subprocess(
"%sexport JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" %
(gradle_opts, self.java_home, self.src_dir, self.gradle, task,
log_file)):
msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (
self.gradle, task)
msg += "=========================================================================================================\n"
with open(log_file, "r") as f:
msg += "\t"
msg += "\t".join(f.readlines()[-20:])
msg += "=========================================================================================================\n"
msg += "The full build log is available at [%s]." % log_file
raise BuildError(msg)
def _exec(self, task_key):
src_dir = self._config.opts("source", "local.src.dir")
gradle = self._config.opts("build", "gradle.bin")
task = self._config.opts("build", task_key)
log_root = self._config.opts("system", "log.dir")
build_log_dir = self._config.opts("build", "log.dir")
log_dir = "%s/%s" % (log_root, build_log_dir)
logger.info("Executing %s %s..." % (gradle, task))
io.ensure_dir(log_dir)
log_file = "%s/build.%s.log" % (log_dir, task_key)
# we capture all output to a dedicated build log file
if process.run_subprocess("cd %s; %s %s > %s 2>&1" %
(src_dir, gradle, task, log_file)):
msg = "Executing '%s %s' failed. Here are the last 20 lines in the build log file:\n" % (
gradle, task)
msg += "=========================================================================================================\n"
with open(log_file, "r") as f:
msg += "\t"
msg += "\t".join(f.readlines()[-20:])
msg += "=========================================================================================================\n"
msg += "The full build log is available at [%s]." % log_file
raise BuildError(msg)
def run(self, task, override_src_dir=None):
from esrally.utils import jvm
src_dir = self.src_dir if override_src_dir is None else override_src_dir
logger.info("Building from sources in [%s]." % src_dir)
logger.info("Executing %s %s..." % (self.gradle, task))
io.ensure_dir(self.log_dir)
log_file = "%s/build.log" % self.log_dir
# we capture all output to a dedicated build log file
build_cmd = "export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (
self.java_home, src_dir, self.gradle, task, log_file)
logger.info("Running build command [%s]" % build_cmd)
if process.run_subprocess(build_cmd):
msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (
self.gradle, task)
msg += "=========================================================================================================\n"
with open(log_file, "r", encoding="utf-8") as f:
msg += "\t"
msg += "\t".join(f.readlines()[-20:])
msg += "=========================================================================================================\n"
msg += "The full build log is available at [%s]." % log_file
raise BuildError(msg)
def run(self, task):
from esrally.utils import jvm
logger.info("Building from sources in [%s]." % self.src_dir)
logger.info("Executing %s %s..." % (self.gradle, task))
io.ensure_dir(self.log_dir)
log_file = "%s/build.log" % self.log_dir
# we capture all output to a dedicated build log file
jvm_major_version = jvm.major_version(self.java_home)
if jvm_major_version > 8:
logger.info("Detected JVM with major version [%d]. Adjusting JDK module access options for the build." % jvm_major_version)
gradle_opts = "export GRADLE_OPTS=\"%s\"; " % Builder.JAVA_9_GRADLE_OPTS
else:
gradle_opts = ""
if process.run_subprocess("%sexport JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" %
(gradle_opts, self.java_home, self.src_dir, self.gradle, task, log_file)):
msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (self.gradle, task)
msg += "=========================================================================================================\n"
with open(log_file, "r") as f:
msg += "\t"
msg += "\t".join(f.readlines()[-20:])
msg += "=========================================================================================================\n"
msg += "The full build log is available at [%s]." % log_file
raise BuildError(msg)
def clone(self):
src = self.src_dir
remote = self.remote_url
io.ensure_dir(src)
print("Downloading sources from %s to %s." % (remote, src))
# Don't swallow subprocess output, user might need to enter credentials...
if process.run_subprocess("git clone %s %s" % (remote, src)):
raise SupplyError("Could not clone from %s to %s" % (remote, src))
def clone(self):
src = self.src_dir
remote = self.remote_url
io.ensure_dir(src)
print("Downloading sources from %s to %s." % (remote, src))
# Don't swallow subprocess output, user might need to enter credentials...
if process.run_subprocess("git clone %s %s" % (remote, src)):
raise SupplyError("Could not clone from %s to %s" % (remote, src))
def _exec(self, task_key):
src_dir = self._config.opts("source", "local.src.dir")
gradle = self._config.opts("build", "gradle.bin")
task = self._config.opts("build", task_key)
log_root = self._config.opts("system", "log.dir")
build_log_dir = self._config.opts("build", "log.dir")
log_dir = "%s/%s" % (log_root, build_log_dir)
logger.info("Executing %s %s..." % (gradle, task))
io.ensure_dir(log_dir)
log_file = "%s/build.%s.log" % (log_dir, task_key)
# we capture all output to a dedicated build log file
if process.run_subprocess("cd %s; %s %s > %s.tmp 2>&1" % (src_dir, gradle, task, log_file)):
logger.warning("Executing '%s %s' failed" % (gradle, task))
os.rename(("%s.tmp" % log_file), log_file)
def run(self, task):
logger.info("Building Elasticsearch from sources in [%s]." % self.src_dir)
logger.info("Executing %s %s..." % (self.gradle, task))
io.ensure_dir(self.log_dir)
log_file = "%s/build.log" % self.log_dir
# we capture all output to a dedicated build log file
if process.run_subprocess("export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" %
(self.java_home, self.src_dir, self.gradle, task, log_file)):
msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (self.gradle, task)
msg += "=========================================================================================================\n"
with open(log_file, "r") as f:
msg += "\t"
msg += "\t".join(f.readlines()[-20:])
msg += "=========================================================================================================\n"
msg += "The full build log is available at [%s]." % log_file
raise BuildError(msg)
def _exec(self, task_key):
src_dir = self._config.opts("source", "local.src.dir")
gradle = self._config.opts("build", "gradle.bin")
task = self._config.opts("build", task_key)
log_root = self._config.opts("system", "log.dir")
build_log_dir = self._config.opts("build", "log.dir")
log_dir = "%s/%s" % (log_root, build_log_dir)
logger.info("Executing %s %s..." % (gradle, task))
io.ensure_dir(log_dir)
log_file = "%s/build.%s.log" % (log_dir, task_key)
# we capture all output to a dedicated build log file
if process.run_subprocess("cd %s; %s %s > %s 2>&1" % (src_dir, gradle, task, log_file)):
#logger.warning("Executing '%s %s' failed" % (gradle, task))
msg = "Executing '%s %s' failed. Here are the last 20 lines in the build log file:\n" % (gradle, task)
msg += "=========================================================================================================\n"
with open(log_file, "r") as f:
msg += "\t"
msg += "\t".join(f.readlines()[-20:])
msg += "=========================================================================================================\n"
msg += "The full build log is available at [%s]." % log_file
raise BuildError(msg)
def checkout(src_dir, branch="master"):
if process.run_subprocess("git -C {0} checkout --quiet {1}".format(
src_dir, branch)):
raise exceptions.SupplyError("Could not checkout '%s'" % branch)
def clone(src, remote):
io.ensure_dir(src)
# Don't swallow subprocess output, user might need to enter credentials...
if process.run_subprocess("git clone %s %s" % (remote, src)):
raise exceptions.SupplyError("Could not clone from '%s' to '%s'" %
(remote, src))
def fetch(src, remote="origin"):
# Don't swallow output but silence git at least a bit... (--quiet)
if process.run_subprocess("git -C {0} fetch --prune --quiet {1}".format(
src, remote)):
raise exceptions.SupplyError("Could not fetch source tree from '%s'" %
remote)
def clone(src, remote):
io.ensure_dir(src)
# Don't swallow subprocess output, user might need to enter credentials...
if process.run_subprocess("git clone %s %s" % (remote, src)):
raise exceptions.SupplyError("Could not clone from '%s' to '%s'" % (remote, src))
def checkout(src_dir, branch="master"):
if process.run_subprocess("git -C {0} checkout --quiet {1}".format(
src_dir, branch)):
raise exceptions.SupplyError(
"Could not checkout branch [%s]. Do you have uncommitted changes?"
% branch)
def pull(self, remote="origin", branch="master"):
# Don't swallow output but silence git at least a bit... (--quiet)
if process.run_subprocess(
"sh -c 'cd {0}; git checkout --quiet {2} && git fetch --quiet {1} && git rebase --quiet {1}/{2}'"
.format(self.src_dir, remote, branch)):
raise SupplyError("Could not fetch latest source tree")
def fetch(src, remote="origin"):
# Don't swallow output but silence git at least a bit... (--quiet)
if process.run_subprocess(
"git -C {0} fetch --prune --quiet {1}".format(src, remote)):
raise exceptions.SupplyError("Could not fetch source tree from '%s'" % remote)
def pull_revision(self, revision):
if process.run_subprocess(
"sh -c 'cd %s; git fetch --quiet origin && git checkout --quiet %s'"
% (self.src_dir, revision)):
raise SupplyError("Could not fetch source tree for revision %s" %
revision)
def pull_revision(src_dir, revision):
fetch(src_dir)
if process.run_subprocess("git -C {0} checkout --quiet {1}".format(src_dir, revision)):
raise exceptions.SupplyError("Could not checkout source tree for revision [%s]" % revision)
def pull_revision(self, revision):
if process.run_subprocess(
"sh -c 'cd %s; git fetch --quiet origin && git checkout --quiet %s'" % (self.src_dir, revision)):
raise SupplyError("Could not fetch source tree for revision %s" % revision)
def pull_ts(self, ts):
if process.run_subprocess(
"sh -c 'cd %s; git fetch --quiet origin && git checkout --quiet `git rev-list -n 1 --before=\"%s\" "
"--date=iso8601 origin/master`'" % (self.src_dir, ts)):
raise SupplyError(
"Could not fetch source tree for timestamped revision %s" % ts)
def pull_revision(src_dir, revision):
if process.run_subprocess(
"git -C {0} fetch --quiet origin && git -C {0} checkout --quiet {1}"
.format(src_dir, revision)):
raise exceptions.SupplyError(
"Could not fetch source tree for revision %s" % revision)
def rebase(src_dir, remote="origin", branch="master"):
checkout(src_dir, branch)
if process.run_subprocess("git -C {0} rebase --quiet {1}/{2}".format(src_dir, remote, branch)):
raise exceptions.SupplyError("Could not rebase on '%s'" % branch)
def pull_ts(src_dir, ts):
if process.run_subprocess(
"git -C {0} fetch --quiet origin && git -C {0} checkout --quiet `git -C {0} rev-list -n 1 --before=\"{1}\" "
"--date=iso8601 origin/master`".format(src_dir, ts)):
raise exceptions.SupplyError("Could not fetch source tree for timestamped revision %s" % ts)
def rebase(src_dir, remote="origin", branch="master"):
checkout(src_dir, branch)
if process.run_subprocess("git -C {0} rebase --quiet {1}/{2}".format(
src_dir, remote, branch)):
raise exceptions.SupplyError("Could not rebase on '%s'" % branch)
def checkout(src_dir, branch="master"):
if process.run_subprocess(
"git -C {0} checkout --quiet {1}".format(src_dir, branch)):
raise exceptions.SupplyError("Could not checkout '%s'" % branch)
def pull_ts(src_dir, ts):
if process.run_subprocess(
"git -C {0} fetch --quiet origin && git -C {0} checkout --quiet `git -C {0} rev-list -n 1 --before=\"{1}\" "
"--date=iso8601 origin/master`".format(src_dir, ts)):
raise exceptions.SupplyError(
"Could not fetch source tree for timestamped revision %s" % ts)
def pull_revision(src_dir, revision):
if process.run_subprocess(
"git -C {0} fetch --quiet origin && git -C {0} checkout --quiet {1}".format(src_dir, revision)):
raise exceptions.SupplyError("Could not fetch source tree for revision %s" % revision)
def pull(self, remote="origin", branch="master"):
# Don't swallow output but silence git at least a bit... (--quiet)
if process.run_subprocess(
"sh -c 'cd {0}; git checkout --quiet {2} && git fetch --quiet {1} && git rebase --quiet {1}/{2}'"
.format(self.src_dir, remote, branch)):
raise SupplyError("Could not fetch latest source tree")
def pull_ts(self, ts):
if process.run_subprocess(
"sh -c 'cd %s; git fetch --quiet origin && git checkout --quiet `git rev-list -n 1 --before=\"%s\" "
"--date=iso8601 origin/master`'" %
(self.src_dir, ts)):
raise SupplyError("Could not fetch source tree for timestamped revision %s" % ts)
