def http_proxy(): config_dir = os.path.join(os.path.dirname(__file__), "resources", "squid") lines = process.run_subprocess_with_output(f"docker run --rm --name squid -d " f"-v {config_dir}/squidpasswords:/etc/squid/squidpasswords " f"-v {config_dir}/squid.conf:/etc/squid/squid.conf " f"-p 3128:3128 datadog/squid") proxy_container_id = lines[0].strip() proxy = HttpProxy(authenticated_url=f"http://*****:*****@127.0.0.1:3128", anonymous_url=f"http://127.0.0.1:3128") yield proxy process.run_subprocess(f"docker stop {proxy_container_id}")
def run(self, task): try: src_dir = self._config.opts("source", "local.src.dir") except config.ConfigError: logging.exception("Rally is not configured to build from sources") raise SystemSetupError( "Rally is not setup to build from sources. You can either benchmark a binary distribution or " "install the required software and reconfigure Rally with %s --configure." % PROGRAM_NAME) logger.info("Building Elasticsearch from sources in [%s]." % src_dir) gradle = self._config.opts("build", "gradle.bin") java_home = self._config.opts("runtime", "java8.home") log_dir = self._config.opts("system", "log.dir") logger.info("Executing %s %s..." % (gradle, task)) io.ensure_dir(log_dir) log_file = "%s/build.log" % log_dir # we capture all output to a dedicated build log file if process.run_subprocess( "export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (java_home, src_dir, gradle, task, log_file)): msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % ( gradle, task) msg += "=========================================================================================================\n" with open(log_file, "r") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" msg += "The full build log is available at [%s]." % log_file raise BuildError(msg)
def run(self, task): try: src_dir = self._config.opts("source", "local.src.dir") except config.ConfigError: logging.exception("Rally is not configured to build from sources") raise SystemSetupError("Rally is not setup to build from sources. You can either benchmark a binary distribution or " "install the required software and reconfigure Rally with %s --configure." % PROGRAM_NAME) logger.info("Building Elasticsearch from sources in [%s]." % src_dir) gradle = self._config.opts("build", "gradle.bin") java_home = self._config.opts("runtime", "java8.home") log_dir = self._config.opts("system", "log.dir") logger.info("Executing %s %s..." % (gradle, task)) io.ensure_dir(log_dir) log_file = "%s/build.log" % log_dir # we capture all output to a dedicated build log file if process.run_subprocess("export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (java_home, src_dir, gradle, task, log_file)): msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (gradle, task) msg += "=========================================================================================================\n" with open(log_file, "r") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" msg += "The full build log is available at [%s]." % log_file raise BuildError(msg)
def run(self, command, override_src_dir=None): src_dir = self.src_dir if override_src_dir is None else override_src_dir logger.info("Building from sources in [%s].", src_dir) logger.info("Executing %s...", command) io.ensure_dir(self.log_dir) log_file = os.path.join(self.log_dir, "build.log") # we capture all output to a dedicated build log file build_cmd = "export JAVA_HOME={}; cd {}; {} >> {} 2>&1".format( self.java_home, src_dir, command, log_file) logger.info("Running build command [%s]", build_cmd) if process.run_subprocess(build_cmd): msg = "Executing '{}' failed. The last 20 lines in the build log file are:\n".format( command) msg += "=========================================================================================================\n" with open(log_file, "r", encoding="utf-8") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" msg += "The full build log is available at [{}].".format(log_file) raise BuildError(msg)
def run(self, task): from esrally.utils import jvm logger.info("Building from sources in [%s]." % self.src_dir) logger.info("Executing %s %s..." % (self.gradle, task)) io.ensure_dir(self.log_dir) log_file = "%s/build.log" % self.log_dir # we capture all output to a dedicated build log file jvm_major_version = jvm.major_version(self.java_home) if jvm_major_version > 8: logger.info( "Detected JVM with major version [%d]. Adjusting JDK module access options for the build." % jvm_major_version) gradle_opts = "export GRADLE_OPTS=\"%s\"; " % Builder.JAVA_9_GRADLE_OPTS else: gradle_opts = "" if process.run_subprocess( "%sexport JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (gradle_opts, self.java_home, self.src_dir, self.gradle, task, log_file)): msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % ( self.gradle, task) msg += "=========================================================================================================\n" with open(log_file, "r") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" msg += "The full build log is available at [%s]." % log_file raise BuildError(msg)
def _exec(self, task_key): src_dir = self._config.opts("source", "local.src.dir") gradle = self._config.opts("build", "gradle.bin") task = self._config.opts("build", task_key) log_root = self._config.opts("system", "log.dir") build_log_dir = self._config.opts("build", "log.dir") log_dir = "%s/%s" % (log_root, build_log_dir) logger.info("Executing %s %s..." % (gradle, task)) io.ensure_dir(log_dir) log_file = "%s/build.%s.log" % (log_dir, task_key) # we capture all output to a dedicated build log file if process.run_subprocess("cd %s; %s %s > %s 2>&1" % (src_dir, gradle, task, log_file)): msg = "Executing '%s %s' failed. Here are the last 20 lines in the build log file:\n" % ( gradle, task) msg += "=========================================================================================================\n" with open(log_file, "r") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" msg += "The full build log is available at [%s]." % log_file raise BuildError(msg)
def run(self, task, override_src_dir=None): from esrally.utils import jvm src_dir = self.src_dir if override_src_dir is None else override_src_dir logger.info("Building from sources in [%s]." % src_dir) logger.info("Executing %s %s..." % (self.gradle, task)) io.ensure_dir(self.log_dir) log_file = "%s/build.log" % self.log_dir # we capture all output to a dedicated build log file build_cmd = "export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % ( self.java_home, src_dir, self.gradle, task, log_file) logger.info("Running build command [%s]" % build_cmd) if process.run_subprocess(build_cmd): msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % ( self.gradle, task) msg += "=========================================================================================================\n" with open(log_file, "r", encoding="utf-8") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" msg += "The full build log is available at [%s]." % log_file raise BuildError(msg)
def run(self, task): from esrally.utils import jvm logger.info("Building from sources in [%s]." % self.src_dir) logger.info("Executing %s %s..." % (self.gradle, task)) io.ensure_dir(self.log_dir) log_file = "%s/build.log" % self.log_dir # we capture all output to a dedicated build log file jvm_major_version = jvm.major_version(self.java_home) if jvm_major_version > 8: logger.info("Detected JVM with major version [%d]. Adjusting JDK module access options for the build." % jvm_major_version) gradle_opts = "export GRADLE_OPTS=\"%s\"; " % Builder.JAVA_9_GRADLE_OPTS else: gradle_opts = "" if process.run_subprocess("%sexport JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (gradle_opts, self.java_home, self.src_dir, self.gradle, task, log_file)): msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (self.gradle, task) msg += "=========================================================================================================\n" with open(log_file, "r") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" msg += "The full build log is available at [%s]." % log_file raise BuildError(msg)
def clone(self): src = self.src_dir remote = self.remote_url io.ensure_dir(src) print("Downloading sources from %s to %s." % (remote, src)) # Don't swallow subprocess output, user might need to enter credentials... if process.run_subprocess("git clone %s %s" % (remote, src)): raise SupplyError("Could not clone from %s to %s" % (remote, src))
def clone(self): src = self.src_dir remote = self.remote_url io.ensure_dir(src) print("Downloading sources from %s to %s." % (remote, src)) # Don't swallow subprocess output, user might need to enter credentials... if process.run_subprocess("git clone %s %s" % (remote, src)): raise SupplyError("Could not clone from %s to %s" % (remote, src))
def _exec(self, task_key): src_dir = self._config.opts("source", "local.src.dir") gradle = self._config.opts("build", "gradle.bin") task = self._config.opts("build", task_key) log_root = self._config.opts("system", "log.dir") build_log_dir = self._config.opts("build", "log.dir") log_dir = "%s/%s" % (log_root, build_log_dir) logger.info("Executing %s %s..." % (gradle, task)) io.ensure_dir(log_dir) log_file = "%s/build.%s.log" % (log_dir, task_key) # we capture all output to a dedicated build log file if process.run_subprocess("cd %s; %s %s > %s.tmp 2>&1" % (src_dir, gradle, task, log_file)): logger.warning("Executing '%s %s' failed" % (gradle, task)) os.rename(("%s.tmp" % log_file), log_file)
def run(self, task): logger.info("Building Elasticsearch from sources in [%s]." % self.src_dir) logger.info("Executing %s %s..." % (self.gradle, task)) io.ensure_dir(self.log_dir) log_file = "%s/build.log" % self.log_dir # we capture all output to a dedicated build log file if process.run_subprocess("export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (self.java_home, self.src_dir, self.gradle, task, log_file)): msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (self.gradle, task) msg += "=========================================================================================================\n" with open(log_file, "r") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" msg += "The full build log is available at [%s]." % log_file raise BuildError(msg)
def _exec(self, task_key): src_dir = self._config.opts("source", "local.src.dir") gradle = self._config.opts("build", "gradle.bin") task = self._config.opts("build", task_key) log_root = self._config.opts("system", "log.dir") build_log_dir = self._config.opts("build", "log.dir") log_dir = "%s/%s" % (log_root, build_log_dir) logger.info("Executing %s %s..." % (gradle, task)) io.ensure_dir(log_dir) log_file = "%s/build.%s.log" % (log_dir, task_key) # we capture all output to a dedicated build log file if process.run_subprocess("cd %s; %s %s > %s 2>&1" % (src_dir, gradle, task, log_file)): #logger.warning("Executing '%s %s' failed" % (gradle, task)) msg = "Executing '%s %s' failed. Here are the last 20 lines in the build log file:\n" % (gradle, task) msg += "=========================================================================================================\n" with open(log_file, "r") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" msg += "The full build log is available at [%s]." % log_file raise BuildError(msg)
def checkout(src_dir, branch="master"): if process.run_subprocess("git -C {0} checkout --quiet {1}".format( src_dir, branch)): raise exceptions.SupplyError("Could not checkout '%s'" % branch)
def clone(src, remote): io.ensure_dir(src) # Don't swallow subprocess output, user might need to enter credentials... if process.run_subprocess("git clone %s %s" % (remote, src)): raise exceptions.SupplyError("Could not clone from '%s' to '%s'" % (remote, src))
def fetch(src, remote="origin"): # Don't swallow output but silence git at least a bit... (--quiet) if process.run_subprocess("git -C {0} fetch --prune --quiet {1}".format( src, remote)): raise exceptions.SupplyError("Could not fetch source tree from '%s'" % remote)
def clone(src, remote): io.ensure_dir(src) # Don't swallow subprocess output, user might need to enter credentials... if process.run_subprocess("git clone %s %s" % (remote, src)): raise exceptions.SupplyError("Could not clone from '%s' to '%s'" % (remote, src))
def checkout(src_dir, branch="master"): if process.run_subprocess("git -C {0} checkout --quiet {1}".format( src_dir, branch)): raise exceptions.SupplyError( "Could not checkout branch [%s]. Do you have uncommitted changes?" % branch)
def pull(self, remote="origin", branch="master"): # Don't swallow output but silence git at least a bit... (--quiet) if process.run_subprocess( "sh -c 'cd {0}; git checkout --quiet {2} && git fetch --quiet {1} && git rebase --quiet {1}/{2}'" .format(self.src_dir, remote, branch)): raise SupplyError("Could not fetch latest source tree")
def fetch(src, remote="origin"): # Don't swallow output but silence git at least a bit... (--quiet) if process.run_subprocess( "git -C {0} fetch --prune --quiet {1}".format(src, remote)): raise exceptions.SupplyError("Could not fetch source tree from '%s'" % remote)
def pull_revision(self, revision): if process.run_subprocess( "sh -c 'cd %s; git fetch --quiet origin && git checkout --quiet %s'" % (self.src_dir, revision)): raise SupplyError("Could not fetch source tree for revision %s" % revision)
def pull_revision(src_dir, revision): fetch(src_dir) if process.run_subprocess("git -C {0} checkout --quiet {1}".format(src_dir, revision)): raise exceptions.SupplyError("Could not checkout source tree for revision [%s]" % revision)
def pull_revision(self, revision): if process.run_subprocess( "sh -c 'cd %s; git fetch --quiet origin && git checkout --quiet %s'" % (self.src_dir, revision)): raise SupplyError("Could not fetch source tree for revision %s" % revision)
def pull_ts(self, ts): if process.run_subprocess( "sh -c 'cd %s; git fetch --quiet origin && git checkout --quiet `git rev-list -n 1 --before=\"%s\" " "--date=iso8601 origin/master`'" % (self.src_dir, ts)): raise SupplyError( "Could not fetch source tree for timestamped revision %s" % ts)
def pull_revision(src_dir, revision): if process.run_subprocess( "git -C {0} fetch --quiet origin && git -C {0} checkout --quiet {1}" .format(src_dir, revision)): raise exceptions.SupplyError( "Could not fetch source tree for revision %s" % revision)
def rebase(src_dir, remote="origin", branch="master"): checkout(src_dir, branch) if process.run_subprocess("git -C {0} rebase --quiet {1}/{2}".format(src_dir, remote, branch)): raise exceptions.SupplyError("Could not rebase on '%s'" % branch)
def pull_ts(src_dir, ts): if process.run_subprocess( "git -C {0} fetch --quiet origin && git -C {0} checkout --quiet `git -C {0} rev-list -n 1 --before=\"{1}\" " "--date=iso8601 origin/master`".format(src_dir, ts)): raise exceptions.SupplyError("Could not fetch source tree for timestamped revision %s" % ts)
def rebase(src_dir, remote="origin", branch="master"): checkout(src_dir, branch) if process.run_subprocess("git -C {0} rebase --quiet {1}/{2}".format( src_dir, remote, branch)): raise exceptions.SupplyError("Could not rebase on '%s'" % branch)
def checkout(src_dir, branch="master"): if process.run_subprocess( "git -C {0} checkout --quiet {1}".format(src_dir, branch)): raise exceptions.SupplyError("Could not checkout '%s'" % branch)
def pull_ts(src_dir, ts): if process.run_subprocess( "git -C {0} fetch --quiet origin && git -C {0} checkout --quiet `git -C {0} rev-list -n 1 --before=\"{1}\" " "--date=iso8601 origin/master`".format(src_dir, ts)): raise exceptions.SupplyError( "Could not fetch source tree for timestamped revision %s" % ts)
def pull_revision(src_dir, revision): if process.run_subprocess( "git -C {0} fetch --quiet origin && git -C {0} checkout --quiet {1}".format(src_dir, revision)): raise exceptions.SupplyError("Could not fetch source tree for revision %s" % revision)
def pull(self, remote="origin", branch="master"): # Don't swallow output but silence git at least a bit... (--quiet) if process.run_subprocess( "sh -c 'cd {0}; git checkout --quiet {2} && git fetch --quiet {1} && git rebase --quiet {1}/{2}'" .format(self.src_dir, remote, branch)): raise SupplyError("Could not fetch latest source tree")
def pull_ts(self, ts): if process.run_subprocess( "sh -c 'cd %s; git fetch --quiet origin && git checkout --quiet `git rev-list -n 1 --before=\"%s\" " "--date=iso8601 origin/master`'" % (self.src_dir, ts)): raise SupplyError("Could not fetch source tree for timestamped revision %s" % ts)