def _run_composite(self, jar: str, result_parent: str, handle_out, handle_err): """Called internally only by this.run()""" cmd = '{} {} -jar {} -m COMPOSITE {}'.format(self.jdk, self.jvm_options, jar, self._spec_opts()) for x in range(int(self.num_runs)): result_dir = self._prerun(result_parent) handle_out(os.linesep) handle_out(os.linesep) handle_out('Starting run {} of {}.'.format(x, self.num_runs)) handle_out('Using command: "{}"'.format(cmd)) handle_out(os.linesep) handle_out(os.linesep) data = self._start_data_collection(result_dir, handle_out) errout = open(os.path.join(result_dir, 'composite.out'), 'w') stdout = open(os.path.join(result_dir, 'composite.log'), 'w') p = Popen(shlex.split(cmd), cwd=result_dir, stdout=PIPE, stderr=PIPE, universal_newlines=True) spec_run._running = True errstream = stream(p.stderr) outstream = stream(p.stdout) while spec_run._running and p.poll() is None: e = errstream.readline() if e != '': errout.write(e) handle_err(e) o = outstream.readline() if o != '': stdout.write(o) handle_out(o) errout.close() errstream.close() stdout.close() outstream.close() if spec_run._running: exitcode = p.wait() else: handle_out(os.linesep) handle_out(os.linesep) handle_out("Canceling benchmark...") exitcode = 0 p.kill() for pf in data: pf.kill() if exitcode != 0 and spec_run._running: return -1 return 0
def _distributed_sut(self, jar: str, result_parent: str, handle_out, handle_err): """Called internally only by this.run()""" ctrl_ip = self.properties.root['specjbb.controller.host'].value pingcmd = 'ping -c 1 {}'.format(ctrl_ip) handle_out(os.linesep) handle_out(os.linesep) handle_out("Checking host {} is online...".format(ctrl_ip)) FNULL = open(os.devnull, 'w') ping = Popen(shlex.split(pingcmd), stderr=FNULL, stdout=FNULL) exitcode = ping.wait() FNULL.close() if exitcode != 0: handle_err( 'ERROR: Failed to ping Controller host (specjbb.controller.host): {}' .format(ctrl_ip)) return 4 handle_out(os.linesep) handle_out("Starting backends, press Ctrl + C to terminate.") handle_out(os.linesep) opts = self._tx_opts() procs = [] result_dir = self._prerun(result_parent) data = self._start_data_collection(result_dir, handle_out) for g in range(int(self.properties.root['specjbb.group.count'].value)): be_name = 'beJVM Group{}.Backend.beJVM.log'.format(g) cmd = '{} {} -jar {} -m BACKEND {} -G={} -J=beJVM'.format( self.jdk, self.jvm_options, jar, opts, g) p = Popen(shlex.split(cmd), cwd=result_dir, stdout=PIPE, stderr=STDOUT, universal_newlines=True) procs.append([ p, open(os.path.join(result_dir, be_name), 'w'), stream(p.stdout) ]) spec_run._running = True while spec_run._running: for p in procs: if p[0].poll() is None: o = p[2].readline() if o != '': handle_out(o) p[1].write(o) for p in procs: p[2].close() p[0].kill() p[1].close() for pf in data: pf.kill() # Each process will continue until manually terminated with ctrl c. return 0
def _rollup(self, path: str, outhandle): """ :param path: Path to a directory containing directories of results :return: The path to the result .csv file """ if not os.path.exists(path): raise FileNotFoundError(path) rollscript = os.path.abspath(os.path.join(__file__, os.pardir)) rollscript = os.path.join(rollscript, "scripts", "Rollup.pl") if not os.path.exists(rollscript): raise FileNotFoundError(rollscript) perlbin = '/usr/bin/perl' if not os.path.exists(perlbin): perlbin = '/bin/perl' if not os.path.exists(perlbin): return outhandle(os.linesep) outhandle(os.linesep) outhandle("Starting analysis on {}...".format(path)) outhandle(os.linesep) p = Popen(shlex.split("{} {} {}".format(perlbin, rollscript, path)), cwd=path, stdout=PIPE, stderr=STDOUT, universal_newlines=True) outstream = stream(p.stdout) while p.poll() is None: o = outstream.readline() if o != '': outhandle(o) p.wait() resname, resext = os.path.splitext(os.path.basename(path)) resname += ".csv" result = os.path.join(os.path.abspath(os.path.join(path, os.pardir)), resname) return result
# parse command line arguments args = parser.parse_args() dict_args = vars(args) # print arguments for k in dict_args: print(k, '=', dict_args[k]) print() # jump line if args.log: # start logging system output log_file = abspath(args.log) mkdir(log_path) if not exists (log_path) else None sys.stdout = sys.stderr = sys.stdin = Logger(log_file, 'a') if args.type == 'stream': stream(query=args.query, post_url=args.url) # elif args.type == 'trends': # trending_topics(query=args.query, # post_url=args.url) else: # get published tweets collect(query=args.query, query_type=args.type, lang=args.lang, geocode=args.geocode, max_id=args.max, since_id=args.since, stop_number=args.number, post_url=args.url, wait_time=args.wait,
def _run_multi(self, jar: str, result_parent: str, handle_out, handle_err): """Called internally only by this.run()""" opts = self._spec_opts() tx_opts = self._tx_opts() has_numa = self._check_numa() and self.numa_nodes > 1 numa_cmd = 'numactl --cpunodebind={} --localalloc' for x in range(int(self.num_runs)): handle_out(os.linesep) result_dir = self._prerun(result_parent) handle_out("Starting run {} of {} in {}...".format( x + 1, self.num_runs, result_dir)) handle_out(os.linesep) handle_out(os.linesep) cont_std = open(os.path.join(result_dir, 'controller.log'), 'w') cont_err = open(os.path.join(result_dir, 'controller.out'), 'w') cmd = '{} {} -jar {} -m MULTICONTROLLER {}'.format( self.jdk, self.jvm_options, jar, opts) controller = Popen(shlex.split(cmd), cwd=result_dir, stdout=PIPE, stderr=PIPE, universal_newlines=True) tx_procs = [] be_procs = [] data = self._start_data_collection(result_dir, handle_out) for g in range( int(self.properties.root['specjbb.group.count'].value)): numa = numa_cmd.format((g - 1) % 4) for j in range(self.properties. root['specjbb.txi.pergroup.count'].value): ti_name = "Group{}.TxInjector.txiJVM{}".format(g, j) if has_numa: cmd = '{} {} {} -jar {} -m TXINJECTOR -G={} -J=txiJVM{} {}'.format( numa, self.jdk, self.jvm_options, jar, g, j, tx_opts) else: cmd = '{} {} -jar {} -m TXINJECTOR -G={} -J=txiJVM{} {}'.format( self.jdk, self.jvm_options, jar, g, j, tx_opts) handle_out('Using command: "{}"'.format(cmd)) p = Popen(shlex.split(cmd), cwd=result_dir, stdout=PIPE, stderr=PIPE, universal_newlines=True) tx_procs.append([ p, open( os.path.join(result_dir, '{}.log'.format(ti_name)), 'w'), open( os.path.join(result_dir, '{}.out'.format(ti_name)), 'w'), stream(p.stderr), stream(p.stdout) ]) be_name = "Group{}.Backend.beJVM".format(g) if has_numa: cmd = '{} {} {} -jar {} -m BACKEND {} -G={} -J=beJVM'.format( numa, self.jdk, self.jvm_options, jar, tx_opts, g) else: cmd = '{} {} -jar {} -m BACKEND {} -G={} -J=beJVM'.format( self.jdk, self.jvm_options, jar, tx_opts, g) handle_out('Using command: "{}"'.format(cmd)) p = Popen(shlex.split(cmd), cwd=result_dir, stdout=PIPE, stderr=PIPE, universal_newlines=True) be_procs.append([ p, open(os.path.join(result_dir, '{}.log'.format(be_name)), 'w'), open(os.path.join(result_dir, '{}.out'.format(be_name)), 'w'), stream(p.stderr), stream(p.stdout) ]) spec_run._running = True errstream = stream(controller.stderr) outstream = stream(controller.stdout) while spec_run._running and controller.poll() is None: e = errstream.readline() if e != '': cont_err.write(e) handle_err(e) o = outstream.readline() if o != '': cont_std.write(o) handle_out("Controller: {}".format(o)) for p in tx_procs: if not spec_run._running: break if p[0].poll() is None: e = p[3].readline() if e != '': p[2].write(e) handle_err(e) o = p[4].readline() if o != '': p[1].write(o) handle_out("TX: {}".format(o)) for p in be_procs: if not spec_run._running: break if p[0].poll() is None: e = p[3].readline() if e != '': p[2].write(e) handle_err(e) o = p[4].readline() if o != '': p[1].write(o) handle_out(os.linesep) handle_out("Benchmark {} of {} ended.".format( x + 1, self.num_runs)) cont_err.close() cont_std.close() if spec_run._running: exitcode = controller.wait() else: handle_out(os.linesep) handle_out("Canceling benchmark...") handle_out(os.linesep) exitcode = 0 controller.kill() for p in tx_procs: p[3].close() p[4].close() p[0].kill() p[1].close() p[2].close() for p in be_procs: p[3].close() p[4].close() p[0].kill() p[1].close() p[2].close() for pf in data: pf.kill() if exitcode != 0 and spec_run._running: return -1 return 0
def _run_distributed_ctrl_txl(self, jar: str, result_parent: str, handle_out, handle_err): """Called internally only by this.run()""" ctrl_ip = self.properties.root['specjbb.controller.host'].value pingcmd = 'ping -c 1 {}'.format(ctrl_ip) handle_out(os.linesep) handle_out(os.linesep) handle_out("Checking host {} is online...".format(ctrl_ip)) FNULL = open(os.devnull, 'w') ping = Popen(shlex.split(pingcmd), stderr=FNULL, stdout=FNULL) exitcode = ping.wait() FNULL.close() if exitcode != 0: handle_err( 'ERROR: Failed to ping Controller host (specjbb.controller.host): {}' .format(ctrl_ip)) return 4 handle_out(os.linesep) handle_out(os.linesep) handle_out("Starting injectors, press Ctrl + C to terminate run.") handle_out(os.linesep) opts = self._spec_opts() tx_opts = self._tx_opts() result_dir = self._prerun(result_parent) cmd = '{} {} -jar {} -m DISTCONTROLLER {}'.format( self.jdk, self.jvm_options, jar, opts) tx_procs = [] cont_std = open(os.path.join(result_dir, 'controller.log'), 'w') cont_err = open(os.path.join(result_dir, 'controller.out'), 'w') controller = Popen(shlex.split(cmd), cwd=result_dir, stdout=PIPE, stderr=PIPE, universal_newlines=True) data = self._start_data_collection(result_dir, handle_out) for g in range(int(self.properties.root['specjbb.group.count'].value)): for j in range( int(self.properties.root['specjbb.txi.pergroup.count']. value)): ti_name = "{}Group{}.TxInjector.txiJVM{}".format( result_dir, g, j) cmd = '{} {} -jar {} -m TXINJECTOR -G={}'.format( self.jdk, self.jvm_options, jar, tx_opts, g) p = Popen(shlex.split(cmd), cwd=result_dir, stdout=PIPE, stderr=PIPE, universal_newlines=True) tx_procs.append([ p, open(os.path.join(result_dir, '{}.log'.format(ti_name)), 'w'), open(os.path.join(result_dir, '{}.out'.format(ti_name)), 'w'), stream(p.stderr), stream(p.stdout) ]) spec_run._running = True errstream = stream(controller.stderr) outstream = stream(controller.stdout) while spec_run._running and controller.poll() is None: e = errstream.readline() if e != '': cont_err.write(e) handle_err(e) o = outstream.readline() if o != '': cont_std.write(o) handle_out(o) for p in tx_procs: if p[0].poll() is None: e = p[3].readline() if e != '': p[2].write(e) handle_err(e) o = p[4].readline() if o != '': p[1].write(o) handle_out(o) cont_err.close() cont_std.close() if spec_run._running: exitcode = controller.wait() else: handle_out(os.linesep) handle_out("Ending benchmark...") exitcode = 0 controller.kill() for p in tx_procs: p[3].close() p[4].close() p[0].kill() p[1].close() p[2].close() for pf in data: pf.kill() if exitcode != 0 and spec_run._running: return -1 return 0