def _run_composite(self, jar: str, result_parent: str, handle_out,
handle_err):
"""Called internally only by this.run()"""
cmd = '{} {} -jar {} -m COMPOSITE {}'.format(self.jdk,
self.jvm_options, jar,
self._spec_opts())
for x in range(int(self.num_runs)):
result_dir = self._prerun(result_parent)
handle_out(os.linesep)
handle_out(os.linesep)
handle_out('Starting run {} of {}.'.format(x, self.num_runs))
handle_out('Using command: "{}"'.format(cmd))
handle_out(os.linesep)
handle_out(os.linesep)
data = self._start_data_collection(result_dir, handle_out)
errout = open(os.path.join(result_dir, 'composite.out'), 'w')
stdout = open(os.path.join(result_dir, 'composite.log'), 'w')
p = Popen(shlex.split(cmd),
cwd=result_dir,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
spec_run._running = True
errstream = stream(p.stderr)
outstream = stream(p.stdout)
while spec_run._running and p.poll() is None:
e = errstream.readline()
if e != '':
errout.write(e)
handle_err(e)
o = outstream.readline()
if o != '':
stdout.write(o)
handle_out(o)
errout.close()
errstream.close()
stdout.close()
outstream.close()
if spec_run._running:
exitcode = p.wait()
else:
handle_out(os.linesep)
handle_out(os.linesep)
handle_out("Canceling benchmark...")
exitcode = 0
p.kill()
for pf in data:
pf.kill()
if exitcode != 0 and spec_run._running:
return -1
return 0
def _distributed_sut(self, jar: str, result_parent: str, handle_out,
handle_err):
"""Called internally only by this.run()"""
ctrl_ip = self.properties.root['specjbb.controller.host'].value
pingcmd = 'ping -c 1 {}'.format(ctrl_ip)
handle_out(os.linesep)
handle_out(os.linesep)
handle_out("Checking host {} is online...".format(ctrl_ip))
FNULL = open(os.devnull, 'w')
ping = Popen(shlex.split(pingcmd), stderr=FNULL, stdout=FNULL)
exitcode = ping.wait()
FNULL.close()
if exitcode != 0:
handle_err(
'ERROR: Failed to ping Controller host (specjbb.controller.host): {}'
.format(ctrl_ip))
return 4
handle_out(os.linesep)
handle_out("Starting backends, press Ctrl + C to terminate.")
handle_out(os.linesep)
opts = self._tx_opts()
procs = []
result_dir = self._prerun(result_parent)
data = self._start_data_collection(result_dir, handle_out)
for g in range(int(self.properties.root['specjbb.group.count'].value)):
be_name = 'beJVM Group{}.Backend.beJVM.log'.format(g)
cmd = '{} {} -jar {} -m BACKEND {} -G={} -J=beJVM'.format(
self.jdk, self.jvm_options, jar, opts, g)
p = Popen(shlex.split(cmd),
cwd=result_dir,
stdout=PIPE,
stderr=STDOUT,
universal_newlines=True)
procs.append([
p,
open(os.path.join(result_dir, be_name), 'w'),
stream(p.stdout)
])
spec_run._running = True
while spec_run._running:
for p in procs:
if p[0].poll() is None:
o = p[2].readline()
if o != '':
handle_out(o)
p[1].write(o)
for p in procs:
p[2].close()
p[0].kill()
p[1].close()
for pf in data:
pf.kill()
# Each process will continue until manually terminated with ctrl c.
return 0
def _rollup(self, path: str, outhandle):
"""
:param path: Path to a directory containing directories of results
:return: The path to the result .csv file
"""
if not os.path.exists(path):
raise FileNotFoundError(path)
rollscript = os.path.abspath(os.path.join(__file__, os.pardir))
rollscript = os.path.join(rollscript, "scripts", "Rollup.pl")
if not os.path.exists(rollscript):
raise FileNotFoundError(rollscript)
perlbin = '/usr/bin/perl'
if not os.path.exists(perlbin):
perlbin = '/bin/perl'
if not os.path.exists(perlbin):
return
outhandle(os.linesep)
outhandle(os.linesep)
outhandle("Starting analysis on {}...".format(path))
outhandle(os.linesep)
p = Popen(shlex.split("{} {} {}".format(perlbin, rollscript, path)),
cwd=path,
stdout=PIPE,
stderr=STDOUT,
universal_newlines=True)
outstream = stream(p.stdout)
while p.poll() is None:
o = outstream.readline()
if o != '':
outhandle(o)
p.wait()
resname, resext = os.path.splitext(os.path.basename(path))
resname += ".csv"
result = os.path.join(os.path.abspath(os.path.join(path, os.pardir)),
resname)
return result
# parse command line arguments args = parser.parse_args() dict_args = vars(args) # print arguments for k in dict_args: print(k, '=', dict_args[k]) print() # jump line if args.log: # start logging system output log_file = abspath(args.log) mkdir(log_path) if not exists (log_path) else None sys.stdout = sys.stderr = sys.stdin = Logger(log_file, 'a') if args.type == 'stream': stream(query=args.query, post_url=args.url) # elif args.type == 'trends': # trending_topics(query=args.query, # post_url=args.url) else: # get published tweets collect(query=args.query, query_type=args.type, lang=args.lang, geocode=args.geocode, max_id=args.max, since_id=args.since, stop_number=args.number, post_url=args.url, wait_time=args.wait,
def _run_multi(self, jar: str, result_parent: str, handle_out, handle_err):
"""Called internally only by this.run()"""
opts = self._spec_opts()
tx_opts = self._tx_opts()
has_numa = self._check_numa() and self.numa_nodes > 1
numa_cmd = 'numactl --cpunodebind={} --localalloc'
for x in range(int(self.num_runs)):
handle_out(os.linesep)
result_dir = self._prerun(result_parent)
handle_out("Starting run {} of {} in {}...".format(
x + 1, self.num_runs, result_dir))
handle_out(os.linesep)
handle_out(os.linesep)
cont_std = open(os.path.join(result_dir, 'controller.log'), 'w')
cont_err = open(os.path.join(result_dir, 'controller.out'), 'w')
cmd = '{} {} -jar {} -m MULTICONTROLLER {}'.format(
self.jdk, self.jvm_options, jar, opts)
controller = Popen(shlex.split(cmd),
cwd=result_dir,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
tx_procs = []
be_procs = []
data = self._start_data_collection(result_dir, handle_out)
for g in range(
int(self.properties.root['specjbb.group.count'].value)):
numa = numa_cmd.format((g - 1) % 4)
for j in range(self.properties.
root['specjbb.txi.pergroup.count'].value):
ti_name = "Group{}.TxInjector.txiJVM{}".format(g, j)
if has_numa:
cmd = '{} {} {} -jar {} -m TXINJECTOR -G={} -J=txiJVM{} {}'.format(
numa, self.jdk, self.jvm_options, jar, g, j,
tx_opts)
else:
cmd = '{} {} -jar {} -m TXINJECTOR -G={} -J=txiJVM{} {}'.format(
self.jdk, self.jvm_options, jar, g, j, tx_opts)
handle_out('Using command: "{}"'.format(cmd))
p = Popen(shlex.split(cmd),
cwd=result_dir,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
tx_procs.append([
p,
open(
os.path.join(result_dir, '{}.log'.format(ti_name)),
'w'),
open(
os.path.join(result_dir, '{}.out'.format(ti_name)),
'w'),
stream(p.stderr),
stream(p.stdout)
])
be_name = "Group{}.Backend.beJVM".format(g)
if has_numa:
cmd = '{} {} {} -jar {} -m BACKEND {} -G={} -J=beJVM'.format(
numa, self.jdk, self.jvm_options, jar, tx_opts, g)
else:
cmd = '{} {} -jar {} -m BACKEND {} -G={} -J=beJVM'.format(
self.jdk, self.jvm_options, jar, tx_opts, g)
handle_out('Using command: "{}"'.format(cmd))
p = Popen(shlex.split(cmd),
cwd=result_dir,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
be_procs.append([
p,
open(os.path.join(result_dir, '{}.log'.format(be_name)),
'w'),
open(os.path.join(result_dir, '{}.out'.format(be_name)),
'w'),
stream(p.stderr),
stream(p.stdout)
])
spec_run._running = True
errstream = stream(controller.stderr)
outstream = stream(controller.stdout)
while spec_run._running and controller.poll() is None:
e = errstream.readline()
if e != '':
cont_err.write(e)
handle_err(e)
o = outstream.readline()
if o != '':
cont_std.write(o)
handle_out("Controller: {}".format(o))
for p in tx_procs:
if not spec_run._running:
break
if p[0].poll() is None:
e = p[3].readline()
if e != '':
p[2].write(e)
handle_err(e)
o = p[4].readline()
if o != '':
p[1].write(o)
handle_out("TX: {}".format(o))
for p in be_procs:
if not spec_run._running:
break
if p[0].poll() is None:
e = p[3].readline()
if e != '':
p[2].write(e)
handle_err(e)
o = p[4].readline()
if o != '':
p[1].write(o)
handle_out(os.linesep)
handle_out("Benchmark {} of {} ended.".format(
x + 1, self.num_runs))
cont_err.close()
cont_std.close()
if spec_run._running:
exitcode = controller.wait()
else:
handle_out(os.linesep)
handle_out("Canceling benchmark...")
handle_out(os.linesep)
exitcode = 0
controller.kill()
for p in tx_procs:
p[3].close()
p[4].close()
p[0].kill()
p[1].close()
p[2].close()
for p in be_procs:
p[3].close()
p[4].close()
p[0].kill()
p[1].close()
p[2].close()
for pf in data:
pf.kill()
if exitcode != 0 and spec_run._running:
return -1
return 0
def _run_distributed_ctrl_txl(self, jar: str, result_parent: str,
handle_out, handle_err):
"""Called internally only by this.run()"""
ctrl_ip = self.properties.root['specjbb.controller.host'].value
pingcmd = 'ping -c 1 {}'.format(ctrl_ip)
handle_out(os.linesep)
handle_out(os.linesep)
handle_out("Checking host {} is online...".format(ctrl_ip))
FNULL = open(os.devnull, 'w')
ping = Popen(shlex.split(pingcmd), stderr=FNULL, stdout=FNULL)
exitcode = ping.wait()
FNULL.close()
if exitcode != 0:
handle_err(
'ERROR: Failed to ping Controller host (specjbb.controller.host): {}'
.format(ctrl_ip))
return 4
handle_out(os.linesep)
handle_out(os.linesep)
handle_out("Starting injectors, press Ctrl + C to terminate run.")
handle_out(os.linesep)
opts = self._spec_opts()
tx_opts = self._tx_opts()
result_dir = self._prerun(result_parent)
cmd = '{} {} -jar {} -m DISTCONTROLLER {}'.format(
self.jdk, self.jvm_options, jar, opts)
tx_procs = []
cont_std = open(os.path.join(result_dir, 'controller.log'), 'w')
cont_err = open(os.path.join(result_dir, 'controller.out'), 'w')
controller = Popen(shlex.split(cmd),
cwd=result_dir,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
data = self._start_data_collection(result_dir, handle_out)
for g in range(int(self.properties.root['specjbb.group.count'].value)):
for j in range(
int(self.properties.root['specjbb.txi.pergroup.count'].
value)):
ti_name = "{}Group{}.TxInjector.txiJVM{}".format(
result_dir, g, j)
cmd = '{} {} -jar {} -m TXINJECTOR -G={}'.format(
self.jdk, self.jvm_options, jar, tx_opts, g)
p = Popen(shlex.split(cmd),
cwd=result_dir,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
tx_procs.append([
p,
open(os.path.join(result_dir, '{}.log'.format(ti_name)),
'w'),
open(os.path.join(result_dir, '{}.out'.format(ti_name)),
'w'),
stream(p.stderr),
stream(p.stdout)
])
spec_run._running = True
errstream = stream(controller.stderr)
outstream = stream(controller.stdout)
while spec_run._running and controller.poll() is None:
e = errstream.readline()
if e != '':
cont_err.write(e)
handle_err(e)
o = outstream.readline()
if o != '':
cont_std.write(o)
handle_out(o)
for p in tx_procs:
if p[0].poll() is None:
e = p[3].readline()
if e != '':
p[2].write(e)
handle_err(e)
o = p[4].readline()
if o != '':
p[1].write(o)
handle_out(o)
cont_err.close()
cont_std.close()
if spec_run._running:
exitcode = controller.wait()
else:
handle_out(os.linesep)
handle_out("Ending benchmark...")
exitcode = 0
controller.kill()
for p in tx_procs:
p[3].close()
p[4].close()
p[0].kill()
p[1].close()
p[2].close()
for pf in data:
pf.kill()
if exitcode != 0 and spec_run._running:
return -1
return 0