def run_benchmark():
os.chdir('%s/benchmarks/scripts' % benchmark_folder)
FNULL = open(os.devnull, 'w')
before_popen = Popen('./example%s.sh' % str(utilities.argv.spark_example),
shell=True,
stdout=FNULL,
stderr=FNULL)
try:
while before_popen.poll() is None:
utilities.wait_animation(
'Running example {0} before. PID {1}'.format(
utilities.argv.spark_example, before_popen.pid))
print('\nexample {0} before complete'.format(
utilities.argv.spark_example))
except KeyboardInterrupt:
before_popen.terminate()
exit()
after_popen = Popen('./example%s-after.sh' %
str(utilities.argv.spark_example),
shell=True,
stdout=FNULL,
stderr=FNULL)
try:
while after_popen.poll() is None:
utilities.wait_animation(
'Running example {0} after. PID {1}'.format(
utilities.argv.spark_example, after_popen.pid))
print('\nexample {0} after complete'.format(
utilities.argv.spark_example))
except KeyboardInterrupt:
after_popen.terminate()
exit()
os.chdir(main_dir)
def run_impala_example():
os.chdir('impala')
FNULL = open(os.devnull, 'w')
if utilities.argv.impala_query == 'all':
impala_popen = Popen("./run_exp.sh tpcds_text_{0} {1}".format(
utilities.argv.dataset_size, utilities.argv.impala_server),
shell=True,
stderr=FNULL,
stdout=FNULL)
else:
if int(utilities.argv.impala_query) < 10:
utilities.argv.impala_query = '{:02d}'.format(
int(utilities.argv.impala_query))
impala_popen = Popen(
"./run_one.sh tpcds_text_{0} query{1}.sql {2}".format(
utilities.argv.dataset_size, utilities.argv.impala_query,
utilities.argv.impala_server),
shell=True,
stderr=FNULL,
stdout=FNULL)
try:
while impala_popen.poll() is None:
utilities.wait_animation('Running Impala query{0}'.format(
utilities.argv.impala_query))
except KeyboardInterrupt:
impala_popen.terminate()
exit()
os.chdir(main_dir)
print('\nImpala query complete')
def run_autoaction():
os.chdir(benchmark_folder + '/auto-actions/demos')
FNULL = open(os.devnull, 'w')
for file in os.listdir(os.getcwd()):
if not os.path.isdir(file) and file != 'run-all-demos':
benchmark_popen = Popen('bash %s' % file, shell=True, stderr=FNULL, stdout=FNULL)
try:
while benchmark_popen.poll() is None:
utilities.wait_animation('{0} is running PID {1}'.format(file, benchmark_popen.pid))
except KeyboardInterrupt:
benchmark_popen.terminate()
return
os.chdir(main_dir)
def run_streaming_demo():
os.chdir(os.path.join(main_dir, benchmark_folder))
FNULL = open(os.devnull, 'w')
streaming_app_popen = Popen('./streaming-app.sh', stdout=FNULL, stderr=FNULL, shell=True)
producer_popen = Popen('./producer.sh', stdout=FNULL, stderr=FNULL, shell=True)
try:
while streaming_app_popen.poll() is None:
utilities.wait_animation('Spark streaming job is running PID {0} {1}'.format(streaming_app_popen.pid, producer_popen.pid))
producer_popen.terminate()
except:
traceback.print_exc()
streaming_app_popen.terminate()
producer_popen.terminate()
os.chdir(main_dir)
print('\nSpark streaming demo completed')
def run_benchmark():
os.chdir(os.path.join(main_dir, benchmark_folder))
FNULL = open(os.devnull, 'w')
benchmark_popen = Popen(
'./run_tagged_wf_sla_input.spark-submit.bash -tmp-hdfs /tmp/abc -wf-name wf-sla-mr-spark -project-name test -dept-name eng',
shell=True,
stdout=FNULL,
stderr=FNULL)
try:
while benchmark_popen.poll() is None:
utilities.wait_animation('Oozie benchmark running PID %s ' %
benchmark_popen.pid)
print('\nOozie benchmark Done')
except KeyboardInterrupt:
benchmark_popen.terminate()
exit()
os.chdir(main_dir)
def run_hive_benchmark():
FNULL = open(os.devnull, 'w')
os.chdir('sample-queries-tpcds')
hive_popen = Popen('hive -i init76Pre.txt -f query76.sql', stderr=FNULL, stdout=FNULL, shell=True)
try:
while hive_popen.poll() is None:
utilities.wait_animation('Hive benchmark before is running PID %s ' % hive_popen.pid)
print('\nHive benchmark before Done')
except KeyboardInterrupt:
hive_popen.terminate()
exit()
hive_popen = Popen('hive -i init76Post.txt -f query76.sql', stderr=FNULL, stdout=FNULL, shell=True)
try:
while hive_popen.poll() is None:
utilities.wait_animation('Hive benchmark after is running PID %s ' % hive_popen.pid)
print('\nHive benchmark after Done')
except KeyboardInterrupt:
hive_popen.terminate()
exit()
os.chdir(main_dir)