def run_benchmark(): os.chdir('%s/benchmarks/scripts' % benchmark_folder) FNULL = open(os.devnull, 'w') before_popen = Popen('./example%s.sh' % str(utilities.argv.spark_example), shell=True, stdout=FNULL, stderr=FNULL) try: while before_popen.poll() is None: utilities.wait_animation( 'Running example {0} before. PID {1}'.format( utilities.argv.spark_example, before_popen.pid)) print('\nexample {0} before complete'.format( utilities.argv.spark_example)) except KeyboardInterrupt: before_popen.terminate() exit() after_popen = Popen('./example%s-after.sh' % str(utilities.argv.spark_example), shell=True, stdout=FNULL, stderr=FNULL) try: while after_popen.poll() is None: utilities.wait_animation( 'Running example {0} after. PID {1}'.format( utilities.argv.spark_example, after_popen.pid)) print('\nexample {0} after complete'.format( utilities.argv.spark_example)) except KeyboardInterrupt: after_popen.terminate() exit() os.chdir(main_dir)
def run_impala_example(): os.chdir('impala') FNULL = open(os.devnull, 'w') if utilities.argv.impala_query == 'all': impala_popen = Popen("./run_exp.sh tpcds_text_{0} {1}".format( utilities.argv.dataset_size, utilities.argv.impala_server), shell=True, stderr=FNULL, stdout=FNULL) else: if int(utilities.argv.impala_query) < 10: utilities.argv.impala_query = '{:02d}'.format( int(utilities.argv.impala_query)) impala_popen = Popen( "./run_one.sh tpcds_text_{0} query{1}.sql {2}".format( utilities.argv.dataset_size, utilities.argv.impala_query, utilities.argv.impala_server), shell=True, stderr=FNULL, stdout=FNULL) try: while impala_popen.poll() is None: utilities.wait_animation('Running Impala query{0}'.format( utilities.argv.impala_query)) except KeyboardInterrupt: impala_popen.terminate() exit() os.chdir(main_dir) print('\nImpala query complete')
def run_autoaction(): os.chdir(benchmark_folder + '/auto-actions/demos') FNULL = open(os.devnull, 'w') for file in os.listdir(os.getcwd()): if not os.path.isdir(file) and file != 'run-all-demos': benchmark_popen = Popen('bash %s' % file, shell=True, stderr=FNULL, stdout=FNULL) try: while benchmark_popen.poll() is None: utilities.wait_animation('{0} is running PID {1}'.format(file, benchmark_popen.pid)) except KeyboardInterrupt: benchmark_popen.terminate() return os.chdir(main_dir)
def run_streaming_demo(): os.chdir(os.path.join(main_dir, benchmark_folder)) FNULL = open(os.devnull, 'w') streaming_app_popen = Popen('./streaming-app.sh', stdout=FNULL, stderr=FNULL, shell=True) producer_popen = Popen('./producer.sh', stdout=FNULL, stderr=FNULL, shell=True) try: while streaming_app_popen.poll() is None: utilities.wait_animation('Spark streaming job is running PID {0} {1}'.format(streaming_app_popen.pid, producer_popen.pid)) producer_popen.terminate() except: traceback.print_exc() streaming_app_popen.terminate() producer_popen.terminate() os.chdir(main_dir) print('\nSpark streaming demo completed')
def run_benchmark(): os.chdir(os.path.join(main_dir, benchmark_folder)) FNULL = open(os.devnull, 'w') benchmark_popen = Popen( './run_tagged_wf_sla_input.spark-submit.bash -tmp-hdfs /tmp/abc -wf-name wf-sla-mr-spark -project-name test -dept-name eng', shell=True, stdout=FNULL, stderr=FNULL) try: while benchmark_popen.poll() is None: utilities.wait_animation('Oozie benchmark running PID %s ' % benchmark_popen.pid) print('\nOozie benchmark Done') except KeyboardInterrupt: benchmark_popen.terminate() exit() os.chdir(main_dir)
def run_hive_benchmark(): FNULL = open(os.devnull, 'w') os.chdir('sample-queries-tpcds') hive_popen = Popen('hive -i init76Pre.txt -f query76.sql', stderr=FNULL, stdout=FNULL, shell=True) try: while hive_popen.poll() is None: utilities.wait_animation('Hive benchmark before is running PID %s ' % hive_popen.pid) print('\nHive benchmark before Done') except KeyboardInterrupt: hive_popen.terminate() exit() hive_popen = Popen('hive -i init76Post.txt -f query76.sql', stderr=FNULL, stdout=FNULL, shell=True) try: while hive_popen.poll() is None: utilities.wait_animation('Hive benchmark after is running PID %s ' % hive_popen.pid) print('\nHive benchmark after Done') except KeyboardInterrupt: hive_popen.terminate() exit() os.chdir(main_dir)