def evaluation(repeats=100): root = os.getcwd() XTERN_ROOT = os.environ["XTERN_ROOT"] bash_path = eval.which('bash')[0] # relink local.options local_options = '%s/apps/ldap/openldap-2.4.33/obj/tests/local.options' % XTERN_ROOT try: os.unlink(local_options) except OSError: pass os.symlink('%s/local.options' % root, local_options) # xtern part cmd = ' '.join( ['PRELOAD_LIB=%s/dync_hook/interpose.so' % XTERN_ROOT, 'make', 'test']) out_dir = '%s/xtern' % root eval.mkdir_p(out_dir) for i in range(repeats): killall('slapd') sys.stderr.write("\tPROGRESS: %5d/%d\r" % (i + 1, int(repeats))) # progress with open('%s/output.%d' % (out_dir, i), 'w', 102400) as log_file: os.chdir('%s/apps/ldap/openldap-2.4.33/obj' % XTERN_ROOT) proc = subprocess.Popen(cmd, stdout=log_file, stderr=subprocess.STDOUT, shell=True, executable=bash_path, bufsize=102400, preexec_fn=os.setsid) proc.wait() os.chdir(root) # move log files into 'xtern' directory #os.renames('%s/apps/ldap/openldap-2.4.33/obj/tests/out-bdb' % XTERN_ROOT, '%s/out.%d' % (out_dir, i)) # non-det part cmd = ' '.join([ 'PRELOAD_LIB=%s/eval/rand-intercept/rand-intercept.so' % XTERN_ROOT, 'make', 'test' ]) out_dir = '%s/non-det' % root eval.mkdir_p(out_dir) for i in range(repeats): killall('slapd') sys.stderr.write("\tPROGRESS: %5d/%d\r" % (i + 1, int(repeats))) # progress with open('%s/output.%d' % (out_dir, i), 'w', 102400) as log_file: os.chdir('%s/apps/ldap/openldap-2.4.33/obj' % XTERN_ROOT) proc = subprocess.Popen(cmd, stdout=log_file, stderr=subprocess.STDOUT, shell=True, executable=bash_path, bufsize=102400, preexec_fn=os.setsid) proc.wait() os.chdir(root) # move log files into 'xtern' directory #os.renames('%s/apps/ldap/openldap-2.4.33/obj/tests/out-bdb' % XTERN_ROOT, '%s/out.%d' % (out_dir, i)) # restore os.unlink(local_options) os.symlink('%s/apps/ldap/local.options' % XTERN_ROOT, local_options) # get stats xtern_cost = [] for i in range(int(repeats)): log_file_name = 'xtern/output.%d' % i for line in reversed(open(log_file_name, 'r').readlines()): if re.search('^real [0-9]+\.[0-9][0-9]$', line): xtern_cost += [float(line.split()[1])] break nondet_cost = [] for i in range(int(repeats)): log_file_name = 'non-det/output.%d' % i for line in reversed(open(log_file_name, 'r').readlines()): if re.search('^real [0-9]+\.[0-9][0-9]$', line): nondet_cost += [float(line.split()[1])] break eval.write_stats(xtern_cost, nondet_cost, int(repeats)) killall('slapd')
def evaluation(repeats = 100): root = os.getcwd() XTERN_ROOT = os.environ["XTERN_ROOT"] bash_path = eval.which('bash')[0] # relink local.options local_options = '%s/apps/ldap/openldap-2.4.33/obj/tests/local.options' % XTERN_ROOT try: os.unlink(local_options) except OSError: pass os.symlink('%s/local.options' % root, local_options) # xtern part cmd = ' '.join(['PRELOAD_LIB=%s/dync_hook/interpose.so' % XTERN_ROOT, 'make', 'test']) out_dir = '%s/xtern' % root eval.mkdir_p(out_dir) for i in range(repeats): killall('slapd') sys.stderr.write("\tPROGRESS: %5d/%d\r" % (i+1, int(repeats))) # progress with open('%s/output.%d' % (out_dir, i), 'w', 102400) as log_file: os.chdir('%s/apps/ldap/openldap-2.4.33/obj' % XTERN_ROOT) proc = subprocess.Popen(cmd, stdout=log_file, stderr=subprocess.STDOUT, shell=True, executable=bash_path, bufsize = 102400, preexec_fn=os.setsid) proc.wait() os.chdir(root) # move log files into 'xtern' directory #os.renames('%s/apps/ldap/openldap-2.4.33/obj/tests/out-bdb' % XTERN_ROOT, '%s/out.%d' % (out_dir, i)) # non-det part cmd = ' '.join(['PRELOAD_LIB=%s/eval/rand-intercept/rand-intercept.so' % XTERN_ROOT, 'make', 'test']) out_dir = '%s/non-det' % root eval.mkdir_p(out_dir) for i in range(repeats): killall('slapd') sys.stderr.write("\tPROGRESS: %5d/%d\r" % (i+1, int(repeats))) # progress with open('%s/output.%d' % (out_dir, i), 'w', 102400) as log_file: os.chdir('%s/apps/ldap/openldap-2.4.33/obj' % XTERN_ROOT) proc = subprocess.Popen(cmd, stdout=log_file, stderr=subprocess.STDOUT, shell=True, executable=bash_path, bufsize = 102400, preexec_fn=os.setsid) proc.wait() os.chdir(root) # move log files into 'xtern' directory #os.renames('%s/apps/ldap/openldap-2.4.33/obj/tests/out-bdb' % XTERN_ROOT, '%s/out.%d' % (out_dir, i)) # restore os.unlink(local_options) os.symlink('%s/apps/ldap/local.options' % XTERN_ROOT, local_options) # get stats xtern_cost = [] for i in range(int(repeats)): log_file_name = 'xtern/output.%d' % i for line in reversed(open(log_file_name, 'r').readlines()): if re.search('^real [0-9]+\.[0-9][0-9]$', line): xtern_cost += [float(line.split()[1])] break nondet_cost = [] for i in range(int(repeats)): log_file_name = 'non-det/output.%d' % i for line in reversed(open(log_file_name, 'r').readlines()): if re.search('^real [0-9]+\.[0-9][0-9]$', line): nondet_cost += [float(line.split()[1])] break eval.write_stats(xtern_cost, nondet_cost, int(repeats)) killall('slapd')
def model_checking(configs, benchmark, args): try: from lxml import etree except ImportError: logging.error("Please install 'lxml' e.g. sudo easy_install-2.7 lxml") sys.exit(1) # get environment variables try: SMT_MC_ROOT = os.environ["SMT_MC_ROOT"] except KeyError as e: logging.error("Please set the environment variable " + str(e)) sys.exit(1) XTERN_ROOT = os.environ["XTERN_ROOT"] APPS = os.path.abspath("%s/apps" % XTERN_ROOT) apps_name, exec_file = eval.extract_apps_exec(benchmark, APPS) # check dbug ARBITER = "%s/mc-tools/dbug/install/bin/dbug-arbiter" % SMT_MC_ROOT if not eval.checkExist(ARBITER): logging.error("%s does not exist" % ARBITER) EXPLORER = "%s/mc-tools/dbug/install/bin/dbug-explorer" % SMT_MC_ROOT if not eval.checkExist(EXPLORER): logging.error("%s does not exist" % EXPLORER) # check wrapper WRAPPER = "%s/xtern/eval/model-chk-wrapper/wrapper" % SMT_MC_ROOT if not eval.checkExist(WRAPPER): logging.error("%s does not exist, please go there and make" % WRAPPER) # generate run.xml client = configs.get(benchmark, 'DBUG_CLIENT') client_inputs = configs.get(benchmark, 'DBUG_CLIENT_INPUTS') arbiter_port = configs.get(benchmark, 'DBUG_ARBITER_PORT') explorer_port = configs.get(benchmark, 'DBUG_EXPLORER_PORT') dbug_timeout = configs.get(benchmark, 'DBUG_TIMEOUT') program_input = configs.get(benchmark, 'DBUG_INPUT') program_output = configs.get(benchmark, 'DBUG_OUTPUT') prefix = configs.get(benchmark, "DBUG_PREFIX") dpor = configs.get(benchmark, "DBUG_DPOR") inputs = configs.get(benchmark, "INPUTS") export = configs.get(benchmark, "EXPORT") if prefix: prefix_filename = "%s.prefix" % benchmark.split()[0] with open(prefix_filename, "w") as prefix_file: prefix_file.write(prefix.replace("\\n", "\n")) dbug_config = etree.Element("config") if prefix: dbug_prefix = etree.SubElement(dbug_config, "prefix") dbug_prefix.set("path", prefix_filename) arbiter = etree.SubElement(dbug_config, "arbiter") explorer = etree.SubElement(dbug_config, "explorer") interposition = etree.SubElement(dbug_config, "interposition") program = etree.SubElement(dbug_config, "program") if client: program2 = etree.SubElement(dbug_config, "program") arbiter.set("port", arbiter_port) #if client: # arbiter.set("command", "%s -l -b 2 -e 3" % ARBITER) #else: # arbiter.set("command", "%s -l" % ARBITER) explorer.set("strategy", "random") explorer.set("dpor", dpor) explorer.set("port", explorer_port) explorer.set("log_dir", ".") explorer.set("timeout", dbug_timeout) interposition.set("command", WRAPPER) command = ' '.join([exec_file] + inputs.split()) program.set("command", command) if program_input: program.set("input", program_input) if program_output: program.set("output", program_output) if client: exec_file2 = os.path.abspath(APPS + '/' + benchmark.split()[0] + '/' + client) command2 = ' '.join([exec_file2] + client_inputs.split()) program2.set("command", command2) with open("run_xtern.xml", "w") as run_xtern_xml: run_xtern_xml.write(etree.tostring(dbug_config, pretty_print=True)) # strip the interposition element since the dbug-only mode does not need it. etree.strip_elements(dbug_config, "interposition") with open("run.xml", "w") as run_xml: run_xml.write(etree.tostring(dbug_config, pretty_print=True)) init_env_cmd = configs.get(benchmark, "INIT_ENV_CMD") if init_env_cmd: os.system(init_env_cmd) # copy files eval.copy_file(os.path.realpath(exec_file), os.path.basename(exec_file)) bash_path = eval.which('bash')[0] dbug_cmd = '%s %s run.xml' % (export, EXPLORER) dbug_xtern_cmd = '%s %s run_xtern.xml' % (export, EXPLORER) if args.generate_xml_only: logging.info(dbug_cmd) logging.info(dbug_xtern_cmd) return local_timeout = int(float(dbug_timeout) * 1.1) if not args.smtmc_only: with open('dbug.log', 'w', 102400) as log_file: signal.signal(signal.SIGALRM, DbugTimeoutHandler) signal.alarm(local_timeout) try: logging.info(dbug_cmd) proc = subprocess.Popen(dbug_cmd, stdout=log_file, stderr=subprocess.STDOUT, shell=True, executable=bash_path, bufsize=102400, preexec_fn=os.setsid) proc.wait() signal.alarm(0) except DbugTimeout: signal.alarm(0) logging.warning( "'%s' with dbug does not stop after %d seconds, kill it..." % (benchmark, local_timeout)) except KeyboardInterrupt: pass try: os.killpg(proc.pid, signal.SIGKILL) proc.kill() except OSError: pass time.sleep(1) if not args.dbug_only: with open('dbug_xtern.log', 'w', 102400) as log_file: signal.signal(signal.SIGALRM, DbugTimeoutHandler) signal.alarm(local_timeout) try: logging.info(dbug_xtern_cmd) proc = subprocess.Popen(dbug_xtern_cmd, stdout=log_file, stderr=subprocess.STDOUT, shell=True, executable=bash_path, bufsize=102400, preexec_fn=os.setsid) proc.wait() signal.alarm(0) except DbugTimeout: signal.alarm(0) logging.warning( "'%s' with dbug_xtern does not stop after %d seconds, kill it..." % (benchmark, local_timeout)) except KeyboardInterrupt: pass try: os.killpg(proc.pid, signal.SIGKILL) proc.kill() except OSError: pass
def model_checking(configs, benchmark, args): try: from lxml import etree except ImportError: logging.error("Please install 'lxml' e.g. sudo easy_install-2.7 lxml") sys.exit(1) # get environment variables try: SMT_MC_ROOT = os.environ["SMT_MC_ROOT"] except KeyError as e: logging.error("Please set the environment variable " + str(e)) sys.exit(1) XTERN_ROOT = os.environ["XTERN_ROOT"] APPS = os.path.abspath("%s/apps" % XTERN_ROOT) apps_name, exec_file = eval.extract_apps_exec(benchmark, APPS) # check dbug ARBITER = "%s/mc-tools/dbug/install/bin/dbug-arbiter" % SMT_MC_ROOT if not eval.checkExist(ARBITER): logging.error("%s does not exist" % ARBITER) EXPLORER = "%s/mc-tools/dbug/install/bin/dbug-explorer" % SMT_MC_ROOT if not eval.checkExist(EXPLORER): logging.error("%s does not exist" % EXPLORER) # check wrapper WRAPPER = "%s/xtern/eval/model-chk-wrapper/wrapper" % SMT_MC_ROOT if not eval.checkExist(WRAPPER): logging.error("%s does not exist, please go there and make" % WRAPPER) # generate run.xml client = configs.get(benchmark, 'DBUG_CLIENT') client_inputs = configs.get(benchmark, 'DBUG_CLIENT_INPUTS') arbiter_port = configs.get(benchmark, 'DBUG_ARBITER_PORT') explorer_port = configs.get(benchmark, 'DBUG_EXPLORER_PORT') dbug_timeout = configs.get(benchmark, 'DBUG_TIMEOUT') program_input = configs.get(benchmark, 'DBUG_INPUT') program_output = configs.get(benchmark, 'DBUG_OUTPUT') prefix = configs.get(benchmark, "DBUG_PREFIX") dpor = configs.get(benchmark, "DBUG_DPOR") inputs = configs.get(benchmark, "INPUTS") export = configs.get(benchmark, "EXPORT") if prefix: prefix_filename = "%s.prefix" % benchmark.split()[0] with open(prefix_filename, "w") as prefix_file: prefix_file.write(prefix.replace("\\n", "\n")) dbug_config = etree.Element("config") if prefix: dbug_prefix = etree.SubElement(dbug_config, "prefix") dbug_prefix.set("path", prefix_filename) arbiter = etree.SubElement(dbug_config, "arbiter") explorer = etree.SubElement(dbug_config, "explorer") interposition = etree.SubElement(dbug_config, "interposition") program = etree.SubElement(dbug_config, "program") if client: program2 = etree.SubElement(dbug_config, "program") arbiter.set("port", arbiter_port) #if client: # arbiter.set("command", "%s -l -b 2 -e 3" % ARBITER) #else: # arbiter.set("command", "%s -l" % ARBITER) explorer.set("strategy", "random") explorer.set("dpor", dpor) explorer.set("port", explorer_port) explorer.set("log_dir", ".") explorer.set("timeout", dbug_timeout) interposition.set("command", WRAPPER) command = ' '.join([exec_file] + inputs.split()) program.set("command", command) if program_input: program.set("input", program_input) if program_output: program.set("output", program_output) if client: exec_file2 = os.path.abspath(APPS + '/' + benchmark.split()[0] + '/' + client) command2= ' '.join([exec_file2] + client_inputs.split()) program2.set("command", command2) with open("run_xtern.xml", "w") as run_xtern_xml: run_xtern_xml.write(etree.tostring(dbug_config, pretty_print=True)) # strip the interposition element since the dbug-only mode does not need it. etree.strip_elements(dbug_config, "interposition") with open("run.xml", "w") as run_xml: run_xml.write(etree.tostring(dbug_config, pretty_print=True)) init_env_cmd = configs.get(benchmark, "INIT_ENV_CMD") if init_env_cmd: os.system(init_env_cmd) # copy files eval.copy_file(os.path.realpath(exec_file), os.path.basename(exec_file)) bash_path = eval.which('bash')[0] dbug_cmd = '%s %s run.xml' % (export, EXPLORER) dbug_xtern_cmd = '%s %s run_xtern.xml' % (export, EXPLORER) if args.generate_xml_only: logging.info(dbug_cmd); logging.info(dbug_xtern_cmd); return local_timeout = int(float(dbug_timeout)*1.1) if not args.smtmc_only: with open('dbug.log', 'w', 102400) as log_file: signal.signal(signal.SIGALRM, DbugTimeoutHandler) signal.alarm(local_timeout) try: logging.info(dbug_cmd) proc = subprocess.Popen(dbug_cmd, stdout=log_file, stderr=subprocess.STDOUT, shell=True, executable=bash_path, bufsize = 102400, preexec_fn=os.setsid) proc.wait() signal.alarm(0) except DbugTimeout: signal.alarm(0) logging.warning("'%s' with dbug does not stop after %d seconds, kill it..." % (benchmark, local_timeout)) except KeyboardInterrupt: pass try: os.killpg(proc.pid, signal.SIGKILL) proc.kill() except OSError: pass time.sleep(1) if not args.dbug_only: with open('dbug_xtern.log', 'w', 102400) as log_file: signal.signal(signal.SIGALRM, DbugTimeoutHandler) signal.alarm(local_timeout) try: logging.info(dbug_xtern_cmd) proc = subprocess.Popen(dbug_xtern_cmd, stdout=log_file, stderr=subprocess.STDOUT, shell=True, executable=bash_path, bufsize = 102400, preexec_fn=os.setsid) proc.wait() signal.alarm(0) except DbugTimeout: signal.alarm(0) logging.warning("'%s' with dbug_xtern does not stop after %d seconds, kill it..." % (benchmark, local_timeout)) except KeyboardInterrupt: pass try: os.killpg(proc.pid, signal.SIGKILL) proc.kill() except OSError: pass