def launch_experiment(): """Launches the experiment. Specifically: - Load the gin configs and bindings. - Initialize the Logger object. - Initialize the environment. - Initialize the observation stacker. - Initialize the agent. - Reload from the latest checkpoint, if available, and initialize the Checkpointer object. - Run the experiment. """ if FLAGS.base_dir == None: raise ValueError('--base_dir is None: please provide a path for ' 'logs and checkpoints.') run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings) experiment_logger = logger.Logger('{}/logs'.format(FLAGS.base_dir)) environment = run_experiment.create_environment() obs_stacker = run_experiment.create_obs_stacker(environment) agent = run_experiment.create_agent(environment, obs_stacker) checkpoint_dir = '{}/checkpoints'.format(FLAGS.base_dir) start_iteration, experiment_checkpointer = ( run_experiment.initialize_checkpointing(agent, experiment_logger, checkpoint_dir, FLAGS.checkpoint_file_prefix)) run_experiment.run_experiment(agent, environment, start_iteration, obs_stacker, experiment_logger, experiment_checkpointer, checkpoint_dir, logging_file_prefix=FLAGS.logging_file_prefix)
def launch_experiment(): """Launches the experiment. Specifically: - Load the gin configs and bindings. - Initialize the Logger object. - Initialize the environment. - Initialize the observation stacker. - Initialize the agent. - Reload from the latest checkpoint, if available, and initialize the Checkpointer object. - Run the experiment. """ if FLAGS.base_dir is None: raise ValueError('--base_dir is None: please provide a path for ' 'logs and checkpoints.') run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings) experiment_logger = logger.Logger('{}/{}'.format(FLAGS.base_dir, FLAGS.logging_dir)) game_type = 'Hanabi-Full-Minimal' if FLAGS.belief_mode == 'replace' else 'Hanabi-Full' environment = run_experiment.create_environment( game_type=game_type, num_players=FLAGS.num_players) belief_model = run_experiment.create_belief_model( FLAGS.belief_level, FLAGS.belief_mode, FLAGS.num_players, num_b0_samples=FLAGS.n_b0_samples, comms_reward=FLAGS.comms_reward, beta=FLAGS.beta) obs_stacker = run_experiment.create_obs_stacker( environment, belief_model, history_size=FLAGS.history_size) agent = run_experiment.create_agent(environment, obs_stacker, agent_type=FLAGS.agent_type, tf_device=FLAGS.tf_device) checkpoint_dir = '{}/{}'.format(FLAGS.base_dir, FLAGS.checkpoint_dir) start_iteration, experiment_checkpointer = ( run_experiment.initialize_checkpointing(agent, experiment_logger, checkpoint_dir, FLAGS.checkpoint_file_prefix)) run_experiment.run_experiment( agent, environment, start_iteration, obs_stacker, belief_model, experiment_logger, experiment_checkpointer, checkpoint_dir, logging_file_prefix=FLAGS.logging_file_prefix, checkpoint_every_n=FLAGS.checkpoint_every_n, log_every_n=FLAGS.log_every_n, num_iterations=FLAGS.num_iterations, mode=FLAGS.mode)
def run_grasp(self, filename): alpha = input(f"What alpha to use? Enter number (default 0.8):{linesep}>>> ") try: alpha = float(alpha) except ValueError: alpha = 0.8 max_time_in_seconds = 0 minutes = input(f"Time to compute? (in minutes):{linesep}>>> ") if len(minutes) > 0: minutes = float(minutes) max_time_in_seconds = minutes * 60 run_experiment(input_filename = filename, alpha = alpha, max_time_in_seconds = max_time_in_seconds)
def run_extender_frn_model(drug): params = get_all_params() return run_experiment( params=params, experiment_name='extender_frn_model', drug=drug, as_indexes=True, max_evals=MAX_EVALS, )
def run_queue(): queue = pathlib.Path("experiments/queue") while len(list(queue.glob("*"))) > 0: locks.lock_queue() runs = set(queue.glob("*")) - {locks.lock_queue_path()} if len(runs) == 0: locks.unlock_queue() return run = list(runs)[0] run_folder_name = run.name + "_" + str(int(time.time())) current_run = shutil.move(str(run), "experiments/in_progress/" + run_folder_name, "-r") locks.unlock_queue() run_experiment(current_run)
def run_extender_robust_model(drug): params = get_all_params() return run_experiment( params=params, experiment_name='extender_robust_model', drug=drug, as_indexes=True, max_evals=MAX_EVALS, )
def run_experiment(self, filename): alpha = input( f"What alpha to use? Enter number (default 0.8):{linesep}>>> ") try: alpha = float(alpha) except ValueError: alpha = 0.8 iterations = input(f"Time to compute? (iterations):{linesep}>>> ") if len(iterations) > 0: iterations = int(iterations) times = input(f"How many times to run experiment?:{linesep}>>> ") if len(times) > 0: times = int(times) experiments_run = 0 while experiments_run < times: experiments_run += 1 run_experiment(input_filename=filename, alpha=alpha, max_iterations=iterations)
def f_start_experiment(experiment_yaml, min_scenario_index, max_scenario_index, concurrent, log_level_print, log_level_file, shuffle_instances=True, overwrite_existing_temporary_scenarios=False, overwrite_existing_intermediate_solutions=False, remove_temporary_scenarios=False, remove_intermediate_solutions=False): """ Executes the experiment according to the execution parameters found in the experiment_yaml. This function is separated from start_experiment so that it can be reused when extending the CLI from outside the alib. :param experiment_yaml: readable file object containing the execution parameters in yml format :param scenario_output_file: path to pickle file to which the resulting scenarios will be written :param threads: number of concurrent threads used for scenario generation :return: """ click.echo('Start Experiment') util.ExperimentPathHandler.initialize() file_basename = os.path.basename( experiment_yaml.name).split(".")[0].lower() log_file = os.path.join( util.ExperimentPathHandler.LOG_DIR, "{}_experiment_execution.log".format(file_basename)) initialize_logger(log_file, log_level_print, log_level_file) run_experiment.run_experiment( experiment_yaml, min_scenario_index, max_scenario_index, concurrent, shuffle_instances, overwrite_existing_temporary_scenarios, overwrite_existing_intermediate_solutions, remove_temporary_scenarios, remove_intermediate_solutions)
args.repeat = 10 output_dir = args.output # Method name, num_repeats, output_folder. settings = [ ["sol_random", 0, "random"], ["sol_random_half", 0, "random_half"], ["sol_repeat_random_half", 10, "repeat_random_half_10"], ["sol_repeat_random_half", 100, "repeat_random_half_100"], ["sol_repeat_random_half", 1000, "repeat_random_half_1000"], ["sol_repeat_random_half", 10000, "repeat_random_half_10000"], ["sol_sort_greedy", 0, "sort_greedy"], ] # Run multiple experiments. for setting in settings: args.method = setting[0] args.num_repeats = setting[1] output_path = os.path.join(output_dir, setting[2]) if os.path.exists(output_path): print( "Don't run experiment {}, because experiment folder already exists." .format(output_path)) else: args.output = output_path run_experiment(args) # Visualize. plot_comparison(output_dir)
from sympy import solve, im from sympy.abc import x N = range(101, 131) T = [10, 100, 1000, 10000] c = 5 h = 15 b = 120 Xs = [] for t in T: print() print("t = ", t) X = [] for n in N: print("n = ", n) lm, mu, a, EQ, alpha = run_experiment(n, t) X.append([EQ, alpha]) Xs.append(X) def method1(EX, n): rhos = solve(EX - n - (1 + x**2) / (2 * (1 - x))) for rho in rhos: if im(rho) == 0 and rho >= 0: return rho * n def method2(EX, n, alpha): rhos = solve(EX - n * x - (x / (1 - x)) * alpha) for rho in rhos: if im(rho) == 0 and rho >= 0: return rho * n
input_filename = [] for instance in range(1, num_instances + 1): print ' instance {}'.format(instance) dataf = data_filename_snr(tmpdir, n, k, snr, instance) gen_input(n, k, dataf, seed=random.randint(0, 2000000000), stats_file=data_stats_filename_snr(tmpdir, n, k, snr, instance), noise_variance=snr_db_to_noise_variance(snr, n, k), randomize_phase=True) input_filename.append(dataf) print ' writing index file ...' indexf = index_filename_snr(tmpdir, n, k, snr) write_index_file(indexf, input_filename) for alg in algs: resultsf = results_filename_snr(tmpdir, alg, n, k, snr) print ' algorithm: {}'.format(alg) r = run_experiment(n, k, indexf, alg, l0_eps, num_trials, seed=random.randint(0, 2000000000), output_file=resultsf) rel_l2_l2 = compute_relative_l2_l2_errors(r) ne = 0 for err in rel_l2_l2: if err > relative_l2_l2_error_threshold: ne += 1 if ne > 0: print ' {} large l2/l2-errors occurred (threshold: {}).'.format(ne, relative_l2_l2_error_threshold) for f in input_filename: os.remove(f) time_results = {} topk_l1_results = {} relative_l2_l2_results = {}
total_epsilon_budget = tot_eps * level_sample_size else: eps_func = lambda x, y: None threshold_func = lambda i, prefix_len, eps, remaining: 5 total_epsilon_budget = None # res = run_experiment.run_experiment(split_dataset['pos_image'], split_dataset['pos_dataset'], level_sample_size=level_sample_size, secagg_round_size=10000, threshold_func=threshold_func, collapse_threshold=None, eps_func=eps_func, total_epsilon_budget=total_epsilon_budget, top_k=5, partial=100, max_levels=10, collapse_func=None, total_size=1024, min_dp_size=None, dropout_rate=None, output_flag=False, quantize=None, save_gif=False, positivity=positivity, start_with_level=start_with_level) results_eps[level_sample_size].append(res) results[tot_eps][i] = results_eps torch.save(results, f'results_pos_{positivity}_{args.name}.pt')
from ErlangC import ErlangC import matplotlib.pyplot as plt from run_experiment import run_experiment N = range(101, 126, 5) T = [10, 100, 1000, 10000] c = 5 h = 15 costs = [] for t in T: print() print("t = ", t) cost = [] for n in N: print("n = ", n) lm, mu, EQ = run_experiment(n, t) cost.append(c * n + h * EQ) costs.append(cost) plt.yscale('log') for cost in costs: plt.plot(N, cost) #lm = 10 #mu = 1 T = [10, 100, 1000, 10000] costs = [] for t in T: lm, mu, Q = run_experiment(101, t) c = 5 h = 15 x = range(101, 126)
def train_model(path_list,dst_folder): train_file = path_list[0] test_file = path_list[1] result_all = {} #random the file if rand_num > 1: rand_file = train_file + '_rand' else: rand_file = train_file rand_file_cache = rand_file + '_cache' for k in range(0,rand_num): if rand_num > 1: print 'shuffle datset...' sol_shuffle.sol_shuffle(train_file, rand_file) cmd_data = dataset.get_cmd_data_by_file(rand_file, test_file, is_cache) dataset.analyze(rand_file); for opt in opt_list: print '-----------------------------------' print ' Experiment on %s' %opt + ' Random %d' %k print '-----------------------------------' if opt == 'vw': result_file = 'vw_result_%d' %k + '.txt' result_once = run_vw.run_vw(rand_file, test_file, ds, result_file, is_cache) elif opt == 'liblinear': result_file = 'liblinear_result_%d' %k + '.txt' result_once = run_liblinear.run_liblinear(rand_file, test_file, ds, result_file) elif opt == 'fgm': result_file = 'fgm_result_%d' %k + '.txt' result_once = run_fgm.run_fgm(rand_file, test_file, ds, result_file) elif opt == 'mRMR': result_file = dst_folder + '/%s' %opt + '_result_%d' %k + '.txt' result_once = run_mRMR.run_mRMR(rand_file, test_file, ds, result_file) print '\nparsing result...' #write the result to file parse_file = dst_folder +'/%s' %opt + '_%d' %k + '.txt' result_once2 = run_util.parse_result(result_file, parse_file); bs_num = len(result_once) if bs_num != len(result_once2): print 'inconsistent parsing result' for m in range(0,bs_num): result_once[m][0] = result_once2[m][0] result_once[m][1] = result_once2[m][1] if result_once[m][2] == 0: result_once[m][2] = result_once2[m][2] if result_once[m][3] == 0: result_once[m][3] = result_once2[m][3] else: result_file = dst_folder + '/%s' %opt + '_result_%d' %k + '.txt' cmd = cmd_data cmd += extra_cmd if is_default_param == False: cmd += dataset.get_model_param(ds, opt) run_experiment.run_experiment(opt,result_file,ds, cmd) print '\nparsing result...' #write the result to file parse_file = dst_folder +'/%s' %opt + '_%d' %k + '.txt' result_once = run_util.parse_result(result_file, parse_file); result_all = add_to_dict(opt,result_all, result_once) #remove previous file if rand_num > 1: os.system('rm -f %s' %rand_file_cache) os.system('rm -f %s' %rand_file) #average the result for opt in opt_list: rows = len(result_all[opt]) cols = len(result_all[opt][0]) for k in range(0,rows): for m in range(0,cols): result_all[opt][k][m] /= rand_num return result_all
for k in kvals: print "k = {}".format(k) print " generating input data ..." input_filename = [] for instance in range(1, num_instances + 1): print " instance {}".format(instance) dataf = data_filename(tmpdir, n, k, instance) gen_input(n, k, dataf, random.randint(0, 2000000000), stats_file=data_stats_filename(tmpdir, n, k, instance)) input_filename.append(dataf) print " writing index file ..." indexf = index_filename(tmpdir, n, k) write_index_file(indexf, input_filename) for alg in algs: resultsf = results_filename(tmpdir, alg, n, k) print " algorithm: {}".format(alg) r = run_experiment(n, k, indexf, alg, l0_eps, num_trials, resultsf) ne = num_l0_errors(r) if ne > 0: print " {} L0-errors occurred.".format(ne) for f in input_filename: os.remove(f) time_results = {} l0_results = {} success_results = {} for alg in algs: time_results[alg] = {} l0_results[alg] = {} success_results[alg] = {}
print(line.replace('filePrefix = DO NOT CHANGE', 'filePrefix = ' + fileprefix_exp2), end='') for line in fileinput.input(name_folder + name_experiment + name_exp2, inplace=True): print(line.replace('recordIOMap_fileName = DO NOT CHANGE', 'recordIOMap_fileName = ' + filename_activityfile), end='') # then we run the experiments runs = 5 for r in list(range(0, runs)): print(['run number ' + str(r)]) # Running experiment run_experiment(cmd[0]) # Getting data from file data.append(pd.read_csv(datafile)) genome.append(pd.read_csv(genomefile)) # Running recorders run_experiment(cmd[1]) # Getting recordings from file activity.append(pd.read_csv(activityfile)) with open(name_folder + name_experiment + 'LOD_data.pkl', 'wb') as f: pickle.dump(data, f) with open(name_folder + name_experiment + 'genome.pkl', 'wb') as f: pickle.dump(genome, f) with open(name_folder + name_experiment + 'activity.pkl', 'wb') as f:
import matplotlib.pyplot as plt from run_experiment import run_experiment x = range(101, 126) # This is the number of servers in the system T = 10 c = 5 h = 15 cost = [] for N in x: lm, mu, EQ = run_experiment(N, T) cost.append(c*N + h*EQ) x1 = range(101, 126) x2 = range(101, 126, 2) x3 = range(101, 126, 3) x4 = range(101, 126, 4) x5 = range(101, 126, 5) xs = [x1, x2, x3, x4, x5] plt.scatter(x1, cost) for x in xs: y = [cost[101-i] for i in x] mymodel = np.poly1d(np.polyfit(x, y, 3)) myline = np.linspace(101, 125, 100) plt.plot(myline, mymodel(myline)) plt.axvline(112, color='k', linestyle='dotted')
from identity import experiment_identities sys.path.append("../../appium_exec") from run_experiment import run_experiment from abstract_experiment import abstract_experiment # weather channel APP_DIR = "/var/www/html/mitm/com.weather.Weather-5.5.6-505060343-minAPI14.apk" class run_weather_channel(abstract_experiment): def __init__(self, driver): self.driver = driver self.identities = experiment_identities() # set start index here def experiment(self): time.sleep(11) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-l", "--log-dir", required=True, help="Where experiment log is be stored") args = parser.parse_args() result = run_experiment(run_weather_channel, app_dir=APP_DIR, num=7500) with open(args.log_dir, 'a') as f: json.dump(result, f)
sec_agg / level_sample_size) * mechanisms.get_std_from_eps( max(eps, remaining)) total_epsilon_budget = tot_eps * level_sample_size else: eps_func = lambda x, y: None threshold_func = lambda i, prefix_len, eps, remaining: 5 total_epsilon_budget = None res = run_experiment.run_experiment( true_image, dataset, level_sample_size=level_sample_size, secagg_round_size=sec_agg, threshold_func=threshold_func, collapse_func=lambda threshold: 1 / 4 * threshold, eps_func=eps_func, total_epsilon_budget=total_epsilon_budget, top_k=5, partial=100, max_levels=10, total_size=1024, min_dp_size=None, dropout_rate=None, output_flag=True, quantize=None, save_gif=False, positivity=False) results[c].append(res) torch.save(results, f'c_results_{args.name}.pt')
import sys import time sys.path.append("../appium_exec") from run_experiment import run_experiment from abstract_experiment import abstract_experiment APP_DIR = "/var/www/html/mitm/Complete Astrology Zodiac_v2.1_apkpure.com.apk" class run_weather_channel(abstract_experiment): def __init__(self, driver): self.driver = driver def experiment(self): time.sleep(15) if __name__ == "__main__": print(run_experiment(run_weather_channel, app_dir=APP_DIR, num=100))
h = 15. dictn = {} for n in x1: for T in [10, 100, 1000, 10000]: dictn[(n, T)] = [] dictn2 = {} for N in [len(x) for x in xs]: for T in [10, 100, 1000, 10000]: dictn2[(N, T)] = [] for i in range(100): for n in x1: #print("Simulating n = ", n) lm, mu, EQ = run_experiment(n, 10000, dictn) print("So far ", len(dictn[(101, 10000)]), " experiements have been run") f = open("/content/drive/My Drive/Summer Research 2020/dictn.txt", "w") f.write(str(dictn)) f.close() for T in [10, 100, 1000, 10000]: for i in range(100): cost = [c * n + h * dictn[(n, T)][i] for n in x1] for x in xs: y = [cost[i - 101] for i in x] mymodel = np.poly1d(np.polyfit(x, y, 3)) cost_pred = mymodel(x1) min_i = np.argmin(cost_pred) min_n = x1[min_i] dictn2[(len(x), T)].append(min_n)
import ximea_cam_aquire_save as xim import run_experiment as runexp import run_analysis as runana import imp import numpy as np import matplotlib.pyplot as plt import imageio import cv2 import schedule import time #settings subject = 'buddy_stationary' task = 'chat_1' exp = 'pre' capture_dir_list = ['./capture] analysis_dir = './analysis' collection_minutes = 1 save_batchsize = 200 imp.reload(runexp) imp.reload(xim) runexp.run_experiment(subject_name=subject, task_name=task, exp_type=exp, save_dirs=capture_dir_list, collection_minutes=collection_minutes, save_batchsize=save_batchsize)
import ximea_cam_aquire_save as xim import run_analysis as ana import numpy as np import os import run_experiment as runexp subject = 'ag' task = 'laptop_1' exp = 'pre' capture_dir_list = ['/home/vasha/Desktop/backpack_computer_aquisition/capture'] path = os.path.join(capture_dir_list[0], subject, task, exp, 'test') if not os.path.exists(path): oldmask = os.umask(000) os.makedirs(path, 777) os.umask(oldmask) collection_minutes = 3 save_batchsize = 200 ncameras = 1 pupil_port = 50020 runexp.run_experiment(subject_name=subject, task_name=task, exp_type=exp, save_dirs=capture_dir_list, collection_minutes=collection_minutes, save_batchsize=save_batchsize, pupil_port=pupil_port, n_cameras=ncameras)
for instance in range(1, num_instances + 1): print ' instance {}'.format(instance) dataf = data_filename(tmpdir, n, k, instance) gen_input(n, k, dataf, random.randint(0, 2000000000), stats_file=data_stats_filename(tmpdir, n, k, instance)) input_filename.append(dataf) print ' writing index file ...' indexf = index_filename(tmpdir, n, k) write_index_file(indexf, input_filename) for alg in algs: resultsf = results_filename(tmpdir, alg, n, k) print ' algorithm: {}'.format(alg) r = run_experiment(n, k, indexf, alg, l0_eps, num_trials, resultsf) ne = num_l0_errors(r) if ne > 0: print ' {} L0-errors occurred.'.format(ne) for f in input_filename: os.remove(f) time_results = {} l0_results = {} success_results = {} for alg in algs: time_results[alg] = {} l0_results[alg] = {} success_results[alg] = {}
def main(): data_gen() run_experiment() post_process()
print_warning( "You do not have the 'logging' module. Please consider" + " installing it, and all other required or optional " + "modules, via the command line \"> pip install -r requirements.txt\"" ) ### Initialize experiment directories and global variables ## Set the experiment name according to a boring structure (but simple is good :D) settings.EXP_NAME = "{}_model_{}".format(settings.EXP_NAME_PREFIX, settings.MODEL) ## Initialize global variables that store the various directories where ## results will be saved for this experiment. Moreover, create them. initialize_directories() ### All systems are go, let's fire up this ship to infinity and beyond! from run_experiment import run_experiment t = datetime.datetime.now() log("") log("============================================================") log("Current stardate: {0}".format(t.strftime("%Y-%m-%d %H:%M:%S"))) log("") log("All output is logged on disk to: {}".format(settings.OUTLOGFILE)) log("All errors are logged on disk to: {}".format(settings.ERRLOGFILE)) log("") run_experiment() ### Graceful exit sys.exit(0)