def launch_experiment():
"""Launches the experiment.
Specifically:
- Load the gin configs and bindings.
- Initialize the Logger object.
- Initialize the environment.
- Initialize the observation stacker.
- Initialize the agent.
- Reload from the latest checkpoint, if available, and initialize the
Checkpointer object.
- Run the experiment.
"""
if FLAGS.base_dir == None:
raise ValueError('--base_dir is None: please provide a path for '
'logs and checkpoints.')
run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
experiment_logger = logger.Logger('{}/logs'.format(FLAGS.base_dir))
environment = run_experiment.create_environment()
obs_stacker = run_experiment.create_obs_stacker(environment)
agent = run_experiment.create_agent(environment, obs_stacker)
checkpoint_dir = '{}/checkpoints'.format(FLAGS.base_dir)
start_iteration, experiment_checkpointer = (
run_experiment.initialize_checkpointing(agent,
experiment_logger,
checkpoint_dir,
FLAGS.checkpoint_file_prefix))
run_experiment.run_experiment(agent, environment, start_iteration,
obs_stacker,
experiment_logger, experiment_checkpointer,
checkpoint_dir,
logging_file_prefix=FLAGS.logging_file_prefix)
def launch_experiment():
"""Launches the experiment.
Specifically:
- Load the gin configs and bindings.
- Initialize the Logger object.
- Initialize the environment.
- Initialize the observation stacker.
- Initialize the agent.
- Reload from the latest checkpoint, if available, and initialize the
Checkpointer object.
- Run the experiment.
"""
if FLAGS.base_dir is None:
raise ValueError('--base_dir is None: please provide a path for '
'logs and checkpoints.')
run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
experiment_logger = logger.Logger('{}/{}'.format(FLAGS.base_dir,
FLAGS.logging_dir))
game_type = 'Hanabi-Full-Minimal' if FLAGS.belief_mode == 'replace' else 'Hanabi-Full'
environment = run_experiment.create_environment(
game_type=game_type, num_players=FLAGS.num_players)
belief_model = run_experiment.create_belief_model(
FLAGS.belief_level,
FLAGS.belief_mode,
FLAGS.num_players,
num_b0_samples=FLAGS.n_b0_samples,
comms_reward=FLAGS.comms_reward,
beta=FLAGS.beta)
obs_stacker = run_experiment.create_obs_stacker(
environment, belief_model, history_size=FLAGS.history_size)
agent = run_experiment.create_agent(environment,
obs_stacker,
agent_type=FLAGS.agent_type,
tf_device=FLAGS.tf_device)
checkpoint_dir = '{}/{}'.format(FLAGS.base_dir, FLAGS.checkpoint_dir)
start_iteration, experiment_checkpointer = (
run_experiment.initialize_checkpointing(agent, experiment_logger,
checkpoint_dir,
FLAGS.checkpoint_file_prefix))
run_experiment.run_experiment(
agent,
environment,
start_iteration,
obs_stacker,
belief_model,
experiment_logger,
experiment_checkpointer,
checkpoint_dir,
logging_file_prefix=FLAGS.logging_file_prefix,
checkpoint_every_n=FLAGS.checkpoint_every_n,
log_every_n=FLAGS.log_every_n,
num_iterations=FLAGS.num_iterations,
mode=FLAGS.mode)
def run_grasp(self, filename):
alpha = input(f"What alpha to use? Enter number (default 0.8):{linesep}>>> ")
try:
alpha = float(alpha)
except ValueError:
alpha = 0.8
max_time_in_seconds = 0
minutes = input(f"Time to compute? (in minutes):{linesep}>>> ")
if len(minutes) > 0:
minutes = float(minutes)
max_time_in_seconds = minutes * 60
run_experiment(input_filename = filename, alpha = alpha, max_time_in_seconds = max_time_in_seconds)
def run_extender_frn_model(drug):
params = get_all_params()
return run_experiment(
params=params,
experiment_name='extender_frn_model',
drug=drug,
as_indexes=True,
max_evals=MAX_EVALS,
)
def run_queue():
queue = pathlib.Path("experiments/queue")
while len(list(queue.glob("*"))) > 0:
locks.lock_queue()
runs = set(queue.glob("*")) - {locks.lock_queue_path()}
if len(runs) == 0:
locks.unlock_queue()
return
run = list(runs)[0]
run_folder_name = run.name + "_" + str(int(time.time()))
current_run = shutil.move(str(run),
"experiments/in_progress/" + run_folder_name,
"-r")
locks.unlock_queue()
run_experiment(current_run)
def run_extender_robust_model(drug):
params = get_all_params()
return run_experiment(
params=params,
experiment_name='extender_robust_model',
drug=drug,
as_indexes=True,
max_evals=MAX_EVALS,
)
def run_experiment(self, filename):
alpha = input(
f"What alpha to use? Enter number (default 0.8):{linesep}>>> ")
try:
alpha = float(alpha)
except ValueError:
alpha = 0.8
iterations = input(f"Time to compute? (iterations):{linesep}>>> ")
if len(iterations) > 0:
iterations = int(iterations)
times = input(f"How many times to run experiment?:{linesep}>>> ")
if len(times) > 0:
times = int(times)
experiments_run = 0
while experiments_run < times:
experiments_run += 1
run_experiment(input_filename=filename,
alpha=alpha,
max_iterations=iterations)
def f_start_experiment(experiment_yaml,
min_scenario_index,
max_scenario_index,
concurrent,
log_level_print,
log_level_file,
shuffle_instances=True,
overwrite_existing_temporary_scenarios=False,
overwrite_existing_intermediate_solutions=False,
remove_temporary_scenarios=False,
remove_intermediate_solutions=False):
"""
Executes the experiment according to the execution parameters found in the experiment_yaml.
This function is separated from start_experiment so that it can be reused when extending the CLI from outside
the alib.
:param experiment_yaml: readable file object containing the execution parameters in yml format
:param scenario_output_file: path to pickle file to which the resulting scenarios will be written
:param threads: number of concurrent threads used for scenario generation
:return:
"""
click.echo('Start Experiment')
util.ExperimentPathHandler.initialize()
file_basename = os.path.basename(
experiment_yaml.name).split(".")[0].lower()
log_file = os.path.join(
util.ExperimentPathHandler.LOG_DIR,
"{}_experiment_execution.log".format(file_basename))
initialize_logger(log_file, log_level_print, log_level_file)
run_experiment.run_experiment(
experiment_yaml, min_scenario_index, max_scenario_index, concurrent,
shuffle_instances, overwrite_existing_temporary_scenarios,
overwrite_existing_intermediate_solutions, remove_temporary_scenarios,
remove_intermediate_solutions)
args.repeat = 10
output_dir = args.output
# Method name, num_repeats, output_folder.
settings = [
["sol_random", 0, "random"],
["sol_random_half", 0, "random_half"],
["sol_repeat_random_half", 10, "repeat_random_half_10"],
["sol_repeat_random_half", 100, "repeat_random_half_100"],
["sol_repeat_random_half", 1000, "repeat_random_half_1000"],
["sol_repeat_random_half", 10000, "repeat_random_half_10000"],
["sol_sort_greedy", 0, "sort_greedy"],
]
# Run multiple experiments.
for setting in settings:
args.method = setting[0]
args.num_repeats = setting[1]
output_path = os.path.join(output_dir, setting[2])
if os.path.exists(output_path):
print(
"Don't run experiment {}, because experiment folder already exists."
.format(output_path))
else:
args.output = output_path
run_experiment(args)
# Visualize.
plot_comparison(output_dir)
from sympy import solve, im
from sympy.abc import x
N = range(101, 131)
T = [10, 100, 1000, 10000]
c = 5
h = 15
b = 120
Xs = []
for t in T:
print()
print("t = ", t)
X = []
for n in N:
print("n = ", n)
lm, mu, a, EQ, alpha = run_experiment(n, t)
X.append([EQ, alpha])
Xs.append(X)
def method1(EX, n):
rhos = solve(EX - n - (1 + x**2) / (2 * (1 - x)))
for rho in rhos:
if im(rho) == 0 and rho >= 0: return rho * n
def method2(EX, n, alpha):
rhos = solve(EX - n * x - (x / (1 - x)) * alpha)
for rho in rhos:
if im(rho) == 0 and rho >= 0: return rho * n
input_filename = []
for instance in range(1, num_instances + 1):
print ' instance {}'.format(instance)
dataf = data_filename_snr(tmpdir, n, k, snr, instance)
gen_input(n, k, dataf, seed=random.randint(0, 2000000000),
stats_file=data_stats_filename_snr(tmpdir, n, k, snr, instance),
noise_variance=snr_db_to_noise_variance(snr, n, k),
randomize_phase=True)
input_filename.append(dataf)
print ' writing index file ...'
indexf = index_filename_snr(tmpdir, n, k, snr)
write_index_file(indexf, input_filename)
for alg in algs:
resultsf = results_filename_snr(tmpdir, alg, n, k, snr)
print ' algorithm: {}'.format(alg)
r = run_experiment(n, k, indexf, alg, l0_eps, num_trials,
seed=random.randint(0, 2000000000), output_file=resultsf)
rel_l2_l2 = compute_relative_l2_l2_errors(r)
ne = 0
for err in rel_l2_l2:
if err > relative_l2_l2_error_threshold:
ne += 1
if ne > 0:
print ' {} large l2/l2-errors occurred (threshold: {}).'.format(ne,
relative_l2_l2_error_threshold)
for f in input_filename:
os.remove(f)
time_results = {}
topk_l1_results = {}
relative_l2_l2_results = {}
total_epsilon_budget = tot_eps * level_sample_size
else:
eps_func = lambda x, y: None
threshold_func = lambda i, prefix_len, eps, remaining: 5
total_epsilon_budget = None
#
res = run_experiment.run_experiment(split_dataset['pos_image'],
split_dataset['pos_dataset'],
level_sample_size=level_sample_size,
secagg_round_size=10000,
threshold_func=threshold_func,
collapse_threshold=None,
eps_func=eps_func,
total_epsilon_budget=total_epsilon_budget,
top_k=5,
partial=100,
max_levels=10,
collapse_func=None,
total_size=1024,
min_dp_size=None,
dropout_rate=None,
output_flag=False,
quantize=None,
save_gif=False,
positivity=positivity,
start_with_level=start_with_level)
results_eps[level_sample_size].append(res)
results[tot_eps][i] = results_eps
torch.save(results, f'results_pos_{positivity}_{args.name}.pt')
from ErlangC import ErlangC
import matplotlib.pyplot as plt
from run_experiment import run_experiment
N = range(101, 126, 5)
T = [10, 100, 1000, 10000]
c = 5
h = 15
costs = []
for t in T:
print()
print("t = ", t)
cost = []
for n in N:
print("n = ", n)
lm, mu, EQ = run_experiment(n, t)
cost.append(c * n + h * EQ)
costs.append(cost)
plt.yscale('log')
for cost in costs:
plt.plot(N, cost)
#lm = 10
#mu = 1
T = [10, 100, 1000, 10000]
costs = []
for t in T:
lm, mu, Q = run_experiment(101, t)
c = 5
h = 15
x = range(101, 126)
def train_model(path_list,dst_folder):
train_file = path_list[0]
test_file = path_list[1]
result_all = {}
#random the file
if rand_num > 1:
rand_file = train_file + '_rand'
else:
rand_file = train_file
rand_file_cache = rand_file + '_cache'
for k in range(0,rand_num):
if rand_num > 1:
print 'shuffle datset...'
sol_shuffle.sol_shuffle(train_file, rand_file)
cmd_data = dataset.get_cmd_data_by_file(rand_file, test_file, is_cache)
dataset.analyze(rand_file);
for opt in opt_list:
print '-----------------------------------'
print ' Experiment on %s' %opt + ' Random %d' %k
print '-----------------------------------'
if opt == 'vw':
result_file = 'vw_result_%d' %k + '.txt'
result_once = run_vw.run_vw(rand_file, test_file, ds, result_file, is_cache)
elif opt == 'liblinear':
result_file = 'liblinear_result_%d' %k + '.txt'
result_once = run_liblinear.run_liblinear(rand_file, test_file, ds, result_file)
elif opt == 'fgm':
result_file = 'fgm_result_%d' %k + '.txt'
result_once = run_fgm.run_fgm(rand_file, test_file, ds, result_file)
elif opt == 'mRMR':
result_file = dst_folder + '/%s' %opt + '_result_%d' %k + '.txt'
result_once = run_mRMR.run_mRMR(rand_file, test_file, ds, result_file)
print '\nparsing result...'
#write the result to file
parse_file = dst_folder +'/%s' %opt + '_%d' %k + '.txt'
result_once2 = run_util.parse_result(result_file, parse_file);
bs_num = len(result_once)
if bs_num != len(result_once2):
print 'inconsistent parsing result'
for m in range(0,bs_num):
result_once[m][0] = result_once2[m][0]
result_once[m][1] = result_once2[m][1]
if result_once[m][2] == 0:
result_once[m][2] = result_once2[m][2]
if result_once[m][3] == 0:
result_once[m][3] = result_once2[m][3]
else:
result_file = dst_folder + '/%s' %opt + '_result_%d' %k + '.txt'
cmd = cmd_data
cmd += extra_cmd
if is_default_param == False:
cmd += dataset.get_model_param(ds, opt)
run_experiment.run_experiment(opt,result_file,ds, cmd)
print '\nparsing result...'
#write the result to file
parse_file = dst_folder +'/%s' %opt + '_%d' %k + '.txt'
result_once = run_util.parse_result(result_file, parse_file);
result_all = add_to_dict(opt,result_all, result_once)
#remove previous file
if rand_num > 1:
os.system('rm -f %s' %rand_file_cache)
os.system('rm -f %s' %rand_file)
#average the result
for opt in opt_list:
rows = len(result_all[opt])
cols = len(result_all[opt][0])
for k in range(0,rows):
for m in range(0,cols):
result_all[opt][k][m] /= rand_num
return result_all
for k in kvals:
print "k = {}".format(k)
print " generating input data ..."
input_filename = []
for instance in range(1, num_instances + 1):
print " instance {}".format(instance)
dataf = data_filename(tmpdir, n, k, instance)
gen_input(n, k, dataf, random.randint(0, 2000000000), stats_file=data_stats_filename(tmpdir, n, k, instance))
input_filename.append(dataf)
print " writing index file ..."
indexf = index_filename(tmpdir, n, k)
write_index_file(indexf, input_filename)
for alg in algs:
resultsf = results_filename(tmpdir, alg, n, k)
print " algorithm: {}".format(alg)
r = run_experiment(n, k, indexf, alg, l0_eps, num_trials, resultsf)
ne = num_l0_errors(r)
if ne > 0:
print " {} L0-errors occurred.".format(ne)
for f in input_filename:
os.remove(f)
time_results = {}
l0_results = {}
success_results = {}
for alg in algs:
time_results[alg] = {}
l0_results[alg] = {}
success_results[alg] = {}
print(line.replace('filePrefix = DO NOT CHANGE',
'filePrefix = ' + fileprefix_exp2),
end='')
for line in fileinput.input(name_folder + name_experiment + name_exp2,
inplace=True):
print(line.replace('recordIOMap_fileName = DO NOT CHANGE',
'recordIOMap_fileName = ' + filename_activityfile),
end='')
# then we run the experiments
runs = 5
for r in list(range(0, runs)):
print(['run number ' + str(r)])
# Running experiment
run_experiment(cmd[0])
# Getting data from file
data.append(pd.read_csv(datafile))
genome.append(pd.read_csv(genomefile))
# Running recorders
run_experiment(cmd[1])
# Getting recordings from file
activity.append(pd.read_csv(activityfile))
with open(name_folder + name_experiment + 'LOD_data.pkl', 'wb') as f:
pickle.dump(data, f)
with open(name_folder + name_experiment + 'genome.pkl', 'wb') as f:
pickle.dump(genome, f)
with open(name_folder + name_experiment + 'activity.pkl', 'wb') as f:
import matplotlib.pyplot as plt from run_experiment import run_experiment x = range(101, 126) # This is the number of servers in the system T = 10 c = 5 h = 15 cost = [] for N in x: lm, mu, EQ = run_experiment(N, T) cost.append(c*N + h*EQ) x1 = range(101, 126) x2 = range(101, 126, 2) x3 = range(101, 126, 3) x4 = range(101, 126, 4) x5 = range(101, 126, 5) xs = [x1, x2, x3, x4, x5] plt.scatter(x1, cost) for x in xs: y = [cost[101-i] for i in x] mymodel = np.poly1d(np.polyfit(x, y, 3)) myline = np.linspace(101, 125, 100) plt.plot(myline, mymodel(myline)) plt.axvline(112, color='k', linestyle='dotted')
from identity import experiment_identities
sys.path.append("../../appium_exec")
from run_experiment import run_experiment
from abstract_experiment import abstract_experiment
# weather channel
APP_DIR = "/var/www/html/mitm/com.weather.Weather-5.5.6-505060343-minAPI14.apk"
class run_weather_channel(abstract_experiment):
def __init__(self, driver):
self.driver = driver
self.identities = experiment_identities() # set start index here
def experiment(self):
time.sleep(11)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"-l", "--log-dir", required=True, help="Where experiment log is be stored")
args = parser.parse_args()
result = run_experiment(run_weather_channel,
app_dir=APP_DIR, num=7500)
with open(args.log_dir, 'a') as f:
json.dump(result, f)
sec_agg / level_sample_size) * mechanisms.get_std_from_eps(
max(eps, remaining))
total_epsilon_budget = tot_eps * level_sample_size
else:
eps_func = lambda x, y: None
threshold_func = lambda i, prefix_len, eps, remaining: 5
total_epsilon_budget = None
res = run_experiment.run_experiment(
true_image,
dataset,
level_sample_size=level_sample_size,
secagg_round_size=sec_agg,
threshold_func=threshold_func,
collapse_func=lambda threshold: 1 / 4 * threshold,
eps_func=eps_func,
total_epsilon_budget=total_epsilon_budget,
top_k=5,
partial=100,
max_levels=10,
total_size=1024,
min_dp_size=None,
dropout_rate=None,
output_flag=True,
quantize=None,
save_gif=False,
positivity=False)
results[c].append(res)
torch.save(results, f'c_results_{args.name}.pt')
import sys
import time
sys.path.append("../appium_exec")
from run_experiment import run_experiment
from abstract_experiment import abstract_experiment
APP_DIR = "/var/www/html/mitm/Complete Astrology Zodiac_v2.1_apkpure.com.apk"
class run_weather_channel(abstract_experiment):
def __init__(self, driver):
self.driver = driver
def experiment(self):
time.sleep(15)
if __name__ == "__main__":
print(run_experiment(run_weather_channel, app_dir=APP_DIR, num=100))
h = 15.
dictn = {}
for n in x1:
for T in [10, 100, 1000, 10000]:
dictn[(n, T)] = []
dictn2 = {}
for N in [len(x) for x in xs]:
for T in [10, 100, 1000, 10000]:
dictn2[(N, T)] = []
for i in range(100):
for n in x1:
#print("Simulating n = ", n)
lm, mu, EQ = run_experiment(n, 10000, dictn)
print("So far ", len(dictn[(101, 10000)]), " experiements have been run")
f = open("/content/drive/My Drive/Summer Research 2020/dictn.txt", "w")
f.write(str(dictn))
f.close()
for T in [10, 100, 1000, 10000]:
for i in range(100):
cost = [c * n + h * dictn[(n, T)][i] for n in x1]
for x in xs:
y = [cost[i - 101] for i in x]
mymodel = np.poly1d(np.polyfit(x, y, 3))
cost_pred = mymodel(x1)
min_i = np.argmin(cost_pred)
min_n = x1[min_i]
dictn2[(len(x), T)].append(min_n)
import ximea_cam_aquire_save as xim
import run_experiment as runexp
import run_analysis as runana
import imp
import numpy as np
import matplotlib.pyplot as plt
import imageio
import cv2
import schedule
import time
#settings
subject = 'buddy_stationary'
task = 'chat_1'
exp = 'pre'
capture_dir_list = ['./capture]
analysis_dir = './analysis'
collection_minutes = 1
save_batchsize = 200
imp.reload(runexp)
imp.reload(xim)
runexp.run_experiment(subject_name=subject,
task_name=task,
exp_type=exp,
save_dirs=capture_dir_list,
collection_minutes=collection_minutes,
save_batchsize=save_batchsize)
import ximea_cam_aquire_save as xim
import run_analysis as ana
import numpy as np
import os
import run_experiment as runexp
subject = 'ag'
task = 'laptop_1'
exp = 'pre'
capture_dir_list = ['/home/vasha/Desktop/backpack_computer_aquisition/capture']
path = os.path.join(capture_dir_list[0], subject, task, exp, 'test')
if not os.path.exists(path):
oldmask = os.umask(000)
os.makedirs(path, 777)
os.umask(oldmask)
collection_minutes = 3
save_batchsize = 200
ncameras = 1
pupil_port = 50020
runexp.run_experiment(subject_name=subject,
task_name=task,
exp_type=exp,
save_dirs=capture_dir_list,
collection_minutes=collection_minutes,
save_batchsize=save_batchsize,
pupil_port=pupil_port,
n_cameras=ncameras)
for instance in range(1, num_instances + 1):
print ' instance {}'.format(instance)
dataf = data_filename(tmpdir, n, k, instance)
gen_input(n,
k,
dataf,
random.randint(0, 2000000000),
stats_file=data_stats_filename(tmpdir, n, k, instance))
input_filename.append(dataf)
print ' writing index file ...'
indexf = index_filename(tmpdir, n, k)
write_index_file(indexf, input_filename)
for alg in algs:
resultsf = results_filename(tmpdir, alg, n, k)
print ' algorithm: {}'.format(alg)
r = run_experiment(n, k, indexf, alg, l0_eps, num_trials, resultsf)
ne = num_l0_errors(r)
if ne > 0:
print ' {} L0-errors occurred.'.format(ne)
for f in input_filename:
os.remove(f)
time_results = {}
l0_results = {}
success_results = {}
for alg in algs:
time_results[alg] = {}
l0_results[alg] = {}
success_results[alg] = {}
def main():
data_gen()
run_experiment()
post_process()
print_warning(
"You do not have the 'logging' module. Please consider" +
" installing it, and all other required or optional " +
"modules, via the command line \"> pip install -r requirements.txt\""
)
### Initialize experiment directories and global variables
## Set the experiment name according to a boring structure (but simple is good :D)
settings.EXP_NAME = "{}_model_{}".format(settings.EXP_NAME_PREFIX,
settings.MODEL)
## Initialize global variables that store the various directories where
## results will be saved for this experiment. Moreover, create them.
initialize_directories()
### All systems are go, let's fire up this ship to infinity and beyond!
from run_experiment import run_experiment
t = datetime.datetime.now()
log("")
log("============================================================")
log("Current stardate: {0}".format(t.strftime("%Y-%m-%d %H:%M:%S")))
log("")
log("All output is logged on disk to: {}".format(settings.OUTLOGFILE))
log("All errors are logged on disk to: {}".format(settings.ERRLOGFILE))
log("")
run_experiment()
### Graceful exit
sys.exit(0)
