for label in range(10):
label_ix = ix[y == label]
perm = np.random.permutation(range(len(label_ix)))[:int(N / 10)]
chosen.append(label_ix[perm])
chosen = np.concatenate(chosen)
np.random.shuffle(chosen)
return chosen
dataset = "cifar"
model = cnn
trials = 1
sizes = [55000]
y = np.load("./datasets/%s_with_val.npz" % dataset)["y_train"]
ix = np.arange(0, len(y), 1).astype("int32")
e = Experiment(model, dataset)
for N in sizes:
accs = []
for T in range(trials):
start = time.time()
print "N=%d, Trial=%d" % (N, T)
acc = e.get_test_acc(train_ix=balanced_random(N),
epochs=100,
lr=0.0001,
train_bs=min(100, max(1, N / 100)),
verbose=True,
save_dir=None)
accs.append(acc)
print "Trial Acc=%s (%.2f s)" % (str(acc), time.time() - start)
methods = [
rc.RumorCenter(),
dc.DistanceCenter(),
jc.JordanCenter(),
ri.ReverseInfection(),
di.DynamicImportance(),
prior_detector8,
gsba.GSBA(prior_detector1),
gsba_bao7.GSBA_coverage_7(prior_detector1),
gsba_bao9.GSBA_coverage_9(prior_detector1)
]
logger = log.Logger(logname='../data/main_1000random_graph0005.log', loglevel=logging.INFO, logger="experiment").get_log()
experiment = Experiment(methods, logger)
experiment.propagation_model = 'SI'
start_time = clock()
print "Starting..."
d = data.Graph("../data/random_graph/1000/1000random_graph0005.txt", weighted=0)
d.debug = False
test_num = 100
print 'Graph size: ', d.graph.number_of_nodes(), d.graph.number_of_edges()
test_category = experiment.RANDOM_TEST
experiment.start(d, test_category, test_num, 20, 350, 40)
# test_category = experiment.FULL_TEST
# experiment.start(d, test_category, test_num, 200, 400,100)
end_time = clock()
row.append(np.mean(results_df['P'] < sig_level))
row.append(np.median(results_df['P']))
power_array.append(row)
df = pd.DataFrame(
columns=['Estimators'] + \
['P(reject), $\\alpha='+str(sl)+'$' for sl in sig_levels] + \
['median p-val'],
data=power_array)
print(df)
print()
df.to_csv(tsvfilename(s), sep='\t', index=False)
with open(latexfilename(s), 'w') as f:
f.write(df.to_latex(index=False, escape=False))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--exp-name', help='name of the json experiment file')
parser.add_argument('--sig-levels',
nargs='+',
type=float,
help='significance level')
parser.add_argument('--exclude',
nargs='+',
default=[],
help='prettynames of estimators to ignore')
args = parser.parse_args()
exp = Experiment(args.exp_name)
get_power(exp, args.sig_levels, exclude=args.exclude)
counter += 1
# if counter > 20:
# break
experiment_windows_273_1024_2047 = Experiment(
# Paths to tools
r"D:\git\BitFunnel\build-msvc\tools\BitFunnel\src\Release\BitFunnel.exe",
r"D:\git\mg4j-workbench",
r"/home/mhop/git/partitioned_elias_fano/bin",
# The directory containing all indexes and the basename for this index
r"D:\temp\indexes",
r"273-1024-2047",
# The directory with the gov2 chunks and the regular expression pattern
# used to determine which chunks will be used for this experiment.
r"d:\sigir\chunks-1024-2047",
r"GX.*", # Use all chunks
# The query log to be used for this experiment.
r"D:\sigir\queries\06.efficiency_topics.all",
# Min and max thread counts
8,
1,
8)
# generate_query_log(experiment_windows_273_1024_2047, 20000)
measure_quadwords(experiment_windows_273_1024_2047, 9)
analyze_quadwords(experiment_windows_273_1024_2047, 9)
import argparse
import os
from experiment import Experiment
parser = argparse.ArgumentParser(description='Run an experiment')
parser.add_argument('fname')
parser.add_argument('--n_jobs',
help='Number of trials to run in parallel',
default=1,
type=int)
def parse_config(fname):
data = json.load(open(fname, 'r'))
return data
if __name__ == '__main__':
args = parser.parse_args()
fname = args.fname
n_jobs = args.n_jobs
base = os.path.basename(fname)
noext = os.path.splitext(base)[0]
cfg = parse_config(fname)
exp = Experiment(cfg)
exp.run(n_jobs=n_jobs)
exp.save(noext)
from gevent import monkey monkey.patch_all() from flask import Flask, render_template, session, request from flask.ext.socketio import SocketIO, emit, join_room, leave_room print "Starting to build experiment..." from experiment import Experiment experiment = Experiment() from double_auction_experiment import DoubleAuctionExperiment experiment = DoubleAuctionExperiment() app = Flask(__name__) app.debug = True app.config['SECRET_KEY'] = 'secret!' socketio = SocketIO(app) experiment.set_emitter(socketio.emit) import willow.views import willow.admin_views
def make_figure(figurename,
figpath=None,
Culture=FIGURE_CULTURE,
with_background_image=False):
# Load electrode coordinates and calculate neighborhood
pos = load_positions(mea='hidens')
x = pos.x
y = pos.y
distances = electrode_distances()
all_triggers, all_AIS, all_axonal_delays, all_dendritic_return_currents = Experiment(
Culture).compartments()
all_Neurons = []
all_Coefficents = []
all_Count = []
all_Delay = []
all_N_gaps = []
print(FIGURE_NEURONS)
for neuron in FIGURE_NEURONS:
# delay ~ distance
delay = all_axonal_delays[neuron]
index_AIS = all_AIS[neuron]
axon = np.isfinite(delay)
delay_AIS = delay[index_AIS]
delay = delay[axon]
min_delay = min(delay) if not np.isfinite(delay_AIS) else delay_AIS
delay = delay - min_delay
r = distances[index_AIS, axon]
# Polynomial fit to delay ~ r
Coefficents = np.polyfit(r, delay, 1)
print(Coefficents)
# Sholl analysis
R, Count = sholl_analysis(r)
R, Delay = sholl_analysis(r, delay)
# Number of large gaps indicative of myelinated parts of axons
N_gaps = len(axon_gaps(axon, distances))
# Collect
all_Neurons.append(neuron)
all_Coefficents.append(Coefficents)
all_Count.append(Count)
all_Delay.append(Delay)
all_N_gaps.append(N_gaps)
from collections import Counter
# print(all_N_gaps)
# print(Counter(all_N_gaps))
# # Colors for neurons
# cNorm = mpl.colors.Normalize(vmin=1, vmax=max(FIGURE_NEURONS))
# scalarMap = plt.cm.ScalarMappable(norm=cNorm, cmap=plt.get_cmap('hsv'))
# color = scalarMap.to_rgba(neuron)
# color = 'black'
fig = plt.figure(figurename, figsize=(9, 5))
ax1 = plt.subplot(121)
ax1.plot(R, np.array(all_Count).T, '-')
ax1.set_xlim((0, 1400))
ax1.set_ylabel('number of electrodes with axonal signal')
ax1.set_xlabel(r'distance from AIS in $\mathsf{\mu m}$')
adjust_position(ax1, xshrink=0.02)
plt.title('a', loc='left', fontsize=18)
ax2 = plt.subplot(122)
ax2.plot(R, np.array(all_Delay).T, '-')
ax2.set_xlim((0, 1400))
ax2.set_ylabel(r'axonal delay $\tau_{axon}$ in ms')
ax2.set_xlabel(r'distance from AIS in $\mathsf{\mu m}$')
adjust_position(ax2, xshrink=0.02)
plt.title('b', loc='left', fontsize=18)
# from sklearn.decomposition import NMF
# model = NMF(n_components=2)
# model.fit(all_Count)
#
# x, y = zip(*model.transform(all_Count))
# ax2.plot(np.log(np.array(x)+1),np.log(np.array(y)+1),'.')
show_or_savefig(figpath, figurename)
from experiment import Experiment
from algorithm.SMOTE import Smote
if __name__ == "__main__":
total_number = 10000
ratio = 0.9
dg1 = DataGenerator(total_number=total_number, ratio=ratio)
data_train, label_train = dg1.generate()
dg2 = DataGenerator(total_number=total_number, ratio=ratio)
data_test, label_test = dg2.generate()
# before smote
n_neighbors = [1, 3, 5, 7, 9, 11, 13, 15, 17]
for n in n_neighbors:
print("n_neighbors:", n)
print("before smote")
exp = Experiment(data=data_train, label=label_train)
true_posi, false_posi, true_neg, false_neg = exp.get_confusion_matrix(
data=data_test, label=label_test, n_neighbors=n)
print("true_posi:", true_posi, "false_posi:", false_posi, "true_neg:",
true_neg, "false_neg:", false_neg)
# smote
minority_samples = []
for i in range(len(label_train)):
if label_train[i] == 1.0:
minority_samples.append(data_train[i])
smote = Smote(sample=minority_samples, N=100, k=5)
smote.over_sampling()
synthetics = smote.synthetic
data_smote = list(data_train)
data_smote.extend(synthetics)
label_smote = list(label_train)
paramMap["r3"] = 0.3 # Default third row radius
# absolute path to the mpb executable
mpb = "/Users/sean/documents/mpb-1.5/mpb/mpb"
# absolute path to the input ctl
inputFile = "/Users/sean/UniversityOfOttawa/Photonics/PCWO/W1_2D_v04.ctl.txt"
# absolute path to the output ctl
outputFile = "/Users/sean/UniversityOfOttawa/Photonics/PCWO/optimizerTestFile.txt"
# we define a general experiment object
# that we reuse whenever we need to make a command-line mpb call
# see experiment.py for functionality
experiment = Experiment(mpb, inputFile, outputFile)
# ex.setParams(paramVector)
experiment.setCalculationType('4') # accepts an int from 0 to 5
experiment.setBand(23)
constraintFunctions = [constraints.latticeConstraintsLD]
pcw = PhCWDesign(paramMap, 0, constraintFunctions)
max_generation = 10 # number of iterations of the DE alg
population_size = 20 # number of solutions to consider in DE
random_update = 0.2 # chance of updating vector fields in DE alg
elite_size = 10 # number of solutions to store in DE, and use for GD
# do knn_expriment on raw data without any techniques
# metrics are f1 and accuracy
# if __name__ == "__main__":
# total_number = 10000
# ratio = 0.9
# n_neighbors = [1, 3, 5, 9, 14, 20, 25, 30, 35, 40, 50]
# cv = [3, 5, 7, 11, 13, 15, 17, 19, 21]
# data, label = DataGenerator(total_number=total_number, ratio=ratio).generate()
# for i in cv:
# for j in n_neighbors:
# exp = Experiment(data, label)
# result = exp.do_knn_experiment(i, j)
# print("cv:", i, " n_neighbors:", j, " acc:", np.mean(result["accuracy"]), " f1:", np.mean(result["f1"]))
if __name__ == "__main__":
total_number = 10000
ratio = 0.9
n_neighbors = [1, 2, 3, 5, 7, 10, 13, 15, 17, 19]
cv = [3, 5, 7, 9]
data_train, label_train = DataGenerator(total_number=total_number,
ratio=ratio).generate()
data_predict, label_predict = DataGenerator(total_number=total_number,
ratio=ratio).generate()
exp = Experiment(data=data_train, label=label_train)
for n in n_neighbors:
true_posi, false_posi, true_neg, false_neg = exp.get_confusion_matrix(
data=data_predict, label=label_predict, n_neighbors=n)
print("n:", n, "true_po:", true_posi, " false_p:", false_posi,
" true_neg:", true_neg, " fasle_neg:", false_neg)
parser = argparse.ArgumentParser(description="A way to define variables for \
training, tests and models")
parser.add_argument('--metrics_dir', type=str, help="Path to save metrics file \
\nDisabled if save_metrics flag is False.\n\nDefault value \
is './'")
parser.add_argument('--save_metrics', type=bool, help='Boolean flag determines \
whether metrics should be saved.\n\n Default value is True')
parser.add_argument('--custom_config', type=str, help='The path to the \
configuration file to be used. If equal to None, then \
the config located in the same folder will be used.',
default=None)
args = parser.parse_args()
cfg = Config()
with Experiment('', cfg) as exp:
if args.metrics_dir:
cfg.metrics_dir = args.metrics_dir
if args.custom_config:
try:
spec = importlib.util.spec_from_file_location(args.custom_config)
config_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(config_module)
cfg = config_module.Config()
except:
raise ValueError(f"Cannot import config module from {args.custom_config}")
for log_name, dataset in datasets.items():
for bench_func, epoch, framework in zip(BENCHMARK_FUNCTIONS,
EPOCHS,
FRAMEWORKS):
import argparse
from experiment import Experiment
import Limited_GP
parser = argparse.ArgumentParser()
parser.add_argument('--number-of-batches', type=int, default=1)
parser.add_argument('--current-batch', type=int, default=1)
parser.add_argument('--budget', type=int, default=10000)
parser.add_argument('--suite-name', default='bbob')
args = parser.parse_args()
e = Experiment(suite_name=args.suite_name,
solver=Limited_GP.solve,
algorithm_name='Limited-GP')
e.run(budget=args.budget,
current_batch=args.current_batch,
number_of_batches=args.number_of_batches)
# alC = ActiveLearnerCertainty(dtst, 'certainty', model, batchSize)
# alBC100 = ActiveLearnerBlockCertainty(dtst, 'block-certainty-100', model, batchSize, 100)
# alBC10 = ActiveLearnerBlockCertainty(dtst, 'block-certainty-10', model, batchSize, 10)
# alBC1 = ActiveLearnerBlockCertainty(dtst, 'block-certainty-1', model, batchSize, 1)
# alBC01 = ActiveLearnerBlockCertainty(dtst, 'block-certainty-01', model, batchSize, 0.1)
# alBC001 = ActiveLearnerBlockCertainty(dtst, 'block-certainty-001', model, batchSize, 0.01)
# alQ = ActiveLearnerQUIRE(dtst, 'quire', model, batchSize, 1.0, 1., 'rbf', 1., 3)
# alLALindepend = ActiveLearnerLAL(dtst, 'lal-rand', model, batchSize, lalModel1)
# alLALiterative = ActiveLearnerLAL(dtst, 'lal-iter', model, batchSize, lalModel2)
## alHSVM = ActiveLearnerHintSVM(dtst, nEstimators, 'hint-SVM', model, batchSize, K, 0.1, 0.1, .5, None, 'linear', 'rbf', 3, 0.1, 0., 1e-3, 1, 100., 0)
alMinExp = ActiveLearnerMinExpError(dtst, 'minExpError', model, batchSize)
# als = [alR, alU, alC, alBC100, alBC10, alBC1, alBC01, alBC001, alQ, alLALindepend, alLALiterative, alMinExp]
als = [alMinExp]
#
exp = Experiment(nIterations, quality_metrics, dtst, als, maxVoteCount, 'here we can put a comment about the current experiments')
# # the Results class helps to add, save and plot results of the experiments
res = Results(exp, nExperiments)
print("Running AL experiments...")
for i in range(nExperiments):
print('\n experiment #'+str(i+1))
# run an experiment
performance = exp.run()
res.addPerformance(performance)
# reset the experiment (including sampling a new starting state for the dataset)
exp.reset()
print("Done!")
res.saveResults('rte_combined_minExp')
# -*- coding: utf-8 -*-
"""
Created on Mon Jan 7 16:06:17 2019
@author: bastien
"""
from experiment import Experiment
from actions_builder import Action
# variables
list_action = [
'MOVE_BACKWARD', 'MOVE_LEFT', 'SPEED', 'TURN_LEFT', 'ATTACK', 'TURN_RIGHT',
'MOVE_FORWARD', 'MOVE_RIGHT', 'test'
]
game_features = ['frag_count', 'health']
game_variables = ['ENNEMY']
scenario = 'basic'
action_builder = Action(list_action)
e = Experiment(scenario,
action_builder,
game_features=game_features,
visible=True)
#e.start(2)
#screen, variables, game_features = e.observe_state(game_variables, game_features)
#action = action_builder.doom_action[9]
#tics = 1
#r = e.game.make_action(action, tics)
import argparse
from experiment import Experiment
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--device', type=str, default='cuda:0')
parser.add_argument('--epochs', type=int, default=10)
parser.add_argument('--batch_size', type=int, default=128)
parser.add_argument('--n_workers', type=int, default=4)
parser.add_argument('--log_freq', type=int, default=10000)
args = None
Experiment(args).run()
def test_probability_of_a_head():
e = Experiment()
p = e.flip_coin_x_times(100_000)
assert p == approx(0.5, rel=0.01)
def add_experiment(experiment_json):
experiment_id = experiment['experiment_id']
experiment = Experiment(experiment_json)
experiment.start()
experiments[experiment_id] = experiment
return str(experiment_id) + " has been added successfully\n"
def _reset_blend_func(self, instruction):
'''Reset of the Gabor blending properties for creation of new stimuli
glBlendFunc(starting RGBA values, desired RGBA values)'''
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
if __name__ == '__main__':
from experiment import Experiment
from state import UntilDone, Wait, Parallel, Serial
from keyboard import KeyPress
from video import Label
exp = Experiment(background_color="#4F33FF")
g = Grating(width=250, height=250, contrast=0.1)
with UntilDone():
KeyPress()
g.update(bottom=exp.screen.center)
KeyPress()
g = Grating(width=500,
height=500,
envelope='Gaussian',
frequency=75,
phase=11.0,
color_one='blue',
color_two='red',
contrast=0.25)
if __name__ == "__main__":
exps = {
"batch_4":
["batch_4", "batch_4-2", "batch_4-3", "batch_4-4", "batch_4-5"],
"batch_8":
["batch_8", "batch_8-run-2", "batch_8-3", "batch_8-4", "batch_8-5"],
"5e-4_full": ["5e-4-1", "5e-4-2", "5e-4-3", "5e-4-4", "5e-4-5"],
"unet_lite_1e-4": [
"unet-1e-4-1", "unet-1e-4-2", "unet-1e-4-3", "unet-1e-4-4",
"unet-1e-4-5"
],
"unet_lite_5e-4": [
"unet_lite_0.4_5e-4-1", "unet_lite_0.4_5e-4-2",
"unet_lite_0.4_5e-4-3", "unet_lite_0.4_5e-4-4"
]
}
results = {}
for exp in exps:
for i in exps[exp]:
print(exp, i)
e = Experiment(exp, i)
r = e.get_perf_stats()
results[i] = r
write_to_file_in_dir('../experiment_data', 'results8.json',
results)
log_to_file_in_dir('../experiment_data', 'results8-pretty.log',
get_pretty_string(i, r))
print("Finished")
# Day 2: GPi ON
for trial in exp.task.block(0):
exp.model.process(exp.task, trial)
records[1] = exp.task.records
return records
mdl = "model-topalidou.json"
tsk = "tasks/task-topalidou.json"
rslt = folderRes + "control.npy"
rprt = folderRep + "control.txt"
experiment = Experiment(model=mdl,
task=tsk,
result=rslt,
report=rprt,
n_session=25,
n_block=2,
seed=123)
records = experiment.run(session, "Topalidou Control")
# Textual results
# -----------------------------------------------------------------------------
P = np.squeeze(records["best"][:, 0, :25])
P = P.mean(axis=len(P.shape) - 1)
print("D1 start: %.3f ± %.3f" % (P.mean(), P.std()))
P = np.squeeze(records["best"][:, 0, -25:])
P = P.mean(axis=len(P.shape) - 1)
print("D1 end: %.3f ± %.3f" % (P.mean(), P.std()))
P = np.squeeze(records["RT"][:, 0])
self._widget.start()
super(MovingDots, self).show()
def unshow(self):
# custom unshow so that the widget doesn't run when not onscreen
super(MovingDots, self).unshow()
self._widget.stop()
if __name__ == '__main__':
from experiment import Experiment
from state import UntilDone, Meanwhile, Wait, Loop, Debug
from keyboard import KeyPress
exp = Experiment(background_color=("purple", .3))
Wait(.5)
g = MovingDots(radius=300,
scale=10,
num_dots=4,
motion_props=[{"coherence": 0.25, "direction": 0,
"direction_variance": 0},
{"coherence": 0.25, "direction": 90,
"direction_variance": 0},
{"coherence": 0.25, "direction": 180,
"direction_variance": 0},
{"coherence": 0.25, "direction": 270,
"direction_variance": 0}])
with UntilDone():
b = []
with open(wpath, 'rb') as csvfile:
rows = csv.reader(csvfile)
for row in rows:
w.append(row[1:])
with open(bpath, 'rb') as csvfile:
rows = csv.reader(csvfile)
for row in rows:
b.append(row[1:])
return w, b
def q2_123(data):
case = int(sys.argv[1])
model = three.ThreeLayers(case)
exp = Experiment(model, data, 500, 1)
exp.train()
print_graph(exp.loss_log)
def q2_4(data, path):
case = int(sys.argv[1])
wfile = "w-100-40-4.csv"
bfile = "b-100-40-4.csv"
if case == 2:
wfile = "w-28-6-4.csv"
bfile = "b-28-6-4.csv"
elif case == 3:
wfile = "w-14-28-4.csv"
bfile = "b-14-28-4.csv"
w, b = loadwb(path + wfile, path + bfile)
def main_routine_func(ms1_error = 5, ms2_error = 20, ms_tol_ppm = True,
plot_XIC=False,plot_ms2=False,aggregate_results=True,
search_negative_ions = True,search_positive_ions = True,
score_ms1_error=True,score_ms2_error=True,score_hypergeom=True,
score_intensity_expl=True,score_pred_tr=True,
min_intensity_explained=5.0,min_hypergeom=2.0,
filter_head_spec=True,filter_fa_spec=True,
rt_file = "rt_pred/MASSTRPLAN_preds_l3_116.csv",mzml_files_loc="",
db_file = "db/filtered_db_noether_backup.msp",
n_chunks = 32, n_cores = 8, gui=None,output_dir=""):
if os.path.isfile(mzml_files_loc):
mzml_files = [mzml_files_loc]
else:
mzml_files = [os.path.join(mzml_files_loc,file) for file in os.listdir(mzml_files_loc) if file.lower().endswith(".mzml")]
mgf_files = [os.path.join(mzml_files_loc,file) for file in os.listdir(mzml_files_loc) if file.lower().endswith(".mgf")]
mzml_files.extend(mgf_files)
tot_chunk_count = 0
logging.basicConfig(filename="prec_filter.log",
level=logging.DEBUG,
filemode="w",
format="%(levelname)s:%(created)f:%(asctime)s:%(message)s")
chunk_ranges = determine_chunks_prec(db_file,chunks=n_chunks)
head_spec = [hs.rstrip() for hs in open("head_spec.txt")]
fa_spec = [fs.rstrip() for fs in open("fa_spec.txt").readlines()]
param_dict = {"ms1_error":ms1_error,"ms2_error":ms2_error,"ms_tol_ppm":ms_tol_ppm,"search_negative_ions":search_negative_ions,
"search_positive_ions":search_positive_ions,"score_ms1_error":score_ms1_error,"score_ms2_error":score_ms2_error,
"score_hypergeom":score_hypergeom,"score_intensity_expl":score_intensity_expl,"score_pred_tr":score_pred_tr,
"min_intensity_explained":min_intensity_explained,"min_hypergeom":min_hypergeom,"filter_head_spec":filter_head_spec,
"filter_fa_spec":filter_fa_spec,"head_spec":head_spec,"fa_spec":fa_spec,"db_file":db_file}
iso_dists = Isotopic_dists()
iso_filt = isotope_filters()
iso_align = IsotopeAlignment()
scoring_funct = ScoringFunction()
outfile_high_confid = open("high_confident_ident.csv","w")
scan_num_to_precf_res = {}
aggregated_results = IdentificationResults()
xic_movingavg_length = 4
for mzml_file in mzml_files:
if len(output_dir) == 0: base_path = os.path.join(os.path.abspath(__file__),".".join(mzml_file.split(".")[:-1]))
else: base_path = output_dir
results = IdentificationResults()
if not os.path.exists(base_path):
os.makedirs(base_path)
if not os.path.exists(os.path.join(base_path,"MS2")):
os.makedirs(os.path.join(base_path,"MS2"))
if not os.path.exists(os.path.join(base_path,"XIC")):
os.makedirs(os.path.join(base_path,"XIC"))
if mzml_file.endswith(".mzML"): exp = Experiment(mzml_file)
elif mzml_file.endswith(".mgf"): exp = Experiment(mzml_file,mgf=True)
pool = Pool(processes=8)
chunk_results = {}
for lower_limit_mz,upper_limit_mz in chunk_ranges:
chunk_results[lower_limit_mz] = pool.apply_async(perform_search, args = (mzml_file,exp,lower_limit_mz,upper_limit_mz,scoring_funct),kwds=param_dict)
pool.close()
#pool.join()
for lower_limit_mz,cr in chunk_results.items():
tot_chunk_count += 1
if gui: gui.update_progress2((tot_chunk_count/(float(n_chunks)*len(mzml_files)))*100)
for chunked_res in cr.get():
results.add_result(*chunked_res)
if aggregate_results:
aggregated_results.add_result(*chunked_res)
# TODO change name "res" is stupid; change to res_ranked....
res,ranked_lipids = results.get_best_ranked_score()
print(base_path)
print(results)
print(res)
results.write_res_to_file(outfile_name=os.path.join(base_path,"ranked_res.csv"))
results.write_res_to_file(outfile_name=os.path.join(base_path,"best_res.csv"),best=True)
if plot_XIC:
prepared_plotting_vars = []
for n_exp,n_lip in res:
n_lip_entry = results.lipid_identifier_to_entry[n_lip]
mz_list_entry = [mz for mz,intens,name in n_lip_entry]
intens_list_entry = [intens for mz,intens,name in n_lip_entry]
scan_num = n_exp.split("|")[2]
aligned_peaks = results.get_aligned_peaks(n_exp,n_lip)
xic_list = exp.get_XIC(exp.scan_to_spectrum[scan_num].prec_mass,
positive_mode=False,
negative_mode=exp.scan_to_spectrum[scan_num].negative_scan)
xic_movingavg = moving_average([y[1] for x,y in enumerate(xic_list)],n=xic_movingavg_length)
xic_movingavg_time = [y[0] for x,y in enumerate(xic_list)][:len(xic_movingavg)]
xic_peaks = get_peaks(list(zip(xic_movingavg_time,xic_movingavg)))
try: closest_peak = min(xic_peaks,key=lambda x: abs(float(x[0]) - exp.scan_to_spectrum[scan_num].scan_start_time))
except ValueError: closest_peak = exp.scan_to_spectrum[scan_num].scan_start_time
plt.plot(np.array(xic_movingavg_time),xic_movingavg)
try: plt.axvline(closest_peak[0],color="green")
except: pass
plt.axvline(exp.scan_to_spectrum[scan_num].scan_start_time,color="red")
plt.axvline(exp.scan_to_spectrum[exp.ms2_to_ms1[scan_num]].scan_start_time,color="blue")
# Pipes or double points not allowed by windows...
plt.savefig(os.path.join(base_path,"XIC/%s_%s.png" % (n_exp.replace("|","+"),n_lip.split("|")[0].replace(":","-"))))
plt.close()
prepared_plotting_vars.append([mz_list_entry,intens_list_entry,scan_num,n_exp,n_lip,aligned_peaks])
plot_ms2 = True
if plot_ms2:
pool = Pool(processes=8)
chunk_results = {}
for mz_list_entry,intens_list_entry,scan_num,n_exp,n_lip,aligned_peaks in prepared_plotting_vars:
chunk_results[n_exp+n_lip] = pool.apply_async(plot_aligned_spec, args = (mz_list_entry,intens_list_entry,exp.scan_to_spectrum[scan_num].mz_array,exp.scan_to_spectrum[scan_num].intensity_array,os.path.join(base_path,"MS2/%s_%s.pdf" % (n_exp.replace("|","_").replace(":","_"),n_lip.replace("|","_").replace(":","_"))),aligned_peaks))
pool.close()
pool.join()
for lower_limit_mz,cr in chunk_results.items():
cr.get()
if aggregate_results:
aggregated_results.write_res_to_file(outfile_name=os.path.join(base_path,"aggr_ranked_res.csv"))
aggregated_results.write_res_to_file(outfile_name=os.path.join(base_path,"aggr_best_res.csv"),best=True)
res,ranked_lipids = aggregated_results.get_best_ranked_score()
poss_lipids = list(set([n_lip for n_exp,n_lip in res]))
for lip_ident in poss_lipids:
get_rt_list = [float(n_exp.split("|")[-1]) for n_exp,n_lip in res if n_lip == lip_ident]
if len(get_rt_list) > 3: #and np.std(get_rt_list) < 2.5:
print(median(get_rt_list))
rt_lip_median = median([float(n_exp.split("|")[-1]) for n_exp,n_lip in res if n_lip == lip_ident])
outfile_high_confid.write("%s\t%s\n" % (lip_ident,rt_lip_median))
outfile_high_confid.flush()
type=int,
nargs='+',
default=None,
help='the patch size for prediction')
opt = parser.parse_args()
torch.manual_seed(2020)
if not torch.cuda.is_available():
opt.cuda = False
if opt.cuda:
torch.cuda.manual_seed_all(2020)
cudnn.benchmark = True
cudnn.deterministic = True
opt.patch_size = opt.image_size if opt.patch_size is None else opt.patch_size
if __name__ == '__main__':
experiment = Experiment(opt)
train_dir = opt.data_dir / 'train'
val_dir = opt.data_dir / 'val'
test_dir = val_dir
if opt.epochs > 0:
if opt.epochs > 0:
experiment.train(train_dir,
val_dir,
opt.patch_stride,
opt.batch_size,
num_workers=opt.num_workers,
epochs=opt.epochs)
experiment.test(test_dir, opt.patch_size, num_workers=opt.num_workers)
# Log the pulse to a pulse-specific log
Log(name='pulse',
pulse_on=pulse.pulse_on,
pulse_code=pulse.code,
pulse_off=pulse.pulse_off,
pulse_port=pulse.port,
pulse_width=pulse.width)
if __name__ == '__main__':
from experiment import Experiment
from state import Meanwhile, Debug
# set up default experiment
exp = Experiment()
# test running pulses whilst the rest of the experiment is going
with Meanwhile():
with Loop():
pulse = Pulse(code='S1')
Wait(duration=1.0, jitter=1.0)
Log(name='pulse',
pulse_on=pulse.pulse_on,
pulse_code=pulse.code,
pulse_off=pulse.pulse_off)
# First wait for a bit and send some pulses
Wait(10)
# print something
'choose_random_topic': args.choose_random_topic,
'save_feature': args.save_feature
}
training_args = {
'num_epochs': args.num_epochs,
'batch_size': 256,
'model_width': 1024,
'model_height': 2,
'learning_rate': 0.00025,
'clip_delta': 1.0,
'update_rule': 'deepmind_rmsprop'
}
reinforce_args = {
'steps_per_epoch': 1000,
'replay_start_size': 500,
'replay_memory_size': 10000,
'epsilon_decay': 100000,
'epsilon_min': 0.1,
'epsilon_start': 1.0,
'freeze_interval': 500,
'update_frequency': 1
}
###############################
# Run Experiment #
###############################
exp = Experiment(retrieval_args, training_args, reinforce_args)
exp.run()
###########################################################################
experiment_windows_273_150_100 = Experiment(
# Paths to tools
r"D:\git\BitFunnel\build-msvc\tools\BitFunnel\src\Release\BitFunnel.exe",
r"D:\git\mg4j-workbench",
r"/home/mhop/git/partitioned_elias_fano/bin",
# The directory containing all indexes and the basename for this index
r"D:\temp\indexes",
100,
150,
r"273-150-100",
# The directory with the gov2 chunks and the regular expression pattern
# used to determine which chunks will be used for this experiment.
r"d:\sigir\chunks-100-150",
r"GX.*", # Use all chunks
# The query log to be used for this experiment.
r"D:\sigir\queries\06.efficiency_topics.all",
# BitFunnel density
0.15,
# Min and max thread counts
8,
1,
8)
experiment_windows_273_64_127 = Experiment(
fig.show()
def plot_mae_increase(x, maes):
increase = []
for m in maes:
increase.append(((m[0] - m[1]) / m[0]) * 100)
fig = plt.figure()
plt.plot(x, increase, label='算法提升率')
plt.grid(axis="y")
plt.xlabel('用户度范围')
plt.ylabel('算法提升率')
plt.title('不同用户度范围下多准则算法相较于单准则算法mae提升率')
fig.legend()
# fig.savefig(filepath)
fig.show()
dirs = './yahoodata(interval-2)/'
readpath = os.listdir(dirs)
readpath.sort(key=lambda x: int(x.split('-')[0]))
maes = []
ks = [i for i in range(1, 22)]
for i in range(len(readpath)):
test = Experiment(no_of_criteria=5, combin_func='total_reg')
test.readDatas(dirs + readpath[i] + '/all.txt')
s_mae, m_mae = test.SVD()
maes.append((s_mae, m_mae))
plot_mae(readpath, maes)
plot_mae_increase(readpath, maes)
def main(args):
# Get name of output directory, and create it if it doesn't exist
param_str = (
"input_dim = %i, output_dim = %i, n_train = %i, t_lim = %.2f, "
"num_repeats = %i, find_best_params = %s"
% (
args.input_dim,
args.output_dim,
args.n_train,
args.t_lim,
args.n_repeats,
args.find_best_params,
)
)
current_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.join(
current_dir,
"Outputs",
"Gradient descent",
param_str,
)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
# Initialise data set
np.random.seed(6763)
sin_data = data.Sinusoidal(
args.input_dim,
args.output_dim,
args.n_train,
x_lo=-2,
x_hi=2,
freq=(1 if args.input_dim == 1 else None),
)
# Define function to be run for each experiment
def run_experiment(
num_units,
num_layers,
log10_s0,
alpha,
beta,
act_func,
max_steps,
batch_size,
batch_replace,
plot_preds=False,
):
# Initialise network and batch getter
model = models.NeuralNetwork(
input_dim=args.input_dim,
output_dim=args.output_dim,
num_hidden_units=[num_units for _ in range(num_layers)],
act_funcs=[act_func, models.activations.identity],
)
if (batch_size is None) or (batch_size >= args.n_train):
batch_getter = optimisers.batch.FullTrainingSet()
else:
batch_getter = optimisers.batch.ConstantBatchSize(
batch_size,
batch_replace,
)
# Perform gradient descent
result = optimisers.gradient_descent(
model,
sin_data,
terminator=optimisers.Terminator(t_lim=args.t_lim),
evaluator=optimisers.Evaluator(t_interval=args.t_eval),
line_search=optimisers.LineSearch(
s0=pow(10, log10_s0),
alpha=alpha,
beta=beta,
max_its=max_steps,
),
batch_getter=batch_getter,
)
# If specified, plot the final model predictions
if plot_preds:
print("Plotting final predictions...")
plotting.plot_data_predictions(
plot_name="Final predictions",
dir_name=output_dir,
dataset=sin_data,
output_dim=args.output_dim,
model=model,
)
# Return the final test error
TestError = optimisers.results.columns.TestError
final_test_error = result.get_values(TestError)[-1]
return final_test_error
# Initialise the Experiment object, and add parameters
experiment = Experiment(run_experiment, output_dir, args.n_repeats)
addp = lambda *args: experiment.add_parameter(Parameter(*args))
addp("num_units", 10, [5, 10, 15, 20] )
addp("num_layers", 1, [1, 2, 3] )
addp("log10_s0", 0, np.linspace(-1, 3, 5) )
addp("alpha", 0.5, np.linspace(0.1, 1, 9, endpoint=False) )
addp("beta", 0.5, np.linspace(0.1, 1, 9, endpoint=False) )
addp("max_steps", 10, [5, 10, 15, 20] )
addp("batch_size", 100, [25, 50, 75, 100, 150, 200, 300, None] )
addp("batch_replace", True, [True, False] )
addp(
"act_func",
models.activations.gaussian,
[
models.activations.gaussian,
models.activations.cauchy,
models.activations.logistic,
models.activations.relu,
],
)
# Call warmup function
optimisers.warmup()
# Call function to run all experiments
if args.find_best_params:
experiment.find_best_parameters()
else:
experiment.sweep_all_parameters()
# Write the results of all experiments to a text file
experiment.save_results_as_text()
# Open the output plot directory
os.system("explorer \"%s\"" % output_dir)
# Plot the predictions using the model with the optimal hyper-parameters
default_param_dict = experiment.get_default_param_dictionary()
run_experiment(**default_param_dict, plot_preds=True)
def test_memory_rnn_gradient(self):
# Data setup
memory_size = 20
word_size = 4
batch_size = 1
patch_size = 10
patch_width = memory_size + 5
sequence_length = 10
header = ExperimentHeader(
params={
"word_size": word_size,
"memory_size": memory_size,
"patch_size": patch_size,
"patch_width": patch_width
})
experiment = Experiment("test_memory_cell", header, Args(batch_size))
pb = NTMBase(experiment)
patch = Input((patch_size, patch_width), name="patch")
memory_tm1 = Input((memory_size, word_size), name="memory")
memory_t = memory_tm1
flat_patch = Reshape((patch_size * patch_width, ))(patch)
write_word = Dense(word_size)(flat_patch)
erase_word = Dense(word_size)(flat_patch)
ptr = Dense(patch_size)(flat_patch)
address = pb.resolve_address(ptr, patch)
memory_t = pb.erase(memory_t, address, erase_word)
ptr = Dense(patch_size)(flat_patch)
address = pb.resolve_address(ptr, patch)
memory_t = pb.write(memory_t, address, write_word)
ptr = Dense(patch_size)(flat_patch)
address = pb.resolve_address(ptr, patch)
read = pb.read(memory_t, address)
out = Dense(3)(read)
rnn = RecurrentModel(input=patch,
output=out,
initial_states=[memory_tm1],
final_states=[memory_t])
a = Input((sequence_length, patch_size, patch_width), name="patch_seq")
b = rnn(a)
model = Model(a, b)
model.compile(loss='mse', optimizer='sgd')
model.fit(
{
"patch_seq":
np.random.random(
(batch_size, sequence_length, patch_size, patch_width)),
# "memory": np.random.random((batch_size, memory_size, word_size)),
},
np.random.random((batch_size, 3)))
model.predict({
"patch_seq":
np.zeros((batch_size, sequence_length, patch_size, patch_width)),
# "memory": np.zeros((batch_size, memory_size, word_size)),
})
