def search_with_vanguard(options):
"""maintains a list of extended_graphs. Each extended graph is mutated,
and the best {shape[0]} mutants are kept. We maintain a tree structure of nested lists of graphs.
This structure is called the 'vanguard,' the lowest-level lists are called 'platoons.'
We 'retreat' the vanguard so that no more than {shape[1]} levels are present.
This allows us to use a form of backtracking, which mitigates the loss of genetic diversity
encountered when we mutate the vertices one-at-a-time.
options = branch_factor, meta_pop, pop_per_mu, iterations_per_mu,
elite_percent, crossover_percent, meta_elite_percent, make_unique,meta_select_proc
"""
pop_size = options["meta_pop"]
#independence_number =3
#g = ExtendedGraph([(i,i+1) for i in range(4)] + [(4,0),(5,4),(5,2)])
g = options["start_graph"]
pop = [g]
mutation_options = {
"branch_factor": options["branch_factor"],
"pop_per_mu": options["pop_per_mu"],
"iterations_per_mu": options["iterations_per_mu"],
"elite_percent": options["elite_percent"],
"crossover_percent": options["crossover_percent"]
}
genetic_alg1 = GA(FUN.fit,
curry_add_vertex_and_mutate(mutation_options),
None,
0.0,
options["meta_elite_percent"],
pop_size=options["meta_pop"],
make_unique=options["make_unique"])
#pop = [g]
results = genetic_alg1.run(pop, 31, meta_select_proc=True)
print([FUN.fit(r) for r in results])
#return sorted(results, key=FUN.fit, reverse = True)[0]
return results[0]
def main():
start_time = time()
print("---------- main1 --------------")
f0 = gzip.open('/home/luca/data/mnist/train-images-idx3-ubyte.gz', 'r')
f1 = gzip.open('/home/luca/data/mnist/t10k-images-idx3-ubyte.gz', 'r')
l0 = gzip.open('/home/luca/data/mnist/train-labels-idx1-ubyte.gz', 'r')
l1 = gzip.open('/home/luca/data/mnist/t10k-labels-idx1-ubyte.gz', 'r')
X_train = np.frombuffer(f0.read(), dtype=np.uint8,
offset=16).reshape(-1, 28 * 28)
X_test = np.frombuffer(f1.read(), dtype=np.uint8,
offset=16).reshape(-1, 28 * 28)
y_train = np.frombuffer(l0.read(), dtype=np.uint8, offset=8)
y_test = np.frombuffer(l1.read(), dtype=np.uint8, offset=8)
y_train = one_hot_encoding(y_train)
y_label = one_hot_encoding(y_test)
mean = np.mean(X_train)
std = np.std(X_train)
X_train, X_test = X_train - mean, X_test - mean
X_train, X_test = X_train / std, X_test / std
model = neural_network((89, 'TanH'), (10, 'Sigmoid'),
input_nodes=784,
seed=20190119)
model = fit(x_train=X_train,
y_train=y_train,
x_test=X_test,
y_test=y_label,
model=model,
optimizer=sgd(epochs=50,
eta=0.35,
etaN=0.15,
decay_type='exponential'),
batch_size=60,
eval_every=5,
early_stop=True,
seed=20190119)
validate_accuracy(x_test=X_test, y_test=y_test, model=model)
# print(model[0][0][0].shape)
# print(np.sum(model[0][0][0]))
# print(model[0][0][1].shape)
# print(np.sum(model[0][0][1]))
#
# print(model[1][0][0].shape)
# print(np.sum(model[1][0][0]))
# print(model[1][0][1].shape)
# print(np.sum(model[1][0][1]))
# print()
print("--- %s seconds ---" % (time() - start_time))
def main():
start_time = time()
print("---------- main5 --------------")
f0 = gzip.open('/home/luca/data/mnist/train-images-idx3-ubyte.gz', 'r')
f1 = gzip.open('/home/luca/data/mnist/t10k-images-idx3-ubyte.gz', 'r')
l0 = gzip.open('/home/luca/data/mnist/train-labels-idx1-ubyte.gz', 'r')
l1 = gzip.open('/home/luca/data/mnist/t10k-labels-idx1-ubyte.gz', 'r')
X_train = np.frombuffer(f0.read(), dtype=np.uint8,
offset=16).reshape(-1, 28 * 28)
X_test = np.frombuffer(f1.read(), dtype=np.uint8,
offset=16).reshape(-1, 28 * 28)
y_train = np.frombuffer(l0.read(), dtype=np.uint8, offset=8)
y_test = np.frombuffer(l1.read(), dtype=np.uint8, offset=8)
y_train = one_hot_encoding(y_train)
y_label = one_hot_encoding(y_test)
mean = np.mean(X_train)
std = np.std(X_train)
X_train, X_test = X_train - mean, X_test - mean
X_train, X_test = X_train / std, X_test / std
model = neural_network((89, 'TanH'), (10, 'Softmax'),
input_nodes=784,
seed=20190119,
weight_init='scaled')
model = fit(x_train=X_train,
y_train=y_train,
x_test=X_test,
y_test=y_label,
model=model,
optimizer=sgd(epochs=50,
eta=0.15,
etaN=0.05,
decay_type='exponential',
beta=0.85),
batch_size=60,
eval_every=5,
early_stop=True,
loss_function='cross-entropy',
seed=20190119,
dropout=0.8)
validate_accuracy(x_test=X_test, y_test=y_test, model=model)
print("--- %s seconds ---" % (time() - start_time))
import matplotlib.pyplot as plt
filepath = r"data/pict.dat"
pictures_matrix = ld.get_pictures(filepath)
# for i in range(pictures_matrix.shape[0]):
# plt.imshow(pictures_matrix[i])
# plt.show()
number_training = 3
number_of_distorted = 2
training = np.zeros(
(number_training, pictures_matrix.shape[1] * pictures_matrix.shape[2]))
for i in range(number_training):
training[i] = np.ravel(pictures_matrix[i])
weights = fun.fit(training)
#Taking distorted patterns
distorted = np.zeros(
(number_of_distorted, pictures_matrix.shape[1] * pictures_matrix.shape[2]))
distorted[0] = np.ravel(pictures_matrix[9])
distorted[1] = np.ravel(pictures_matrix[10])
#Energy at attractors
print('Energy of Attractors')
for i in range(number_training):
energy = fun.compute_lyapunov_energy(training[i].reshape(1, -1), weights)
print(energy)
print('-----------')
#Energy at distorted patterns
os.chdir(workingPath)
ne = meanValues.getNumberOfEvents()
time = meanValues.getTimes()
T1 = 1 / (kb * temperatures)
command = "ne/time"
y = eval(command)
plt.ylabel(command)
command = "1/kb/temperatures" #+np.log(flux**2.5)"
x = eval(command)
plt.xlabel(command)
try:
plt.semilogy(x, y, ".", label=folder + " " + str(j))
if hex:
a, b = f.fit(x, y, 0, 12)
plt.semilogy(x, f.exp(x, a, b), label="fit low " + str(b))
a, b = f.fit(x, y, 12, 17)
plt.semilogy(x, f.exp(x, a, b), label="fit middle " + str(b))
a, b = f.fit(x, y, 17, 30)
plt.semilogy(x, f.exp(x, a, b), label="fit middle " + str(b))
plt.ylim(1e9, 1e14)
else:
a, b = f.fit(x, y, 0, 8)
plt.semilogy(x, f.exp(x, a, b), label="fit low " + str(b))
a, b = f.fit(x, y, 8, 16)
plt.semilogy(x, f.exp(x, a, b), label="fit middle " + str(b))
a, b = f.fit(x, y, 17, 27)
plt.semilogy(x, f.exp(x, a, b), label="fit high " + str(b))
plt.legend(loc='best', prop={'size': 6})
plt.savefig("ne.png")
def __call__(self):
pandda_start_time = time.time()
working_phil = parse_phil_args(master_phil=pandda_phil,
args=self.args,
blank_arg_prepend=None,
home_scope=None)
# # TODO: remove
# print("printing welcome")
# sys.stdout.flush()
# # welcome()
#
# # TODO: remove
# print("getting phil")
# sys.stdout.flush()
# p = working_phil.extract()
#
# # TODO: remove
# print("making directories")
# sys.stdout.flush()
# out_dir = easy_directory(os.path.abspath(p.pandda.output.out_dir))
# _log_dir = easy_directory(os.path.join(out_dir, 'logs'))
# # TODO: remove
# print("made directories")
# sys.stdout.flush()
#
# _log_filename = 'pandda-{}.log'.format(time.strftime("%Y-%m-%d-%H%M", time.gmtime()))
#
# # TODO: remove
# print("got log fileename")
# sys.stdout.flush()
# _def_filename = _log_filename.replace('.log', '.def')
# _eff_filename = _log_filename.replace('.log', '.eff')
#
# # TODO: remove
# print("makeing log")
# sys.stdout.flush()
# log = Log(log_file=os.path.join(_log_dir, _log_filename), verbose=p.settings.verbose)
#
# # TODO: remove
# print("args processor")
# sys.stdout.flush()
# pandda_arg_processor = PanddaArgsProcessor(log, p.pandda)
# args = pandda_arg_processor()
# params = args.params
# settings = p.settings
# # TODO: remove
# print("got settings")
# sys.stdout.flush()
#
# pickled_dataset_meta = Meta({'number_of_datasets': 0, 'dataset_labels': [], 'dataset_pickle_list': []})
################################################################################################################
# Maps options to code abstractions
pandda_config = PanDDAConfig(working_phil)
# Get Dataset
pandda_dataset = mdc.dataset.dataset.MultiCrystalDataset(
dataloader=pandda_config.dataloader,
sample_loader=pandda_config.sample_loader)
reference = pandda_config.get_reference(pandda_dataset.datasets)
pandda_dataset.sample_loader.reference = reference
# transform dataset
dataset = chain(pandda_dataset, [
pandda_config.check_data, pandda_config.scale_diffraction,
pandda_config.filter_structure, pandda_config.filter_wilson,
pandda_config.align
])
# Get grid
grid = pandda_config.get_grid(reference)
# Get file tree
tree = pandda_config.pandda_output(dataset)
# Define event Model
pandda_event_model = PanDDAEventModel(
pandda_config.statistical_model,
pandda_config.clusterer,
pandda_config.event_finder,
bdc_calculator=pandda_config.bdc_calculator,
statistics=[],
map_maker=pandda_config.map_maker,
event_table_maker=pandda_config.event_table_maker,
cpus=pandda_config["args"]["cpus"],
tree=tree)
# Get partitions
partitions = pandda_config.partitioner(dataset)
# instatiate a dataloader for the datasets
# dataloader = DefaultPanDDADataloader(min_train_datasets=60,
# max_test_datasets=60)
# Get the datasets to iterate over
ds = [(idx, d) for idx, d in pandda_config.dataloader(dataset)]
# Iterate over resolution shells
for shell_num, shell_dataset in ds:
client = get_client()
# ###############################################
# Get resolution
# ###############################################
resolutions_test = max([
dts.data.summary.high_res for dtag, dts in
shell_dataset.partition_datasets("test").items()
])
resolutions_train = max([
dts.data.summary.high_res for dtag, dts in
shell_dataset.partition_datasets("train").items()
])
max_res = max(resolutions_test, resolutions_train)
# ###############################################
# Instantiate sheel variable names
# ###############################################
# Dataset names
dtags = set(
shell_dataset.partition_datasets("test").keys() +
shell_dataset.partition_datasets("train").keys())
dask_dtags = {
"{}".format(dtag.replace("-", "_")): dtag
for dtag in dtags
}
train_dtags = [
dtag for dtag in dask_dtags
if (dask_dtags[dtag] in shell_dataset.partition_datasets(
"train").keys())
]
test_dtags = [
dtag for dtag in dask_dtags
if (dask_dtags[dtag] in shell_dataset.partition_datasets(
"test").keys())
]
# ###############################################
# Truncate datasets
# ###############################################
# TODO: move to imports section
truncated_reference, truncated_datasets = PanddaDiffractionDataTruncater(
)(shell_dataset.datasets, reference)
# ###############################################
# Load computed variables into dask
# ###############################################
# Rename trucnated datasets
for ddtag, dtag in dask_dtags.items():
truncated_datasets[ddtag] = truncated_datasets[dtag]
# record max res of shell datasets
shell_max_res = max_res
# ###############################################
# Generate maps
# ###############################################
# Generate reference map for shell
shell_ref_map = delayed(get_reference_map)(
pandda_config.reference_map_getter, reference, shell_max_res,
grid)
# Load maps
xmaps = {}
for dtag in dask_dtags:
xmaps[dtag] = delayed(load_sample)(pandda_config.map_loader,
truncated_datasets[dtag],
grid, shell_ref_map,
shell_max_res)
# ###############################################
# Fit statistical model to trianing sets
# ###############################################
xmaps_persisted_futures = client.persist(
[xmaps[dtag] for dtag in dask_dtags])
xmaps_computed = {
dtag: client.compute(xmaps_persisted_futures[i]).result()
for i, dtag in enumerate(dask_dtags)
}
shell_fit_model = fit(
pandda_config.statistical_model,
[xmaps_computed[dtag] for dtag in train_dtags],
[xmaps_computed[dtag] for dtag in test_dtags])
shell_fit_model_scattered = client.scatter(shell_fit_model)
xmaps_scattered = client.scatter(
[xmaps_computed[dtag] for dtag in dask_dtags])
xmaps_scattered_dict = {
dtag: xmaps_scattered[i]
for i, dtag in enumerate(dask_dtags)
}
grid_scattered = client.scatter(grid)
# ###############################################
# Find events
# ###############################################
zmaps = {}
clusters = {}
events = {}
bdcs = {}
for dtag in dask_dtags:
# Get z maps by evaluating model on maps
zmaps[dtag] = delayed(evaluate_model)(
shell_fit_model_scattered, xmaps_scattered_dict[dtag])
# Cluster outlying points in z maps
clusters[dtag] = delayed(cluster_outliers)(
pandda_config.clusterer, truncated_datasets[dtag],
zmaps[dtag], grid_scattered)
# Find events by filtering the clusters
events[dtag] = delayed(filter_clusters)(
pandda_config.event_finder, truncated_datasets[dtag],
clusters[dtag], grid_scattered)
events_persisted_futures = client.persist(
[events[dtag] for dtag in dask_dtags])
events_computed = {
dtag: client.compute(events_persisted_futures[i]).result()
for i, dtag in enumerate(dask_dtags)
}
events_scattered = client.scatter(
[events_computed[dtag] for dtag in dask_dtags])
events_scattered_dict = {
dtag: xmaps_scattered[i]
for i, dtag in enumerate(dask_dtags)
}
# Calculate background correction factors
for dtag in dask_dtags:
bdcs[dtag] = delayed(estimate_bdcs)(
pandda_config.bdc_calculator, truncated_datasets[dtag],
xmaps_scattered_dict[dtag], shell_ref_map, events[dtag],
grid_scattered)
# Criticise each indiidual dataset (generate statistics, event map and event table)
event_maps = {}
for dtag in dask_dtags:
event_maps[dtag] = delayed(make_event_map)(
pandda_config.map_maker, tree, pandda_config.map_loader,
truncated_datasets[dtag], shell_ref_map, events[dtag],
bdcs[dtag])
event_maps_persisted_futures = client.persist(
[event_maps[dtag] for dtag in dask_dtags])
event_maps_computed = {
dtag: client.compute(event_maps_persisted_futures[i]).result()
for i, dtag in enumerate(dask_dtags)
}
shell_maps = delayed(make_shell_maps)(pandda_config.map_maker,
tree, shell_num, reference,
shell_ref_map)
shell_maps_persisted_futures = client.persist(shell_maps)
shell_maps_computed = shell_maps_persisted_futures.result()
event_table = delayed(make_event_table)(
pandda_config.event_table_maker, tree, shell_num,
shell_dataset, events_computed)
event_table_persisted_future = client.persist(event_table)
event_table_computed = event_table_persisted_future.result()
client.close()
#%% Rosenbluth
"""
Simulating the polymer with the Rosenbluth algorithm
"""
start_time = time.clock()
R_save, distance_save, w_save, W_save = f.Iterative_simulation(
Group_size, N, epsilon, sig, R_save, distance_save, w_save, W_save)
print('rosenbluth')
print(time.clock() - start_time)
P_save = np.cumprod(W_save[0, :, :], 0)
Rpol_avg = f.modify(P_save, Rpol_avg, N, Group_size, distance_save)
sigma = f.sigma(N, P_save, Group_size, distance_save**2)
a = f.fit(N, Rpol_avg[:, 0]**2)
fit = (a[0] * (np.linspace(1, N, N) - 1)**0.75)**2
pf.chain(R_save[:, :, 2], 5)
fignr = 1
pf.length(Rpol_avg, N, sigma, fit, fignr, "Rosenbluth")
#%% Perm
"""
Simulating the polymer with the PERM algorithm
"""
# R_perm = np.zeros((2,N,1))
# W_perm = np.zeros((N-2,1))
# P_perm = np.zeros((N-2,1))