def run_all(fiducial, simulation_runs, face, statistics, savename,
multicore=True, ncores=10, verbose=True):
fiducial_timesteps = np.sort([os.path.join(fiducial, x)
for x in os.listdir(fiducial)
if os.path.isdir(os.path.join(fiducial, x))])
timesteps_labels = [x[-8:] for x in fiducial_timesteps]
if verbose:
print "Simulation runs to be analyzed: %s" % (simulation_runs)
print "Started at "+str(datetime.now())
# Distances will be stored in an array of dimensions
# # statistics x # sim runs x # timesteps
# The +1 in the second dimensions is to include the
# fiducial case against itself.
distances_storage = np.zeros((len(statistics),
len(simulation_runs),
len(fiducial_timesteps)))
for i, run in enumerate(simulation_runs):
timesteps = np.sort([os.path.join(run, x) for x in os.listdir(run)
if os.path.isdir(os.path.join(run, x))])
if verbose:
print "On Simulation %s/%s" % (i+1, len(simulation_runs))
print str(datetime.now())
if multicore:
pool = Pool(processes=ncores)
distances = pool.map(single_input, izip(fiducial_timesteps,
timesteps,
repeat(statistics)))
pool.close()
pool.join()
distances_storage[:, i, :] = \
sort_distances(statistics, distances).T
else:
for ii, timestep in enumerate(timesteps):
fiducial_dataset = fromfits(fiducial_timesteps[ii], keywords)
testing_dataset = fromfits(timestep, keywords)
if i == 0:
distances, fiducial_models = \
stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = \
stats_wrapper(fiducial_dataset, testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics)
distances = [distances]
distances_storage[:, i, ii:ii+1] = \
sort_distances(statistics, distances).T
return distances_storage, timesteps_labels
def timestep_wrapper(fiducial_timestep, testing_timestep, statistics,
add_noise, rms_noise):
# Derive the property arrays assuming uniform noise (for sims)
fiducial_dataset = load_and_reduce(fiducial_timestep,
add_noise=add_noise,
rms_noise=rms_noise)
testing_dataset = load_and_reduce(testing_timestep,
add_noise=add_noise,
rms_noise=rms_noise)
if add_noise:
vca_break = 1.5
vcs_break = -0.5
else:
vca_break = None
vcs_break = -0.8
distances = stats_wrapper(fiducial_dataset,
testing_dataset,
statistics=statistics,
multicore=True,
vca_break=vca_break,
vcs_break=vcs_break)
return distances
def timestep_wrapper(files_list,
pos,
statistics,
noise=False,
rms_noise=0.001):
pos1, pos2 = pos
if files_list[pos1] == None or files_list[pos2] == None:
return None, pos1, pos2
print "On " + str(datetime.now()) + " running %s %s" % (pos1, pos2)
print "Files: %s %s" % (files_list[pos1], files_list[pos2])
# Derive the property arrays assuming uniform noise (for sims)
dataset1 = load_and_reduce(files_list[pos1],
add_noise=noise,
rms_noise=rms_noise)
dataset2 = load_and_reduce(files_list[pos2],
add_noise=noise,
rms_noise=rms_noise)
distances = stats_wrapper(dataset1,
dataset2,
statistics=statistics,
multicore=True)
return distances, pos1, pos2
def run_all(fiducial,
simulation_runs,
face,
statistics,
savename,
multicore=True,
ncores=10,
verbose=True):
if verbose:
print "Simulation runs to be analyzed: %s" % (simulation_runs)
print "Started at " + str(datetime.now())
# Distances will be stored in an array of dimensions
# # statistics x # sim runs x # timesteps
# The +1 in the second dimensions is to include the
# fiducial case against itself.
distances_storage = np.zeros((len(statistics), len(simulation_runs)))
if multicore:
pool = Pool(processes=ncores)
distances = pool.map(
single_input,
izip(repeat(fiducial), simulation_runs, repeat(statistics)))
pool.close()
pool.join()
distances_storage = sort_distances(statistics, distances).T
else:
for i, design in enumerate(simulation_runs):
fiducial_dataset = fromfits(fiducial, keywords)
testing_dataset = fromfits(design, keywords)
if i == 0:
distances, fiducial_models = \
stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = \
stats_wrapper(fiducial_dataset, testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics)
distances = [distances]
distances_storage[:, i:i+1] = \
sort_distances(statistics, distances).T
return distances_storage
def timestep_wrapper(fiducial_timestep, testing_timestep, statistics):
fiducial_dataset = fromfits(fiducial_timestep, keywords)
testing_dataset = fromfits(testing_timestep, keywords)
distances = stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics, multicore=True,
filenames=[fiducial_timestep, testing_timestep])
return distances
def timestep_wrapper(fiducial, design, statistics):
fiducial_dataset = fromfits(fiducial, keywords)
design_dataset = fromfits(design, keywords)
distances = stats_wrapper(fiducial_dataset, design_dataset,
statistics=statistics, multicore=True,
filenames=[fiducial, design])
return distances
def run_all(fiducial, simulation_runs, face, statistics, savename,
multicore=True, ncores=10, verbose=True):
if verbose:
print "Simulation runs to be analyzed: %s" % (simulation_runs)
print "Started at "+str(datetime.now())
# Distances will be stored in an array of dimensions
# # statistics x # sim runs x # timesteps
# The +1 in the second dimensions is to include the
# fiducial case against itself.
distances_storage = np.zeros((len(statistics),
len(simulation_runs)))
if multicore:
pool = Pool(processes=ncores)
distances = pool.map(single_input, izip(repeat(fiducial),
simulation_runs,
repeat(statistics)))
pool.close()
pool.join()
distances_storage = sort_distances(statistics, distances).T
else:
for i, design in enumerate(simulation_runs):
fiducial_dataset = fromfits(fiducial, keywords)
testing_dataset = fromfits(design, keywords)
if i == 0:
distances, fiducial_models = \
stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = \
stats_wrapper(fiducial_dataset, testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics)
distances = [distances]
distances_storage[:, i:i+1] = \
sort_distances(statistics, distances).T
return distances_storage
def timestep_wrapper(fiducial, design, statistics):
fiducial_dataset = fromfits(fiducial, keywords)
design_dataset = fromfits(design, keywords)
distances = stats_wrapper(fiducial_dataset,
design_dataset,
statistics=statistics,
multicore=True,
filenames=[fiducial, design])
return distances
def timestep_wrapper(fid_files, des_files, pos, statistics, noise=False,
rms_noise=0.001):
pos1, pos2 = pos
# Derive the property arrays assuming uniform noise (for sims)
dataset1 = load_and_reduce(fid_files[pos1], add_noise=noise,
rms_noise=rms_noise)
dataset2 = load_and_reduce(des_files[pos2], add_noise=noise,
rms_noise=rms_noise)
distances = stats_wrapper(dataset1, dataset2,
statistics=statistics, multicore=True)
return distances, pos1, pos2
def timestep_wrapper(fiducial_timestep, testing_timestep, statistics, noise_added):
# Derive the property arrays assuming uniform noise (for sims)
fiducial_dataset = load_and_reduce(fiducial_timestep)
testing_dataset = load_and_reduce(testing_timestep)
if noise_added:
vca_break = 1.5
vcs_break = -0.5
else:
vca_break = None
vcs_break = -0.8
distances = stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics, multicore=True,
vca_break=vca_break, vcs_break=vcs_break)
return distances
def timestep_wrapper(files_list, pos, statistics, noise=False,
rms_noise=0.001):
pos1, pos2 = pos
if files_list[pos1] == None or files_list[pos2] == None:
return None, pos1, pos2
print "On "+str(datetime.now())+" running %s %s" % (pos1, pos2)
print "Files: %s %s" % (files_list[pos1], files_list[pos2])
# Derive the property arrays assuming uniform noise (for sims)
dataset1 = load_and_reduce(files_list[pos1], add_noise=noise,
rms_noise=rms_noise)
dataset2 = load_and_reduce(files_list[pos2], add_noise=noise,
rms_noise=rms_noise)
distances = stats_wrapper(dataset1, dataset2,
statistics=statistics, multicore=True)
return distances, pos1, pos2
def run_comparison(fits, statistics, add_noise):
fits1, fits2 = fits
# Derive the property arrays assuming uniform noise (for sims)
fiducial_dataset = load_and_reduce(fits1)
testing_dataset = load_and_reduce(fits2)
if add_noise:
vca_break = 1.5
vcs_break = -0.5
else:
vca_break = None
vcs_break = -0.8
distances = stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics, multicore=True,
vca_break=vca_break, vcs_break=vcs_break)
return distances, fits1, fits2
def run_all(fiducial, simulation_runs, statistics, savename,
pool=None, verbose=True,
multi_timesteps=False, add_noise=False, rms_noise=0.001):
'''
Given a fiducial set and a series of sets to compare to, loop
through and compare all sets and their time steps. Return an array of
the distances.
Parameters
----------
verbose : bool, optional
Prints out the time when completing a set.
multi_timesteps : bool, optional
If multiple timesteps are given for each simulation run, parallelize
over the timesteps. If only one is given, parallelize over the
simulation runs.
'''
if verbose:
# print "Simulation runs to be analyzed: %s" % (simulation_runs)
print "Started at "+str(datetime.now())
if multi_timesteps:
# Distances will be stored in an array of dimensions
# # statistics x # sim runs x # timesteps
distances_storage = np.zeros((len(statistics),
len(simulation_runs),
len(fiducial)))
print distances_storage.shape
for i, key in enumerate(simulation_runs.keys()):
timesteps = simulation_runs[key]
if verbose:
print "On Simulation %s/%s" % (i+1, len(simulation_runs))
print str(datetime.now())
if pool is not None:
distances = pool.map(single_input, zip(fiducial,
timesteps,
repeat(statistics),
repeat(add_noise),
repeat(rms_noise)))
# If there aren't the maximum number of timesteps, pad the
# output to match the max.
if len(distances) < len(fiducial):
diff = len(fiducial) - len(distances)
for d in range(diff):
distances.append(dict.fromkeys(statistics, np.NaN))
distances_storage[:, i, :] = \
sort_distances(statistics, distances).T
else:
for ii, timestep in enumerate(timesteps):
fiducial_dataset = load_and_reduce(fiducial[ii],
add_noise=add_noise,
rms_noise=rms_noise)
testing_dataset = load_and_reduce(timestep,
add_noise=add_noise,
rms_noise=rms_noise)
if i == 0:
distances, fiducial_models = \
stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = \
stats_wrapper(fiducial_dataset, testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics)
distances = [distances]
distances_storage[:, i, ii:ii+1] = \
sort_distances(statistics, distances).T
else:
distances_storage = np.zeros((len(statistics),
len(simulation_runs)))
if pool is not None:
# print zip(repeat(fiducial), simulation_runs.values(),
# repeat(statistics))
# print blah
distances = pool.map(single_input, zip(repeat(fiducial),
simulation_runs.values(),
repeat(statistics),
repeat(add_noise),
repeat(rms_noise)))
distances_storage = sort_distances(statistics, distances).T
else:
# Load the fiducial in
fiducial_dataset = load_and_reduce(fiducial, add_noise=add_noise,
rms_noise=rms_noise)
for i, key in enumerate(simulation_runs.keys()):
testing_dataset = \
load_and_reduce(simulation_runs[key][comp_face],
add_noise=add_noise,
rms_noise=rms_noise)
if i == 0:
distances, fiducial_models = \
stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = \
stats_wrapper(fiducial_dataset, testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics)
distances = [distances]
distances_storage[:, i:i+1] = \
sort_distances(statistics, distances).T
return distances_storage
def run_all(fiducial,
simulation_runs,
statistics,
savename,
pool=None,
verbose=True,
multi_timesteps=False,
add_noise=False,
rms_noise=0.001):
'''
Given a fiducial set and a series of sets to compare to, loop
through and compare all sets and their time steps. Return an array of
the distances.
Parameters
----------
verbose : bool, optional
Prints out the time when completing a set.
multi_timesteps : bool, optional
If multiple timesteps are given for each simulation run, parallelize
over the timesteps. If only one is given, parallelize over the
simulation runs.
'''
if verbose:
# print "Simulation runs to be analyzed: %s" % (simulation_runs)
print "Started at " + str(datetime.now())
if multi_timesteps:
# Distances will be stored in an array of dimensions
# # statistics x # sim runs x # timesteps
distances_storage = np.zeros(
(len(statistics), len(simulation_runs), len(fiducial)))
print distances_storage.shape
for i, key in enumerate(simulation_runs.keys()):
timesteps = simulation_runs[key]
if verbose:
print "On Simulation %s/%s" % (i + 1, len(simulation_runs))
print str(datetime.now())
if pool is not None:
distances = pool.map(
single_input,
zip(fiducial, timesteps, repeat(statistics),
repeat(add_noise), repeat(rms_noise)))
# If there aren't the maximum number of timesteps, pad the
# output to match the max.
if len(distances) < len(fiducial):
diff = len(fiducial) - len(distances)
for d in range(diff):
distances.append(dict.fromkeys(statistics, np.NaN))
distances_storage[:, i, :] = \
sort_distances(statistics, distances).T
else:
for ii, timestep in enumerate(timesteps):
fiducial_dataset = load_and_reduce(fiducial[ii],
add_noise=add_noise,
rms_noise=rms_noise)
testing_dataset = load_and_reduce(timestep,
add_noise=add_noise,
rms_noise=rms_noise)
if i == 0:
distances, fiducial_models = \
stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = \
stats_wrapper(fiducial_dataset, testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics)
distances = [distances]
distances_storage[:, i, ii:ii+1] = \
sort_distances(statistics, distances).T
else:
distances_storage = np.zeros((len(statistics), len(simulation_runs)))
if pool is not None:
# print zip(repeat(fiducial), simulation_runs.values(),
# repeat(statistics))
# print blah
distances = pool.map(
single_input,
zip(repeat(fiducial), simulation_runs.values(),
repeat(statistics), repeat(add_noise), repeat(rms_noise)))
distances_storage = sort_distances(statistics, distances).T
else:
# Load the fiducial in
fiducial_dataset = load_and_reduce(fiducial,
add_noise=add_noise,
rms_noise=rms_noise)
for i, key in enumerate(simulation_runs.keys()):
testing_dataset = \
load_and_reduce(simulation_runs[key][comp_face],
add_noise=add_noise,
rms_noise=rms_noise)
if i == 0:
distances, fiducial_models = \
stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = \
stats_wrapper(fiducial_dataset, testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics)
distances = [distances]
distances_storage[:, i:i+1] = \
sort_distances(statistics, distances).T
return distances_storage
def run_all(fiducial, simulation_runs, face, statistics, savename,
multicore=True, ncores=10, verbose=True, comp_face=None,
multi_timesteps=False):
'''
Given a fiducial set and a series of sets to compare to, loop
through and compare all sets and their timesteps. Return an array of
the distances.
Parameters
----------
verbose : bool, optional
Prints out the time when completing a set.
comp_face : int or None, optional
Face to compare to. If None, this is set to the fiducial face.
multi_timesteps : bool, optional
If multiple timesteps are given for each simulation run, parallelize
over the timesteps. If only one is given, parallelize over the
simulation runs.
'''
if comp_face is None:
comp_face = face
fiducial_timesteps = fiducial[face]
if verbose:
# print "Simulation runs to be analyzed: %s" % (simulation_runs)
print "Started at "+str(datetime.now())
if multi_timesteps:
# Distances will be stored in an array of dimensions
# # statistics x # sim runs x # timesteps
distances_storage = np.zeros((len(statistics),
len(simulation_runs),
len(fiducial_timesteps)))
print distances_storage.shape
for i, key in enumerate(simulation_runs.keys()):
timesteps = simulation_runs[key][comp_face]
if verbose:
print "On Simulation %s/%s" % (i+1, len(simulation_runs))
print str(datetime.now())
if multicore:
pool = Pool(processes=ncores)
distances = pool.map(single_input, izip(fiducial_timesteps,
timesteps,
repeat(statistics)))
pool.close()
pool.join()
distances_storage[:, i, :] = \
sort_distances(statistics, distances).T
else:
for ii, timestep in enumerate(timesteps):
fiducial_dataset = load_and_reduce(fiducial_timesteps[ii])
testing_dataset = load_and_reduce(timestep)
if i == 0:
distances, fiducial_models = \
stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = \
stats_wrapper(fiducial_dataset, testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics)
distances = [distances]
distances_storage[:, i, ii:ii+1] = \
sort_distances(statistics, distances).T
else:
distances_storage = np.zeros((len(statistics),
len(simulation_runs)))
if multicore:
pool = Pool(processes=ncores)
distances = pool.map(single_input, izip(repeat(fiducial),
simulation_runs,
repeat(statistics)))
pool.close()
pool.join()
distances_storage = sort_distances(statistics, distances).T
else:
for i, key in enumerate(simulation_runs.keys()):
fiducial_dataset = load_and_reduce(fiducial[face])
testing_dataset = \
load_and_reduce(simulation_runs[key][comp_face])
if i == 0:
distances, fiducial_models = \
stats_wrapper(fiducial_dataset, testing_dataset,
statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = \
stats_wrapper(fiducial_dataset, testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics)
distances = [distances]
distances_storage[:, i:i+1] = \
sort_distances(statistics, distances).T
return distances_storage
def run_all(
fiducial,
simulation_runs,
face,
statistics,
savename,
multicore=True,
ncores=10,
verbose=True,
comp_face=None,
multi_timesteps=False,
):
"""
Given a fiducial set and a series of sets to compare to, loop
through and compare all sets and their timesteps. Return an array of
the distances.
Parameters
----------
verbose : bool, optional
Prints out the time when completing a set.
comp_face : int or None, optional
Face to compare to. If None, this is set to the fiducial face.
multi_timesteps : bool, optional
If multiple timesteps are given for each simulation run, parallelize
over the timesteps. If only one is given, parallelize over the
simulation runs.
"""
if comp_face is None:
comp_face = face
fiducial_timesteps = fiducial[face]
if verbose:
# print "Simulation runs to be analyzed: %s" % (simulation_runs)
print "Started at " + str(datetime.now())
if multi_timesteps:
# Distances will be stored in an array of dimensions
# # statistics x # sim runs x # timesteps
distances_storage = np.zeros((len(statistics), len(simulation_runs), len(fiducial_timesteps)))
print distances_storage.shape
for i, key in enumerate(simulation_runs.keys()):
timesteps = simulation_runs[key][comp_face]
if verbose:
print "On Simulation %s/%s" % (i + 1, len(simulation_runs))
print str(datetime.now())
if multicore:
pool = Pool(processes=ncores)
distances = pool.map(single_input, izip(fiducial_timesteps, timesteps, repeat(statistics)))
pool.close()
pool.join()
distances_storage[:, i, :] = sort_distances(statistics, distances).T
else:
for ii, timestep in enumerate(timesteps):
fiducial_dataset = load_and_reduce(fiducial_timesteps[ii])
testing_dataset = load_and_reduce(timestep)
if i == 0:
distances, fiducial_models = stats_wrapper(
fiducial_dataset, testing_dataset, statistics=statistics
)
all_fiducial_models = fiducial_models
else:
distances = stats_wrapper(
fiducial_dataset,
testing_dataset,
fiducial_models=all_fiducial_models,
statistics=statistics,
)
distances = [distances]
distances_storage[:, i, ii : ii + 1] = sort_distances(statistics, distances).T
else:
distances_storage = np.zeros((len(statistics), len(simulation_runs)))
if multicore:
pool = Pool(processes=ncores)
distances = pool.map(single_input, izip(repeat(fiducial), simulation_runs, repeat(statistics)))
pool.close()
pool.join()
distances_storage = sort_distances(statistics, distances).T
else:
for i, key in enumerate(simulation_runs.keys()):
fiducial_dataset = load_and_reduce(fiducial[face])
testing_dataset = load_and_reduce(simulation_runs[key][comp_face])
if i == 0:
distances, fiducial_models = stats_wrapper(fiducial_dataset, testing_dataset, statistics=statistics)
all_fiducial_models = fiducial_models
else:
distances = stats_wrapper(
fiducial_dataset, testing_dataset, fiducial_models=all_fiducial_models, statistics=statistics
)
distances = [distances]
distances_storage[:, i : i + 1] = sort_distances(statistics, distances).T
return distances_storage
