def vol_renders():
"""Volume render the simulations."""
import os
import yt
yt.enable_parallelism()
results_base = 'results/approximate/'
plots_base = 'plots/'
for mass_P in ['0.90']:
for mass_S in ['0.60', '0.90']:
for roche in ['0.90', '1.00']:
for rot in ['0', '1']:
for hybrid in ['0', '1']:
for ncell in ['256', '512', '1024']:
mass_string = "_m_P_" + mass_P + "_m_S_" + mass_S
results_dir = 'mass_P_' + mass_P + '/mass_S_' + mass_S + '/roche' + roche + '/rot' + rot + '/hybrid' + hybrid + '/n' + ncell + '/'
output_dir = results_base + results_dir + '/output/'
if not os.path.exists(output_dir):
continue
plot_list = wdmerger.get_plotfiles(output_dir, prefix='smallplt')
plot_list = [output_dir + plot for plot in plot_list]
plot_dir = plots_base + results_dir
if not os.path.exists(plot_dir) and is_root():
os.makedirs(plot_dir)
ts = yt.DatasetSeries(plot_list)
for ds in ts.piter():
pltname = ds.basename
outfile_name = plot_dir + 'approximate' + mass_string + '_roche_' + roche + '_rot_' + rot + '_hybrid_' + hybrid + '_n_' + ncell + '_' + pltname + '.png'
if os.path.isfile(outfile_name):
continue
wdmerger.vol_render_density(outfile_name, ds)
'''
This code uses yt's clump finding feature to detect regions of high density of a particualr field. The bounds provided are the default as seen within the documentation, consider tweaking these as desired for better results.
'''
import yt
from yt.analysis_modules.level_sets.api import *
import numpy as np
import os
yt.enable_parallelism()
d_path = '/mnt/research/galaxies-REU/sims/isolated-galaxies/MW_1638kpcBox_800pcCGM_200pcDisk_lowres/DD????/DD????'
d_list = []
lower = 3
upper = 1776
make_dir = False
fields = ['H_p0_number_density']
for i in range(lower, upper):
path = "DD%04d" % i
path = d_path + path + "/" + path
d_list.append(path)
d_series = yt.load(d_path)
axes = ['x', 'z']
storage = {}
if make_dir:
for axis in axes:
default='/tigress/changgoo/',
help='base working directory')
parser.add_argument('-d',
'--directory',
type=str,
default='',
help='working directory')
parser.add_argument('-i', '--id', type=str, help='id of dataset')
parser.add_argument('-p',
'--parallel',
action='store_true',
help='parallel')
parser.add_argument('-ph',
'--phase',
action='store_true',
help='phase diagram analysis')
parser.add_argument('-ro',
'--rotation',
type=float,
default=28.,
help='rotational velocity')
parser.add_argument('-r',
'--range',
type=str,
default='',
help='time range, start:end:skip')
args = parser.parse_args()
if vars(args)['parallel']: yt.enable_parallelism()
main(**vars(args))
line = [line for line in lines if '#a =' in line]
scale_in_file = round(float(line[0].split('=')[-1]), 4)
if scale == scale_in_file:
halo_file = this_file
break
return halo_file
if __name__ == "__main__":
args = parse()
import yt
from yt.analysis_modules.sunrise_export import sunrise_octree_exporter
from yt.analysis_modules.halo_finding.halo_objects import RockstarHaloList
yt.enable_parallelism()
if yt.is_root():
print '/nStarting '+ sys.argv[0]
print 'Parsed arguments: '
print args
print
# Get parsed values
sim_dirs, snap_base = args['sim_dirs'], args['snap_base']
print 'Analyzing ', sim_dirs
out_dir = args['out_dir']
modify_outdir = 0
if 'sim_dir' in out_dir:
out_dir = out_dir.replace('sim_dir','')
from galaxy_analysis.analysis import time_average_phase_diagram as tapd
from galaxy_analysis.utilities import utilities
from galaxy_analysis.analysis import Galaxy
from galaxy_analysis.yt_fields import field_generators as fg
import yt
import numpy as np
import glob as glob
import os, sys
print(yt.enable_parallelism(), " ------------------------")
def plot_time_average_PD(wdir,
t_min,
t_max,
nbin=100,
plots=['nT', 'G_o', 'Q_o'],
outdir=None):
x = utilities.select_data_by_time(wdir, tmin=0.0, tmax=np.inf)
sim_files = {'files': x[0], 'Time': x[1]}
# 'DD_files' : np.sort(glob.glob(wdir + '/DD????/DD????'))}
#iremove = len(sim_files['files'])
#sim_files['DD_files'] = sim_files['DD_files'][-iremove:]
# --- get data sets associated with output files (this is gross - I am sorry)
sim_files['DD_files'] = np.array([
x.split('_galaxy_data')[0] + '/' + x.split('_galaxy_data')[0][-6:]
for x in sim_files['files']
])
# --- make sure they exist
sim_files['Time'] = np.array([
import sys
import os
matplotlib.use('Agg')
matplotlib.rcParams['font.family'] = 'stixgeneral'
matplotlib.rcParams['figure.dpi'] = 150
import matplotlib.pyplot as plt
import yt
yt.mylog.setLevel("INFO")
from mpi4py import MPI
from mpl_toolkits.axes_grid1 import make_axes_locatable
from particles.particle_filters import *
comm = MPI.COMM_WORLD
yt.enable_parallelism(communicator=comm)
dir = './'
ls = {
(0, 10): ['solid', 2],
(10, 20): ['dotted', 2],
(20, 30): ['dashed', 1],
(30, 60): ['solid', 1],
(60, 100): ['dotted', 1]
}
rmin, rmax = -10, 100
try:
ind = int(sys.argv[1])
# You must run this job in parallel.
# There are several mpi flags which can be useful in order for it to work OK.
# It requires at least 3 processors in order to run because of the way in which
# rockstar divides up the work. Make sure you have mpi4py installed as per
# http://yt-project.org/docs/dev/analyzing/parallel_computation.html#setting-up-parallel-yt
# Usage: mpirun -np <num_procs> --mca btl ^openib python this_script.py
import yt
from yt.extensions.astro_analysis.halo_analysis.halo_catalog import HaloCatalog
from yt.data_objects.particle_filters import add_particle_filter
from yt.extensions.astro_analysis.halo_finding.rockstar.api import RockstarHaloFinder
yt.enable_parallelism() # rockstar halofinding requires parallelism
# Create a dark matter particle filter
# This will be code dependent, but this function here is true for enzo
def DarkMatter(pfilter, data):
filter = data[("all", "particle_type")] == 1 # DM = 1, Stars = 2
return filter
add_particle_filter("dark_matter", function=DarkMatter, filtered_type='all', \
requires=["particle_type"])
# First, we make sure that this script is being run using mpirun with
# at least 3 processors as indicated in the comments above.
assert (yt.communication_system.communicators[-1].size >= 3)
# Load the dataset and apply dark matter filter
fn = "Enzo_64/DD0043/data0043"
def createProfWithTimeForX(Param_Dict, worker):
"""Make a profile plot having the time as the x-Axis.
Parameters:
Param_Dict: For the fields and DataSets to be plotted.
Returns:
arr: list containing two YTArrays having the time as the first and the
y-field as second entry
"""
GUILogger.info("This may take some...time...")
ts = Param_Dict["DataSeries"]
timeMin = Param_Dict["XMin"] # they should already be converted to xunit.
timeMax = Param_Dict["XMax"]
times = []
datasets = []
emitStatus(worker, "Gathering time data")
for ds in ts:
# use the times we have already calculated for each dataset
time = Param_Dict["DataSetDict"][str(ds) + "Time"].to_value(Param_Dict["XUnit"])
timecompare = float("{:.3g}".format(time))
if timeMin <= timecompare <= timeMax:
times.append(time)
datasets.append(str(ds))
GUILogger.log(29, "Iterating over the whole series from {:.3g} to {:.3g} {}..."
.format(timeMin, timeMax, Param_Dict["XUnit"]))
calcQuan = getCalcQuanName(Param_Dict)
field = Param_Dict["YAxis"]
calcQuanString = getCalcQuanString(Param_Dict)
storage = {}
i = 0
length = len(times)
if Param_Dict["YAxis"] in Param_Dict["NewDerFieldDict"].keys():
for ds in ts:
if str(ds) in datasets:
try:
yResult = Param_Dict["DataSetDict"][str(ds) + field + calcQuan]
except KeyError:
ad = ds.all_data()
yResult = eval(calcQuanString)
# save the plotpoints for later use
value = yt.YTQuantity(yResult, Param_Dict["YUnit"]).to_value(Param_Dict["FieldUnits"][field])
Param_Dict["DataSetDict"][str(ds) + field + calcQuan] = value
storage[str(i)] = yResult # this is kind of clunky, but this way we don't run into problems later
i += 1
progString = f"{i}/{length} data points calculated"
emitStatus(worker, progString)
if i % ceil(length/10) == 0: # maximum of 10 updates
GUILogger.info(f"Progress: {progString}.")
else: # We want to use parallel iteration if possible
yt.enable_parallelism(suppress_logging=True)
newTS = yt.load(Param_Dict["Directory"] + "/" + Param_Dict["Seriesname"])
for store, ds in newTS.piter(storage=storage):
try:
yResult = Param_Dict["DataSetDict"][str(ds) + field + calcQuan]
except KeyError:
ad = ds.all_data() # This is needed for the following command
yResult = eval(calcQuanString)
# save the plotpoints for later use
value = yt.YTQuantity(yResult, Param_Dict["YUnit"]).to_value(Param_Dict["FieldUnits"][field])
Param_Dict["DataSetDict"][str(ds) + field + calcQuan] = value
store.result = yResult
i += 1
progString = f"{i}/{length} data points calculated"
emitStatus(worker, progString)
if i % ceil(length/10) == 0: # maximum of 10 updates
GUILogger.info(f"Progress: {progString}.")
labels = [field]
# Convert the storage dictionary values to an array, so they can be
# easily plotted
arr_x = yt.YTArray(times, Param_Dict["XUnit"])
arr_y = yt.YTArray(list(storage.values()), Param_Dict["YUnit"])
arr = [arr_x, arr_y]
# print(arr)
return arr, labels
def createMultipleProfiles(Param_Dict, worker):
"""Make a profile plot for each of the requested times and return them so
they can be plotted.
Parameters:
Param_Dict: For the fields and DataSets to be plotted.
Returns:
arr: list containing YTArrays having the x-field as the first and the
y-fields as second and following entries
labels: the labels for the rows.
"""
GUILogger.info("This may take some time.")
onlyEvery = Param_Dict["ProfOfEvery"]
if onlyEvery == 1:
numString = ""
else:
suf = lambda n: "%d%s "%(n,{1:"st",2:"nd",3:"rd"}.get(n if n<20 else n%10,"th"))
numString = suf(onlyEvery)
GUILogger.log(29, "Creating a profile for every {}dataset of the series...".format(numString))
# the user can input to only plot every nth file:
yt.enable_parallelism(suppress_logging=True)
storage = {}
labels = []
i = 0
ts = Param_Dict["DataSeries"]
length = ceil(len(ts)/onlyEvery)
if Param_Dict["YAxis"] in Param_Dict["NewDerFieldDict"].keys():
for ds in ts:
if i % onlyEvery == 0:
# Create a data container to hold the whole dataset.
ad = ds.all_data()
# Create a 1d profile of xfield vs. yfield:
prof = yt.create_profile(ad, Param_Dict["XAxis"],
fields=[Param_Dict["YAxis"]],
weight_field=Param_Dict["WeightField"])
# Add labels
time = Param_Dict["DataSetDict"][str(ds) + "Time"]
label = "{} at {:.3g} ".format(Param_Dict["YAxis"], time.value)
label += str(time.units)
labels.append(label)
storage[str(i)] = prof[Param_Dict["YAxis"]]
progString = f"{int(i/onlyEvery+1)}/{length} profiles done"
emitStatus(worker, progString)
if i % ceil(length/10) == 0: # maximum of 10 updates
GUILogger.info(f"Progress: {progString}.")
i += 1
else: # We want to use parallel iteration if possible
ts = yt.load(Param_Dict["Directory"] + "/" + Param_Dict["Seriesname"])
for store, ds in ts.piter(storage=storage):
if i % onlyEvery == 0:
ad = ds.all_data()
prof = yt.create_profile(ad, Param_Dict["XAxis"],
fields=[Param_Dict["YAxis"]],
weight_field=Param_Dict["WeightField"])
# Add labels
time = Param_Dict["DataSetDict"][str(ds) + "Time"]
label = "{} at {:.3g} ".format(Param_Dict["YAxis"], time.value)
label += str(time.units)
labels.append(label)
store.result = prof[Param_Dict["YAxis"]]
progString = f"{int(i/onlyEvery+1)}/{length} profiles done"
emitStatus(worker, progString)
GUILogger.info(f"Progress: {progString}.")
i += 1
# Convert the storage dictionary values to an array with x-axis as first
# row and then the results of y-field as following rows.
arr_x = prof.x
arr = [arr_x]
for arr_y in storage.values():
if arr_y is not None:
arr.append(arr_y)
return arr, labels
#!usr/bin/env python
import sys
import os
import yt
yt.mylog.setLevel("INFO")
import yt_synchrotron_emissivity as sync
yt.enable_parallelism(suppress_logging=True)
dir = './'
ptype = 'lobe'
proj_axis = 'x'
#proj_axis = [1,0,2]
extend_cells = 8
nus = [(nu, 'MHz') for nu in [100, 1400, 8000]]
try:
ind = int(sys.argv[1])
#ts = yt.DatasetSeries(os.path.join(dir,'*_hdf5_plt_cnt_%04d' % ind), parallel=1, setup_function=sync.setup_part_file)
# This works for 1 file at a time. Thus parallel=1
ts = yt.DatasetSeries(os.path.join(dir, 'data/*_hdf5_plt_cnt_%04d' % ind),
parallel=1)
except IndexError:
ts = yt.DatasetSeries(os.path.join(dir, 'data/*_hdf5_plt_cnt_????'),
parallel=1)
for ds in ts.piter():
if '0000' in ds.basename: continue
for nu in nus:
# The two projection axes cannot be completed at the same time
anlsT = tst.anlzT(ts, anlzD, outdir="/my/plot/path", plotT=plotT, trypkl=True)
######### END OF EXAMPLE SCRIPT #########
"""
import os
import sys
import numpy as np
if (sys.version_info > (3, 0)):
# Python 3 code in this block
import pickle
else:
# Python 2 code in this block
import cPickle as pickle
import yt
yt.enable_parallelism() # Tap into yt's mpi4py parallelism (e.g. now can call via mpirun -np 10 python <blah>.py)
yt.funcs.mylog.setLevel(30) # This sets the output notification threshold to 30, WARNING. Default is 20, INFO.
from mpi4py import MPI
import numbers
import inspect
comm = MPI.COMM_WORLD
rank = comm.rank
# TODO: Incorporate this into the new analysis schema
def tsinf(ts):
""" Walks through the time series and returns a dict with some basic info like time steps """
numfiles = len(ts)
ts_inf = {}
ts_inf['times_ns'] = np.zeros(numfiles)
# You must run this job in parallel.
# There are several mpi flags which can be useful in order for it to work OK.
# It requires at least 3 processors in order to run because of the way in which
# rockstar divides up the work. Make sure you have mpi4py installed as per
# http://yt-project.org/docs/dev/analyzing/parallel_computation.html#setting-up-parallel-yt
# Usage: mpirun -np <num_procs> --mca btl ^openib python this_script.py
import yt
from yt.analysis_modules.halo_analysis.halo_catalog import HaloCatalog
from yt.data_objects.particle_filters import add_particle_filter
from yt.analysis_modules.halo_finding.rockstar.api import RockstarHaloFinder
yt.enable_parallelism() # rockstar halofinding requires parallelism
# Create a dark matter particle filter
# This will be code dependent, but this function here is true for enzo
def DarkMatter(pfilter, data):
filter = data[("all", "particle_type")] == 1 # DM = 1, Stars = 2
return filter
add_particle_filter("dark_matter", function=DarkMatter, filtered_type='all', \
requires=["particle_type"])
# First, we make sure that this script is being run using mpirun with
# at least 3 processors as indicated in the comments above.
assert(yt.communication_system.communicators[-1].size >= 3)
# Load the dataset and apply dark matter filter
fn = "Enzo_64/DD0043/data0043"
ds = yt.load(fn)
def run_sightlines(outputfilename,save_after_num,parallel,\
simulation_dest = None,run = 'default',throwerrors = 'warn'):
if run not in ['default', 'test']:
print('unknown option for "run" %s.'%run+\
' Please restart with "run = default"'+\
' or "run = test".')
#do not print out anything from yt (it prints plenty)
yt.funcs.mylog.setLevel(50)
if parallel:
yt.enable_parallelism()
readvalsoutput = simulation_quasar_sphere.read_values(outputfilename)
#by creating a QuasarSphere, it knows all its metadata and other
#information from simparams and scanparams (first lines of file at
#'filename')
q = simulation_quasar_sphere.SimQuasarSphere(
start_up_info_packet=readvalsoutput)
if q.simparams[6] is None:
if simulation_dest:
q.simparams[6] = simulation_dest
else:
raise NoSimulationError('Simulation file location unknown, '+\
'run with "simulation_dest" to process')
else:
simulation_dest = q.simparams[6]
ds,fields_to_keep = code_specific_setup.load_and_setup(simulation_dest,\
q.fullname,ions = q.ions,\
redshift = q.redshift)
set_up_general(ds, q.code, q.center, q.bulk_velocity, q.Rvir)
code_specific_setup.check_redshift(ds, outputfilename=outputfilename)
num_bin_vars = q.gasbins.get_length()
#Can start at a position further than 0 if reached
starting_point = q.length_reached
bins = np.append(np.arange(starting_point, q.length, save_after_num),
q.length)
#first for loop is non-parallel. If 32 processors available, it will break up
#into bins of size 32 at a time for example. At end, saves data from all 32.
#this is (~12 bins) in usual circumstances
for i in range(0, len(bins) - 1):
current_info = q.info[bins[i]:bins[i + 1]]
if yt.is_root():
tprint("%s-%s /%s" % (bins[i], bins[i + 1], len(q.info)))
my_storage = {}
#2nd for loop is parallel. Each vector goes to a different processor, and creates
#a separate trident sightline (~32 sightlines [in a bin]).
#the longest processing step is ray = trident.make_simple_ray, and it's
#the only step which actually takes any time (below two for loops go by fast)
for sto, in_vec in yt.parallel_objects(current_info,
storage=my_storage):
vector = np.copy(in_vec)
index = vector[0]
toprint = "line %s, (r = %.0f) densities " % (str(
int(index)), vector[3])
tprint("<line %d, starting process> " % index)
ident = str(index)
start = ds.arr(tuple(vector[5:8]), 'unitary')
end = ds.arr(tuple(vector[8:11]), 'unitary')
try:
ray = trident.make_simple_ray(ds,
start_position=start,
end_position=end,
data_filename="ray" + ident +
".h5",
fields=fields_to_keep,
ftype='gas')
except KeyboardInterrupt:
print('skipping sightline %s ...' % index)
print('Interrupt again within 5 seconds to *actually* end')
time.sleep(5)
continue
except ValueError as e:
throw_errors_if_allowed(
e, throwerrors,
'ray has shape %s - %s, but size 0' % (start, end))
except Exception as e:
throw_errors_if_allowed(e, throwerrors,
'problem with making ray')
continue
trident.add_ion_fields(ray, q.ions)
field_data = ray.all_data()
dl = field_data['gas', 'dl']
#3rd for loop is for processing each piece of info about each ion
#including how much that ion is in each bin according to gasbinning
#here just process topline data (column densities and ion fractions)
#(~10 ions)
for j in range(len(q.ions)):
ion = q.ions[j]
ionfield = field_data["gas", ion_to_field_name(ion)]
cdens = np.sum((ionfield * dl).in_units('cm**-2')).value
vector[11 + j * (num_bin_vars + 2)] = cdens
total_nucleus = np.sum(ionfield[ionfield>0]/\
field_data["gas",ion_to_field_name(ion,'ion_fraction')][ionfield>0]\
* dl[ionfield>0])
vector[11 + j * (num_bin_vars + 2) + 1] = cdens / total_nucleus
#4th for loop is processing each gasbin for the current ion
#(~20 bins)
for k in range(num_bin_vars):
try:
variable_name, edges, units = q.gasbins.get_field_binedges_for_num(
k, ion)
if variable_name is None:
vector[11 + j * (num_bin_vars + 2) + k +
2] = np.nan
elif variable_name in ray.derived_field_list:
if units:
data = field_data[variable_name].in_units(
units)
else:
data = field_data[variable_name]
abovelowerbound = data > edges[0]
belowupperbound = data < edges[1]
withinbounds = np.logical_and(
abovelowerbound, belowupperbound)
coldens_in_line = (ionfield[withinbounds]) * (
dl[withinbounds])
coldens_in_bin = np.sum(coldens_in_line)
vector[11 + j * (num_bin_vars + 2) + k +
2] = coldens_in_bin / cdens
else:
print(
str(variable_name) +
" not in ray.derived_field_list")
except Exception as e:
throw_errors_if_allowed(
e, throwerrors,
'Could not bin into %s with edges %s' %
(variable_name, edges))
toprint += "%s:%e " % (ion, cdens)
#gets some more information from the general sightline.
#metallicity, average density (over the whole sightline)
#mass-weighted temperature
try:
if ('gas', "H_nuclei_density") in ray.derived_field_list:
Z = np.sum(field_data['gas',"metal_density"]*dl)/ \
np.sum(field_data['gas',"H_nuclei_density"]*mh*dl)
else:
Z = np.sum(field_data['gas',"metal_density"]*dl)/ \
np.sum(field_data['gas',"number_density"]*mh*dl)
vector[-1] = Z
except Exception as e:
throw_errors_if_allowed(e, throwerrors,
'problem with average metallicity')
try:
n = np.sum(
field_data['gas', 'number_density'] * dl) / np.sum(dl)
vector[-2] = n
except Exception as e:
throw_errors_if_allowed(e, throwerrors,
'problem with average density')
try:
T = np.average(field_data['gas','temperature'],\
weights=field_data['gas','density']*dl)
vector[-3] = T
except Exception as e:
throw_errors_if_allowed(e, throwerrors,
'problem with average temperature')
try:
os.remove("ray" + ident + ".h5")
except:
pass
tprint(toprint)
#'vector' now contains real data, not just '-1's
sto.result_id = index
sto.result = vector
#save all parallel sightlines after they finish (every 32 lines are saved at once)
if yt.is_root():
keys = my_storage.keys()
for key in keys:
q.info[int(key)] = my_storage[key]
q.scanparams[6] += (bins[i + 1] - bins[i])
q.length_reached = q.scanparams[6]
if run != 'test':
outputfilename = q.save_values(oldfilename=outputfilename)
tprint("file saved to " + outputfilename + ".")
This script additionally contains a couple of simple example usages for
both a typical phase diagram (n,T) and a spatial phase diagram (e.g.
a gas profile in a galaxy disk).
"""
__author__ = "Andrew Emerick"
from galaxy_analysis import Galaxy
import numpy as np
import yt
import glob
import os
# attempt to enable parallelism
# result is True if this works
parallel_on = yt.enable_parallelism()
def _mag_z(field, data):
return np.abs(data['cylindrical_z'].to('pc'))
yt.add_field(("gas", "mag_z"), function=_mag_z, units="pc")
def _create_region(ds, region_type, prop):
"""
Helper function to construct a region in yt given some arguments. This
is used to ensure consistent regions across data set. Mainly parses user
input into appropriate region type and args/kwargs with some
assumptions on defaults if certain fields are not provided.
