def get_varlist(self, pos=False, particle=False):
"""Get a list of all existing VAR# file names.
pos = False: give full list
pos = 'last'/'first': give latest/first var file
pos = 'lastXXX' / 'firstXXX' give last/first XXX varfiles
pos = list of numbers: give varfiles at this positions
particle = True: return PVAR- instead of VAR-list"""
import glob
from os.path import join as join
from os.path import basename
from pencil.math import natural_sort
key = 'VAR'
if particle == True: key = 'PVAR'
varlist = natural_sort([
basename(i)
for i in glob.glob(join(self.datadir, 'proc0') + '/' + key + '*')
])
#if particle: varlist = ['P'+i for i in varlist]
if pos == False: return varlist
if pos == 'first': return [varlist[0]]
if pos == 'last': return [varlist[-1]]
if pos.startswith('last'): return varlist[-int(pos[4:]):]
if pos.startswith('first'): return varlist[:int(pos[4:])]
if type(pos) == type([]):
if pos[0].startswith('VAR'): pos = [i[3:] for i in pos]
if pos[0].startswith('PVAR'): pos = [i[3:] for i in pos]
return [varlist[int(i)] for i in pos]
return varlist
def divisors(factors, nmax, nmin=8):
divisors = list()
nn = len(factors)
n = factors[0]
divisors.append(n)
for i in range(0,nn-1):
n = factors[i-1] * n
if n <= nmax/nmin and not n in divisors and np.mod(nmax,n)==0:
divisors.append(n)
for j in range(i,nn):
m = factors[j]*factors[i]
if m <= nmax/nmin and not m in divisors and np.mod(nmax,m)==0:
divisors.append(m)
for k in range(j+1,nn):
p = factors[k]*factors[i-1]*factors[j]
if p <= nmax/nmin and not p in divisors and np.mod(nmax,p)==0:
divisors.append(p)
if 1 not in divisors:
divisors.append(1)
return natural_sort(divisors)
def group(simulations, groupby, sort=True, only_started=False, reverse=False):
"""Group simulation by a quantity. Each Simulation object can only be part of one group.
Args:
simulations: put here a Simulations object or a list of simulations [sim1, sim2, ...]
groupby: put here the heyword after which the grouping shall happen
sort: set True to sort returned dictionary naturally
only_started: only group simulations that already has started
Return:
a dictionary with keywords are the group entries and values are lists of simulations in that group
"""
from collections import OrderedDict
from pencil.math import natural_sort
sim_dict_grouped = {}
if type(simulations) == type(['list']):
sim_list = simulations
#elif type(simulations) == Simulations:
# sim_list = simulations.sims
else:
print('!! ERROR: Dont know how to interprated simulations argument..')
return False
# sort out simulations that has not started
if only_started == True: sim_list = [s for s in sim_list if s.started()]
# special cases:
if groupby in ['Lx', 'Ly', 'Lz']:
if groupby[-1] == 'x': ii = 0
elif groupby[-1] == 'y': ii = 1
elif groupby[-1] == 'z': ii = 2
for sim in sim_list:
q = str(sim.param['lxyz'][ii])
if (not q in sim_dict_grouped.keys()):
sim_dict_grouped[q] = [sim]
else:
sim_dict_grouped[q].append(sim)
elif groupby in ['nx', 'ny', 'nz']:
for sim in sim_list:
q = str(getattr(sim.dim, groupby))
if (not q in sim_dict_grouped.keys()):
sim_dict_grouped[q] = [sim]
else:
sim_dict_grouped[q].append(sim)
# case the groupby-keyword can be found via __simulation__.get_value
elif sim_list[0].get_value(groupby) != None:
for sim in sim_list:
q = str(sim.get_value(groupby))
if (not q in sim_dict_grouped.keys()):
sim_dict_grouped[q] = [sim]
else:
sim_dict_grouped[q].append(sim)
else:
print(
'!! ERROR: Coudnt group simulations, no fitting groupby-keyword has been found to match "'
+ groupby + '"!')
return False
if sort:
sim_dict_grouped_n_sorted = OrderedDict()
keys = sim_dict_grouped.keys()
for key in natural_sort(keys, reverse=reverse):
sim_dict_grouped_n_sorted[key] = sim_dict_grouped[key]
return sim_dict_grouped_n_sorted
return sim_dict_grouped
import pencil as pc
from pencil.math import natural_sort
from pencil.calc import calc_shocktube
import os
from os.path import join, exists
import matplotlib.pyplot as plt
from matplotlib import colors
from matplotlib import cm
from matplotlib import artist
import glob
import string
os.chdir('/scratch/project_2001062/sod_tests') # root dir of multiple sims
wrkdir = os.getcwd() # variable used for chdir
sims_list = natural_sort(glob.glob('*')) # ordered list of sub dirs
sims = np.array(sims_list) # convert list to np array
indx = [0,1,2] # list of the subset of sims which will be used, can be revised for each action
# create objects for each sim which will be used
for sim in sims:
print('processing parameters for'+sim)
os.chdir(join(wrkdir,sim)) # cd to sim dir
globals()[sim+'par'] = pc.read.param()
globals()[sim+'gd'] = pc.read.grid(quiet=True,trim=True)
globals()[sim+'dim'] = pc.read.dim()
for sim in sims[:]:
print('processing varfiles for'+sim)
def group(simulations, groupby, sort=True, only_started=False, reverse=False):
"""
group(simulations, groupby, sort=True, only_started=False, reverse=False)
Group simulation by a quantity. Each Simulation object can only be part of one group.
Parameters
----------
simulations : list of sim objects
Put here a Simulations object or a list of simulations [sim1, sim2, ...].
groupby : string
Put here the heyword after which the grouping shall happen.
sort : bool
Set True to sort returned dictionary naturally.
only_started : bool
Only group simulations that already has started.
reverse : bool
Flag for reverse grouping.
Returns
-------
Dictionary with keywords are the group entries and values are lists of simulations in that group.
"""
from collections import OrderedDict
from pencil.math import natural_sort
sim_dict_grouped = {}
if type(simulations) == type(["list"]):
sim_list = simulations
# elif type(simulations) == Simulations:
# sim_list = simulations.sims
else:
print("!! ERROR: Dont know how to interprated simulations argument..")
return False
# sort out simulations that has not started
if only_started == True:
sim_list = [s for s in sim_list if s.started()]
# special cases:
if groupby in ["Lx", "Ly", "Lz"]:
if groupby[-1] == "x":
ii = 0
elif groupby[-1] == "y":
ii = 1
elif groupby[-1] == "z":
ii = 2
for sim in sim_list:
q = str(sim.param["lxyz"][ii])
if not q in sim_dict_grouped.keys():
sim_dict_grouped[q] = [sim]
else:
sim_dict_grouped[q].append(sim)
elif groupby in ["nx", "ny", "nz"]:
for sim in sim_list:
q = str(getattr(sim.dim, groupby))
if not q in sim_dict_grouped.keys():
sim_dict_grouped[q] = [sim]
else:
sim_dict_grouped[q].append(sim)
# case the groupby-keyword can be found via __simulation__.get_value
elif sim_list[0].get_value(groupby) != None:
for sim in sim_list:
q = str(sim.get_value(groupby))
if not q in sim_dict_grouped.keys():
sim_dict_grouped[q] = [sim]
else:
sim_dict_grouped[q].append(sim)
else:
print(
'!! ERROR: Coudnt group simulations, no fitting groupby-keyword has been found to match "'
+ groupby + '"!')
return False
if sort:
sim_dict_grouped_n_sorted = OrderedDict()
keys = sim_dict_grouped.keys()
for key in natural_sort(keys, reverse=reverse):
sim_dict_grouped_n_sorted[key] = sim_dict_grouped[key]
return sim_dict_grouped_n_sorted
return sim_dict_grouped
def sim2h5(
newdir=".",
olddir=".",
varfile_names=None,
todatadir="data/allprocs",
fromdatadir="data",
precision="d",
nghost=3,
lpersist=True,
x=None,
y=None,
z=None,
lshear=False,
snap_by_proc=False,
aver_by_proc=False,
lremove_old_snapshots=False,
lremove_old_slices=False,
lread_all_videoslices=False,
vlarge=100000000,
lremove_old_averages=False,
execute=False,
quiet=True,
l2D=True,
lvars=True,
lvids=True,
laver=True,
laver2D=False,
lremove_deprecated_vids=False,
lsplit_slices=False,
):
"""
Copy a simulation object written in Fortran binary to hdf5.
The default is to copy all snapshots from/to the current simulation
directory. Optionally the old files can be removed to
call signature:
sim2h5(newdir='.', olddir='.', varfile_names=None,
todatadir='data/allprocs', fromdatadir='data',
precision='d', nghost=3, lpersist=False,
x=None, y=None, z=None, lshear=False,
snap_by_proc=False, aver_by_proc=False,
lremove_old_snapshots=False,
lremove_old_slices=False, lread_all_videoslices=True,
lremove_old_averages=False, execute=False, quiet=True,
l2D=True, lvars=True, lvids=True, laver=True)
Keyword arguments:
*olddir*:
String path to simulation source directory.
Path may be relative or absolute.
*newdir*:
String path to simulation destination directory.
Path may be relative or absolute.
*varfile_names*:
A list of names of the snapshot files to be written, e.g. VAR0
If None all varfiles in olddir+'/data/proc0/' will be converted
*todatadir*:
Directory to which the data is stored.
*fromdatadir*:
Directory from which the data is collected.
*precision*:
Single 'f' or double 'd' precision for new data.
*nghost*:
Number of ghost zones.
TODO: handle switching size of ghost zones.
*lpersist*:
option to include persistent variables from snapshots.
*xyz*:
xyz arrays of the domain with ghost zones.
This will normally be obtained from Grid object, but facility to
redefine an alternative grid value.
*lshear*:
Flag for the shear.
*execute*:
optional confirmation required if lremove_old.
*lremove_old_snapshots*:
If True the old snapshot data will be deleted once the new h5 data
has been saved.
*lremove_old_slices*:
If True the old video slice data will be deleted once the new h5 data
has been saved.
*lremove_old_averages*:
If True the old averages data will be deleted once the new h5 data
has been saved.
*aver_by_proc*
Option to read old binary files by processor and write in
parallel
*laver2D*
If True apply to each plane_list 'y', 'z' and load each variable
sequentially
*l_mpi*:
Applying MPI parallel process
*driver*:
HDF5 file io driver either None or mpio
*comm*:
MPI library calls
*rank*:
Integer ID of processor
*size*:
Number of MPI processes
"""
import glob
import numpy as np
import os
from os.path import exists, join
import subprocess as sub
import sys
from .. import read
from .. import sim
from . import write_h5_grid
from pencil.util import is_sim_dir
try:
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
driver = "mpio"
l_mpi = True
l_mpi = l_mpi and (size != 1)
except ImportError:
comm = None
driver = None
rank = 0
size = 1
l_mpi = False
if not l_mpi:
comm = None
driver = None
print("rank {} and size {}".format(rank, size))
sys.stdout.flush()
if rank == size - 1:
print("l_mpi", l_mpi)
sys.stdout.flush()
# test if simulation directories
if newdir == ".":
newdir = os.getcwd()
if olddir == ".":
olddir = os.getcwd()
os.chdir(olddir)
if not is_sim_dir():
if rank == 0:
print("ERROR: Directory (" + olddir + ") needs to be a simulation")
sys.stdout.flush()
return -1
if newdir != olddir:
if not exists(newdir):
cmd = "pc_newrun -s " + newdir
if rank == size - 1:
process = sub.Popen(cmd.split(), stdout=sub.PIPE)
output, error = process.communicate()
print(cmd, output, error)
# os.system(cmd)
if comm:
comm.Barrier()
os.chdir(newdir)
if not is_sim_dir():
if rank == 0:
print("ERROR: Directory (" + newdir +
") needs to be a simulation")
sys.stdout.flush()
return -1
#
lremove_old = lremove_old_snapshots or lremove_old_slices or lremove_old_averages
if lremove_old:
if not execute:
os.chdir(olddir)
if rank == 0:
print("WARNING: Are you sure you wish to remove the Fortran" +
" binary files from \n" + os.getcwd() + ".\n" +
"Set execute=True to proceed.")
sys.stdout.flush()
return -1
os.chdir(olddir)
if lvars:
if varfile_names == None:
os.chdir(fromdatadir + "/proc0")
lVARd = False
varfiled_names = natural_sort(glob.glob("VARd*"))
if len(varfiled_names) > 0:
varfile_names = natural_sort(glob.glob("VAR*"))
for iv in range(len(varfile_names) - 1, -1, -1):
if "VARd" in varfile_names[iv]:
varfile_names.remove(varfile_names[iv])
lVARd = True
else:
varfile_names = natural_sort(glob.glob("VAR*"))
os.chdir(olddir)
else:
lVARd = False
if isinstance(varfile_names, list):
varfile_names = varfile_names
else:
varfile_names = [varfile_names]
varfiled_names = []
tmp_names = []
for varfile_name in varfile_names:
if "VARd" in varfile_names:
varfiled_names.append(varfile_name)
lVARd = True
else:
tmp_names.append(varfile_name)
varfile_names = tmp_names
gkeys = [
"x",
"y",
"z",
"Lx",
"Ly",
"Lz",
"dx",
"dy",
"dz",
"dx_1",
"dy_1",
"dz_1",
"dx_tilde",
"dy_tilde",
"dz_tilde",
]
grid = None
if rank == size - 1:
grid = read.grid(quiet=True)
if l_mpi:
grid = comm.bcast(grid, root=size - 1)
if not quiet:
print(rank, grid)
sys.stdout.flush()
for key in gkeys:
if not key in grid.__dict__.keys():
if rank == 0:
print("ERROR: key " + key + " missing from grid")
sys.stdout.flush()
return -1
# obtain the settings from the old simulation
settings = {}
skeys = [
"l1",
"l2",
"m1",
"m2",
"n1",
"n2",
"nx",
"ny",
"nz",
"mx",
"my",
"mz",
"nprocx",
"nprocy",
"nprocz",
"maux",
"mglobal",
"mvar",
"precision",
]
if rank == 0:
olddim = read.dim()
for key in skeys:
settings[key] = np.array(olddim.__getattribute__(key))
olddim = None
settings["nghost"] = np.array(nghost)
settings["precision"] = precision.encode()
if l_mpi:
settings = comm.bcast(settings, root=0)
if snap_by_proc:
nprocs = settings["nprocx"] * settings["nprocy"] * settings["nprocz"]
if np.mod(nprocs, size) != 0:
print("WARNING: efficiency requires cpus to divide ncpus")
sys.stdout.flush()
if not quiet:
print(rank, grid)
sys.stdout.flush()
# obtain physical units from old simulation
ukeys = [
"length",
"velocity",
"density",
"magnetic",
"time",
"temperature",
"flux",
"energy",
"mass",
"system",
]
param = read.param(quiet=True)
param.__setattr__("unit_mass", param.unit_density * param.unit_length**3)
param.__setattr__("unit_energy", param.unit_mass * param.unit_velocity**2)
param.__setattr__("unit_time", param.unit_length / param.unit_velocity)
param.__setattr__("unit_flux", param.unit_mass / param.unit_time**3)
param.unit_system = param.unit_system.encode()
# index list for variables in f-array
if not quiet:
print(rank, param)
sys.stdout.flush()
indx = None
if rank == 0:
indx = read.index()
if l_mpi:
indx = comm.bcast(indx, root=0)
# check consistency between Fortran binary and h5 data
os.chdir(newdir)
dim = None
if is_sim_dir():
if rank == size - 1:
if exists(join(newdir, "data", "dim.dat")):
try:
dim = read.dim()
except ValueError:
pass
if l_mpi:
dim = comm.bcast(dim, root=size - 1)
if dim:
if not quiet:
print(rank, dim)
sys.stdout.flush()
try:
dim.mvar == settings["mvar"]
dim.mx == settings["mx"]
dim.my == settings["my"]
dim.mz == settings["mz"]
except ValueError:
if rank == size - 1:
print("ERROR: new simulation dimensions do not match.")
sys.stdout.flush()
return -1
dim = None
os.chdir(olddir)
if rank == size - 1:
print("precision is ", precision)
sys.stdout.flush()
if laver2D:
aver2h5(
newdir,
olddir,
todatadir="data/averages",
fromdatadir="data",
l2D=False,
precision=precision,
quiet=quiet,
laver2D=laver2D,
lremove_old_averages=False,
aver_by_proc=aver_by_proc,
l_mpi=l_mpi,
driver=driver,
comm=comm,
rank=rank,
size=size,
)
l2D = False
# copy snapshots
if lvars and len(varfile_names) > 0:
var2h5(
newdir,
olddir,
varfile_names,
todatadir,
fromdatadir,
snap_by_proc,
precision,
lpersist,
quiet,
nghost,
settings,
param,
grid,
x,
y,
z,
lshear,
lremove_old_snapshots,
indx,
l_mpi=l_mpi,
driver=driver,
comm=comm,
rank=rank,
size=size,
)
# copy downsampled snapshots if present
if lvars and lVARd:
var2h5(
newdir,
olddir,
varfiled_names,
todatadir,
fromdatadir,
False,
precision,
lpersist,
quiet,
nghost,
settings,
param,
grid,
x,
y,
z,
lshear,
lremove_old_snapshots,
indx,
trimall=True,
l_mpi=l_mpi,
driver=driver,
comm=comm,
rank=rank,
size=size,
)
if lvars:
var2h5(
newdir,
olddir,
[
"var.dat",
],
todatadir,
fromdatadir,
snap_by_proc,
precision,
lpersist,
quiet,
nghost,
settings,
param,
grid,
x,
y,
z,
lshear,
lremove_old_snapshots,
indx,
l_mpi=l_mpi,
driver=driver,
comm=comm,
rank=rank,
size=size,
)
# copy old video slices to new h5 sim
if lvids:
if lremove_deprecated_vids:
for ext in [
"bb.", "uu.", "ux.", "uy.", "uz.", "bx.", "by.", "bz."
]:
cmd = "rm -f " + join(olddir, fromdatadir, "proc*",
"slice_" + ext + "*")
if rank == 0:
process = sub.Popen(cmd.split(), stdout=sub.PIPE)
output, error = process.communicate()
print(cmd, output, error)
cmd = "rm -f " + join(fromdatadir, "slice_" + ext + "*")
if rank == 0:
process = sub.Popen(cmd.split(), stdout=sub.PIPE)
output, error = process.communicate()
print(cmd, output, error)
if comm:
comm.Barrier()
cmd = "src/read_all_videofiles.x"
if rank == size - 1 and lread_all_videoslices:
process = sub.Popen(cmd.split(), stdout=sub.PIPE)
output, error = process.communicate()
print(cmd, output, error)
if comm:
comm.Barrier()
slices2h5(
newdir,
olddir,
grid,
todatadir="data/slices",
fromdatadir=fromdatadir,
precision=precision,
quiet=quiet,
vlarge=vlarge,
lsplit_slices=lsplit_slices,
lremove_old_slices=lremove_old_slices,
l_mpi=l_mpi,
driver=driver,
comm=comm,
rank=rank,
size=size,
)
# copy old averages data to new h5 sim
if laver:
aver2h5(
newdir,
olddir,
todatadir="data/averages",
fromdatadir=fromdatadir,
l2D=l2D,
precision=precision,
quiet=quiet,
aver_by_proc=False,
lremove_old_averages=lremove_old_averages,
l_mpi=l_mpi,
driver=driver,
comm=comm,
rank=rank,
size=size,
)
# check some critical sim files are present for new sim without start
# construct grid.h5 sim information if requied for new h5 sim
os.chdir(newdir)
if l_mpi:
comm.Barrier()
if rank == 0:
write_h5_grid(
file_name="grid",
datadir="data",
precision=precision,
nghost=nghost,
settings=settings,
param=param,
grid=grid,
unit=None,
quiet=quiet,
)
source_file = join(olddir, fromdatadir, "proc0/varN.list")
target_file = join(newdir, todatadir, "varN.list")
if exists(source_file):
cmd = "cp " + source_file + " " + target_file
process = sub.Popen(cmd.split(), stdout=sub.PIPE)
output, error = process.communicate()
print(cmd, output, error)
items = [
"def_var.pro",
"index.pro",
"jobid.dat",
"param.nml",
"particle_index.pro",
"pc_constants.pro",
"pointmass_index.pro",
"pt_positions.dat",
"sn_series.dat",
"svnid.dat",
"time_series.dat",
"tsnap.dat",
"tspec.dat",
"tvid.dat",
"t2davg.dat",
"var.general",
"variables.pro",
"varname.dat",
]
for item in items:
source_file = join(olddir, fromdatadir, item)
target_file = join(newdir, fromdatadir, item)
if exists(source_file):
if not exists(target_file):
cmd = "cp " + source_file + " " + target_file
process = sub.Popen(cmd.split(), stdout=sub.PIPE)
output, error = process.communicate()
print(cmd, output, error)
print("Simulation Fortran to h5 completed on rank {}.".format(rank))
sys.stdout.flush()
import matplotlib.pyplot as plt
from matplotlib import colors
from matplotlib import cm
from matplotlib import artist
import glob
import string
_sod_tests = "/scratch/project_2001062/sod_tests"
if not os.path.isdir(_sod_tests):
print("This Python file requires the directory {} to exist".format(
_sod_tests))
sys.exit(0)
os.chdir(_sod_tests) # root dir of multiple sims
wrkdir = os.getcwd() # variable used for chdir
sims_list = natural_sort(glob.glob("*")) # ordered list of sub dirs
sims = np.array(sims_list) # convert list to np array
indx = [
0,
1,
2,
] # list of the subset of sims which will be used, can be revised for each action
# create objects for each sim which will be used
for sim in sims:
print("processing parameters for" + sim)
os.chdir(join(wrkdir, sim)) # cd to sim dir
globals()[sim + "par"] = pc.read.param()
globals()[sim + "gd"] = pc.read.grid(quiet=True, trim=True)
globals()[sim + "dim"] = pc.read.dim()
import sys
import matplotlib.pyplot as plt
from matplotlib import colors
from matplotlib import cm
from matplotlib import artist
import glob
import string
_sod_tests = '/scratch/project_2001062/sod_tests'
if not os.path.isdir(_sod_tests):
print("This Python file requires the directory {} to exist".format(_sod_tests))
sys.exit(0)
os.chdir(_sod_tests) # root dir of multiple sims
wrkdir = os.getcwd() # variable used for chdir
sims_list = natural_sort(glob.glob('*')) # ordered list of sub dirs
sims = np.array(sims_list) # convert list to np array
indx = [0,1,2] # list of the subset of sims which will be used, can be revised for each action
# create objects for each sim which will be used
for sim in sims:
print('processing parameters for'+sim)
os.chdir(join(wrkdir,sim)) # cd to sim dir
globals()[sim+'par'] = pc.read.param()
globals()[sim+'gd'] = pc.read.grid(quiet=True,trim=True)
globals()[sim+'dim'] = pc.read.dim()
for sim in sims[:]:
print('processing varfiles for'+sim)