def mount_stats(grid=False):
""" Print statistics on mounted tar files"""
print("-"*40)
print(BB("statistics on mounted tar files"))
print(f"mounting point: {dirmounted}")
for subd in subdomains:
if grid:
destdir = dirgrid.format(subd=subd)
else:
destdir = dirhis.format(subd=subd)
if os.path.exists(destdir):
filelist = os.listdir(f"{destdir}")
nbfiles = len(filelist)
if nbfiles > 0:
tiles = set([int(f.split(".")[-2]) for f in filelist])
nbtiles = len(tiles)
tile = list(tiles)[0]
fs = [f for f in filelist if f"{tile:04}.nc" in f]
if grid:
msg = f" - {subd:02} : {nbtiles:03} tiles"
else:
_hisdate = read_toc(destdir, subd)
# dateindex = sorted([int(f.split(".")[-3]) for f in fs])
# msg = f" - {subd:02} : {nbtiles:03} tiles x {dateindex} dateindex"
bbhisdate = BB(_hisdate)
msg = f" - {subd:02} : {bbhisdate} with {nbtiles:03} tiles"
else:
msg = f" - {subd:02} : empty"
else:
warning = BB("destroyed")
msg = f" - {subd:02} : {warning}"
print(msg)
def __repr__(self):
string = []
if hasattr(self, "dims"):
# {d: self.shape[k] for k, d in enumerate(self.dims)}
shape = [(d, self.sizes[d]) for d in self.dims]
dims_str = [f"{dim}: {length}" for dim, length in shape]
string += [MA("<M-array (" + ", ".join(dims_str) + ")>")]
else:
ndim = self.ndim
string += [
MA(f"<M-array ({ndim}-dimensional with unnamed dimensions)>")
]
string += [BB("* Data:")]
# string += [np.core.arrayprint.array2string(self)]
string += [np.asarray(self).__repr__()]
if hasattr(self, "attrs") and len(self.attrs.keys()) > 0:
string += [BB("* Attributes:")]
maxlen = max([len(k) for k in self.attrs.keys()])
maxlen += 1
for k, v in self.attrs.items():
string += [f" - {k:{maxlen}s}: {v}"]
else:
string += [BB("* Attributes:") + " None"]
if hasattr(self, "stagg"):
stagg_str = [f"{dim}: {val}" for dim, val in self.stagg.items()]
stagg_str = "(" + ", ".join(stagg_str) + ")"
else:
stagg_str = "None"
string += [BB("* Staggering: ") + stagg_str]
return "\n".join(string)
def __repr__(self):
string = [
"<Param: paragridded parameters from " + BB(f"{paramfile}") + ">"
]
for key in self.toc:
val = getattr(self, key)
string += [BB(f"{key}: ") + f"{val}"]
return "\n".join(string)
def __repr__(self):
dims = self.infos["dims"]
halow = self.infos["halow"]
shape = self.shape
string = []
string += [VA("<class 'paragridded.nctools.Variable'>")]
string += [BB("* name: ") + f"{self.varname}{dims}"]
string += [BB("* attrs:")]
string += [f" - {k}: {v}" for k, v in self.attrs.items()]
string += [BB("* shape : ") + f"{shape}"]
string += [BB("* halo width: ") + f"{halow}"]
return "\n".join(string)
def mount_tar(source, tarfile, destdir):
"""
source: str, directory of the tar files
template: str, template name for the tar file containing "{subd"
subd: int, index of the subdomain (0<=subd<=13)
destdir: str, directory where to archivemount
"""
srcfile = f"{source}/{tarfile}"
#print(f"mount {srcfile} on {destdir}")
assert os.path.isfile(srcfile), f"{srcfile} does not exsit"
sqlitefile = get_sqlitefilename(srcfile)
home = os.path.expanduser("~")
ratardirsqlite = f"{home}/.ratarmount"
if os.path.isfile(f"{ratardirsqlite}/{sqlitefile}"):
# nothing to do
pass
else:
if os.path.isfile(f"{sqlitesdir}/{sqlitefile}"):
command = f"cp {sqlitesdir}/{sqlitefile} {ratardirsqlite}/"
os.system(command)
assert len(ratarmount) > 0, BB("You forgot to set the ratarmount path")
command = f"{ratarmount} {srcfile} {destdir}"
os.system(command)
if os.path.isfile(f"{sqlitesdir}/{sqlitefile}"):
# nothing to do
pass
else:
command = f"cp {ratardirsqlite}/{sqlitefile} {sqlitesdir}/"
os.system(command)
def __repr__(self):
partition = self.blocks["partition"]
string = []
string += [MD("<class 'paragridded.nctools.MDataset'>")]
if self.missing.all():
string += ["empty dataset, all tiles are missing"]
else:
tiles = self.tiles.copy()
tiles[self.missing] = -1
tile0 = get_one_tile(tiles, self.missing)
ncfile = getncfile(self.nctemplate, tile0, **self.kwargs)
string += [BB("* filename: ") + f"(for tile=={tile0})"]
string += [f" - {ncfile}"]
string += [BB("* partition: ") + f"{partition}"]
string += [BB("* tiles:") + f" {self.tiles.shape}"]
string += [tiles.__repr__()]
string += [BB("* dims: ") + f"{self.dims}"]
string += [BB("* sizes:")]
string += [f" - {k}: {v}" for k, v in self.sizes.items()]
string += [BB("* variables:")]
string += [f" - {k}: {v}" for k, v in self.toc.items()]
string += [BB("* Halo width: ") + f"{self.halow}"]
return "\n".join(string)
def __repr__(self):
string = []
string += [GR("<class 'paragridded.grid.grid'>")]
string += [BB("* Dimensions:") + f"{self.dims}"]
string += [BB("* Coordinates:")]
maxlen = max([len(v) for v in self.coords.keys()])
maxlen += 1
for k, v in self.coords.items():
string += [f" - {k:{maxlen}s} : {v.sizes}"]
string += [BB("* Corners:")]
y = getattr(self, self.dims[-2])()
x = getattr(self, self.dims[-1])()
string += [f" - West-South : {y[0,0]}, {x[0,0]}"]
string += [f" - East-South : {y[0,-1]}, {x[0,-1]}"]
string += [f" - West-North : {y[-1,0]}, {x[-1,0]}"]
string += [f" - East-North : {y[-1,-1]}, {x[-1,-1]}"]
string += [BB("* Halo width: ")+f"{self.halow}"]
return "\n".join(string)
def set_ratarmount():
mount = "ratarmount"
options = "" # "-c -gs 160000"
ratarmount = os.popen(f"which {mount}").read()
if len(ratarmount) > 0:
# remove the trailing "\n"
ratarmount = ratarmount[:-1]
print(f"found ratarmount in : {ratarmount}")
else:
if rank == 0:
print("")
print(BB("warning").center(20, "*"))
print(f"{mount} is not installed or cannot be found")
print("you can set it manually with")
print("giga.ratarmount = /path/to/bin/ratarmount")
return ratarmount
def __init__(self, param, subd):
self.datadir = param.dirgigabin
self.subd = subd
yamfile = f"{param.dirmodule}/data/giga_{subd:02}.yaml"
with open(yamfile) as f:
data = yaml.load(f, yaml.RoundTripLoader)
levels = data["levels"]
nlevels = len(levels)
filename = data["filename"]
atom = tuple(data["atom"])
atomsize = np.prod(atom)
headersize = data["header"] * atomsize
variable_level = levels.index("variable")
variables = data["definitions"]["variable"]
stripes = data["stripes"]
varsizes = [data["variable_size"][var] * atomsize for var in variables]
offset = {}
assert levels[-1] == "variable"
assert levels[0] == stripes["level"]
for level in levels[::-1]:
nelem = len(data["definitions"][level])
if level == "variable":
varsize = data["variable_size"]
sizes = [varsize[var] * atomsize for var in variables]
variable_sizes = sizes
elif level == stripes["level"]:
# roundup prev size to upper value
unit = stripes["unit"]
nelemperstripe = stripes["nelements"]
currentsize = headersize + prev_size * nelemperstripe
# transform in bytes then back in number of float
stripesize = roundup(currentsize * floatsize,
unit) // floatsize
paddingsize = stripesize - currentsize
sizes = [prev_size] * nelemperstripe
nstripes = nelem // nelemperstripe
nelem_laststripe = nelem % nelemperstripe
if nelem_laststripe > 0:
nstripes += 1
filesize = nstripes * stripesize * floatsize
if debug:
print(BB(f" stripe properties:"))
print(f"size is rounded up: {currentsize} -> {stripesize}")
print(f" padding size: {paddingsize}")
print(
f" stripesize: {stripesize*floatsize} B ({stripesize*floatsize//MB:.2f} MB)"
)
print(f" number of stripes: {nstripes}")
print(BB(f" filesize: {filesize/GB:.2f} GB"))
assert (stripesize * floatsize) < 4 * GB
else:
sizes = [prev_size] * nelem
offset[level] = [0] + list(np.cumsum(sizes))
prev_size = offset[level][-1]
variable_shape = {}
for varname in variables:
natoms = data["variable_size"][varname]
if natoms == 1:
shape = atom
else:
shape = (natoms, ) + atom
variable_shape[varname] = shape
self.levels = levels
self.data = data
self.atom = atom
self.stripes = stripes
self.nelemperstripe = nelemperstripe
self.stripesize = stripesize
self.headersize = headersize
self.offset = offset
self.variable_shape = variable_shape
self.variables = variables