def RePack(output_file, input_files):
"""Write a new data pack to |output_file| based on a list of filenames
(|input_files|)"""
resources = {}
encoding = None
for filename in input_files:
new_content = ReadDataPack(filename)
# Make sure we have no dups.
duplicate_keys = set(new_content.resources.keys()) & set(
resources.keys())
if len(duplicate_keys) != 0:
raise exceptions.KeyError("Duplicate keys: " +
str(list(duplicate_keys)))
# Make sure encoding is consistent.
if encoding in (None, BINARY):
encoding = new_content.encoding
elif new_content.encoding not in (BINARY, encoding):
raise exceptions.KeyError("Inconsistent encodings: " +
str(encoding) + " vs " +
str(new_content.encoding))
resources.update(new_content.resources)
# Encoding is 0 for BINARY, 1 for UTF8 and 2 for UTF16
if encoding is None:
encoding = BINARY
WriteDataPack(resources, output_file, encoding)
def RePackFromDataPackStrings(inputs,
whitelist,
suppress_removed_key_output=False):
"""Returns a data pack string that combines the resources from inputs.
Args:
inputs: a list of data pack strings that need to be combined.
whitelist: a list of resource IDs that should be kept in the output string
or None to include all resources.
suppress_removed_key_output: Do not print removed keys.
Returns:
DataPackContents: a tuple containing the new combined data pack and its
encoding.
Raises:
KeyError: if there are duplicate keys or resource encoding is
inconsistent.
"""
resources = {}
encoding = None
for content in inputs:
# Make sure we have no dups.
duplicate_keys = set(content.resources.keys()) & set(resources.keys())
if duplicate_keys:
raise exceptions.KeyError('Duplicate keys: ' +
str(list(duplicate_keys)))
# Make sure encoding is consistent.
if encoding in (None, BINARY):
encoding = content.encoding
elif content.encoding not in (BINARY, encoding):
raise exceptions.KeyError('Inconsistent encodings: ' +
str(encoding) + ' vs ' +
str(content.encoding))
if whitelist:
whitelisted_resources = dict([(key, content.resources[key])
for key in content.resources.keys()
if key in whitelist])
resources.update(whitelisted_resources)
removed_keys = [
key for key in content.resources.keys() if key not in whitelist
]
if not suppress_removed_key_output:
for key in removed_keys:
print 'RePackFromDataPackStrings Removed Key:', key
else:
resources.update(content.resources)
# Encoding is 0 for BINARY, 1 for UTF8 and 2 for UTF16
if encoding is None:
encoding = BINARY
return DataPackContents(resources, encoding)
def readNameList(self, params, fail=False, verbose=False):
"""
Read parameter value from registered namelist
:param fail: If True fail if param not found
:param verbose (Optional -- default False). Provide verbose information.
:param params -- a list of parameters.
:example self.readNameList(['RHCRIT', 'VF1'])
:return:An OrderedDict with the values indexed by the param names
"""
result = collections.OrderedDict()
for param in params:
# have it as meta function?
if param in self._metaFn: # have a meta funcion -- takes priority.
result[param] = self.readMetaNameList(param, verbose=verbose)
elif param in self._convNameList: # in the conversion index
nlValue = self.readNameListVar(self._convNameList[param],
verbose=verbose)
if len(nlValue) != 1:
raise exceptions.ValueError("Should only have one key")
for k in nlValue.keys(
): # in this case shoudl only have one key and we don't want it as a list
result[param] = nlValue[k]
# just want the value and as single param SHOULD return
elif fail: # not found it and want to fail
raise exceptions.KeyError("Param %s not found" % param)
else:
pass
return result # return the result.
def _setKernelProperty(self, name, value):
import exceptions
#logging.info('_setKernelProperty: %s %s',name,str(value))
if Interface.setPropertyKernel(self.get(),name,str(value)):
return
msg = 'Geant4Kernel::SetProperty [Unhandled]: Cannot set Kernel.'+name+' = '+str(value)
raise exceptions.KeyError(msg)
def __getitem__(self, var):
"""Fake the remote server 'isRunning' option"""
if var == 'isRunning':
return self.isRunning()
elif var == 'frame_number':
return self.idx
raise exceptions.KeyError(var)
def _set(self, name, value):
import exceptions
#logging.info('_set: %s %s',name,str(value))
a = Interface.toAction(self)
if Interface.setProperty(a,name,str(value)):
return
msg = 'Geant4Action::SetProperty [Unhandled]: Cannot set '+a.name()+'.'+name+' = '+str(value)
raise exceptions.KeyError(msg)
def question(self, question_id):
"""A getter for a question with a known `_id`."""
oid = bson.ObjectId(question_id)
for question in self.questions:
if question._id == oid:
return question
raise exceptions.KeyError("Question not found.")
def _getKernelProperty(self, name):
import exceptions
#print '_getKernelProperty:',str(type(self)),name
ret = Interface.getPropertyKernel(self.get(), name)
if ret.status > 0:
return ret.data
elif hasattr(self.get(), name):
return getattr(self.get(), name)
elif hasattr(self, name):
return getattr(self, name)
msg = 'Geant4Kernel::GetProperty [Unhandled]: Cannot access Kernel.' + name
raise exceptions.KeyError(msg)
def _get(self, name):
import exceptions, traceback
#logging.info('_get: %s %s',str(type(self)),name)
a = Interface.toAction(self)
ret = Interface.getProperty(a,name)
if ret.status > 0:
return ret.data
elif hasattr(self.action,name):
return getattr(self.action,name)
elif hasattr(a,name):
return getattr(a,name)
#elif hasattr(self,name):
# return getattr(self,name)
#traceback.print_stack()
msg = 'Geant4Action::GetProperty [Unhandled]: Cannot access property '+a.name()+'.'+name
raise exceptions.KeyError(msg)
def RePack(output_file, input_files):
"""Write a new data pack to |output_file| based on a list of filenames
(|input_files|)"""
resources = {}
for filename in input_files:
new_resources = data_pack.ReadDataPack(filename)
# Make sure we have no dups.
duplicate_keys = set(new_resources.keys()) & set(resources.keys())
if len(duplicate_keys) != 0:
raise exceptions.KeyError("Duplicate keys: " +
str(list(duplicate_keys)))
resources.update(new_resources)
data_pack.WriteDataPack(resources, output_file)
def fopen(self, path, lock=LOCK_EX):
"""Open a file in path, handling the locking"""
self._lock.acquire()
fd = self.cache.get(path, -1)
if self.remap(fd, path, lock):
return self.mm, self.fd
# Initialize the indexed file, if exclusive lock was required
# (writing)
ex = os.path.exists(path) and os.path.exists(path + '.lk')
if not ex:
if lock in [LOCK_EX, LOCK_NBEX]:
self.init(path)
else:
self._lock.release()
raise exceptions.KeyError(
'Non-existent FileBuffer for reading: ' + path)
# Lock must precede mmap
try:
locker.lock(path, lock)
except:
self._lock.release()
raise
self.path = path
# Open the file descriptor
self.fd = os.open(path, flags)
# Create the memory map
self.mm = mmap.mmap(self.fd, length=0)
# Manage caching
if self.cache_len > 0:
clean_cache(self)
self.cache[path] = self.fd
self.info = self._get_info()
return self.mm, self.fd
def write_lonlatdata_in_COARDS_format(ncfile, data, metadata, **writeopt):
# dump data(nlon,nlat) corresponding to data on a positively oriented,
# uniformly spaced lon-lat grid to open netcdf file ncfile
# data is transposed on writing, corresponding to COARDS axis requirements (order TZYX)
# metadata is a dictionary that must contain at least these entries:
# lon: longitude grid ([degrees_E])
# lat: latitude grid ([degrees_N])
# [depth: depth grid ] ([m], optional)
# [time: time grid ] ([sec since offset], optional)
# [time_offset: time_offset ] (datetime like, optional)
# [time_unit: unit ] (as string, optional)
# variable_name: variablename to be used for data
# long_name: long title for variable_name (attribute associated with variable)
# units: units for data (standard conformance not assessed; attribute associated with variable)
# Global attributes must be in entry global_attributes - all other attributes are assumed to be variable attributes
# writeopt must contain "data_layout" (permutation of 'tzyx', corresponding to axis in data)
# and possibly "time_frame_number" to write specific time frames
# variable_name is declared according to appearence of (lon,lat,depth,time) in metadata in COARDS order
# (lon,lat,depth,time) also becomes netcdf dimensions with same name
#
# COARDS notes: dimensions should appear in the relative order T, then Z, then Y, then X in the CDL
# -----------------------------------------------------------------------------------------
# --- mandatory dimension lon ---
if ncfile.dimensions.has_key('lon'): # lon dimension exist, check size
assert ncfile.dimensions['lon'].size == size(metadata["lon"])
else:
ncfile.createDimension('lon', size(metadata["lon"]))
# --- mandatory dimension lat --
if ncfile.dimensions.has_key('lat'): # lat dimension exist, check size
assert ncfile.dimensions['lat'].size == size(metadata["lat"])
else:
ncfile.createDimension('lat', size(metadata["lat"]))
# ---- optional dimensions time, depth --
# for time, signal unlimited dimension by None, if requested
if ncfile.dimensions.has_key('depth'): # lat dimension exist, check size
assert ncfile.dimensions['depth'].size == size(metadata["depth"])
elif metadata.has_key("depth"):
ncfile.createDimension('depth', size(metadata["depth"]))
if metadata.has_key("time") and not ncfile.dimensions.has_key('time'):
if metadata["time"] is None:
ncfile.createDimension('time', None) # unlimited
else:
ncfile.createDimension('time',
size(metadata["time"])) # specific size
# -------- create axis variables as needed --------
if not ncfile.variables.has_key(
'lon'): # if exist, assume OK and do not redefine
lon = ncfile.createVariable('lon', 'd', ('lon', ))
lon.long_name = "Uniformly spaced longitudes"
lon.cartesian_axis = "X"
lon.units = "degrees_E"
lon.ipositive = 1
lon[:] = metadata['lon']
# ----
if not ncfile.variables.has_key(
'lat'): # if exist, assume OK and do not redefine
lat = ncfile.createVariable('lat', 'd', ('lat', ))
lat.long_name = "Uniformly spaced latitudes"
lat.cartesian_axis = "Y"
lat.units = "degrees_N"
lat.ipositive = 1
lat[:] = metadata['lat']
# ----
if metadata.has_key("depth") and not ncfile.variables.has_key('depth'):
depth = ncfile.createVariable('depth', 'd', ('depth', ))
depth.long_name = "depth below sea surface"
depth.units = "m"
depth[:] = metadata['depth']
# ----
if metadata.has_key("time") and not ncfile.variables.has_key('time'):
time = ncfile.createVariable('time', 'i', ('time', ))
if metadata.has_key("time_offset"):
time_offset = metadata["time_offset"].strftime(
'%Y-%m-%d %H:%M:%S') # datetime has strftime method
else:
time_offset = "undeclared offset"
if metadata.has_key("time_unit"):
tunit = metadata["time_unit"]
else:
tunit = "undeclared unit" # default
time.long_name = "time (since %s) corresponding to this time frame" % time_offset
time.units = "%s since %s" % (tunit, time_offset)
# do not set, if time dimension is unlimited
if metadata["time"] is not None:
time[:] = metadata['time']
# -------- create data variable, if needed --------
vardims = () # observe COARDS order
for dim in ('time', 'depth', 'lat', 'lon'):
if ncfile.dimensions.has_key(dim):
vardims = vardims + (dim, )
if not ncfile.variables.has_key(
metadata['variable_name']
): # if exist, assume OK and do not redefine
var = ncfile.createVariable(metadata['variable_name'], data.dtype,
vardims)
else:
var = ncfile.variables[
metadata['variable_name']] # variable already exist
# ---- dump data as full array or time frame ----
# do not check data rank, to allow a declaration cycle and e.g. setting data to zero
reorder = reorder_axes_to_COARDS(writeopt["data_layout"])
if writeopt.has_key("time_frame_number"):
it = writeopt["time_frame_number"]
var[it, :] = transpose(data, axes=reorder)
# store time for this time frame, if it is provided as a single number
if metadata.has_key("time") and isinstance(metadata['time'],
numbers.Number):
ncfile.variables['time'][it] = metadata['time']
else:
var[:] = transpose(data, axes=reorder)
# dump meta data
# check mandatory metadata - remaining mandatory keys generate other exceptions, if not present
if not metadata.has_key('units'):
raise exceptions.KeyError("metadata has no unit entry")
if not metadata.has_key('long_name'):
raise exceptions.KeyError("metadata has no long_name entry")
# ------ global attributes ------
ncfile.Conventions = "COARDS"
if metadata.has_key('global_attributes'):
gattr = metadata['global_attributes']
for key in gattr.keys():
setattr(ncfile, key, gattr[key])
# ------ variable attributes ------
for key in metadata.keys():
if key in ('lon', 'lat', 'depth', 'time', 'time_unit', 'time_offset',
'variable_name',
'global_attributes'): # these are handled above
continue
if key == 'units':
var.units = metadata['units']
continue
if key == 'long_name':
var.long_name = metadata['long_name']
continue
# all other metadata stored as global attributes
setattr(var, key, metadata[key])
def writeNameList(self, verbose=False, fail=False, **params):
"""
Modify existing namelist files using information generated via genConversion
Existing files will be copied to .bak
:param verbose (optional -- default is False). If True provide more information on what is going on.
:param fail (optional default is False). If True fail if a parameter not found.
:keyword arguments are parameters and values.
:return: ordered dict of parameters and values used.
"""
if self._readOnly:
raise exceptions.IOError("Model is read only")
params_used = collections.OrderedDict() #
files = collections.OrderedDict() # list of files to be modified.
for param, value in params.iteritems(
): # extract data from conversion indexed by file --
# could this code be moved into genVarToNameList as really a different view of the same data.
# NO as we would need to do this only once we've finished generating namelist translate tables.
# potential optimisation might be to cache this and trigger error in writeNameList if called after genNameList
# search functions first
if param in self._metaFn: # got a meta function.
if verbose:
"Running function %s" % self._metaFn[param].func_name
metaFnValues = self._metaFn[param](
value) # call the meta param function which returns a dict
params_used[param] = metaFnValues # and update return var
for conv, v in metaFnValues.iteritems(
): # iterate over result of fn.
if conv.file not in files:
files[conv.file] = [
] # if not come across the file set it to empty list
files[conv.file].append(
(v, conv)) # append the value & conversion info.
elif param in self._convNameList: # got it in convNameList ?
for conv in self._convNameList[param]:
if conv.file not in files:
files[conv.file] = [
] # if not come across the file set it to empty list
files[conv.file].append(
(value, conv)) # append the value & conversion
params_used[param] = value # and update return var
elif fail:
raise exceptions.KeyError(
"Failed to find %s in metaFn or convNameList " % param)
else:
pass
# now have conversion tuples ordered by file so let's process the files
for file in files.keys(): # iterate over files
# need to create backup? Only do if no back up exists. This allows generateNameList to be run multiple times
# doing updates. First time it runs we assume we have a directory ready to be modified.
filePath = os.path.join(self.dirPath,
file) # full path to namelist file
# check file exists if not raise exception
if not os.path.isfile(filePath):
#raise exceptions.IOError("file %s does not exist"%(filePath))
continue # skip this file.
backup_file = filePath + "_nl.bak" # and full path to backup fie.
if not os.path.isfile(backup_file):
shutil.copyfile(filePath, backup_file)
# now create the namelist file. Need a temp file
with tempfile.NamedTemporaryFile(dir=self.dirPath,
delete=False) as tmpNL:
# Now construct the patch for the namelist file for all conversion tuples.
nlPatch = collections.OrderedDict(
) # path to exisiting namelist file
for (value, conv) in files[file]:
if conv.namelist not in nlPatch:
nlPatch[conv.namelist] = collections.OrderedDict(
) # dom't have ordered dict so make it
if type(
value
) is np.ndarray: # convert numpy array to list for writing.
value = value.tolist()
elif isinstance(value, unicode):
value = str(
value
) # f90nml can't cope with unicode so convert it to string.
nlPatch[conv.namelist][conv.var] = copy.copy(
value
) # copy the variable to be stored rather than the name.
if verbose:
print "Setting %s,%s to %s in %s" % (
conv.namelist, conv.var, value, filePath)
try:
p = f90nml.patch(filePath, nlPatch,
tmpNL.name) # patch the namelist file
tmpNL.close() # close the temp file once done.
except StopIteration:
print "Problem in f90nml for %s writing to %s" % (
filePath, tmpNL.name), nlPatch
raise # raise exception.
if verbose: print "Patched %s to %s" % (filePath, tmpNL.name)
shutil.move(
tmpNL.name,
filePath) # and copy the patched file back in place.
return params_used
def __getitem__(self, key):
if key not in self.__dict__.keys():
raise exceptions.KeyError()
return self.__dict__[key]
def __normalize_ix(self, ix):
if ix < 0:
ix = len(self) + ix
if ix < 0:
raise exceptions.KeyError('Not enough items in list')
return ix
def get(self, key, defval=UNDEFINED):
if self.has_key(key):
return self[key]
if defval is UNDEFINED:
raise exceptions.KeyError("Key %r is not present." % key)
return defval
