def __init__(self, ncffile, dimension, oldres, newres,
repeat_method=repeat, condense_method=sum, nthick=0):
from PseudoNetCDF.sci_var import Pseudo2NetCDF
PseudoNetCDFFile.__init__(self)
self.__dimension = array(dimension, ndmin=1)
oldres = array(oldres, ndmin=1)
newres = array(newres, ndmin=1)
self.__mesh = newres / oldres.astype('f')
self.__condense = condense_method
self.__repeat = repeat_method
self.__file = ncffile
self.__nthick = nthick
if not logical_or((self.__mesh % 1) == 0,
(1. / self.__mesh) % 1 == 0).any():
raise ValueError("One resolution must be a factor of the other.")
Pseudo2NetCDF().addDimensions(self.__file, self)
any_non_time_key = [
k for k in self.__file.variables.keys() if 'TFLAG' not in k][0]
for dk, dfactor in zip(self.__dimension, 1. / self.__mesh):
dimo = self.dimensions[dk]
ndimo = self.createDimension(str(dk), len(dimo) * dfactor)
ndimo.setunlimited(dimo.isunlimited())
v = self.__file.variables[any_non_time_key]
v = self.__method(v)
self.variables = PseudoNetCDFVariables(
self.__variables, self.__file.variables.keys())
def __init__(self, ncffile, dimension, oldres, newres, repeat_method = repeat, condense_method = sum, nthick = 0):
PseudoNetCDFFile.__init__(self)
self.__dimension = array(dimension, ndmin = 1)
oldres = array(oldres, ndmin = 1)
newres = array(newres, ndmin = 1)
self.__mesh = newres / oldres.astype('f')
self.__condense = condense_method
self.__repeat = repeat_method
self.__file = ncffile
self.__nthick = nthick
if not logical_or((self.__mesh % 1) == 0, (1. / self.__mesh) % 1 ==0).any():
raise ValueError("One resolution must be a factor of the other.")
Pseudo2NetCDF().addDimensions(self.__file, self)
any_non_time_key = [k for k in self.__file.variables.keys() if 'TFLAG' not in k][0]
for dk, dfactor in zip(self.__dimension, 1./self.__mesh):
dimo = self.dimensions[dk]
ndimo = self.createDimension(str(dk), len(dimo)*dfactor)
ndimo.setunlimited(dimo.isunlimited())
v = self.__file.variables[any_non_time_key]
v = self.__method(v)
self.variables = PseudoNetCDFVariables(self.__variables, self.__file.variables.keys())
def __init__(self,
path,
keysubs={'/': '_'},
encoding='utf-8',
default_llod_flag=-8888,
default_llod_value='N/A',
default_ulod_flag=-7777,
default_ulod_value='N/A'):
"""
Arguments:
self - implied input (not supplied in call)
path - path to file
keysubs - dictionary of characters to remove from variable keys and
their replacements
encoding - file encoding (utf-8, latin1, cp1252, etc.)
default_llod_flag - flag value for lower limit of detections if not
specified
default_llod_value - default value to use for replacement of llod_flag
default_ulod_flag - flag value for upper limit of detections if not
specified
default_ulod_value - default value to use for replacement of ulod_flag
Returns:
out - PseudoNetCDFFile interface to data in file.
"""
lastattr = None
PseudoNetCDFFile.__init__(self)
f = openf(path, 'rU', encoding=encoding)
missing = []
units = []
line = f.readline()
if ',' in line:
delim = ','
else:
delim = None
def split(s):
return [s_.strip() for s_ in s.split(delim)]
if split(line)[-1] != '1001':
raise TypeError("File is the wrong format. " +
"Expected 1001; got %s" % (split(line)[-1], ))
n, self.fmt = split(line)
# n_user_comments = 0
n_special_comments = 0
self.n_header_lines = int(n)
try:
for li in range(self.n_header_lines - 1):
li += 2
line = f.readline()
LAST_VAR_DESC_LINE = 12 + len(missing)
SPECIAL_COMMENT_COUNT_LINE = LAST_VAR_DESC_LINE + 1
LAST_SPECIAL_COMMENT_LINE = (SPECIAL_COMMENT_COUNT_LINE +
n_special_comments)
USER_COMMENT_COUNT_LINE = (12 + len(missing) + 2 +
n_special_comments)
if li == PI_LINE:
self.PI_NAME = line.strip()
elif li == ORG_LINE:
self.ORGANIZATION_NAME = line.strip()
elif li == PLAT_LINE:
self.SOURCE_DESCRIPTION = line.strip()
elif li == MISSION_LINE:
self.MISSION_NAME = line.strip()
elif li == VOL_LINE:
self.VOLUME_INFO = ', '.join(split(line))
elif li == DATE_LINE:
line = line.replace(',',
' ').replace('-',
' ').replace(' ',
' ').split()
SDATE = ", ".join(line[:3])
WDATE = ", ".join(line[3:])
self.SDATE = SDATE
self.WDATE = WDATE
self._SDATE = datetime.strptime(SDATE, '%Y, %m, %d')
self._WDATE = datetime.strptime(WDATE, '%Y, %m, %d')
elif li == TIME_INT_LINE:
self.TIME_INTERVAL = line.strip()
elif li == UNIT_LINE:
unitstr = line.replace('\n', '').replace('\r', '').strip()
units.append(unitstr)
self.INDEPENDENT_VARIABLE = units[-1]
elif li == SCALE_LINE:
scales = [eval(i) for i in split(line)]
if set([float(s) for s in scales]) != set([1.]):
raise ValueError(
"Unsupported: scaling is unsupported. " +
" data is scaled by %s" % (str(scales), ))
elif li == MISSING_LINE:
missing = [eval(i) for i in split(line)]
elif li > MISSING_LINE and li <= LAST_VAR_DESC_LINE:
nameunit = line.replace('\n', '').split(',')
name = nameunit[0].strip()
if len(nameunit) > 1:
units.append(nameunit[1].strip())
elif re.compile('(.*)\((.*)\)').match(nameunit[0]):
desc_groups = re.compile('(.*)\((.*)\).*').match(
nameunit[0]).groups()
name = desc_groups[0].strip()
units.append(desc_groups[1].strip())
elif '_' in name:
units.append(name.split('_')[1].strip())
else:
warn('Could not find unit in string: "%s"' % line)
units.append(name.strip())
elif li == SPECIAL_COMMENT_COUNT_LINE:
n_special_comments = int(line.replace('\n', ''))
elif (li > SPECIAL_COMMENT_COUNT_LINE
and li <= LAST_SPECIAL_COMMENT_LINE):
colon_pos = line.find(':')
if line[:1] == ' ':
k = lastattr
v = getattr(self, k, '') + line
else:
k = line[:colon_pos].strip()
v = line[colon_pos + 1:].strip()
setattr(self, k, v)
lastattr = k
elif li == USER_COMMENT_COUNT_LINE:
lastattr = None
# n_user_comments = int(line.replace('\n', ''))
elif (li > USER_COMMENT_COUNT_LINE
and li < self.n_header_lines):
colon_pos = line.find(':')
if line[:1] == ' ':
k = lastattr
v = getattr(self, k, '') + line
else:
k = line[:colon_pos].strip()
v = line[colon_pos + 1:].strip()
setattr(self, k, v)
lastattr = k
elif li == self.n_header_lines:
varstr = line.replace(',', ' ').replace(' ', ' ')
variables = varstr.split()
for oc, nc in keysubs.items():
variables = [vn.replace(oc, nc) for vn in variables]
self.TFLAG = variables[0]
except Exception as e:
raise SyntaxError("Error parsing icartt file %s: %s" %
(path, repr(e)))
missing = missing[:1] + missing
scales = [1.] + scales
if hasattr(self, 'LLOD_FLAG'):
llod_values = loddelim.sub('\n', self.LLOD_VALUE).split()
if len(llod_values) == 1:
llod_values *= len(variables)
else:
llod_values = ['N/A'] + llod_values
assert len(llod_values) == len(variables)
llod_values = [get_lodval(llod_val) for llod_val in llod_values]
llod_flags = len(llod_values) * [self.LLOD_FLAG]
llod_flags = [get_lodval(llod_flag) for llod_flag in llod_flags]
else:
llod_flags = [default_llod_flag] * len(scales)
llod_values = [default_llod_value] * len(scales)
if hasattr(self, 'ULOD_FLAG'):
ulod_values = loddelim.sub('\n', self.ULOD_VALUE).split()
if len(ulod_values) == 1:
ulod_values *= len(variables)
else:
ulod_values = ['N/A'] + ulod_values
assert len(ulod_values) == len(variables)
ulod_values = [get_lodval(ulod_val) for ulod_val in ulod_values]
ulod_flags = len(ulod_values) * [self.ULOD_FLAG]
ulod_flags = [get_lodval(ulod_flag) for ulod_flag in ulod_flags]
else:
ulod_flags = [default_ulod_flag] * len(scales)
ulod_values = [default_ulod_value] * len(scales)
data = f.read()
datalines = data.split('\n')
ndatalines = len(datalines)
while datalines[-1] in ('', ' ', '\r'):
ndatalines -= 1
datalines.pop(-1)
data = genfromtxt(StringIO('\n'.join(datalines).encode()),
delimiter=delim,
dtype='d')
data = data.reshape(ndatalines, len(variables))
data = data.swapaxes(0, 1)
self.createDimension('POINTS', ndatalines)
for vi, var in enumerate(variables):
scale = scales[vi]
miss = missing[vi]
unit = units[vi]
dat = data[vi]
llod_flag = llod_flags[vi]
llod_val = llod_values[vi]
ulod_flag = ulod_flags[vi]
ulod_val = ulod_values[vi]
vals = MaskedArray(dat, mask=dat == miss, fill_value=miss)
tmpvar = self.variables[var] = PseudoNetCDFVariable(self,
var,
'd',
('POINTS', ),
values=vals)
tmpvar.units = unit
tmpvar.standard_name = var
tmpvar.missing_value = miss
tmpvar.fill_value = miss
tmpvar.scale = scale
if hasattr(self, 'LLOD_FLAG'):
tmpvar.llod_flag = llod_flag
tmpvar.llod_value = llod_val
if hasattr(self, 'ULOD_FLAG'):
tmpvar.ulod_flag = ulod_flag
tmpvar.ulod_value = ulod_val
def dtime(s):
return timedelta(seconds=int(s), microseconds=(s - int(s)) * 1.E6)
vtime = vectorize(dtime)
tvar = self.variables[self.TFLAG]
self._date_objs = (self._SDATE + vtime(tvar).view(type=ndarray))
def __init__(self,path):
PseudoNetCDFFile.__init__(self)
f = open(path, 'r')
missing = []
units = []
l = f.readline()
if ',' in l:
delim = ','
else:
delim = None
split = lambda s: list(map(str.strip, s.split(delim)))
if split(l)[-1] != '1001':
raise TypeError("File is the wrong format. Expected 1001; got %s" % (split(l)[-1],))
n, self.fmt = split(l)
n_user_comments = 0
n_special_comments = 0
self.n_header_lines = int(n)
try:
for li in range(self.n_header_lines-1):
li += 2
l = f.readline()
LAST_VAR_DESC_LINE = 12+len(missing)
SPECIAL_COMMENT_COUNT_LINE = LAST_VAR_DESC_LINE + 1
LAST_SPECIAL_COMMENT_LINE = SPECIAL_COMMENT_COUNT_LINE + n_special_comments
USER_COMMENT_COUNT_LINE = 12+len(missing)+2+n_special_comments
if li == PI_LINE:
self.PI_NAME = l.strip()
elif li == ORG_LINE:
self.ORGANIZATION_NAME = l.strip()
elif li == PLAT_LINE:
self.SOURCE_DESCRIPTION = l.strip()
elif li == MISSION_LINE:
self.MISSION_NAME = l.strip()
elif li == VOL_LINE:
self.VOLUME_INFO = l.strip()
elif li == DATE_LINE:
l = l.replace(',', '').split()
SDATE = "".join(l[:3])
WDATE = "".join(l[3:])
self.SDATE = SDATE
self.WDATE = WDATE
self._SDATE = datetime.strptime(SDATE, '%Y%m%d')
self._WDATE = datetime.strptime(WDATE, '%Y%m%d')
elif li == TIME_INT_LINE:
self.TIME_INTERVAL = l.strip()
elif li == UNIT_LINE:
units.append(l.replace('\n', '').replace('\r', '').strip())
self.INDEPENDENT_VARIABLE = units[-1]
elif li == SCALE_LINE:
scales = [eval(i) for i in split(l)]
if set([float(s) for s in scales]) != set([1.]):
raise ValueError("Unsupported: scaling is unsupported. data is scaled by %s" % (str(scales),))
elif li == MISSING_LINE:
missing = [eval(i) for i in split(l)]
elif li > MISSING_LINE and li <= LAST_VAR_DESC_LINE:
nameunit = l.replace('\n','').split(',')
name = nameunit[0].strip()
if len(nameunit) > 1:
units.append(nameunit[1].strip())
elif re.compile('(.*)\((.*)\)').match(nameunit[0]):
desc_groups = re.compile('(.*)\((.*)\).*').match(nameunit[0]).groups()
name = desc_groups[0].strip()
units.append(desc_groups[1].strip())
elif '_' in name:
units.append(name.split('_')[1].strip())
else:
warn('Could not find unit in string: "%s"' % l)
units.append(name.strip())
elif li == SPECIAL_COMMENT_COUNT_LINE:
n_special_comments = int(l.replace('\n', ''))
elif li > SPECIAL_COMMENT_COUNT_LINE and li <= LAST_SPECIAL_COMMENT_LINE:
pass
elif li == USER_COMMENT_COUNT_LINE:
n_user_comments = int(l.replace('\n',''))
elif li > USER_COMMENT_COUNT_LINE and li < self.n_header_lines:
colon_pos = l.find(':')
k = l[:colon_pos].strip()
v = l[colon_pos+1:].strip()
setattr(self,k,v)
elif li == self.n_header_lines:
variables = l.replace(',','').split()
self.TFLAG = variables[0]
except Exception as e:
raise SyntaxError("Error parsing icartt file %s: %s" % (path, repr(e)))
missing = missing[:1]+missing
scales = [1.]+scales
if hasattr(self,'LLOD_FLAG'):
llod_values = loddelim.sub('\n', self.LLOD_VALUE).split()
if len(llod_values) == 1:
llod_values *= len(variables)
else:
llod_values = ['N/A']+llod_values
assert len(llod_values) == len(variables)
llod_values = [get_lodval(llod_val) for llod_val in llod_values]
llod_flags = len(llod_values)*[self.LLOD_FLAG]
llod_flags = [get_lodval(llod_flag) for llod_flag in llod_flags]
if hasattr(self,'ULOD_FLAG'):
ulod_values = loddelim.sub('\n', self.ULOD_VALUE).split()
if len(ulod_values) == 1:
ulod_values *= len(variables)
else:
ulod_values = ['N/A']+ulod_values
assert len(ulod_values) == len(variables)
ulod_values = [get_lodval(ulod_val) for ulod_val in ulod_values]
ulod_flags = len(ulod_values)*[self.ULOD_FLAG]
ulod_flags = [get_lodval(ulod_flag) for ulod_flag in ulod_flags]
data = f.read()
datalines = data.split('\n')
ndatalines = len(datalines)
while datalines[-1] in ('', ' ', '\r'):
ndatalines -=1
datalines.pop(-1)
data = genfromtxt(StringIO(bytes('\n'.join(datalines), 'utf-8')), delimiter = delim, dtype = 'd')
data = data.reshape(ndatalines,len(variables))
data = data.swapaxes(0,1)
self.createDimension('POINTS', ndatalines)
for var, scale, miss, unit, dat, llod_flag, llod_val, ulod_flag, ulod_val in zip(variables, scales, missing, units, data, llod_flags, llod_values, ulod_flags, ulod_values):
vals = MaskedArray(dat, mask = dat == miss, fill_value = miss)
tmpvar = self.variables[var] = PseudoNetCDFVariable(self, var, 'd', ('POINTS',), values = vals)
tmpvar.units = unit
tmpvar.standard_name = var
tmpvar.missing_value = miss
tmpvar.fill_value = miss
tmpvar.scale = scale
if hasattr(self,'LLOD_FLAG'):
tmpvar.llod_flag = llod_flag
tmpvar.llod_value = llod_val
if hasattr(self,'ULOD_FLAG'):
tmpvar.ulod_flag = ulod_flag
tmpvar.ulod_value = ulod_val
self._date_objs = self._SDATE + vectorize(lambda s: timedelta(seconds = int(s), microseconds = (s - int(s)) * 1.E6 ))(self.variables[self.TFLAG]).view(type = ndarray)