def __init__(self, ncffile, dimension, oldres, newres, repeat_method=repeat, condense_method=sum, nthick=0): from PseudoNetCDF.sci_var import Pseudo2NetCDF PseudoNetCDFFile.__init__(self) self.__dimension = array(dimension, ndmin=1) oldres = array(oldres, ndmin=1) newres = array(newres, ndmin=1) self.__mesh = newres / oldres.astype('f') self.__condense = condense_method self.__repeat = repeat_method self.__file = ncffile self.__nthick = nthick if not logical_or((self.__mesh % 1) == 0, (1. / self.__mesh) % 1 == 0).any(): raise ValueError("One resolution must be a factor of the other.") Pseudo2NetCDF().addDimensions(self.__file, self) any_non_time_key = [ k for k in self.__file.variables.keys() if 'TFLAG' not in k][0] for dk, dfactor in zip(self.__dimension, 1. / self.__mesh): dimo = self.dimensions[dk] ndimo = self.createDimension(str(dk), len(dimo) * dfactor) ndimo.setunlimited(dimo.isunlimited()) v = self.__file.variables[any_non_time_key] v = self.__method(v) self.variables = PseudoNetCDFVariables( self.__variables, self.__file.variables.keys())
def __init__(self, ncffile, dimension, oldres, newres, repeat_method = repeat, condense_method = sum, nthick = 0): PseudoNetCDFFile.__init__(self) self.__dimension = array(dimension, ndmin = 1) oldres = array(oldres, ndmin = 1) newres = array(newres, ndmin = 1) self.__mesh = newres / oldres.astype('f') self.__condense = condense_method self.__repeat = repeat_method self.__file = ncffile self.__nthick = nthick if not logical_or((self.__mesh % 1) == 0, (1. / self.__mesh) % 1 ==0).any(): raise ValueError("One resolution must be a factor of the other.") Pseudo2NetCDF().addDimensions(self.__file, self) any_non_time_key = [k for k in self.__file.variables.keys() if 'TFLAG' not in k][0] for dk, dfactor in zip(self.__dimension, 1./self.__mesh): dimo = self.dimensions[dk] ndimo = self.createDimension(str(dk), len(dimo)*dfactor) ndimo.setunlimited(dimo.isunlimited()) v = self.__file.variables[any_non_time_key] v = self.__method(v) self.variables = PseudoNetCDFVariables(self.__variables, self.__file.variables.keys())
def __init__(self, path, keysubs={'/': '_'}, encoding='utf-8', default_llod_flag=-8888, default_llod_value='N/A', default_ulod_flag=-7777, default_ulod_value='N/A'): """ Arguments: self - implied input (not supplied in call) path - path to file keysubs - dictionary of characters to remove from variable keys and their replacements encoding - file encoding (utf-8, latin1, cp1252, etc.) default_llod_flag - flag value for lower limit of detections if not specified default_llod_value - default value to use for replacement of llod_flag default_ulod_flag - flag value for upper limit of detections if not specified default_ulod_value - default value to use for replacement of ulod_flag Returns: out - PseudoNetCDFFile interface to data in file. """ lastattr = None PseudoNetCDFFile.__init__(self) f = openf(path, 'rU', encoding=encoding) missing = [] units = [] line = f.readline() if ',' in line: delim = ',' else: delim = None def split(s): return [s_.strip() for s_ in s.split(delim)] if split(line)[-1] != '1001': raise TypeError("File is the wrong format. " + "Expected 1001; got %s" % (split(line)[-1], )) n, self.fmt = split(line) # n_user_comments = 0 n_special_comments = 0 self.n_header_lines = int(n) try: for li in range(self.n_header_lines - 1): li += 2 line = f.readline() LAST_VAR_DESC_LINE = 12 + len(missing) SPECIAL_COMMENT_COUNT_LINE = LAST_VAR_DESC_LINE + 1 LAST_SPECIAL_COMMENT_LINE = (SPECIAL_COMMENT_COUNT_LINE + n_special_comments) USER_COMMENT_COUNT_LINE = (12 + len(missing) + 2 + n_special_comments) if li == PI_LINE: self.PI_NAME = line.strip() elif li == ORG_LINE: self.ORGANIZATION_NAME = line.strip() elif li == PLAT_LINE: self.SOURCE_DESCRIPTION = line.strip() elif li == MISSION_LINE: self.MISSION_NAME = line.strip() elif li == VOL_LINE: self.VOLUME_INFO = ', '.join(split(line)) elif li == DATE_LINE: line = line.replace(',', ' ').replace('-', ' ').replace(' ', ' ').split() SDATE = ", ".join(line[:3]) WDATE = ", ".join(line[3:]) self.SDATE = SDATE self.WDATE = WDATE self._SDATE = datetime.strptime(SDATE, '%Y, %m, %d') self._WDATE = datetime.strptime(WDATE, '%Y, %m, %d') elif li == TIME_INT_LINE: self.TIME_INTERVAL = line.strip() elif li == UNIT_LINE: unitstr = line.replace('\n', '').replace('\r', '').strip() units.append(unitstr) self.INDEPENDENT_VARIABLE = units[-1] elif li == SCALE_LINE: scales = [eval(i) for i in split(line)] if set([float(s) for s in scales]) != set([1.]): raise ValueError( "Unsupported: scaling is unsupported. " + " data is scaled by %s" % (str(scales), )) elif li == MISSING_LINE: missing = [eval(i) for i in split(line)] elif li > MISSING_LINE and li <= LAST_VAR_DESC_LINE: nameunit = line.replace('\n', '').split(',') name = nameunit[0].strip() if len(nameunit) > 1: units.append(nameunit[1].strip()) elif re.compile('(.*)\((.*)\)').match(nameunit[0]): desc_groups = re.compile('(.*)\((.*)\).*').match( nameunit[0]).groups() name = desc_groups[0].strip() units.append(desc_groups[1].strip()) elif '_' in name: units.append(name.split('_')[1].strip()) else: warn('Could not find unit in string: "%s"' % line) units.append(name.strip()) elif li == SPECIAL_COMMENT_COUNT_LINE: n_special_comments = int(line.replace('\n', '')) elif (li > SPECIAL_COMMENT_COUNT_LINE and li <= LAST_SPECIAL_COMMENT_LINE): colon_pos = line.find(':') if line[:1] == ' ': k = lastattr v = getattr(self, k, '') + line else: k = line[:colon_pos].strip() v = line[colon_pos + 1:].strip() setattr(self, k, v) lastattr = k elif li == USER_COMMENT_COUNT_LINE: lastattr = None # n_user_comments = int(line.replace('\n', '')) elif (li > USER_COMMENT_COUNT_LINE and li < self.n_header_lines): colon_pos = line.find(':') if line[:1] == ' ': k = lastattr v = getattr(self, k, '') + line else: k = line[:colon_pos].strip() v = line[colon_pos + 1:].strip() setattr(self, k, v) lastattr = k elif li == self.n_header_lines: varstr = line.replace(',', ' ').replace(' ', ' ') variables = varstr.split() for oc, nc in keysubs.items(): variables = [vn.replace(oc, nc) for vn in variables] self.TFLAG = variables[0] except Exception as e: raise SyntaxError("Error parsing icartt file %s: %s" % (path, repr(e))) missing = missing[:1] + missing scales = [1.] + scales if hasattr(self, 'LLOD_FLAG'): llod_values = loddelim.sub('\n', self.LLOD_VALUE).split() if len(llod_values) == 1: llod_values *= len(variables) else: llod_values = ['N/A'] + llod_values assert len(llod_values) == len(variables) llod_values = [get_lodval(llod_val) for llod_val in llod_values] llod_flags = len(llod_values) * [self.LLOD_FLAG] llod_flags = [get_lodval(llod_flag) for llod_flag in llod_flags] else: llod_flags = [default_llod_flag] * len(scales) llod_values = [default_llod_value] * len(scales) if hasattr(self, 'ULOD_FLAG'): ulod_values = loddelim.sub('\n', self.ULOD_VALUE).split() if len(ulod_values) == 1: ulod_values *= len(variables) else: ulod_values = ['N/A'] + ulod_values assert len(ulod_values) == len(variables) ulod_values = [get_lodval(ulod_val) for ulod_val in ulod_values] ulod_flags = len(ulod_values) * [self.ULOD_FLAG] ulod_flags = [get_lodval(ulod_flag) for ulod_flag in ulod_flags] else: ulod_flags = [default_ulod_flag] * len(scales) ulod_values = [default_ulod_value] * len(scales) data = f.read() datalines = data.split('\n') ndatalines = len(datalines) while datalines[-1] in ('', ' ', '\r'): ndatalines -= 1 datalines.pop(-1) data = genfromtxt(StringIO('\n'.join(datalines).encode()), delimiter=delim, dtype='d') data = data.reshape(ndatalines, len(variables)) data = data.swapaxes(0, 1) self.createDimension('POINTS', ndatalines) for vi, var in enumerate(variables): scale = scales[vi] miss = missing[vi] unit = units[vi] dat = data[vi] llod_flag = llod_flags[vi] llod_val = llod_values[vi] ulod_flag = ulod_flags[vi] ulod_val = ulod_values[vi] vals = MaskedArray(dat, mask=dat == miss, fill_value=miss) tmpvar = self.variables[var] = PseudoNetCDFVariable(self, var, 'd', ('POINTS', ), values=vals) tmpvar.units = unit tmpvar.standard_name = var tmpvar.missing_value = miss tmpvar.fill_value = miss tmpvar.scale = scale if hasattr(self, 'LLOD_FLAG'): tmpvar.llod_flag = llod_flag tmpvar.llod_value = llod_val if hasattr(self, 'ULOD_FLAG'): tmpvar.ulod_flag = ulod_flag tmpvar.ulod_value = ulod_val def dtime(s): return timedelta(seconds=int(s), microseconds=(s - int(s)) * 1.E6) vtime = vectorize(dtime) tvar = self.variables[self.TFLAG] self._date_objs = (self._SDATE + vtime(tvar).view(type=ndarray))
def __init__(self,path): PseudoNetCDFFile.__init__(self) f = open(path, 'r') missing = [] units = [] l = f.readline() if ',' in l: delim = ',' else: delim = None split = lambda s: list(map(str.strip, s.split(delim))) if split(l)[-1] != '1001': raise TypeError("File is the wrong format. Expected 1001; got %s" % (split(l)[-1],)) n, self.fmt = split(l) n_user_comments = 0 n_special_comments = 0 self.n_header_lines = int(n) try: for li in range(self.n_header_lines-1): li += 2 l = f.readline() LAST_VAR_DESC_LINE = 12+len(missing) SPECIAL_COMMENT_COUNT_LINE = LAST_VAR_DESC_LINE + 1 LAST_SPECIAL_COMMENT_LINE = SPECIAL_COMMENT_COUNT_LINE + n_special_comments USER_COMMENT_COUNT_LINE = 12+len(missing)+2+n_special_comments if li == PI_LINE: self.PI_NAME = l.strip() elif li == ORG_LINE: self.ORGANIZATION_NAME = l.strip() elif li == PLAT_LINE: self.SOURCE_DESCRIPTION = l.strip() elif li == MISSION_LINE: self.MISSION_NAME = l.strip() elif li == VOL_LINE: self.VOLUME_INFO = l.strip() elif li == DATE_LINE: l = l.replace(',', '').split() SDATE = "".join(l[:3]) WDATE = "".join(l[3:]) self.SDATE = SDATE self.WDATE = WDATE self._SDATE = datetime.strptime(SDATE, '%Y%m%d') self._WDATE = datetime.strptime(WDATE, '%Y%m%d') elif li == TIME_INT_LINE: self.TIME_INTERVAL = l.strip() elif li == UNIT_LINE: units.append(l.replace('\n', '').replace('\r', '').strip()) self.INDEPENDENT_VARIABLE = units[-1] elif li == SCALE_LINE: scales = [eval(i) for i in split(l)] if set([float(s) for s in scales]) != set([1.]): raise ValueError("Unsupported: scaling is unsupported. data is scaled by %s" % (str(scales),)) elif li == MISSING_LINE: missing = [eval(i) for i in split(l)] elif li > MISSING_LINE and li <= LAST_VAR_DESC_LINE: nameunit = l.replace('\n','').split(',') name = nameunit[0].strip() if len(nameunit) > 1: units.append(nameunit[1].strip()) elif re.compile('(.*)\((.*)\)').match(nameunit[0]): desc_groups = re.compile('(.*)\((.*)\).*').match(nameunit[0]).groups() name = desc_groups[0].strip() units.append(desc_groups[1].strip()) elif '_' in name: units.append(name.split('_')[1].strip()) else: warn('Could not find unit in string: "%s"' % l) units.append(name.strip()) elif li == SPECIAL_COMMENT_COUNT_LINE: n_special_comments = int(l.replace('\n', '')) elif li > SPECIAL_COMMENT_COUNT_LINE and li <= LAST_SPECIAL_COMMENT_LINE: pass elif li == USER_COMMENT_COUNT_LINE: n_user_comments = int(l.replace('\n','')) elif li > USER_COMMENT_COUNT_LINE and li < self.n_header_lines: colon_pos = l.find(':') k = l[:colon_pos].strip() v = l[colon_pos+1:].strip() setattr(self,k,v) elif li == self.n_header_lines: variables = l.replace(',','').split() self.TFLAG = variables[0] except Exception as e: raise SyntaxError("Error parsing icartt file %s: %s" % (path, repr(e))) missing = missing[:1]+missing scales = [1.]+scales if hasattr(self,'LLOD_FLAG'): llod_values = loddelim.sub('\n', self.LLOD_VALUE).split() if len(llod_values) == 1: llod_values *= len(variables) else: llod_values = ['N/A']+llod_values assert len(llod_values) == len(variables) llod_values = [get_lodval(llod_val) for llod_val in llod_values] llod_flags = len(llod_values)*[self.LLOD_FLAG] llod_flags = [get_lodval(llod_flag) for llod_flag in llod_flags] if hasattr(self,'ULOD_FLAG'): ulod_values = loddelim.sub('\n', self.ULOD_VALUE).split() if len(ulod_values) == 1: ulod_values *= len(variables) else: ulod_values = ['N/A']+ulod_values assert len(ulod_values) == len(variables) ulod_values = [get_lodval(ulod_val) for ulod_val in ulod_values] ulod_flags = len(ulod_values)*[self.ULOD_FLAG] ulod_flags = [get_lodval(ulod_flag) for ulod_flag in ulod_flags] data = f.read() datalines = data.split('\n') ndatalines = len(datalines) while datalines[-1] in ('', ' ', '\r'): ndatalines -=1 datalines.pop(-1) data = genfromtxt(StringIO(bytes('\n'.join(datalines), 'utf-8')), delimiter = delim, dtype = 'd') data = data.reshape(ndatalines,len(variables)) data = data.swapaxes(0,1) self.createDimension('POINTS', ndatalines) for var, scale, miss, unit, dat, llod_flag, llod_val, ulod_flag, ulod_val in zip(variables, scales, missing, units, data, llod_flags, llod_values, ulod_flags, ulod_values): vals = MaskedArray(dat, mask = dat == miss, fill_value = miss) tmpvar = self.variables[var] = PseudoNetCDFVariable(self, var, 'd', ('POINTS',), values = vals) tmpvar.units = unit tmpvar.standard_name = var tmpvar.missing_value = miss tmpvar.fill_value = miss tmpvar.scale = scale if hasattr(self,'LLOD_FLAG'): tmpvar.llod_flag = llod_flag tmpvar.llod_value = llod_val if hasattr(self,'ULOD_FLAG'): tmpvar.ulod_flag = ulod_flag tmpvar.ulod_value = ulod_val self._date_objs = self._SDATE + vectorize(lambda s: timedelta(seconds = int(s), microseconds = (s - int(s)) * 1.E6 ))(self.variables[self.TFLAG]).view(type = ndarray)