Ejemplo n.º 1
0
def ConvertCAMxTime(date, time, nvars=1):
    """
    Use camx date and time arrays to produce an
    IOAPI standard TFLAG variable
    """
    f = PseudoNetCDFFile()
    f.dimensions = {'TSTEP': date.shape[0], 'VAR': nvars, 'DATE-TIME': 2}

    a = array([date, time], dtype='i').swapaxes(0, 1)
    if len(a.shape) == 2:
        a = a[:, newaxis, :]
    date = a[:, :, 0]
    if (date < 70000).any():
        date += 2000000
    else:
        date += 1900000
    time = a[:, :, 1]
    while not (time == 0).all() and time.max() < 10000:
        time *= 100
    a = PseudoNetCDFVariable(f,
                             'TFLAG',
                             'i', ('TSTEP', 'VAR', 'DATE-TIME'),
                             values=a[:, [0], :].repeat(nvars, 1))
    a.units = 'DATE-TIME'.ljust(16)
    a.long_name = 'TFLAG'.ljust(16)
    a.var_desc = a.long_name
    return a
Ejemplo n.º 2
0
    def __init__(self, ncffile, dimension, oldres, newres,
                 repeat_method=repeat, condense_method=sum, nthick=0):
        from PseudoNetCDF.sci_var import Pseudo2NetCDF
        PseudoNetCDFFile.__init__(self)
        self.__dimension = array(dimension, ndmin=1)
        oldres = array(oldres, ndmin=1)
        newres = array(newres, ndmin=1)
        self.__mesh = newres / oldres.astype('f')
        self.__condense = condense_method
        self.__repeat = repeat_method
        self.__file = ncffile
        self.__nthick = nthick

        if not logical_or((self.__mesh % 1) == 0,
                          (1. / self.__mesh) % 1 == 0).any():
            raise ValueError("One resolution must be a factor of the other.")

        Pseudo2NetCDF().addDimensions(self.__file, self)
        any_non_time_key = [
            k for k in self.__file.variables.keys() if 'TFLAG' not in k][0]
        for dk, dfactor in zip(self.__dimension, 1. / self.__mesh):
            dimo = self.dimensions[dk]
            ndimo = self.createDimension(str(dk), len(dimo) * dfactor)
            ndimo.setunlimited(dimo.isunlimited())
        v = self.__file.variables[any_non_time_key]
        v = self.__method(v)

        self.variables = PseudoNetCDFVariables(
            self.__variables, self.__file.variables.keys())
Ejemplo n.º 3
0
 def __getattribute__(self, k):
     try:
         return PseudoNetCDFFile.__getattribute__(self, k)
     except AttributeError:
         for f in self.__files:
             try:
                 return getattr(f, k)
             except:
                 pass
         raise AttributeError("%s not found" % k)
Ejemplo n.º 4
0
def ConvertCAMxTime(date,time,nvars=1):
    """
    Use camx date and time arrays to produce an
    IOAPI standard TFLAG variable
    """
    f = PseudoNetCDFFile()
    f.dimensions = {'TSTEP': date.shape[0], 'VAR': nvars, 'DATE-TIME': 2}
    
    a=array([date,time],dtype='i').swapaxes(0,1)
    if len(a.shape)==2:
        a=a[:,newaxis,:]
    date=a[:,:,0]
    if (date<70000).any():
        date+=2000000
    else:
        date+=1900000
    time=a[:,:,1]
    while not (time==0).all() and time.max()<10000:
        time*=100
    a=PseudoNetCDFVariable(f,'TFLAG','i',('TSTEP','VAR','DATE-TIME'),values=a[:,[0],:].repeat(nvars,1))
    a.units='DATE-TIME'.ljust(16)
    a.long_name='TFLAG'.ljust(16)
    a.var_desc=a.long_name
    return a
Ejemplo n.º 5
0
    def __init__(self, ncffile, dimension, oldres, newres, repeat_method = repeat, condense_method = sum, nthick = 0):
        PseudoNetCDFFile.__init__(self)
        self.__dimension = array(dimension, ndmin = 1)
        oldres = array(oldres, ndmin = 1)
        newres = array(newres, ndmin = 1)
        self.__mesh = newres / oldres.astype('f')
        self.__condense = condense_method
        self.__repeat = repeat_method
        self.__file = ncffile
        self.__nthick = nthick
            
        if not logical_or((self.__mesh % 1) == 0, (1. / self.__mesh) % 1 ==0).any():
            raise ValueError("One resolution must be a factor of the other.")

        Pseudo2NetCDF().addDimensions(self.__file, self)
        any_non_time_key = [k for k in self.__file.variables.keys() if 'TFLAG' not in k][0]
        for dk, dfactor in zip(self.__dimension, 1./self.__mesh):
            dimo = self.dimensions[dk]
            ndimo = self.createDimension(str(dk), len(dimo)*dfactor)
            ndimo.setunlimited(dimo.isunlimited())
        v = self.__file.variables[any_non_time_key]
        v = self.__method(v)
        
        self.variables = PseudoNetCDFVariables(self.__variables, self.__file.variables.keys())
Ejemplo n.º 6
0
    def setUp(self):
        from permm.mechanisms import small_strato
        from PseudoNetCDF import PseudoNetCDFFile
        from numpy import arange, newaxis
        from numpy.random import normal, poisson, random, seed
        
        self.mech = small_strato()
        mrg = self.mrg = PseudoNetCDFFile()
        
        mrg.createDimension('TSTEP', 10)
        mrg.createDimension('ROW', 3)
        mrg.createDimension('LAY', 4)
        mrg.createDimension('COL', 5)
        mrg.createDimension('VAR', 10)
        mrg.createDimension('DATE-TIME', 2)
        
        tflag = mrg.createVariable('TFLAG', 'i', ('TSTEP', 'VAR', 'DATE-TIME'))
        tflag.units = '<YYYYJJJ, HHMMSS>'
        tflag.long_name = tflag.var_desc = 'time '
        
        tflag[:, :, 0] = 2004001
        tflag[:, :, 1] = arange(10)[:, newaxis] * 10000
        
        for i in range(1, 11):
            var = mrg.createVariable('IRR_%d' % i, 'f', ('TSTEP', 'LAY', 'ROW', 'COL'))
            var.units = 'ppt'
            var.long_name = var.var_desc = 'Integrated rate for IRR ordinal %d' % i
            seed(1)
            if i % 2 == 0:
                data = arange(3 * i, 3 * i + 10)[:, newaxis, newaxis, newaxis].repeat(4, 1).repeat(3, 2).repeat(5, 3)
            elif i % 1 == 0:
                data = arange(2 * i, 2 * i + 10)[:, newaxis, newaxis, newaxis].repeat(4, 1).repeat(3, 2).repeat(5, 3)
            else:
                data = arange(1 * i, 1 * i + 10)[:, newaxis, newaxis, newaxis].repeat(4, 1).repeat(3, 2).repeat(5, 3)

            var[:] = data * {True: -1, False: 1}[i % 2 == True]

        self.mech.set_mrg(mrg)
Ejemplo n.º 7
0
 def __repr__(self):
     return PseudoNetCDFFile.__repr__(self) + str(self.variables)
Ejemplo n.º 8
0
    def __init__(self,
                 path,
                 keysubs={'/': '_'},
                 encoding='utf-8',
                 default_llod_flag=-8888,
                 default_llod_value='N/A',
                 default_ulod_flag=-7777,
                 default_ulod_value='N/A'):
        """
Arguments:
   self - implied input (not supplied in call)
   path - path to file
   keysubs - dictionary of characters to remove from variable keys and
             their replacements
   encoding - file encoding (utf-8, latin1, cp1252, etc.)
   default_llod_flag - flag value for lower limit of detections if not
                       specified
   default_llod_value - default value to use for replacement of llod_flag
   default_ulod_flag - flag value for upper limit of detections if not
                       specified
   default_ulod_value - default value to use for replacement of ulod_flag
Returns:
   out - PseudoNetCDFFile interface to data in file.
        """
        lastattr = None
        PseudoNetCDFFile.__init__(self)
        f = openf(path, 'rU', encoding=encoding)
        missing = []
        units = []
        line = f.readline()
        if ',' in line:
            delim = ','
        else:
            delim = None

        def split(s):
            return [s_.strip() for s_ in s.split(delim)]

        if split(line)[-1] != '1001':
            raise TypeError("File is the wrong format.  " +
                            "Expected 1001; got %s" % (split(line)[-1], ))

        n, self.fmt = split(line)
        # n_user_comments = 0
        n_special_comments = 0
        self.n_header_lines = int(n)
        try:
            for li in range(self.n_header_lines - 1):
                li += 2
                line = f.readline()
                LAST_VAR_DESC_LINE = 12 + len(missing)
                SPECIAL_COMMENT_COUNT_LINE = LAST_VAR_DESC_LINE + 1
                LAST_SPECIAL_COMMENT_LINE = (SPECIAL_COMMENT_COUNT_LINE +
                                             n_special_comments)
                USER_COMMENT_COUNT_LINE = (12 + len(missing) + 2 +
                                           n_special_comments)
                if li == PI_LINE:
                    self.PI_NAME = line.strip()
                elif li == ORG_LINE:
                    self.ORGANIZATION_NAME = line.strip()
                elif li == PLAT_LINE:
                    self.SOURCE_DESCRIPTION = line.strip()
                elif li == MISSION_LINE:
                    self.MISSION_NAME = line.strip()
                elif li == VOL_LINE:
                    self.VOLUME_INFO = ', '.join(split(line))
                elif li == DATE_LINE:
                    line = line.replace(',',
                                        ' ').replace('-',
                                                     ' ').replace('  ',
                                                                  ' ').split()
                    SDATE = ", ".join(line[:3])
                    WDATE = ", ".join(line[3:])
                    self.SDATE = SDATE
                    self.WDATE = WDATE
                    self._SDATE = datetime.strptime(SDATE, '%Y, %m, %d')
                    self._WDATE = datetime.strptime(WDATE, '%Y, %m, %d')
                elif li == TIME_INT_LINE:
                    self.TIME_INTERVAL = line.strip()
                elif li == UNIT_LINE:
                    unitstr = line.replace('\n', '').replace('\r', '').strip()
                    units.append(unitstr)
                    self.INDEPENDENT_VARIABLE = units[-1]
                elif li == SCALE_LINE:
                    scales = [eval(i) for i in split(line)]
                    if set([float(s) for s in scales]) != set([1.]):
                        raise ValueError(
                            "Unsupported: scaling is unsupported. " +
                            " data is scaled by %s" % (str(scales), ))
                elif li == MISSING_LINE:
                    missing = [eval(i) for i in split(line)]
                elif li > MISSING_LINE and li <= LAST_VAR_DESC_LINE:
                    nameunit = line.replace('\n', '').split(',')
                    name = nameunit[0].strip()
                    if len(nameunit) > 1:
                        units.append(nameunit[1].strip())
                    elif re.compile('(.*)\((.*)\)').match(nameunit[0]):
                        desc_groups = re.compile('(.*)\((.*)\).*').match(
                            nameunit[0]).groups()
                        name = desc_groups[0].strip()
                        units.append(desc_groups[1].strip())
                    elif '_' in name:
                        units.append(name.split('_')[1].strip())
                    else:
                        warn('Could not find unit in string: "%s"' % line)
                        units.append(name.strip())
                elif li == SPECIAL_COMMENT_COUNT_LINE:
                    n_special_comments = int(line.replace('\n', ''))
                elif (li > SPECIAL_COMMENT_COUNT_LINE
                      and li <= LAST_SPECIAL_COMMENT_LINE):
                    colon_pos = line.find(':')
                    if line[:1] == ' ':
                        k = lastattr
                        v = getattr(self, k, '') + line
                    else:
                        k = line[:colon_pos].strip()
                        v = line[colon_pos + 1:].strip()
                    setattr(self, k, v)
                    lastattr = k
                elif li == USER_COMMENT_COUNT_LINE:
                    lastattr = None
                    # n_user_comments = int(line.replace('\n', ''))
                elif (li > USER_COMMENT_COUNT_LINE
                      and li < self.n_header_lines):
                    colon_pos = line.find(':')
                    if line[:1] == ' ':
                        k = lastattr
                        v = getattr(self, k, '') + line
                    else:
                        k = line[:colon_pos].strip()
                        v = line[colon_pos + 1:].strip()
                    setattr(self, k, v)
                    lastattr = k
                elif li == self.n_header_lines:
                    varstr = line.replace(',', ' ').replace('  ', ' ')
                    variables = varstr.split()
                    for oc, nc in keysubs.items():
                        variables = [vn.replace(oc, nc) for vn in variables]
                    self.TFLAG = variables[0]
        except Exception as e:
            raise SyntaxError("Error parsing icartt file %s: %s" %
                              (path, repr(e)))

        missing = missing[:1] + missing
        scales = [1.] + scales

        if hasattr(self, 'LLOD_FLAG'):
            llod_values = loddelim.sub('\n', self.LLOD_VALUE).split()
            if len(llod_values) == 1:
                llod_values *= len(variables)
            else:
                llod_values = ['N/A'] + llod_values

            assert len(llod_values) == len(variables)
            llod_values = [get_lodval(llod_val) for llod_val in llod_values]

            llod_flags = len(llod_values) * [self.LLOD_FLAG]
            llod_flags = [get_lodval(llod_flag) for llod_flag in llod_flags]
        else:
            llod_flags = [default_llod_flag] * len(scales)
            llod_values = [default_llod_value] * len(scales)

        if hasattr(self, 'ULOD_FLAG'):
            ulod_values = loddelim.sub('\n', self.ULOD_VALUE).split()
            if len(ulod_values) == 1:
                ulod_values *= len(variables)
            else:
                ulod_values = ['N/A'] + ulod_values

            assert len(ulod_values) == len(variables)
            ulod_values = [get_lodval(ulod_val) for ulod_val in ulod_values]

            ulod_flags = len(ulod_values) * [self.ULOD_FLAG]
            ulod_flags = [get_lodval(ulod_flag) for ulod_flag in ulod_flags]
        else:
            ulod_flags = [default_ulod_flag] * len(scales)
            ulod_values = [default_ulod_value] * len(scales)

        data = f.read()
        datalines = data.split('\n')
        ndatalines = len(datalines)
        while datalines[-1] in ('', ' ', '\r'):
            ndatalines -= 1
            datalines.pop(-1)

        data = genfromtxt(StringIO('\n'.join(datalines).encode()),
                          delimiter=delim,
                          dtype='d')
        data = data.reshape(ndatalines, len(variables))
        data = data.swapaxes(0, 1)
        self.createDimension('POINTS', ndatalines)
        for vi, var in enumerate(variables):
            scale = scales[vi]
            miss = missing[vi]
            unit = units[vi]
            dat = data[vi]
            llod_flag = llod_flags[vi]
            llod_val = llod_values[vi]
            ulod_flag = ulod_flags[vi]
            ulod_val = ulod_values[vi]
            vals = MaskedArray(dat, mask=dat == miss, fill_value=miss)
            tmpvar = self.variables[var] = PseudoNetCDFVariable(self,
                                                                var,
                                                                'd',
                                                                ('POINTS', ),
                                                                values=vals)
            tmpvar.units = unit
            tmpvar.standard_name = var
            tmpvar.missing_value = miss
            tmpvar.fill_value = miss
            tmpvar.scale = scale

            if hasattr(self, 'LLOD_FLAG'):
                tmpvar.llod_flag = llod_flag
                tmpvar.llod_value = llod_val

            if hasattr(self, 'ULOD_FLAG'):
                tmpvar.ulod_flag = ulod_flag
                tmpvar.ulod_value = ulod_val

        def dtime(s):
            return timedelta(seconds=int(s), microseconds=(s - int(s)) * 1.E6)

        vtime = vectorize(dtime)
        tvar = self.variables[self.TFLAG]
        self._date_objs = (self._SDATE + vtime(tvar).view(type=ndarray))
Ejemplo n.º 9
0
    def setUp(self):
        from PseudoNetCDF import PseudoNetCDFFile
        self.checkval = """time,layer,latitude,longitude,test
0.0,0.0,0.0,0.0,0.0
0.0,0.0,0.0,1.0,1.0
0.0,0.0,0.0,2.0,2.0
0.0,0.0,0.0,3.0,3.0
0.0,0.0,0.0,4.0,4.0
0.0,0.0,1.0,0.0,5.0
0.0,0.0,1.0,1.0,6.0
0.0,0.0,1.0,2.0,7.0
0.0,0.0,1.0,3.0,8.0
0.0,0.0,1.0,4.0,9.0
0.0,0.0,2.0,0.0,10.0
0.0,0.0,2.0,1.0,11.0
0.0,0.0,2.0,2.0,12.0
0.0,0.0,2.0,3.0,13.0
0.0,0.0,2.0,4.0,14.0
0.0,0.0,3.0,0.0,15.0
0.0,0.0,3.0,1.0,16.0
0.0,0.0,3.0,2.0,17.0
0.0,0.0,3.0,3.0,18.0
0.0,0.0,3.0,4.0,19.0
0.0,1.0,0.0,0.0,20.0
0.0,1.0,0.0,1.0,21.0
0.0,1.0,0.0,2.0,22.0
0.0,1.0,0.0,3.0,23.0
0.0,1.0,0.0,4.0,24.0
0.0,1.0,1.0,0.0,25.0
0.0,1.0,1.0,1.0,26.0
0.0,1.0,1.0,2.0,27.0
0.0,1.0,1.0,3.0,28.0
0.0,1.0,1.0,4.0,29.0
0.0,1.0,2.0,0.0,30.0
0.0,1.0,2.0,1.0,31.0
0.0,1.0,2.0,2.0,32.0
0.0,1.0,2.0,3.0,33.0
0.0,1.0,2.0,4.0,34.0
0.0,1.0,3.0,0.0,35.0
0.0,1.0,3.0,1.0,36.0
0.0,1.0,3.0,2.0,37.0
0.0,1.0,3.0,3.0,38.0
0.0,1.0,3.0,4.0,39.0
0.0,2.0,0.0,0.0,40.0
0.0,2.0,0.0,1.0,41.0
0.0,2.0,0.0,2.0,42.0
0.0,2.0,0.0,3.0,43.0
0.0,2.0,0.0,4.0,44.0
0.0,2.0,1.0,0.0,45.0
0.0,2.0,1.0,1.0,46.0
0.0,2.0,1.0,2.0,47.0
0.0,2.0,1.0,3.0,48.0
0.0,2.0,1.0,4.0,49.0
0.0,2.0,2.0,0.0,50.0
0.0,2.0,2.0,1.0,51.0
0.0,2.0,2.0,2.0,52.0
0.0,2.0,2.0,3.0,53.0
0.0,2.0,2.0,4.0,54.0
0.0,2.0,3.0,0.0,55.0
0.0,2.0,3.0,1.0,56.0
0.0,2.0,3.0,2.0,57.0
0.0,2.0,3.0,3.0,58.0
0.0,2.0,3.0,4.0,59.0
1.0,0.0,0.0,0.0,60.0
1.0,0.0,0.0,1.0,61.0
1.0,0.0,0.0,2.0,62.0
1.0,0.0,0.0,3.0,63.0
1.0,0.0,0.0,4.0,64.0
1.0,0.0,1.0,0.0,65.0
1.0,0.0,1.0,1.0,66.0
1.0,0.0,1.0,2.0,67.0
1.0,0.0,1.0,3.0,68.0
1.0,0.0,1.0,4.0,69.0
1.0,0.0,2.0,0.0,70.0
1.0,0.0,2.0,1.0,71.0
1.0,0.0,2.0,2.0,72.0
1.0,0.0,2.0,3.0,73.0
1.0,0.0,2.0,4.0,74.0
1.0,0.0,3.0,0.0,75.0
1.0,0.0,3.0,1.0,76.0
1.0,0.0,3.0,2.0,77.0
1.0,0.0,3.0,3.0,78.0
1.0,0.0,3.0,4.0,79.0
1.0,1.0,0.0,0.0,80.0
1.0,1.0,0.0,1.0,81.0
1.0,1.0,0.0,2.0,82.0
1.0,1.0,0.0,3.0,83.0
1.0,1.0,0.0,4.0,84.0
1.0,1.0,1.0,0.0,85.0
1.0,1.0,1.0,1.0,86.0
1.0,1.0,1.0,2.0,87.0
1.0,1.0,1.0,3.0,88.0
1.0,1.0,1.0,4.0,89.0
1.0,1.0,2.0,0.0,90.0
1.0,1.0,2.0,1.0,91.0
1.0,1.0,2.0,2.0,92.0
1.0,1.0,2.0,3.0,93.0
1.0,1.0,2.0,4.0,94.0
1.0,1.0,3.0,0.0,95.0
1.0,1.0,3.0,1.0,96.0
1.0,1.0,3.0,2.0,97.0
1.0,1.0,3.0,3.0,98.0
1.0,1.0,3.0,4.0,99.0
1.0,2.0,0.0,0.0,100.0
1.0,2.0,0.0,1.0,101.0
1.0,2.0,0.0,2.0,102.0
1.0,2.0,0.0,3.0,103.0
1.0,2.0,0.0,4.0,104.0
1.0,2.0,1.0,0.0,105.0
1.0,2.0,1.0,1.0,106.0
1.0,2.0,1.0,2.0,107.0
1.0,2.0,1.0,3.0,108.0
1.0,2.0,1.0,4.0,109.0
1.0,2.0,2.0,0.0,110.0
1.0,2.0,2.0,1.0,111.0
1.0,2.0,2.0,2.0,112.0
1.0,2.0,2.0,3.0,113.0
1.0,2.0,2.0,4.0,114.0
1.0,2.0,3.0,0.0,115.0
1.0,2.0,3.0,1.0,116.0
1.0,2.0,3.0,2.0,117.0
1.0,2.0,3.0,3.0,118.0
1.0,2.0,3.0,4.0,119.0
"""
        testfile = self.testfile = PseudoNetCDFFile()
        testfile.createDimension('time', 2)
        testfile.createDimension('layer', 3)
        testfile.createDimension('latitude', 4)
        testfile.createDimension('longitude', 5)
        for dk, dv in testfile.dimensions.items():
            var = testfile.createVariable(dk, 'f', (dk, ))
            var[:] = np.arange(len(dv), dtype='f')
        var = testfile.createVariable(
            'test', 'f', ('time', 'layer', 'latitude', 'longitude'))
        var[:] = np.arange(2 * 3 * 4 * 5).reshape(2, 3, 4, 5)
Ejemplo n.º 10
0
def mrgidx(ipr_paths, irr_paths, idx):
    if isinstance(irr_paths,str):
        irrf = NetCDFFile(irr_paths)
    else:
        irrf = file_master([NetCDFFile(irr_path) for irr_path in irr_paths])
    
    if isinstance(ipr_paths,str):
        iprf = NetCDFFile(ipr_paths)
    else:
        iprf = file_master([NetCDFFile(ipr_path) for ipr_path in ipr_paths])
        
    
    # Process and Reaction keys should exclude TFLAG
    pr_keys = [pr for pr in iprf.variables.keys() if pr not in ('TFLAG',)]
    rr_keys = [rr for rr in irrf.variables.keys() if rr not in ('TFLAG',)]
    
    # Attempt to order reactions by number
    # this is not necessary, but is nice and clean
    try:
        rr_keys = [(int(rr.split('_')[1]), rr) for rr in rr_keys]
        rr_keys.sort()
        rr_keys = [rr[1] for rr in rr_keys]
    except:
        warn("Cannot sort reaction keys")
    
    # Processes are predicated by a delimiter
    prcs = list(set(['_'.join(pr.split('_')[:-1]) for pr in pr_keys]))
    # Species are preceded by a delimiter
    spcs = list(set(['_'.join(pr.split('_')[-1:]) for pr in pr_keys]))
    
    # Select a dummy variable for extracting properties
    pr_tmp = iprf.variables[pr_keys[0]]
    
    # Create an empty file and decorate
    # it as necessary
    outf = PseudoNetCDFFile()
    outf.Species = "".join([spc.ljust(16) for spc in spcs])
    outf.Process = "".join([prc.ljust(16) for prc in prcs])
    outf.Reactions = "".join([rr_key.ljust(16) for rr_key in rr_keys])
    outf.createDimension("PROCESS", len(prcs))
    outf.createDimension("SPECIES", len(spcs))
    outf.createDimension("RXN", len(rr_keys))
    outf.createDimension("TSTEP", pr_tmp[:,0,0,0].shape[0])
    outf.createDimension("TSTEP_STAG", len(outf.dimensions["TSTEP"])+1)
    outf.createDimension("ROW", 1)
    outf.createDimension("LAY", 1)
    outf.createDimension("COL", 1)
    outf.createDimension("VAR", 3)
    outf.createDimension("DATE-TIME", 2)
    tflag = outf.createVariable("TFLAG", "i", ('TSTEP', 'VAR', 'DATE-TIME'))
    tflag.__dict__.update(dict(units = "<YYYYJJJ,HHDDMM>", var_desc = 'TFLAG'.ljust(16), long_name = 'TFLAG'.ljust(16)))
    tflag[:,:,:] = iprf.variables['TFLAG'][:][:,[0],:]
    shape = outf.createVariable("SHAPE", "i", ("TSTEP", "LAY", "ROW", "COL"))
    shape.__dict__.update(dict(units = "ON/OFF", var_desc = "SHAPE".ljust(16), long_name = "SHAPE".ljust(16)))
    shape[:] = 1
    irr = outf.createVariable("IRR", "f", ("TSTEP", "RXN"))
    irr.__dict__.update(dict(units = pr_tmp.units, var_desc = "IRR".ljust(16), long_name = "IRR".ljust(16)))
    ipr = outf.createVariable("IPR", "f", ("TSTEP", "SPECIES", "PROCESS"))
    irr.__dict__.update(dict(units = pr_tmp.units, var_desc = "IPR".ljust(16), long_name = "IPR".ljust(16)))

    for rr, var in zip(rr_keys,irr.swapaxes(0,1)):
        var[:] = irrf.variables[rr][:][idx]
        
    for prc, prcvar in zip(prcs,ipr.swapaxes(0,2)):
        for spc, spcvar in zip(spcs,prcvar):
            try:
                spcvar[:] = iprf.variables['_'.join([prc,spc])][:][idx]
            except KeyError as es:
                warn(str(es))

    return outf
Ejemplo n.º 11
0
            print(point, isin)

varkeys = ['temperature', 'windDir', 'windSpeed', 'dewpoint', 'altimeter']
vardds = [k + 'DD' for k in varkeys]

if args.verbose > 1:
    print('Subset variables')

getvarkeys = varkeys + vardds + \
    ['stationName', 'timeObs', 'timeNominal', 'elevation', 'latitude', 'longitude']

if args.verbose > 1:
    print('Slicing files')

p2p = Pseudo2NetCDF(verbose=0)
outfile = PseudoNetCDFFile()
p2p.addDimensions(ncff, outfile)
outfile.createDimension('recNum', len(found_point_ids))
p2p.addGlobalProperties(ncff, outfile)

for vark in getvarkeys:
    p2p.addVariable(ncff, outfile, vark, data=False)

for vark in getvarkeys:
    invar = ncff.variables[vark]
    outvar = outfile.variables[vark]
    recid = list(invar.dimensions).index('recNum')
    outvar[:] = invar[:].take(found_point_ids, recid)

if args.humidity:
    varkeys.append('specificHumidity')
Ejemplo n.º 12
0
def mrgidx(ipr_paths, irr_paths, idx):
    if isinstance(irr_paths, str):
        irrf = NetCDFFile(irr_paths)
    else:
        irrf = file_master([NetCDFFile(irr_path) for irr_path in irr_paths])

    if isinstance(ipr_paths, str):
        iprf = NetCDFFile(ipr_paths)
    else:
        iprf = file_master([NetCDFFile(ipr_path) for ipr_path in ipr_paths])

    # Process and Reaction keys should exclude TFLAG
    pr_keys = [pr for pr in iprf.variables.keys() if pr not in ('TFLAG', )]
    rr_keys = [rr for rr in irrf.variables.keys() if rr not in ('TFLAG', )]

    # Attempt to order reactions by number
    # this is not necessary, but is nice and clean
    try:
        rr_keys = [(int(rr.split('_')[1]), rr) for rr in rr_keys]
        rr_keys.sort()
        rr_keys = [rr[1] for rr in rr_keys]
    except:
        warn("Cannot sort reaction keys")

    # Processes are predicated by a delimiter
    prcs = list(set(['_'.join(pr.split('_')[:-1]) for pr in pr_keys]))
    # Species are preceded by a delimiter
    spcs = list(set(['_'.join(pr.split('_')[-1:]) for pr in pr_keys]))

    # Select a dummy variable for extracting properties
    pr_tmp = iprf.variables[pr_keys[0]]

    # Create an empty file and decorate
    # it as necessary
    outf = PseudoNetCDFFile()
    outf.Species = "".join([spc.ljust(16) for spc in spcs])
    outf.Process = "".join([prc.ljust(16) for prc in prcs])
    outf.Reactions = "".join([rr_key.ljust(16) for rr_key in rr_keys])
    outf.createDimension("PROCESS", len(prcs))
    outf.createDimension("SPECIES", len(spcs))
    outf.createDimension("RXN", len(rr_keys))
    outf.createDimension("TSTEP", pr_tmp[:, 0, 0, 0].shape[0])
    outf.createDimension("TSTEP_STAG", len(outf.dimensions["TSTEP"]) + 1)
    outf.createDimension("ROW", 1)
    outf.createDimension("LAY", 1)
    outf.createDimension("COL", 1)
    outf.createDimension("VAR", 3)
    outf.createDimension("DATE-TIME", 2)
    tflag = outf.createVariable("TFLAG", "i", ('TSTEP', 'VAR', 'DATE-TIME'))
    tflag.__dict__.update(
        dict(units="<YYYYJJJ,HHDDMM>",
             var_desc='TFLAG'.ljust(16),
             long_name='TFLAG'.ljust(16)))
    tflag[:, :, :] = iprf.variables['TFLAG'][:][:, [0], :]
    shape = outf.createVariable("SHAPE", "i", ("TSTEP", "LAY", "ROW", "COL"))
    shape.__dict__.update(
        dict(units="ON/OFF",
             var_desc="SHAPE".ljust(16),
             long_name="SHAPE".ljust(16)))
    shape[:] = 1
    irr = outf.createVariable("IRR", "f", ("TSTEP", "RXN"))
    irr.__dict__.update(
        dict(units=pr_tmp.units,
             var_desc="IRR".ljust(16),
             long_name="IRR".ljust(16)))
    ipr = outf.createVariable("IPR", "f", ("TSTEP", "SPECIES", "PROCESS"))
    irr.__dict__.update(
        dict(units=pr_tmp.units,
             var_desc="IPR".ljust(16),
             long_name="IPR".ljust(16)))

    for rr, var in zip(rr_keys, irr.swapaxes(0, 1)):
        var[:] = irrf.variables[rr][:][idx]

    for prc, prcvar in zip(prcs, ipr.swapaxes(0, 2)):
        for spc, spcvar in zip(spcs, prcvar):
            try:
                spcvar[:] = iprf.variables['_'.join([prc, spc])][:][idx]
            except KeyError as es:
                warn(str(es))

    return outf
Ejemplo n.º 13
0
    def __init__(self, rf, multi=False, **props):
        """
        Initialization included reading the header and learning
        about the format.
        
        see __readheader and __gettimestep() for more info

        Keywords (i.e., props) for projection: P_ALP, P_BET, P_GAM, XCENT, YCENT, XORIG, YORIG, XCELL, YCELL
        """
        self.__rffile = OpenRecordFile(rf)
        self.__readheader()
        self.__ipr_record_type = {
            24:
            dtype(
                dict(names=[
                    'SPAD', 'DATE', 'TIME', 'SPC', 'PAGRID', 'NEST', 'I', 'J',
                    'K', 'INIT', 'CHEM', 'EMIS', 'PTEMIS', 'PIG', 'WADV',
                    'EADV', 'SADV', 'NADV', 'BADV', 'TADV', 'DIL', 'WDIF',
                    'EDIF', 'SDIF', 'NDIF', 'BDIF', 'TDIF', 'DDEP', 'WDEP',
                    'AERCHEM', 'FCONC', 'UCNV', 'AVOL', 'EPAD'
                ],
                     formats=[
                         '>i', '>i', '>f', '>S10', '>i', '>i', '>i', '>i',
                         '>i', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f',
                         '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f',
                         '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>i'
                     ])),
            26:
            dtype(
                dict(names=[
                    'SPAD', 'DATE', 'TIME', 'SPC', 'PAGRID', 'NEST', 'I', 'J',
                    'K', 'INIT', 'CHEM', 'EMIS', 'PTEMIS', 'PIG', 'WADV',
                    'EADV', 'SADV', 'NADV', 'BADV', 'TADV', 'DIL', 'WDIF',
                    'EDIF', 'SDIF', 'NDIF', 'BDIF', 'TDIF', 'DDEP', 'WDEP',
                    'INORGACHEM', 'ORGACHEM', 'AQACHEM', 'FCONC', 'UCNV',
                    'AVOL', 'EPAD'
                ],
                     formats=[
                         '>i', '>i', '>f', '>S10', '>i', '>i', '>i', '>i',
                         '>i', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f',
                         '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f',
                         '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f',
                         '>i'
                     ]))
        }[len(self.prcnames)]

        prcs = [
            'SPAD', 'DATE', 'TIME', 'PAGRID', 'NEST', 'I', 'J', 'K', 'INIT',
            'CHEM', 'EMIS', 'PTEMIS', 'PIG', 'WADV', 'EADV', 'SADV', 'NADV',
            'BADV', 'TADV', 'DIL', 'WDIF', 'EDIF', 'SDIF', 'NDIF', 'BDIF',
            'TDIF', 'DDEP', 'WDEP'
        ] + {
            24: ['AERCHEM'],
            26: ['INORGACHEM', 'ORGACHEM', 'AQACHEM']
        }[len(self.prcnames)] + ['FCONC', 'UCNV', 'AVOL', 'EPAD']
        varkeys = ['_'.join(i) for i in cartesian(prcs, self.spcnames)]
        varkeys += [
            'SPAD', 'DATE', 'TIME', 'PAGRID', 'NEST', 'I', 'J', 'K', 'TFLAG'
        ]
        self.groups = {}
        NSTEPS = len([i_ for i_ in self.timerange()])
        NVARS = len(varkeys)
        self.createDimension('VAR', NVARS)
        self.createDimension('DATE-TIME', 2)
        self.createDimension('TSTEP', NSTEPS)
        padatatype = []
        pavarkeys = []
        for di, domain in enumerate(self.padomains):
            dk = 'PA%02d' % di
            prefix = dk + '_'
            grp = self.groups[dk] = PseudoNetCDFFile()
            pavarkeys.extend([prefix + k for k in varkeys])
            grp.createDimension('VAR', NVARS)
            grp.createDimension('DATE-TIME', 2)
            grp.createDimension('TSTEP', NSTEPS)
            grp.createDimension('COL', domain['iend'] - domain['istart'] + 1)
            grp.createDimension('ROW', domain['jend'] - domain['jstart'] + 1)
            grp.createDimension('LAY', domain['tlay'] - domain['blay'] + 1)
            padatatype.append(
                (dk, self.__ipr_record_type, (len(grp.dimensions['ROW']),
                                              len(grp.dimensions['COL']),
                                              len(grp.dimensions['LAY']))))
            if len(self.padomains) == 1:
                self.createDimension('COL',
                                     domain['iend'] - domain['istart'] + 1)
                self.createDimension('ROW',
                                     domain['jend'] - domain['jstart'] + 1)
                self.createDimension('LAY',
                                     domain['tlay'] - domain['blay'] + 1)
            exec(
                """def varget(k):
                return self._ipr__variables('%s', k)""" % dk, dict(self=self),
                locals())
            if len(self.padomains) == 1:
                self.variables = PseudoNetCDFVariables(varget, varkeys)
            else:
                grp.variables = PseudoNetCDFVariables(varget, varkeys)

        self.__memmaps = memmap(self.__rffile.infile.name, dtype(padatatype),
                                'r', self.data_start_byte).reshape(
                                    NSTEPS, len(self.spcnames))
        for k, v in props.items():
            setattr(self, k, v)
        try:
            add_cf_from_ioapi(self)
        except:
            pass
Ejemplo n.º 14
0
 def __repr__(self):
     return PseudoNetCDFFile.__repr__(self) + str(self.variables)
Ejemplo n.º 15
0
    def __init__(self,path):
        PseudoNetCDFFile.__init__(self)
        f = open(path, 'r')
        missing = []
        units = []
        l = f.readline()
        if ',' in l:
            delim = ','
        else:
            delim = None
        split = lambda s: list(map(str.strip, s.split(delim)))

        if split(l)[-1] != '1001':
            raise TypeError("File is the wrong format.  Expected 1001; got %s" % (split(l)[-1],))
        
        n, self.fmt = split(l)
        n_user_comments = 0
        n_special_comments = 0
        self.n_header_lines = int(n)
        try:
            for li in range(self.n_header_lines-1):
                li += 2
                l = f.readline()
                LAST_VAR_DESC_LINE = 12+len(missing)
                SPECIAL_COMMENT_COUNT_LINE = LAST_VAR_DESC_LINE + 1
                LAST_SPECIAL_COMMENT_LINE = SPECIAL_COMMENT_COUNT_LINE + n_special_comments
                USER_COMMENT_COUNT_LINE = 12+len(missing)+2+n_special_comments
                if li == PI_LINE:
                    self.PI_NAME = l.strip()
                elif li == ORG_LINE:
                    self.ORGANIZATION_NAME = l.strip()
                elif li == PLAT_LINE:
                    self.SOURCE_DESCRIPTION = l.strip()
                elif li == MISSION_LINE:
                    self.MISSION_NAME = l.strip()
                elif li == VOL_LINE:
                    self.VOLUME_INFO = l.strip()
                elif li == DATE_LINE:
                    l = l.replace(',', '').split()
                    SDATE = "".join(l[:3])
                    WDATE = "".join(l[3:])
                    self.SDATE = SDATE
                    self.WDATE = WDATE
                    self._SDATE = datetime.strptime(SDATE, '%Y%m%d')
                    self._WDATE = datetime.strptime(WDATE, '%Y%m%d')
                elif li == TIME_INT_LINE:
                    self.TIME_INTERVAL = l.strip()
                elif li == UNIT_LINE:
                    units.append(l.replace('\n', '').replace('\r', '').strip())
                    self.INDEPENDENT_VARIABLE = units[-1]
                elif li == SCALE_LINE:
                    scales = [eval(i) for i in split(l)]
                    if set([float(s) for s in scales]) != set([1.]):
                        raise ValueError("Unsupported: scaling is unsupported.  data is scaled by %s" % (str(scales),))
                elif li == MISSING_LINE:
                    missing = [eval(i) for i in split(l)]
                elif li > MISSING_LINE and li <= LAST_VAR_DESC_LINE:
                    nameunit = l.replace('\n','').split(',')
                    name = nameunit[0].strip()
                    if len(nameunit) > 1:
                        units.append(nameunit[1].strip())
                    elif re.compile('(.*)\((.*)\)').match(nameunit[0]):
                        desc_groups = re.compile('(.*)\((.*)\).*').match(nameunit[0]).groups()
                        name = desc_groups[0].strip()
                        units.append(desc_groups[1].strip())
                    elif '_' in name:
                        units.append(name.split('_')[1].strip())
                    else:
                        warn('Could not find unit in string: "%s"' % l)
                        units.append(name.strip())
                elif li == SPECIAL_COMMENT_COUNT_LINE:
                    n_special_comments = int(l.replace('\n', ''))
                elif li > SPECIAL_COMMENT_COUNT_LINE and li <= LAST_SPECIAL_COMMENT_LINE:
                    pass
                elif li == USER_COMMENT_COUNT_LINE:
                    n_user_comments = int(l.replace('\n',''))
                elif li > USER_COMMENT_COUNT_LINE and li < self.n_header_lines:
                    colon_pos = l.find(':')
                    k = l[:colon_pos].strip()
                    v = l[colon_pos+1:].strip()
                    setattr(self,k,v)
                elif li == self.n_header_lines:
                    variables = l.replace(',','').split()
                    self.TFLAG = variables[0]
        except Exception as e:
            raise SyntaxError("Error parsing icartt file %s: %s" % (path, repr(e)))

        missing = missing[:1]+missing
        scales = [1.]+scales
        
        if hasattr(self,'LLOD_FLAG'):
            llod_values = loddelim.sub('\n', self.LLOD_VALUE).split()
            if len(llod_values) == 1:
                llod_values *= len(variables)
            else:
                llod_values = ['N/A']+llod_values
            
            assert len(llod_values) == len(variables)
            llod_values = [get_lodval(llod_val) for llod_val in llod_values]
            
            llod_flags = len(llod_values)*[self.LLOD_FLAG]
            llod_flags = [get_lodval(llod_flag) for llod_flag in llod_flags]
        
        if hasattr(self,'ULOD_FLAG'):
            ulod_values = loddelim.sub('\n', self.ULOD_VALUE).split()
            if len(ulod_values) == 1:
                ulod_values *= len(variables)
            else:
                ulod_values = ['N/A']+ulod_values

            assert len(ulod_values) == len(variables)
            ulod_values = [get_lodval(ulod_val) for ulod_val in ulod_values]
            
            ulod_flags = len(ulod_values)*[self.ULOD_FLAG]
            ulod_flags = [get_lodval(ulod_flag) for ulod_flag in ulod_flags]
        
        data = f.read()
        datalines = data.split('\n')
        ndatalines = len(datalines)
        while datalines[-1] in ('', ' ', '\r'):
            ndatalines -=1
            datalines.pop(-1)
        data = genfromtxt(StringIO(bytes('\n'.join(datalines), 'utf-8')), delimiter = delim, dtype = 'd')
        data = data.reshape(ndatalines,len(variables))
        data = data.swapaxes(0,1)
        self.createDimension('POINTS', ndatalines)
        for var, scale, miss, unit, dat, llod_flag, llod_val, ulod_flag, ulod_val in zip(variables, scales, missing, units, data, llod_flags, llod_values, ulod_flags, ulod_values):
            vals = MaskedArray(dat, mask = dat == miss, fill_value = miss)
            tmpvar = self.variables[var] = PseudoNetCDFVariable(self, var, 'd', ('POINTS',), values = vals)
            tmpvar.units = unit
            tmpvar.standard_name = var
            tmpvar.missing_value = miss
            tmpvar.fill_value = miss
            tmpvar.scale = scale

            if hasattr(self,'LLOD_FLAG'):
                tmpvar.llod_flag = llod_flag
                tmpvar.llod_value = llod_val

            if hasattr(self,'ULOD_FLAG'):
                tmpvar.ulod_flag = ulod_flag
                tmpvar.ulod_value = ulod_val

        
        self._date_objs = self._SDATE + vectorize(lambda s: timedelta(seconds = int(s), microseconds = (s - int(s)) * 1.E6 ))(self.variables[self.TFLAG]).view(type = ndarray)