Exemple #1
0
    def bbox(self, header: PoseHeader):
        data = ma.transpose(self.data, axes=POINTS_DIMS)

        # Split data by components, `ma` doesn't support ".split"
        components = []
        idx = 0
        for component in header.components:
            components.append(data[list(range(idx,
                                              idx + len(component.points)))])
            idx += len(component.points)

        boxes = [
            ma.stack([ma.min(c, axis=0), ma.max(c, axis=0)])
            for c in components
        ]
        boxes_cat = ma.concatenate(boxes)
        if type(boxes_cat.mask
                ) == np.bool_:  # Sometimes, it doesn't concatenate the mask...
            boxes_mask = ma.concatenate([b.mask for b in boxes])
            boxes_cat = ma.array(boxes_cat, mask=boxes_mask)

        new_data = ma.transpose(boxes_cat, axes=POINTS_DIMS)

        confidence_mask = np.split(new_data.mask, [-1], axis=3)[0]
        confidence_mask = np.squeeze(confidence_mask, axis=-1)
        confidence = np.where(confidence_mask == True, 0, 1)

        return NumPyPoseBody(self.fps, new_data, confidence)
Exemple #2
0
    def get_points(self, indexes: List[int]):
        data = ma.transpose(self.data, axes=POINTS_DIMS)
        new_data = ma.transpose(data[indexes], axes=POINTS_DIMS)

        confidence_reshape = (2, 1, 0)
        confidence = np.transpose(self.confidence, axes=confidence_reshape)
        new_confidence = np.transpose(confidence[indexes], axes=confidence_reshape)

        return NumPyPoseBody(self.fps, new_data, new_confidence)
Exemple #3
0
def _out2csv(path, fheader, f_name, ignore_fields):
    print(f_name)
    ignore_fields = ignore_fields if ignore_fields else []
    with open(os.path.join(path, f_name + '.csv'), "w") as csvfile:
        c = csv.writer(csvfile, delimiter=',')
        csv_header = sorted(
            list(set(fheader.variables.keys()) - set(ignore_fields)))
        insert_indices = {
            "time": csv_header.index("time"),
            "lat": csv_header.index("lat"),
            "lon": csv_header.index("lon")
        }
        c.writerow(csv_header)

        var_matrix = []
        set_mask = False
        mask_indices = np.array([])
        length_dic = []
        for var in csv_header:
            if var not in ['time', 'lat', 'lon'] and var not in ignore_fields:
                if not set_mask:
                    # we will have lon as the row and lat as the column for kfold split vertically
                    mask_indices = ma.where(~ma.getmaskarray(
                        ma.transpose(fheader.variables[var][:], (0, 2, 1))))
                    set_mask = True
                compressed = ma.transpose(fheader.variables[var][:],
                                          (0, 2, 1)).compressed()
                length_dic.append(len(compressed))
                var_matrix.append(compressed)

        assert all(length_dic[0] == length for length in length_dic), \
            "Should always be called after all variables are masked to be overlapped"

        dim_indices = {
            "time": mask_indices[0],
            "lon": mask_indices[1],
            "lat": mask_indices[2]
        }  # order of indices changes correspondingly
        for dim_name in [
                _[0] for _ in sorted(insert_indices.items(),
                                     key=operator.itemgetter(1))
        ]:
            var_matrix.insert(insert_indices[dim_name], [
                fheader.variables[dim_name][i] for i in dim_indices[dim_name]
            ])

        num = 0
        for row in zip(*var_matrix):
            c.writerow(row)
            num += 1
Exemple #4
0
 def get_rank_minimum_matrix(self, sorted_sample_population):
     return sum([
         multiply((self.get_steped_difference(sorted_sample) *
                   transpose(self.get_steped_difference(sorted_sample))),
                  weight) for weight, sorted_sample in zip(
                      self.get_adjusted_weights(sorted_sample_population),
                      sorted_sample_population)
     ])
Exemple #5
0
 def findEout(self, numSamples=1000):
     e_out = 0
     dataSamples = [self.createDataPoint() for _ in range(numSamples)]
     for x, y in dataSamples:
         e_out += log(1 +
                      exp(-1 * multiply(y, np.dot(transpose(self.w), x))))
     e_out /= float(numSamples)
     return e_out
Exemple #6
0
    def __init__(self, data, useAttributeLabels=False):
        self.data = data
        self.useAttributeLabels = useAttributeLabels
        self.attr_labels, self.data_classes = self._data_info(data)
        self.attributes = [attr for attr in self.data.domain.attributes if attr.varType in [orange.VarTypes.Continuous, orange.VarTypes.Discrete]]
        self.classes = np.array(self.attr_labels if useAttributeLabels else self.data_classes)
        self.keys = range(len(data)) if useAttributeLabels else self.attributes
        self.array, _, _ = data.toNumpyMA()
        if self.useAttributeLabels:
            self.array = ma.transpose(self.array)
#        self.dim = 1 if useAttributeLabels else 0  
        self.dim = 0
Exemple #7
0
    def __init__(self, data, useAttributeLabels=False):
        self.data = data
        self.useAttributeLabels = useAttributeLabels
        self.attr_labels, self.data_classes = self._data_info(data)
        self.attributes = [attr for attr in self.data.domain.attributes \
            if isinstance(attr, (ContinuousVariable, DiscreteVariable))]
        self.classes = np.array(self.attr_labels if useAttributeLabels else self.data_classes)
        self.keys = list(range(len(data))) if useAttributeLabels else self.attributes
        self.array, _, _ = data.toNumpyMA()
        if self.useAttributeLabels:
            self.array = ma.transpose(self.array)
#        self.dim = 1 if useAttributeLabels else 0  
        self.dim = 0
Exemple #8
0
 def runLogisticRegression(self):
     diff = 1
     while diff > 0.01:
         permutation = np.random.permutation(self.N)
         newWeights = self.w.copy()
         for i in permutation:
             x, y = self.trainingData[i]
             gradient = divide(
                 multiply(-1.0, multiply(x, y)),
                 (1.0 + exp(multiply(y, np.dot(transpose(self.w), x)))))
             newWeights = subtract(newWeights,
                                   multiply(self.learningRate, gradient))
         self.epoch += 1
         diff = norm(self.w - newWeights)
         self.w = newWeights
Exemple #9
0
 def test_testTakeTransposeInnerOuter(self):
     # Test of take, transpose, inner, outer products
     x = arange(24)
     y = np.arange(24)
     x[5:6] = masked
     x = x.reshape(2, 3, 4)
     y = y.reshape(2, 3, 4)
     assert_(eq(np.transpose(y, (2, 0, 1)), transpose(x, (2, 0, 1))))
     assert_(eq(np.take(y, (2, 0, 1), 1), take(x, (2, 0, 1), 1)))
     assert_(eq(np.inner(filled(x, 0), filled(y, 0)), inner(x, y)))
     assert_(eq(np.outer(filled(x, 0), filled(y, 0)), outer(x, y)))
     y = array(['abc', 1, 'def', 2, 3], object)
     y[2] = masked
     t = take(y, [0, 3, 4])
     assert_(t[0] == 'abc')
     assert_(t[1] == 2)
     assert_(t[2] == 3)
Exemple #10
0
    def __init__(self, data, useAttributeLabels=False):
        self.data = data
        self.useAttributeLabels = useAttributeLabels
        self.attr_labels, self.data_classes = self._data_info(data)
        self.attributes = [attr for attr in self.data.domain.attributes \
            if isinstance(attr, (ContinuousVariable, DiscreteVariable))]
        self.classes = np.array(
            self.attr_labels if useAttributeLabels else self.data_classes)
        self.keys = list(range(
            len(data))) if useAttributeLabels else self.attributes
        self.array, _, _ = data.toNumpyMA()
        if self.useAttributeLabels:
            self.array = ma.transpose(self.array)


#        self.dim = 1 if useAttributeLabels else 0
        self.dim = 0
 def test_testTakeTransposeInnerOuter(self):
     # Test of take, transpose, inner, outer products
     x = arange(24)
     y = np.arange(24)
     x[5:6] = masked
     x = x.reshape(2, 3, 4)
     y = y.reshape(2, 3, 4)
     assert_(eq(np.transpose(y, (2, 0, 1)), transpose(x, (2, 0, 1))))
     assert_(eq(np.take(y, (2, 0, 1), 1), take(x, (2, 0, 1), 1)))
     assert_(eq(np.inner(filled(x, 0), filled(y, 0)), inner(x, y)))
     assert_(eq(np.outer(filled(x, 0), filled(y, 0)), outer(x, y)))
     y = array(["abc", 1, "def", 2, 3], object)
     y[2] = masked
     t = take(y, [0, 3, 4])
     assert_(t[0] == "abc")
     assert_(t[1] == 2)
     assert_(t[2] == 3)
Exemple #12
0
#
#  This is the ScientificPython method for opening a netCDF file.
#
# ice1     = NetCDFFile(data_dir + "/cdf/fice.nc","r")

#
#  Create a masked array to accommodate missing values in the fice variable.
#
fice = ice1.variables["fice"]  # fice[120,49,100]
ficea = fice[:, :, :]
fill_value = None
if (hasattr(fice, "missing_value")):
    fill_value = fice.missing_value
elif (hasattr(fice, "_FillValue")):
    fill_value = fice._FillValue
fice_masked = MA.transpose(MA.masked_values(ficea, fill_value), (1, 2, 0))

hlat = ice1.variables["hlat"]  # hlat[49]
hlon = ice1.variables["hlon"]  # hlon[100]

dimf = fice.shape  # Define an array to hold long-term monthly means.
ntime = fice.shape[0]
nhlat = fice.shape[1]
nhlon = fice.shape[2]

nmo = 0
month = nmo + 1

icemon = MA.zeros((nhlat, nhlon), dtype=float)
for i in range(fice_masked.shape[0]):
    for j in range(fice_masked.shape[1]):
import numpy.ma as ma

x = ma.arange(4).reshape((2, 2))
x[1, 1] = ma.masked
ma.transpose(x)
Exemple #14
0
 def get_rank_one_matrix(self):
     return multiply(self.anisotropic_evolution_path,
                     transpose(self.anisotropic_evolution_path))
        cmapdiff = CyBuWhRdYl

        if var in ['NO3_excess','FG_CO2','STF_O2','FvPER_DIC']:
            cmap = CyBuWhRdYl
        else:
            cmap = pl.cm.jet

        for r in range(nreg):
            if reg_lname[r] in region_list:
                if POPDIAGPY == 'TRUE':
                  outfile = os.path.join(outdir,'zavg_%s_%s_diff.png'%(var,reg_sname[r]))
                else:
                  outfile = os.path.join(outdir,'zavg_%04d-%04d_%s_%s_diff.png'%(
                    yrstart,yrend,var,reg_sname[r]))
                print('plotting %s'%(os.path.basename(outfile)))
                datam = MA.transpose(varsm[var][:,r,:])
                datao = MA.transpose(varso[var][:,r,:])

                # calculate difference ------------------------------------------
                differnce = datam - datao

                # plot ----------------------------------------------------------

                fig = pl.figure(figsize=(8,11))

                # plot experiment ------------------------------------------------
                ax  = fig.add_subplot(211)
                CF  = ax.contourf(mon,lat,datam,variables[var]['clev'],cmap=cmap,
                        norm=colors.BoundaryNorm(variables[var]['clev'],cmap.N),extend='both')
		CF.cmap.set_under('black')
		CF.cmap.set_over('black')
Exemple #16
0
    def fit(self, A):
        """
        Fit model parameters U, V.
        :param X:
            Data matrix of shape (m, n)
            Unknown values are assumed to take the value of zero (0).
        """
        start_time = time.time()
        # permute matrix A, columns minimum increasing
        columns_perm = np.argsort(np.min(A, axis = 0))
        A = A[:, columns_perm]
        
        m = A.shape[0]  # number of rows
        n = A.shape[1]  # number of columns

        # initialization of U matrix
        U_initial = self.initialize_U(A, m)
        V = min_plus(ma.transpose(np.negative(U_initial)), A)
        U = min_plus(A, ma.transpose(np.negative(V)))
        D = np.subtract(A, max_plus(U, V))

        # initialization of f values needed for convergence test
        f_old = 1
        f_new = self.b_norm(D)
        f = f_new

        i_list, j_list, k_list = range(m), range(n), range(self.rank)
        U_new, V_new = U, V
        comb = get_coordinates(A)
        iterations = 0

        while abs(f_old - f_new) > self.epsilon:
            f = f_new
            for x in comb:
                i = x[0]
                j = x[1]
                temp = False
                temp_1 = False
                for k in k_list:
                    iterations += 1
                    V_new = copy.deepcopy(V)
                    V_new[k, j] = A[i, j] - U[i, k]
                    U_new = min_plus(A, np.transpose(np.negative(V_new)))
                    V_new = min_plus(np.transpose(np.negative(U_new)), A)
                    f_new = self.b_norm(np.subtract(A, max_plus(U_new, V_new)))
                    if self.criterion == 'iterations' and iterations >= self.max_iter:
                        self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
                        return
                    if f_new < f:
                        temp_1 = True
                        break
                if temp_1:
                    break  # only executed if the inner loop DID break
                for k in k_list:
                    iterations += 1
                    U_new = copy.deepcopy(U)
                    U_new[i, k] = A[i, j] - V[k, j]
                    V_new = min_plus(np.transpose(np.negative(U_new)), A)
                    U_new = min_plus(A, np.transpose(np.negative(V_new)))
                    f_new = self.b_norm(np.subtract(A, max_plus(U_new, V_new)))
                    if self.criterion == 'iterations' and iterations >= self.max_iter:
                        self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
                        return
                    if f_new < f:
                        temp = True
                        break
                if temp:
                    break  # only executed if the inner loop DID break
            if f_new < f:
                U = U_new
                V = V_new
                f_old = f
                f = f_new
                if self.criterion == 'iterations' and iterations >= self.max_iter:
                    self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
                    return
            else:
                print("no solution found!")
                self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
                return
        self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
Exemple #17
0
#
#  This is the ScientificPython method for opening a netCDF file.
#
# ice1     = NetCDFFile(data_dir + "/cdf/fice.nc","r")

#
#  Create a masked array to accommodate missing values in the fice variable.
#
fice = ice1.variables["fice"]  # fice[120,49,100]
ficea = fice[:,:,:]
fill_value = None
if (hasattr(fice,"missing_value")):
  fill_value = fice.missing_value
elif (hasattr(fice,"_FillValue")):
  fill_value = fice._FillVlaue
fice_masked = MA.transpose(MA.masked_values(ficea,fill_value),(1,2,0))

hlat = ice1.variables["hlat"]  # hlat[49]
hlon = ice1.variables["hlon"]  # hlon[100]


dimf     = fice.shape  # Define an array to hold long-term monthly means.
ntime    = fice.shape[0]
nhlat    = fice.shape[1]
nhlon    = fice.shape[2]

nmo    = 0
month  = nmo+1

icemon = MA.zeros((nhlat,nhlon),dtype=float)
for i in xrange(fice_masked.shape[0]):
        # plot figures --------------------------------------------------------
        if var in ['NO3_excess','FG_CO2','STF_O2','FvPER_DIC']:
            cmap = CyBuWhRdYl
        else:
            cmap = pl.cm.jet

        for r in range(nreg):
            if reg_lname[r] in region_list:
                if POPDIAGPY == 'TRUE':
                  outfile = os.path.join(outdir,'zavg_%s_%s.png'%(var,reg_sname[r]))
                else:
                  outfile = os.path.join(outdir,'zavg_%04d-%04d_%s_%s.png'%(
                    yrstart,yrend,var,reg_sname[r]))
                print('plotting %s'%(os.path.basename(outfile)))
                data = MA.transpose(vars()[var][:,r,:])

                fig = pl.figure()
                ax  = fig.add_subplot(111)

                if var in vintlist: # vertically integrated
                   CF  = ax.contourf(mon,lat,data,variables[var]['cklev'],cmap=cmap,
                           norm=colors.BoundaryNorm(variables[var]['cklev'],cmap.N),extend='both')
                   CF.cmap.set_under('black')
                   CF.cmap.set_over('black')
                   CS  = ax.contour(mon,lat,data,CF.levels,colors='k',
                           linewidths=0.25)
                   CB  = fig.colorbar(CF,ax=ax,drawedges=True,aspect=15,
                       ticks=variables[var]['cklev'][1:-1],format='%G')
                   CB.set_label(variables[var]['kunits'])
                   for line in CB.ax.get_yticklines(): # remove tick lines
Exemple #19
0
def plot2D(filein,varname,coef=None,units='',lev=None,levunits=None,tmin=None,tmax=None,dtlabel='1h',namefig=None,lbias=False,refdataset=None,error=None,**kwargs):
    """
       Do a 2D plot of varname for several MUSC file
    """

    title0 = kwargs['title']

    data = OrderedDict()
    levax = {}
    time = {}
    timeax = {}
    if coef is None:
      coef = {}
      for k in filein.keys():
        coef[k] = 1.
    elif isinstance(coef,int) or isinstance(coef,float):
      tmp = coef
      coef = {}
      for k in filein.keys():
        coef[k] = tmp
    
    if lev is None:
      lev = {}
      levunits = {}
      for k in filein.keys():
        lev[k] = 'zh'
        levunits[k] = 'km'
    elif isinstance(lev,str):
      tmp = lev
      lev = {}
      for k in filein.keys():
        lev[k] = tmp

    if levunits is None:
      levunits = {}
      for k in filein.keys():
        if lev[k] == 'zh':
          levunits[k] = 'km'
        elif lev[k] in['ph','pf']:
          levunits[k] = 'hPa'
        else:
          print 'lev={} not coded yet'.format(lev[k])
          sys.exit()
    elif isinstance(levunits,str):
      tmp = levunits
      levunits = {}
      for k in filein.keys():
        levunits[k] = tmp

    datasets = filein.keys()
    if lbias:
      if refdataset is None:
          print 'please provide reference dataset (keyword refdataset) to compute bias)'
          sys.exit()
      else: 
          f = cdms2.open(filein[refdataset])
          dataref = f(varname[refdataset],squeeze=1)*coef[refdataset]
          f.close()
          tmp = []
          for dd in datasets:
              if not(dd == refdataset):
                  tmp.append(dd)
          datasets = tmp


    for k in datasets: #filein.keys():
      #print k
      f = cdms2.open(filein[k])
      try:
        data[k] = f(varname[k],squeeze=1)*coef[k]
        if lbias:
            nt,nlev = data[k].shape
            tmp = data[k].getTime()
            data[k] = data[k]-dataref[0:nt,:]
            data[k].setAxis(0,tmp)
      except cdms2.error.CDMSError as e:
        data[k] = None
        f.close()          
        if verbose:
          print 'Variable {2} probably unknown in dataset {0} (file={1})'.format(k,filein[k],varname[k])
          print 'Raised error: cdms2.error.CDMSError', e
        if error is not None:
          if isinstance(error,dict):
            if error.has_key(k):
              error[k].append(varname[k])
            else:
              error[k] = [varname[k],]
          else:
            print 'type of error unexpected:', type(error)
            print 'error should be a dictionnary'
            sys.exit()            
      except:
        raise
      
      if data[k] is not None:
        try:
          levax[k] = f(lev[k],squeeze=1)
        except:
          if lev[k] == 'zh':
            levax[k] = f('zf')
          if lev[k] == 'ph':
            levax[k] = f('pf')            
        f.close()

        if lev[k] == 'zh':
          if levunits[k] == 'm':
            pass
          elif levunits[k] == 'km':
             levax[k] = levax[k]/1000.
          else:
            print 'levunits={} for lev=zh not coded yet'.format(levunits[k])
            sys.exit()
        elif lev[k] == 'ph':
          if levunits[k] == 'Pa':
            pass
          elif levunits[k] == 'hPa':
            levax[k] = levax[k]/100.
          else:
            print 'levunits={} for lev=ph not coded yet'.format(levunits[k])
            sys.exit()
        else:
          print 'lev={} not coded yet'.format(lev)
          sys.exit()
     
        time[k] = data[k].getTime()

      
        if len(levax[k].shape) == 2:
          nt,nlev = levax[k].shape
          timeax[k] = np.tile(time[k][:],(nlev,1))
          X = timeax[k]
          Y = np.transpose(levax[k])
        else:
          nlev, = levax[k].shape
          nt, = time[k].shape
          timeax[k] = np.tile(time[k][:],(nlev,1))
          levax[k] = np.tile(levax[k],(nt,1))
          X = np.array(timeax[k][:])
          Y = np.transpose(levax[k][:])

        if tmin is None:
          tmin = cdtime.reltime(time[k][0],time.units)
        if tmax is None:
          tmax = cdtime.reltime(time[k][-1],time.units)

        tt = []
        tlabels = []

        tminloc = tmin.tocomp()
        if dtlabel == '1h':
          tmin0 = cdtime.comptime(tminloc.year,tminloc.month,tminloc.day,tminloc.hour)

          t0 = tmin0.add(0,cdtime.Hour)
          while t0.cmp(tmax) <= 0:
            if t0.cmp(tmin) >= 0: 
              tt.append(t0.torel(time[k].units).value)
              tlabels.append('{0}'.format(t0.hour))
            t0 = t0.add(1,cdtime.Hour)
        elif dtlabel == '2h':
          tmin0 = cdtime.comptime(tminloc.year,tminloc.month,tminloc.day,tminloc.hour)

          t0 = tmin0.add(0,cdtime.Hour)
          while t0.cmp(tmax) <= 0:
            if t0.cmp(tmin) >= 0: 
              tt.append(t0.torel(time[k].units).value)
              tlabels.append('{0}'.format(t0.hour))
            t0 = t0.add(2,cdtime.Hour)
        elif dtlabel == '6h':
          tmin0 = cdtime.comptime(tminloc.year,tminloc.month,tminloc.day,tminloc.hour)

          t0 = tmin0.add(0,cdtime.Hour)
          while t0.cmp(tmax) <= 0:
            if t0.cmp(tmin) >= 0: 
              tt.append(t0.torel(time[k].units).value)
              tlabels.append('{0}'.format(t0.hour))
            t0 = t0.add(6,cdtime.Hour) 
        elif dtlabel == '10d':
          tmin0 = cdtime.comptime(tminloc.year,tminloc.month,tminloc.day,tminloc.hour)

          t0 = tmin0.add(0,cdtime.Hour)
          while t0.cmp(tmax) <= 0:
            if t0.cmp(tmin) >= 0: 
              tt.append(t0.torel(time[k].units).value)
              tlabels.append('{0}/{1}'.format(t0.month,t0.day))
            if t0.day == 1:
              t0 = cdtime.comptime(t0.year,t0.month,10,0)
            elif t0.day == 10:
              t0 = cdtime.comptime(t0.year,t0.month,20,0)
            elif t0.day == 20:
              if t0.month == 12:
                t0 = cdtime.comptime(t0.year+1,1,1,0)
              else:
                t0 = cdtime.comptime(t0.year,t0.month+1,1,0)
            else:
              t0 = t0.add(10,cdtime.Day)
              #print 't0 unexpected',t0
              #sys.exit()
            
        else:
          print 'dtlabel={} not coded yet'.format(dtlabel)
          sys.exit()

        tlabels = tt,tlabels

        if isinstance(namefig,str):
            tmp = k + '_' + namefig
        elif isinstance(namefig,dict):
            tmp = namefig[k]
        elif namefig is None:
            tmp = None
        else:
            print 'namefig type unexpected:', namefig
            sys.exit()

        kwargs['title'] = '{0} - {1}'.format(title0,k)

        plotutils.plot2D(X,Y,ma.transpose(data[k]),\
            xmin = tmin.torel(time[k].units).value,\
            xmax = tmax.torel(time[k].units).value,\
            xlabels=tlabels,\
            namefig=tmp,\
            **kwargs)
Exemple #20
0
def load_GPM_IMERG_files_with_spatial_filter(file_path=None,
                         filename_pattern=None,
                         filelist=None,
                         variable_name='precipitationCal',
                         user_mask_file=None,
                         mask_variable_name='mask',
                         user_mask_values=[10],
                         longitude_name='lon',
                         latitude_name='lat'):
    ''' Load multiple GPM Level 3 IMEGE files containing calibrated \
        precipitation and generate a two-dimensional array \
        for the masked grid points.
    :param file_path: Directory to the HDF files to load.
    :type file_path: :mod:`string`
    :param filename_pattern: Path to the HDF files to load.
    :type filename_pattern: :mod:`string`
    :param filelist: A list of filenames
    :type filelist: :mod:`string`
    :param variable_name: The variable name to load from the HDF file.
    :type variable_name: :mod:`string`
    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`
    :user_mask_file: user's own gridded mask file(a netCDF file name)
    :type name: :mod:`string`
    :mask_variable_name: mask variables in user_mask_file
    :type name: :mod:`string`
    :longitude_name: longitude variable in user_mask_file
    :type name: :mod:`string`
    :latitude_name: latitude variable in user_mask_file
    :type name: :mod:`string`
    :param user_mask_values: grid points where mask_variable == user_mask_value will be extracted.
    :type user_mask_values: list of strings
    :returns: A two-dimensional array with the requested variable's MASKED data from \
        the HDF file.
    :rtype: :class:`dataset.Dataset`
    :raises ValueError:
    '''

    if not filelist:
        GPM_files = []
        for pattern in filename_pattern:
            GPM_files.extend(glob(file_path + pattern))
    else:
        GPM_files = [line.rstrip('\n') for line in open(filelist)]

    GPM_files.sort()

    file_object_first = h5py.File(GPM_files[0])
    lats = file_object_first['Grid']['lat'][:]
    lons = file_object_first['Grid']['lon'][:]

    lons, lats = numpy.meshgrid(lons, lats)

    nfile = len(GPM_files)
    for ifile, file in enumerate(GPM_files):
        if ifile == 0 and user_mask_file:
            file_object = netCDF4.Dataset(user_mask_file)
            mask_variable = file_object.variables[mask_variable_name][:]
            mask_longitude = file_object.variables[longitude_name][:]
            mask_latitude = file_object.variables[latitude_name][:]
            spatial_mask = utils.regrid_spatial_mask(lons,lats,
                                                     mask_longitude, mask_latitude,
                                                     mask_variable,
                                                     user_mask_values)
            y_index, x_index = numpy.where(spatial_mask == 0)
        print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
        file_object = h5py.File(file)
        values0 = ma.transpose(ma.masked_less(
            file_object['Grid'][variable_name][:], 0.))
        values_masked = values0[y_index, x_index]
        values_masked = ma.expand_dims(values_masked, axis=0)
        if ifile == 0:
            values = values_masked
        else:
            values = ma.concatenate((values, values_masked))
        file_object.close()
    return values
Exemple #21
0
    def __init__(self, filename):
        '''
        Handles setting up a reader.
        '''
        self.fields = {}
        self.info = {}

        self.nc_dataset = Dataset(filename)
        self.filename = filename

        self.time = common.ncvar_to_dict(self.nc_dataset.variables['Time'])
        self.time['data'] = ma.array(
            get_datetime_from_epoch(self.time['data'][:]))
        self.time['units'] = "Datetime objects"
        del self.time['_FillValue']

        self.fields['Nd'] = common.ncvar_to_dict(
            self.nc_dataset.variables['VolumetricDrops'])
        self.fields['Nd']['data'] = ma.transpose(
            np.power(10, self.fields['Nd']['data']))
        ma.set_fill_value(self.fields['Nd']['data'], 0.)
        self.fields['Nd']['data'] = self.fields['Nd']['data'].filled()
        del self.fields['Nd']['_FillValue']
        self.fields['Nd']['units'] = "1/m^3 1/mm"

        self.fields['RR'] = common.ncvar_to_dict(
            self.nc_dataset.variables['ParsivelIntensity'])
        self.fields['RR']['data'] = ma.masked_array(self.fields['RR']['data'])
        ma.set_fill_value(self.fields['RR']['data'],
                          self.fields['RR']['_FillValue'])
        del self.fields['RR']['_FillValue']

        self.fields['Zh'] = common.ncvar_to_dict(
            self.nc_dataset.variables['Reflectivity'])
        del self.fields['Zh']['_FillValue']

        self.fields['num_particles'] = common.ncvar_to_dict(
            self.nc_dataset.variables['RawDrops'])
        self.fields['num_particles']['data'] = ma.masked_array(
            self.fields['num_particles']['data'])
        ma.set_fill_value(self.fields['num_particles']['data'],
                          self.fields['num_particles']['_FillValue'])
        del self.fields['num_particles']['_FillValue']

        self.fields['terminal_velocity'] = \
            common.ncvar_to_dict(self.nc_dataset.variables['VelocityDrops'])
        self.fields['terminal_velocity']['data'] = \
            ma.transpose(ma.masked_array(self.fields['terminal_velocity']['data']))
        ma.set_fill_value(self.fields['terminal_velocity']['data'],
                          self.fields['terminal_velocity']['_FillValue'])
        del self.fields['terminal_velocity']['_FillValue']

        self.fields['Precip_Code'] = common.ncvar_to_dict(
            self.nc_dataset.variables['PrecipCode'])

        diameter = ma.array([
            0.0625, 0.1875, 0.3125, 0.4375, 0.5625, 0.6875, 0.8125, 0.9375,
            1.0625, 1.1875, 1.375, 1.625, 1.875, 2.125, 2.375, 2.75, 3.25,
            3.75, 4.25, 4.75, 5.5, 6.5, 7.5, 8.5, 9.5, 11., 13., 15., 17., 19.,
            21.5, 24.5
        ])

        spread = ma.array([
            0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125,
            0.125, 0.250, 0.250, 0.250, 0.250, 0.250, 0.500, 0.500, 0.500,
            0.500, 0.500, 1.000, 1.000, 1.000, 1.000, 1.000, 2.000, 2.000,
            2.000, 2.000, 2.000, 3.000, 3.000
        ])

        # velocity = ma.array([0.05, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 1.1, 1.3,
        #                      1.5, 1.7, 1.9, 2.2, 2.6, 3.0, 3.4, 3.8, 4.4, 5.2, 6.0, 6.8, 7.6, 8.8,
        #                     10.4, 12.0, 13.6, 15.2, 17.6, 20.8])

        self.bin_edges = common.var_to_dict(
            'bin_edges', np.hstack((0, diameter + np.array(spread) / 2)), 'mm',
            'Boundaries of bin sizes')
        self.spread = common.var_to_dict('spread', spread, 'mm',
                                         'Bin size spread of bins')
        self.diameter = common.var_to_dict('diameter', diameter, 'mm',
                                           'Particle diameter of bins')

        for key in self.nc_dataset.ncattrs():
            self.info[key] = self.nc_dataset.getncattr(key)
Exemple #22
0
def load_GPM_IMERG_files(file_path=None,
                         filename_pattern=None,
                         filelist=None,
                         variable_name='precipitationCal',
                         name='GPM_IMERG'):
    ''' Load multiple GPM Level 3 IMEGE files containing calibrated \
        precipitation and generate an OCW Dataset obejct.

    :param file_path: Directory to the HDF files to load.
    :type file_path: :mod:`string`

    :param filename_pattern: Path to the HDF files to load.
    :type filename_pattern: :mod:`string`

    :param filelist: A list of filenames
    :type filelist: :mod:`string`

    :param variable_name: The variable name to load from the HDF file.
    :type variable_name: :mod:`string`

    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`

    :returns: An OCW Dataset object with the requested variable's data from \
        the HDF file.
    :rtype: :class:`dataset.Dataset`

    :raises ValueError:
    '''

    if not filelist:
        GPM_files = []
        for pattern in filename_pattern:
            GPM_files.extend(glob(file_path + pattern))
    else:
        GPM_files = [line.rstrip('\n') for line in open(filelist)]

    GPM_files.sort()

    file_object_first = h5py.File(GPM_files[0])
    lats = file_object_first['Grid']['lat'][:]
    lons = file_object_first['Grid']['lon'][:]

    lons, lats = numpy.meshgrid(lons, lats)

    variable_unit = "mm/hr"

    times = []
    nfile = len(GPM_files)
    for ifile, file in enumerate(GPM_files):
        print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
        file_object = h5py.File(file)
        time_struct_parsed = strptime(file[-39:-23], "%Y%m%d-S%H%M%S")
        times.append(datetime(*time_struct_parsed[:6]))
        values0 = ma.transpose(
            ma.masked_less(file_object['Grid'][variable_name][:], 0.))
        values0 = ma.expand_dims(values0, axis=0)
        if ifile == 0:
            values = values0
        else:
            values = ma.concatenate((values, values0))
        file_object.close()
    times = numpy.array(times)
    return Dataset(lats,
                   lons,
                   times,
                   values,
                   variable_name,
                   units=variable_unit,
                   name=name)
        if var in ['NO3_excess', 'FG_CO2', 'STF_O2', 'FvPER_DIC']:
            cmap = CyBuWhRdYl
        else:
            cmap = pl.cm.jet

        for r in range(nreg):
            if reg_lname[r] in region_list:
                if POPDIAGPY == 'TRUE':
                    outfile = os.path.join(
                        outdir, 'zavg_%s_%s_diff.png' % (var, reg_sname[r]))
                else:
                    outfile = os.path.join(
                        outdir, 'zavg_%04d-%04d_%s_%s_diff.png' %
                        (yrstart, yrend, var, reg_sname[r]))
                print('plotting %s' % (os.path.basename(outfile)))
                datam = MA.transpose(varsm[var][:, r, :])
                datao = MA.transpose(varso[var][:, r, :])

                # calculate difference ------------------------------------------
                differnce = datam - datao

                # plot ----------------------------------------------------------

                fig = pl.figure(figsize=(8, 11))

                # plot experiment ------------------------------------------------
                ax = fig.add_subplot(211)

                if var in vintlist:  # vertically integrated
                    CF = ax.contourf(mon,
                                     lat,
Exemple #24
0
 def points_perspective(self):
     return ma.transpose(self.data, axes=POINTS_DIMS)
Exemple #25
0
def load_GPM_IMERG_files_with_spatial_filter(file_path=None,
                                             filename_pattern=None,
                                             filelist=None,
                                             variable_name='precipitationCal',
                                             user_mask_file=None,
                                             mask_variable_name='mask',
                                             user_mask_values=[10],
                                             longitude_name='lon',
                                             latitude_name='lat'):
    ''' Load multiple GPM Level 3 IMEGE files containing calibrated \
        precipitation and generate a two-dimensional array \
        for the masked grid points.
    :param file_path: Directory to the HDF files to load.
    :type file_path: :mod:`string`
    :param filename_pattern: Path to the HDF files to load.
    :type filename_pattern: :mod:`string`
    :param filelist: A list of filenames
    :type filelist: :mod:`string`
    :param variable_name: The variable name to load from the HDF file.
    :type variable_name: :mod:`string`
    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`
    :user_mask_file: user's own gridded mask file(a netCDF file name)
    :type name: :mod:`string`
    :mask_variable_name: mask variables in user_mask_file    
    :type name: :mod:`string`
    :longitude_name: longitude variable in user_mask_file    
    :type name: :mod:`string`
    :latitude_name: latitude variable in user_mask_file    
    :type name: :mod:`string`
    :param user_mask_values: grid points where mask_variable == user_mask_value will be extracted.
    :type user_mask_values: list of strings
    :returns: A two-dimensional array with the requested variable's MASKED data from \
        the HDF file.
    :rtype: :class:`dataset.Dataset`
    :raises ValueError:
    '''

    if not filelist:
        GPM_files = []
        for pattern in filename_pattern:
            GPM_files.extend(glob(file_path + pattern))
    else:
        GPM_files = [line.rstrip('\n') for line in open(filelist)]

    GPM_files.sort()

    file_object_first = h5py.File(GPM_files[0])
    lats = file_object_first['Grid']['lat'][:]
    lons = file_object_first['Grid']['lon'][:]

    lons, lats = numpy.meshgrid(lons, lats)

    nfile = len(GPM_files)
    for ifile, file in enumerate(GPM_files):
        if ifile == 0 and user_mask_file:
            file_object = netCDF4.Dataset(user_mask_file)
            mask_variable = file_object.variables[mask_variable_name][:]
            mask_longitude = file_object.variables[longitude_name][:]
            mask_latitude = file_object.variables[latitude_name][:]
            spatial_mask = utils.regrid_spatial_mask(lons, lats,
                                                     mask_longitude,
                                                     mask_latitude,
                                                     mask_variable,
                                                     user_mask_values)
            y_index, x_index = numpy.where(spatial_mask == 0)
        print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
        file_object = h5py.File(file)
        values0 = ma.transpose(
            ma.masked_less(file_object['Grid'][variable_name][:], 0.))
        values_masked = values0[y_index, x_index]
        values_masked = ma.expand_dims(values_masked, axis=0)
        if ifile == 0:
            values = values_masked
        else:
            values = ma.concatenate((values, values_masked))
        file_object.close()
    return values
Exemple #26
0
def load_GPM_IMERG_files(file_path=None,
                         filename_pattern=None,
                         filelist=None,
                         variable_name='precipitationCal',
                         name='GPM_IMERG'):
    ''' Load multiple GPM Level 3 IMEGE files containing calibrated \
        precipitation and generate an OCW Dataset obejct.

    :param file_path: Directory to the HDF files to load.
    :type file_path: :mod:`string`

    :param filename_pattern: Path to the HDF files to load.
    :type filename_pattern: :mod:`string`

    :param filelist: A list of filenames
    :type filelist: :mod:`string`

    :param variable_name: The variable name to load from the HDF file.
    :type variable_name: :mod:`string`

    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`

    :returns: An OCW Dataset object with the requested variable's data from \
        the HDF file.
    :rtype: :class:`dataset.Dataset`

    :raises ValueError:
    '''

    if not filelist:
        GPM_files = []
        for pattern in filename_pattern:
            GPM_files.extend(glob(file_path + pattern))
    else:
        GPM_files = [line.rstrip('\n') for line in open(filelist)]

    GPM_files.sort()

    file_object_first = h5py.File(GPM_files[0])
    lats = file_object_first['Grid']['lat'][:]
    lons = file_object_first['Grid']['lon'][:]

    lons, lats = numpy.meshgrid(lons, lats)

    variable_unit = "mm/hr"

    times = []
    nfile = len(GPM_files)
    for ifile, file in enumerate(GPM_files):
        print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
        file_object = h5py.File(file)
        time_struct_parsed = strptime(file[-39:-23], "%Y%m%d-S%H%M%S")
        times.append(datetime(*time_struct_parsed[:6]))
        values0 = ma.transpose(ma.masked_less(
            file_object['Grid'][variable_name][:], 0.))
        values0 = ma.expand_dims(values0, axis=0)
        if ifile == 0:
            values = values0
        else:
            values = ma.concatenate((values, values0))
        file_object.close()
    times = numpy.array(times)
    return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)