def load_map(fullpath, ftype='xy', delimiter='\t', masked=False, topomask=None, xlim=None, ylim=None, lon=None, lat=None, tm=None, lon180=False, pad=False): """Loads an individual data file. PARAMETERS fullpath (string) : ftype (string, optional) : delimiter (string, optional) : masked (boolean, optional) : topomask (string, optional) : Topography mask. xlim, ylim (array like, optional) : List containing the upper and lower zonal and meridional limits, respectivelly. lon, lat, tm (array like, optional): lon180 (boolean, optional): pad (boolean, optional): Pads edges with NaN's for maps without distorsions. RETURNS lon, lat, tm, z """ if topomask != None: masked = True dat = numpy.loadtxt('%s' % (fullpath), delimiter=delimiter) if ftype == 'xy': x = dat[0, 1:] y = dat[1:, 0] t = dat[0, 0] elif ftype == 'xt': x = dat[0, 1:] y = dat[0, 0] t = dat[1:, 0] elif ftype == 'ty': x = dat[0, 0] y = dat[1:, 0] t = dat[0, 1:] else: raise Warning, 'Type \'%s\' not recognised' % (ftype) # Pads edges with NaN's to avoid distortions when generating maps. if pad: dx, dy = x[1] - x[0], y[1] - y[0] if (lon == None) and (ftype in ['xt', 'xt']): lon = numpy.concatenate([[x[0] - dx], x, [x[-1] + dx]]) if (lat == None) and (ftype in ['xy', 'ty']): lat = numpy.concatenate([[y[0] - dy], y, [y[-1] + dy]]) if masked: z = numpy.ma.asarray(dat[1:, 1:]) z.mask = numpy.isnan(z) else: z = dat[1:, 1:] # Put data in appropriate place. if (lon != None) or (lat != None) or (tm != None): if lon == None: lon = x if lat == None: lat = y if tm == None: tm = t lon180 = common.lon180(lon) x180 = common.lon180(x) if ftype == 'xy': k, l, m = len(lon), len(lat), 1 if masked: Z = numpy.ma.empty((l, k)) * numpy.nan Z.mask = True else: Z = numpy.empty((l, k)) * numpy.nan # u = numpy.unique(set(lon180) & set(x180)) v = numpy.unique(set(lat) & set(y)) if (len(u) == 0) or (len(v) == 0): return False i = [pylab.find(x180 == a)[0] for a in u] j = [pylab.find(y == b)[0] for b in v] mx, my = numpy.meshgrid(i, j) k = [pylab.find(lon180 == a)[0] for a in u] l = [pylab.find(lat == b)[0] for b in v] nx, ny = numpy.meshgrid(k, l) Z[ny, nx] = z[my, mx] if ftype == 'xt': k, l, m = len(lon), 1, len(tm) if masked: Z = numpy.ma.empty((m, k)) * numpy.nan Z.mask = True else: Z = numpy.empty((m, k)) * numpy.nan # u = numpy.unique(set(lon180) & set(x180)) if len(u) == 0: return False i = [pylab.find(x180 == a)[0] for a in u] k = [pylab.find(lon180 == a)[0] for a in u] v = set(tm) & set(t) if len(v) == 0: return False j = [pylab.find(t == b)[0] for b in v] l = [pylab.find(tm == b)[0] for b in v] mx, my = numpy.meshgrid(i, j) nx, ny = numpy.meshgrid(k, l) Z[ny, nx] = z[my, mx] if ftype == 'ty': raise Warning, ('Loading of temporal-meridional files not ' 'implemented yet.') else: lon = x lat = y tm = t Z = z if (xlim != None) or (ylim != None): if xlim == None: xlim = [lon.min(), lon.max()] if ylim == None: ylim = [lat.min(), lat.max()] selx = pylab.find((lon >= min(xlim)) & (lon <= max(xlim))) sely = pylab.find((lat >= min(ylim)) & (lat <= max(ylim))) lon = lon[selx] lat = lat[sely] i, j = numpy.meshgrid(selx, sely) Z = Z[j, i] if topomask != None: # Interpolates topography into data grid. ezi, _, _ = interpolate.nearest([common.etopo.x, common.etopo.y], common.etopo.z, [lon, lat]) if topomask == 'ocean': Z.mask = Z.mask | (ezi > 0) elif topomask == 'land': Z.mask = Z.mask | (ezi < 0) if masked: Z.data[Z.mask] = 0 return lon, lat, tm, Z
def load_map(fullpath, ftype='xy', delimiter='\t', masked=False, topomask=None, xlim=None, ylim=None, lon=None, lat=None, tm=None, lon180=False, pad=False): """Loads an individual data file. PARAMETERS fullpath (string) : ftype (string, optional) : delimiter (string, optional) : masked (boolean, optional) : topomask (string, optional) : Topography mask. xlim, ylim (array like, optional) : List containing the upper and lower zonal and meridional limits, respectivelly. lon, lat, tm (array like, optional): lon180 (boolean, optional): pad (boolean, optional): Pads edges with NaN's for maps without distorsions. RETURNS lon, lat, tm, z """ if topomask != None: masked = True dat = numpy.loadtxt('%s' % (fullpath), delimiter=delimiter) if ftype == 'xy': x = dat[0, 1:] y = dat[1:, 0] t = dat[0, 0] elif ftype == 'xt': x = dat[0, 1:] y = dat[0, 0] t = dat[1:, 0] elif ftype == 'ty': x = dat[0, 0] y = dat[1:, 0] t = dat[0, 1:] else: raise Warning, 'Type \'%s\' not recognised' % (ftype) # Pads edges with NaN's to avoid distortions when generating maps. if pad: dx, dy = x[1] - x[0], y[1] - y[0] if (lon == None) and (ftype in ['xt', 'xy']): lon = numpy.concatenate([[x[0] - dx], x, [x[-1] + dx]]) if (lat == None) and (ftype in ['xy', 'ty']): lat = numpy.concatenate([[y[0] - dy], y, [y[-1] + dy]]) if masked: z = numpy.ma.asarray(dat[1:, 1:]) z.mask = numpy.isnan(z) else: z = dat[1:, 1:] # Put data in appropriate place. if (lon != None) or (lat != None) or (tm != None): if lon == None: lon = x if lat == None: lat = y if tm == None: tm = t lon180 = common.lon180(lon) x180 = common.lon180(x) if ftype == 'xy': k, l, m = len(lon), len(lat), 1 if masked: Z = numpy.ma.empty((l, k)) * numpy.nan Z.mask = True else: Z = numpy.empty((l, k)) * numpy.nan # u = numpy.unique(set(lon180) & set(x180)) v = numpy.unique(set(lat) & set(y)) if (len(u) == 0) or (len(v) == 0): return False i = [pylab.find(x180 == a)[0] for a in u] j = [pylab.find(y == b)[0] for b in v] mx, my = numpy.meshgrid(i, j) k = [pylab.find(lon180 == a)[0] for a in u] l = [pylab.find(lat == b)[0] for b in v] nx, ny = numpy.meshgrid(k, l) Z[ny, nx] = z[my, mx] if ftype == 'xt': k, l, m = len(lon), 1, len(tm) if masked: Z = numpy.ma.empty((m, k)) * numpy.nan Z.mask = True else: Z = numpy.empty((m, k)) * numpy.nan # u = numpy.unique(set(lon180) & set(x180)) if len(u) == 0: return False i = [pylab.find(x180 == a)[0] for a in u] k = [pylab.find(lon180 == a)[0] for a in u] v = set(tm) & set(t) if len(v) == 0: return False j = [pylab.find(t == b)[0] for b in v] l = [pylab.find(tm == b)[0] for b in v] mx, my = numpy.meshgrid(i, j) nx, ny = numpy.meshgrid(k, l) Z[ny, nx] = z[my, mx] if ftype == 'ty': raise Warning, ('Loading of temporal-meridional files not ' 'implemented yet.') else: lon = x lat = y tm = t Z = z if (xlim != None) or (ylim != None): if xlim == None: xlim = [lon.min(), lon.max()] if ylim == None: ylim = [lat.min(), lat.max()] selx = pylab.find((lon >= min(xlim)) & (lon <= max(xlim))) sely = pylab.find((lat >= min(ylim)) & (lat <= max(ylim))) lon = lon[selx] lat = lat[sely] i, j = numpy.meshgrid(selx, sely) Z = Z[j, i] if topomask != None: # Interpolates topography into data grid. ezi, _, _ = interpolate.nearest([common.etopo.x, common.etopo.y], common.etopo.z, [lon, lat]) if topomask == 'ocean': Z.mask = Z.mask | (ezi > 0) elif topomask == 'land': Z.mask = Z.mask | (ezi < 0) if masked: Z.data[Z.mask] = 0 # TODO: change to numpy.nan? return lon, lat, tm, Z
def load_dataset(path, pattern='(.*)', ftype='xy', flist=None, delimiter='\t', var_from_name=False, masked=False, xlim=None, ylim=None, lon=None, lat=None, tm=None, topomask=None, verbose=False): """Loads an entire dataset. It uses the numpy.loadtxt function and therefore accepts regular ASCII files or GZIP compressed ones. PARAMETERS path (string) : The path in which the data files are located. pattern (string, optional) : Regular expression pattern correspondig to valid file names to be loaded. ftype (string, optional) : Specifies the file type that is loaded. The accepted values are 'xy', 'xt' and 'ty'. For 'xy', or map, files, the first line contains the longitude coordinates, the first column contains the latitude coordinates and the rest contains the data in matrix style. If var_from_name is set to True, it assumes that the time is given at the upper left cell. For 'xt', or zonal-temporal, files, the first line contains the longitude coordinates, the first column contains the time and the rest contains the data in matrix style. If var_from_name is set to True, it assumes that the latitude is given at the upper left cell. For 'ty', or temporal-meridional, files, the first line contains the time, the first column contains the longitude and the rest contains the data in matrix style. If var_from_name is set to True, it assumes that the latitude is given at the upper left cell. flist (array like, optional) : Lists the files to be loaded in path. If set, it ignores the pattern. delimiter (string, optional) : Specifies the data delimiter used while loading the data. The default value is '\t' (tab) var_from_name (boolean, optional) : If set to true, it tries to infer eather the time, latitude or longitude from the first match in pattern according to the chosen file type. If set to true, the pattern has to be set in such a way that the last matches contain the value and the hemisphere ('N', 'S', 'E' or 'W') if appropriate. masked (boolean, optional) : Returnes masked array. Default is False. xlim, ylim (array like, optional) : List containing the upper and lower zonal and meridional limits, respectivelly. lon, lat, tm (array like, optional): topomask (string, optional) : Topography mask. verbose (boolean, optional) : If set to true, does not print anything on screen. RETURNS lon (array like) : Longitude. lat (array like) : Latitude. t (array like) : Time. z (array like) : Loaded variable. """ t0 = time() if topomask != None: masked = True S = 'Preparing data' s = '%s...' % (S) if not verbose: os.sys.stdout.write(s) os.sys.stdout.flush() # Generates list of files and tries to match them to the pattern if flist == None: flist = os.listdir(path) flist, match = common.reglist(flist, pattern) # Loads all the data from file list to create arrays N = len(flist) if N == 0: raise Warning, 'No files to be loaded.' # Initializes the set of array limits Lon = set() Lat = set() Tm = set() # Walks through the file loading process twice. At the first step loads # all the files to get all the geographical and temporal boundaries. At the # second step, reloads all files and fits them to the initialized data # arrays for step in range(2): t1 = time() for n, fname in enumerate(flist): t2 = time() if (lon != None) and (lat != None) and (tm !=None): continue x, y, t, z = load_map('%s/%s' % (path, fname), ftype=ftype, delimiter=delimiter, lon=lon, lat=lat, tm=tm, masked=masked, topomask=topomask) if var_from_name: if (ftype == 'xt') | (ftype == 'ty'): var = atof(match[n][-2]) # Gets coordinate out of ... rav = match[n][-1].upper() # ... match and also its ... if (rav == 'S' | rav == 'W'): # ... hemisphere. var *= -1 if ftype == 'xt': y = var else: x = var elif ftype == 'xy': t = atof(match[n][-1]) # Gets time out of last match. if numpy.isnan(t).all(): t = 0 if type(x).__name__ in ['int', 'long', 'float', 'float64']: x = [x] if type(y).__name__ in ['int', 'long', 'float', 'float64']: y = [y] if type(t).__name__ in ['int', 'long', 'float', 'float64']: t = [t] ################################################################### # FIRST STEP ################################################################### if step == 0: Lon.update(x) Lat.update(y) Tm.update(t) ################################################################### # SECOND STEP ################################################################### elif step == 1: selx = [pylab.find(Lon == i)[0] for i in x] sely = [pylab.find(Lat == i)[0] for i in y] selt = [pylab.find(Tm == i)[0] for i in t] i, j, k = common.meshgrid2(selx, sely, selt) if ftype == 'xt': a, b, c = i.shape z = z.reshape((a, 1, c)) # Makes sure only to overwrite values not previously assigned. if masked: Z[k, j, i] = numpy.ma.where(~Z[k, j, i].mask, Z[k, j, i], z) else: Z[k, j, i] = numpy.where(~numpy.isnan(Z[k, j, i]), Z[k, j, i], z) ################################################################### # PROFILING ################################################################### if not verbose: os.sys.stdout.write(len(s) * '\b') s = '%s (%s)... %s ' % (S, fname, common.profiler(N, n + 1, t0, t1, t2)) if not verbose: os.sys.stdout.write(s) os.sys.stdout.flush() # if not verbose: os.sys.stdout.write('\n') # Now creates data array based on input parameters xlim, ylim and # the loaded coordinate sets. if step == 0: if lon == None: Lon = numpy.asarray(list(Lon)) else: Lon = lon if lat == None: Lat = numpy.asarray(list(Lat)) else: Lat = lat if tm == None: Tm = numpy.asarray(list(Tm)) else: Tm = tm Lon.sort() Lat.sort() Tm.sort() # Makes sure that all the coordinates are continuous, equally # spaced and that they are inside the coordinate limits. dx, dy, dt = numpy.diff(Lon), numpy.diff(Lat), numpy.diff(Tm) if len(dx) == 0: dx = numpy.array([1.]) if len(dy) == 0: dy = numpy.array([1.]) if len(dt) == 0: dt = numpy.array([1.]) #if ((not (dx == dx[0]).all()) or (not (dy == dy[0]).all()) or # (not (dt == dt[0]).all())): # raise Warning, 'One or more coordinates are not evenly spaced.' dx = dx[0] dy = dy[0] dt = dt[0] if xlim == None: xlim = [Lon.min(), Lon.max()] if ylim == None: ylim = [Lat.min(), Lat.max()] selx = pylab.find((Lon >= min(xlim)) & (Lon <= max(xlim))) Lon = Lon[selx] sely = pylab.find((Lat >= min(ylim)) & (Lat <= max(ylim))) Lat = Lat[sely] # Pads edges with NaN's to avoid distortions when generating maps. if lon == None: Lon = numpy.concatenate([[Lon[0] - dx], Lon, [Lon[-1] + dx]]) if lat == None: Lat = numpy.concatenate([[Lat[0] - dy], Lat, [Lat[-1] + dy]]) # Initializes data arrays a, b, c = Lon.size, Lat.size, Tm.size if masked: Z = numpy.ma.empty([c, b, a], dtype=float) * numpy.nan Z.mask = True else: Z = numpy.empty([c, b, a], dtype=float) * numpy.nan lon, lat = numpy.array(Lon), numpy.array(Lat) # Now everything might be ready for the second step in the loop, # filling in the data array. S, s = 'Loading data', '' # Interpolates topography into data grid. if topomask != None: if not verbose: print 'Masking topographic features...' ezi, _, _ = interpolate.nearest([common.etopo.x, common.etopo.y], common.etopo.z, [Lon, Lat]) if topomask == 'ocean': tmask = (ezi > 0) elif topomask == 'land': tmask = (ezi < 0) # tmask = tmask.reshape([1, b, a]) tmask = tmask.repeat(c, axis=0) # Z.mask = Z.mask | tmask if masked: Z.mask = Z.mask | numpy.isnan(Z.data) Z.data[Z.mask] = 0 return Lon, Lat, Tm, Z
def load_dataset(path, pattern='(.*)', ftype='xy', flist=None, delimiter='\t', var_from_name=False, masked=False, xlim=None, ylim=None, lon=None, lat=None, tm=None, topomask=None, verbose=False, dummy=False): """Loads an entire dataset. It uses the numpy.loadtxt function and therefore accepts regular ASCII files or GZIP compressed ones. PARAMETERS path (string) : The path in which the data files are located. pattern (string, optional) : Regular expression pattern correspondig to valid file names to be loaded. ftype (string, optional) : Specifies the file type that is loaded. The accepted values are 'xy', 'xt' and 'ty'. For 'xy', or map, files, the first line contains the longitude coordinates, the first column contains the latitude coordinates and the rest contains the data in matrix style. If var_from_name is set to True, it assumes that the time is given at the upper left cell. For 'xt', or zonal-temporal, files, the first line contains the longitude coordinates, the first column contains the time and the rest contains the data in matrix style. If var_from_name is set to True, it assumes that the latitude is given at the upper left cell. For 'ty', or temporal-meridional, files, the first line contains the time, the first column contains the longitude and the rest contains the data in matrix style. If var_from_name is set to True, it assumes that the latitude is given at the upper left cell. flist (array like, optional) : Lists the files to be loaded in path. If set, it ignores the pattern. delimiter (string, optional) : Specifies the data delimiter used while loading the data. The default value is '\t' (tab) var_from_name (boolean, optional) : If set to true, it tries to infer eather the time, latitude or longitude from the first match in pattern according to the chosen file type. If set to true, the pattern has to be set in such a way that the last matches contain the value and the hemisphere ('N', 'S', 'E' or 'W') if appropriate. masked (boolean, optional) : Returnes masked array. Default is False. xlim, ylim (array like, optional) : List containing the upper and lower zonal and meridional limits, respectivelly. lon, lat, tm (array like, optional): topomask (string, optional) : Topography mask. verbose (boolean, optional) : If set to true, does not print anything on screen. dummy (boolean, optional) : If set to true, does not load data and returns blank data array for test purposes. RETURNS lon (array like) : Longitude. lat (array like) : Latitude. t (array like) : Time. z (array like) : Loaded variable. """ t0 = time() if topomask != None: masked = True S = 'Preparing data' s = '%s...' % (S) if not verbose: os.sys.stdout.write(s) os.sys.stdout.flush() # Generates list of files and tries to match them to the pattern if flist == None: flist = os.listdir(path) flist, match = common.reglist(flist, pattern) # Loads all the data from file list to create arrays N = len(flist) if N == 0: raise Warning, 'No files to be loaded.' # Initializes the set of array limits Lon = set() Lat = set() Tm = set() # Walks through the file loading process twice. At the first step loads # all the files to get all the geographical and temporal boundaries. At the # second step, reloads all files and fits them to the initialized data # arrays if dummy: step_range = 1 else: step_range = 2 for step in range(step_range): t1 = time() for n, fname in enumerate(flist): t2 = time() if (lon != None) and (lat != None) and (tm != None): continue x, y, t, z = load_map('%s/%s' % (path, fname), ftype=ftype, delimiter=delimiter, lon=lon, lat=lat, tm=tm, masked=masked, topomask=None) if var_from_name: if (ftype == 'xt') | (ftype == 'ty'): var = atof(match[n][-2]) # Gets coordinate out of ... rav = match[n][-1].upper() # ... match and also its ... if (rav == 'S' | rav == 'W'): # ... hemisphere. var *= -1 if ftype == 'xt': y = var else: x = var elif ftype == 'xy': t = atof(match[n][-1]) # Gets time out of last match. if numpy.isnan(t).all(): t = 0 if type(x).__name__ in ['int', 'long', 'float', 'float64']: x = [x] if type(y).__name__ in ['int', 'long', 'float', 'float64']: y = [y] if type(t).__name__ in ['int', 'long', 'float', 'float64']: t = [t] ################################################################### # FIRST STEP ################################################################### if step == 0: Lon.update(x) Lat.update(y) Tm.update(t) ################################################################### # SECOND STEP ################################################################### elif step == 1: selx = [pylab.find(Lon == i)[0] for i in x] sely = [pylab.find(Lat == i)[0] for i in y] selt = [pylab.find(Tm == i)[0] for i in t] #print len(selx), len(sely), len(selt) i, j, k = common.meshgrid2(selx, sely, selt) if ftype == 'xt': a, b, c = i.shape z = z.reshape((a, 1, c)) # Makes sure only to overwrite values not previously assigned. if masked: Z[k, j, i] = numpy.ma.where(~Z[k, j, i].mask, Z[k, j, i], z) else: Z[k, j, i] = numpy.where(~numpy.isnan(Z[k, j, i]), Z[k, j, i], z) ################################################################### # PROFILING ################################################################### if not verbose: os.sys.stdout.write(len(s) * '\b') s = '%s (%s)... %s ' % (S, fname, common.profiler(N, n + 1, t0, t1, t2)) if not verbose: os.sys.stdout.write(s) os.sys.stdout.flush() # if not verbose: os.sys.stdout.write('\n') # Now creates data array based on input parameters xlim, ylim and # the loaded coordinate sets. if step == 0: if lon == None: Lon = numpy.asarray(list(Lon)) else: Lon = lon if lat == None: Lat = numpy.asarray(list(Lat)) else: Lat = lat if tm == None: Tm = numpy.asarray(list(Tm)) else: Tm = tm Lon.sort() Lat.sort() Tm.sort() # Makes sure that all the coordinates are continuous, equally # spaced and that they are inside the coordinate limits. dx, dy, dt = numpy.diff(Lon), numpy.diff(Lat), numpy.diff(Tm) if len(dx) == 0: dx = numpy.array([1.]) if len(dy) == 0: dy = numpy.array([1.]) if len(dt) == 0: dt = numpy.array([1.]) #if ((not (dx == dx[0]).all()) or (not (dy == dy[0]).all()) or # (not (dt == dt[0]).all())): # raise Warning, 'One or more coordinates are not evenly spaced.' dx = dx[0] dy = dy[0] dt = dt[0] if xlim == None: xlim = [Lon.min(), Lon.max()] if ylim == None: ylim = [Lat.min(), Lat.max()] selx = pylab.find((Lon >= min(xlim)) & (Lon <= max(xlim))) Lon = Lon[selx] sely = pylab.find((Lat >= min(ylim)) & (Lat <= max(ylim))) Lat = Lat[sely] # Pads edges with NaN's to avoid distortions when generating maps. if lon == None: Lon = numpy.concatenate([[Lon[0] - dx], Lon, [Lon[-1] + dx]]) if lat == None: Lat = numpy.concatenate([[Lat[0] - dy], Lat, [Lat[-1] + dy]]) # Initializes data arrays a, b, c = Lon.size, Lat.size, Tm.size if masked: Z = numpy.ma.empty([c, b, a], dtype=float) * numpy.nan Z.mask = True else: Z = numpy.empty([c, b, a], dtype=float) * numpy.nan lon, lat = numpy.array(Lon), numpy.array(Lat) # Now everything might be ready for the second step in the loop, # filling in the data array. S, s = 'Loading data', '' # Interpolates topography into data grid. if topomask != None: if not verbose: print 'Masking topographic features...' ezi, _, _ = interpolate.nearest([common.etopo.x, common.etopo.y], common.etopo.z, [Lon, Lat]) if topomask == 'ocean': tmask = (ezi > 0) elif topomask == 'land': tmask = (ezi < 0) # tmask = tmask.reshape([1, b, a]) tmask = tmask.repeat(c, axis=0) # Z.mask = Z.mask | tmask if masked: Z.mask = Z.mask | numpy.isnan(Z.data) Z.data[Z.mask] = 0 return Lon, Lat, Tm, Z