def verbosalize_logger(log, verbosity): if verbosity <= 0: return if verbosity >= 8: while True: log.critical('MASTML' * random.randint(3, 2**(verbosity - 4)))
def verbosalize_logger(log, verbosity): if verbosity <= 0: return if verbosity >= 8: while True: log.critical('MSATML' * random.randint(3, 2**(verbosity - 4))) old_log = log._log def new_log(level, msg, *args, **kwargs): old_log(level, [ None, None, to_upper, to_full_width, to_leet, deep_fry, deep_fry_2, emojify ][verbosity](msg), *args, **kwargs) log._log = new_log
def _generic_convert_dem_from_ascii2netcdf(self, name_in, name_out=None, quantity_name=None, verbose=False): """Read raster from the following ASCII format (.asc) Internal function. See public function convert_dem_from_ascii2netcdf for details. """ import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] # Read Meta data if verbose: log.critical("Reading METADATA from %s" % (root + ".prj")) metadatafile = open(root + ".prj") metalines = metadatafile.readlines() metadatafile.close() L = metalines[0].strip().split() assert L[0].strip().lower() == "projection" projection = L[1].strip() # TEXT L = metalines[1].strip().split() assert L[0].strip().lower() == "zone" zone = int(L[1].strip()) L = metalines[2].strip().split() assert L[0].strip().lower() == "datum" datum = L[1].strip() # TEXT L = metalines[3].strip().split() assert L[0].strip().lower() == "zunits" # IGNORE zunits = L[1].strip() # TEXT L = metalines[4].strip().split() assert L[0].strip().lower() == "units" units = L[1].strip() # TEXT L = metalines[5].strip().split() assert L[0].strip().lower() == "spheroid" # IGNORE spheroid = L[1].strip() # TEXT L = metalines[6].strip().split() assert L[0].strip().lower() == "xshift" false_easting = float(L[1].strip()) L = metalines[7].strip().split() assert L[0].strip().lower() == "yshift" false_northing = float(L[1].strip()) if name_in[-4:] != ".asc": raise IOError("Input file %s should be of type .asc." % name_in) # Read DEM data datafile = open(name_in) if verbose: log.critical("Reading raster from %s" % (name_in)) lines = datafile.readlines() datafile.close() if verbose: log.critical("Got %d lines" % len(lines)) ncols = int(lines[0].split()[1].strip()) nrows = int(lines[1].split()[1].strip()) # Do cellsize (line 4) before line 2 and 3 cellsize = float(lines[4].split()[1].strip()) # Checks suggested by Joaquim Luis # Our internal representation of xllcorner # and yllcorner is non-standard. xref = lines[2].split() if xref[0].strip() == "xllcorner": xllcorner = float(xref[1].strip()) # + 0.5*cellsize # Correct offset elif xref[0].strip() == "xllcenter": xllcorner = float(xref[1].strip()) else: msg = "Unknown keyword: %s" % xref[0].strip() raise Exception, msg yref = lines[3].split() if yref[0].strip() == "yllcorner": yllcorner = float(yref[1].strip()) # + 0.5*cellsize # Correct offset elif yref[0].strip() == "yllcenter": yllcorner = float(yref[1].strip()) else: msg = "Unknown keyword: %s" % yref[0].strip() raise Exception, msg NODATA_value = int(float(lines[5].split()[1].strip())) assert len(lines) == nrows + 6 if name_out == None: netcdfname = name_in[:-4] + ".dem" else: netcdfname = name_out + ".dem" if verbose: log.critical("Store to NetCDF file %s" % netcdfname) # NetCDF file definition fid = NetCDFFile(netcdfname, netcdf_mode_w) # Create new file fid.institution = "Geoscience Australia" fid.description = "NetCDF DEM format for compact and portable storage " "of spatial point data" fid.ncols = ncols fid.nrows = nrows fid.xllcorner = xllcorner fid.yllcorner = yllcorner fid.cellsize = cellsize fid.NODATA_value = NODATA_value fid.zone = zone fid.false_easting = false_easting fid.false_northing = false_northing fid.projection = projection fid.datum = datum fid.units = units # dimension definitions fid.createDimension("number_of_rows", nrows) fid.createDimension("number_of_columns", ncols) # variable definitions fid.createVariable(quantity_name, netcdf_float, ("number_of_rows", "number_of_columns")) # Get handles to the variables elevation = fid.variables[quantity_name] # Store data import numpy datafile = open(name_in) elevation[:, :] = numpy.loadtxt(datafile, skiprows=6) datafile.close() # n = len(lines[6:]) # for i, line in enumerate(lines[6:]): # fields = line.split() # if verbose and i % ((n+10)/10) == 0: # log.critical('Processing row %d of %d' % (i, nrows)) # # if len(fields) != ncols: # msg = 'Wrong number of columns in file "%s" line %d\n' % (name_in, i) # msg += 'I got %d elements, but there should have been %d\n' % (len(fields), ncols) # raise Exception, msg # # elevation[i, :] = num.array([float(x) for x in fields]) fid.close()
def _generic_dem2pts( self, name_in, name_out=None, quantity_name=None, verbose=False, easting_min=None, easting_max=None, northing_min=None, northing_max=None, ): """Read raster from the following NetCDF format (.dem) Internal function. See public function generic_dem2pts for details. """ # FIXME: Can this be written feasibly using write_pts? import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] if name_in[-4:] == ".asc": intermediate = root + ".dem" if verbose: log.critical("Preconvert %s from asc to %s" % (name_in, intermediate)) asc2dem(name_in) name_in = intermediate elif name_in[-4:] != ".dem": raise IOError("Input file %s should be of type .asc or .dem." % name_in) if name_out != None and basename_out[-4:] != ".pts": raise IOError("Input file %s should be of type .pts." % name_out) # Get NetCDF infile = NetCDFFile(name_in, netcdf_mode_r) if verbose: log.critical("Reading raster from %s" % (name_in)) ncols = int(infile.ncols) nrows = int(infile.nrows) xllcorner = float(infile.xllcorner) # Easting of lower left corner yllcorner = float(infile.yllcorner) # Northing of lower left corner cellsize = float(infile.cellsize) NODATA_value = float(infile.NODATA_value) dem_elevation = infile.variables[quantity_name] zone = int(infile.zone) false_easting = float(infile.false_easting) false_northing = float(infile.false_northing) # print ncols, nrows, xllcorner,yllcorner, cellsize, NODATA_value, zone # Text strings projection = infile.projection datum = infile.datum units = infile.units # print projection, datum, units # Get output file if name_out == None: ptsname = root + ".pts" else: ptsname = name_out if verbose: log.critical("Store to NetCDF file %s" % ptsname) # NetCDF file definition outfile = NetCDFFile(ptsname, netcdf_mode_w) # Create new file outfile.institution = "Geoscience Australia" outfile.description = "NetCDF pts format for compact and portable " "storage of spatial point data" # Assign default values if easting_min is None: easting_min = xllcorner if easting_max is None: easting_max = xllcorner + ncols * cellsize if northing_min is None: northing_min = yllcorner if northing_max is None: northing_max = yllcorner + nrows * cellsize # print easting_min, easting_max, northing_min, northing_max # Compute offsets to update georeferencing easting_offset = xllcorner - easting_min northing_offset = yllcorner - northing_min # Georeferencing outfile.zone = zone outfile.xllcorner = easting_min # Easting of lower left corner outfile.yllcorner = northing_min # Northing of lower left corner outfile.false_easting = false_easting outfile.false_northing = false_northing outfile.projection = projection outfile.datum = datum outfile.units = units # Grid info (FIXME: probably not going to be used, but heck) outfile.ncols = ncols outfile.nrows = nrows dem_elevation_r = num.reshape(dem_elevation, (nrows, ncols)) totalnopoints = nrows * ncols # ======================================== # Do the preceeding with numpy # ======================================== y = num.arange(nrows, dtype=num.float) y = yllcorner + (nrows - 1) * cellsize - y * cellsize x = num.arange(ncols, dtype=num.float) x = xllcorner + x * cellsize xx, yy = num.meshgrid(x, y) xx = xx.flatten() yy = yy.flatten() flag = num.logical_and( num.logical_and((xx <= easting_max), (xx >= easting_min)), num.logical_and((yy <= northing_max), (yy >= northing_min)), ) dem = dem_elevation[:].flatten() id = num.where(flag)[0] xx = xx[id] yy = yy[id] dem = dem[id] clippednopoints = len(dem) # print clippedpoints # print xx # print yy # print dem data_flag = dem != NODATA_value data_id = num.where(data_flag) xx = xx[data_id] yy = yy[data_id] dem = dem[data_id] nn = clippednopoints - len(dem) nopoints = len(dem) if verbose: log.critical("There are %d values in the raster" % totalnopoints) log.critical("There are %d values in the clipped raster" % clippednopoints) log.critical("There are %d NODATA_values in the clipped raster" % nn) outfile.createDimension("number_of_points", nopoints) outfile.createDimension("number_of_dimensions", 2) # This is 2d data # Variable definitions outfile.createVariable("points", netcdf_float, ("number_of_points", "number_of_dimensions")) outfile.createVariable(quantity_name, netcdf_float, ("number_of_points",)) # Get handles to the variables points = outfile.variables["points"] elevation = outfile.variables[quantity_name] points[:, 0] = xx - easting_min points[:, 1] = yy - northing_min elevation[:] = dem infile.close() outfile.close()
def _generic_convert_dem_from_ascii2netcdf(self, name_in, name_out = None, quantity_name = None, verbose = False): """ Read raster from the following ASCII format (.asc) Internal function. See public function convert_dem_from_ascii2netcdf for details. """ import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] # Read Meta data if verbose: log.critical('Reading METADATA from %s' % (root + '.prj')) metadatafile = open(root + '.prj') metalines = metadatafile.readlines() metadatafile.close() L = metalines[0].strip().split() assert L[0].strip().lower() == 'projection' projection = L[1].strip() #TEXT L = metalines[1].strip().split() assert L[0].strip().lower() == 'zone' zone = int(L[1].strip()) L = metalines[2].strip().split() assert L[0].strip().lower() == 'datum' datum = L[1].strip() #TEXT L = metalines[3].strip().split() assert L[0].strip().lower() == 'zunits' #IGNORE zunits = L[1].strip() #TEXT L = metalines[4].strip().split() assert L[0].strip().lower() == 'units' units = L[1].strip() #TEXT L = metalines[5].strip().split() assert L[0].strip().lower() == 'spheroid' #IGNORE spheroid = L[1].strip() #TEXT L = metalines[6].strip().split() assert L[0].strip().lower() == 'xshift' false_easting = float(L[1].strip()) L = metalines[7].strip().split() assert L[0].strip().lower() == 'yshift' false_northing = float(L[1].strip()) if name_in[-4:] != '.asc': raise IOError('Input file %s should be of type .asc.' % name_in) #Read DEM data datafile = open(name_in) if verbose: log.critical('Reading raster from %s' % (name_in)) lines = datafile.readlines() datafile.close() if verbose: log.critical('Got %d lines' % len(lines)) ncols = int(lines[0].split()[1].strip()) nrows = int(lines[1].split()[1].strip()) # Do cellsize (line 4) before line 2 and 3 cellsize = float(lines[4].split()[1].strip()) # Checks suggested by Joaquim Luis # Our internal representation of xllcorner # and yllcorner is non-standard. xref = lines[2].split() if xref[0].strip() == 'xllcorner': xllcorner = float(xref[1].strip()) # + 0.5*cellsize # Correct offset elif xref[0].strip() == 'xllcenter': xllcorner = float(xref[1].strip()) else: msg = 'Unknown keyword: %s' % xref[0].strip() raise Exception, msg yref = lines[3].split() if yref[0].strip() == 'yllcorner': yllcorner = float(yref[1].strip()) # + 0.5*cellsize # Correct offset elif yref[0].strip() == 'yllcenter': yllcorner = float(yref[1].strip()) else: msg = 'Unknown keyword: %s' % yref[0].strip() raise Exception, msg NODATA_value = int(float(lines[5].split()[1].strip())) assert len(lines) == nrows + 6 if name_out == None: netcdfname = name_in[:-4]+'.dem' else: netcdfname = name_out + '.dem' if verbose: log.critical('Store to NetCDF file %s' % netcdfname) # NetCDF file definition fid = NetCDFFile(netcdfname, netcdf_mode_w) #Create new file fid.institution = 'Geoscience Australia' fid.description = 'NetCDF DEM format for compact and portable storage ' \ 'of spatial point data' fid.ncols = ncols fid.nrows = nrows fid.xllcorner = xllcorner fid.yllcorner = yllcorner fid.cellsize = cellsize fid.NODATA_value = NODATA_value fid.zone = zone fid.false_easting = false_easting fid.false_northing = false_northing fid.projection = projection fid.datum = datum fid.units = units # dimension definitions fid.createDimension('number_of_rows', nrows) fid.createDimension('number_of_columns', ncols) # variable definitions fid.createVariable(quantity_name, netcdf_float, ('number_of_rows', 'number_of_columns')) # Get handles to the variables elevation = fid.variables[quantity_name] #Store data import numpy datafile = open(name_in) elevation[:,:] = numpy.loadtxt(datafile, skiprows=6) datafile.close() fid.close()
def _generic_dem2npy(self, name_in, name_out=None, quantity_name=None, verbose=False, easting_min=None, easting_max=None, northing_min=None, northing_max=None): """ Read raster from the following NetCDF format (.dem) Internal function. See public function generic_dem2npy for details. """ import os from anuga.file.netcdf import NetCDFFile root = name_in[:-4] if name_in[-4:] == '.asc': intermediate = root + '.dem' if verbose: log.critical('Preconvert %s from asc to %s' % \ (name_in, intermediate)) self.generic_asc2dem(name_in) name_in = intermediate elif name_in[-4:] != '.dem': raise IOError('Input file %s should be of type .asc or .dem.' % name_in) # Get NetCDF infile = NetCDFFile(name_in, netcdf_mode_r) if verbose: log.critical('Reading raster from %s' % (name_in)) ncols = int(infile.ncols) nrows = int(infile.nrows) xllcorner = float(infile.xllcorner) # Easting of lower left corner yllcorner = float(infile.yllcorner) # Northing of lower left corner cellsize = float(infile.cellsize) NODATA_value = float(infile.NODATA_value) dem_elevation = infile.variables[quantity_name] # Assign default values if easting_min is None: easting_min = xllcorner if easting_max is None: easting_max = xllcorner + ncols*cellsize if northing_min is None: northing_min = yllcorner if northing_max is None: northing_max = yllcorner + nrows*cellsize #======================================== # Do the preceeding with numpy #======================================== y = num.arange(nrows,dtype=num.float) y = yllcorner + (nrows-1)*cellsize - y*cellsize x = num.arange(ncols,dtype=num.float) x = xllcorner + x*cellsize xx,yy = num.meshgrid(x,y) xx = xx.flatten() yy = yy.flatten() flag = num.logical_and(num.logical_and((xx <= easting_max),(xx >= easting_min)), num.logical_and((yy <= northing_max),(yy >= northing_min))) dem = dem_elevation[:].flatten() id = num.where(flag)[0] xx = xx[id] yy = yy[id] dem = dem[id] data_flag = dem != NODATA_value data_id = num.where(data_flag) points = num.zeros((len(data_id[0]),3)) points[:,0] = xx[data_id] - easting_min points[:,1] = yy[data_id] - northing_min points[:,2] = dem[data_id] x_ = num.arange(0, num.ceil(points[:,0].max())) y_ = num.arange(0, num.ceil(points[:,1].max())) grid_x, grid_y = num.meshgrid(x_,y_) grid_z = num.zeros_like(grid_x) rloc = num.round(points[:,1]).astype(int) cloc = num.round(points[:,0]).astype(int) grid_z[rloc, cloc] = points[:,2] points = num.vstack((grid_x.flatten() + easting_min, grid_y.flatten() + northing_min, grid_z.flatten())).T infile.close() return points