def write_asc_file(filename, data, xsize, ysize, geotransform, nodata_value): """Output Raster to ASCII file. Args: filename: output ASCII filename. data: 2D array data. xsize: Col count. ysize: Row count. geotransform: geographic transformation. nodata_value: nodata_flow value. """ UtilClass.mkdir(os.path.dirname(FileClass.get_file_fullpath(filename))) header = 'NCOLS %d\n' \ 'NROWS %d\n' \ 'XLLCENTER %f\n' \ 'YLLCENTER %f\n' \ 'CELLSIZE %f\n' \ 'NODATA_VALUE %f' % (xsize, ysize, geotransform[0] + 0.5 * geotransform[1], geotransform[3] - (ysize - 0.5) * geotransform[1], geotransform[1], nodata_value) with open(filename, 'w', encoding='utf-8') as f: f.write(header) for i in range(0, ysize): for j in range(0, xsize): f.write('%s\t' % repr(data[i][j])) f.write('\n') f.close()
def write_gtiff_file(f_name, n_rows, n_cols, data, geotransform, srs, nodata_value, gdal_type=GDT_Float32): """Output Raster to GeoTiff format file. Args: f_name: output gtiff file name. n_rows: Row count. n_cols: Col count. data: 2D array data. geotransform: geographic transformation. srs: coordinate system. nodata_value: nodata value. gdal_type (:obj:`pygeoc.raster.GDALDataType`): output raster data type, GDT_Float32 as default. """ UtilClass.mkdir(os.path.dirname(FileClass.get_file_fullpath(f_name))) driver = gdal_GetDriverByName(str('GTiff')) try: ds = driver.Create(f_name, n_cols, n_rows, 1, gdal_type) except Exception: print('Cannot create output file %s' % f_name) return ds.SetGeoTransform(geotransform) try: ds.SetProjection(srs.ExportToWkt()) except AttributeError or Exception: ds.SetProjection(srs) ds.GetRasterBand(1).SetNoDataValue(nodata_value) # if data contains numpy.nan, then replaced by nodata_value if isinstance(data, numpy.ndarray) and data.dtype in [numpy.dtype('int'), numpy.dtype('float')]: data = numpy.where(numpy.isnan(data), nodata_value, data) ds.GetRasterBand(1).WriteArray(data) ds = None
def write_asc_file(filename, data, xsize, ysize, geotransform, nodata_value): """Output Raster to ASCII file. Args: filename: output ASCII filename. data: 2D array data. xsize: Col count. ysize: Row count. geotransform: geographic transformation. nodata_value: nodata_flow value. """ UtilClass.mkdir(os.path.dirname(FileClass.get_file_fullpath(filename))) header = 'NCOLS %d\n' \ 'NROWS %d\n' \ 'XLLCENTER %f\n' \ 'YLLCENTER %f\n' \ 'CELLSIZE %f\n' \ 'NODATA_VALUE %f' % (xsize, ysize, geotransform[0] + 0.5 * geotransform[1], geotransform[3] - (ysize - 0.5) * geotransform[1], geotransform[1], nodata_value) with open(filename, 'w', encoding='utf-8') as f: f.write(header) for i in range(0, ysize): for j in range(0, xsize): f.write('%s\t' % repr(data[i][j])) f.write('\n') f.close()
def write_gtiff_file(f_name, n_rows, n_cols, data, geotransform, srs, nodata_value, gdal_type=GDT_Float32): """Output Raster to GeoTiff format file. Args: f_name: output gtiff file name. n_rows: Row count. n_cols: Col count. data: 2D array data. geotransform: geographic transformation. srs: coordinate system. nodata_value: nodata value. gdal_type (:obj:`pygeoc.raster.GDALDataType`): output raster data type, GDT_Float32 as default. """ UtilClass.mkdir(os.path.dirname(FileClass.get_file_fullpath(f_name))) driver = gdal_GetDriverByName(str('GTiff')) try: ds = driver.Create(f_name, n_cols, n_rows, 1, gdal_type) except Exception: print('Cannot create output file %s' % f_name) return ds.SetGeoTransform(geotransform) try: ds.SetProjection(srs.ExportToWkt()) except AttributeError or Exception: ds.SetProjection(srs) ds.GetRasterBand(1).SetNoDataValue(nodata_value) # if data contains numpy.nan, then replaced by nodata_value if isinstance(data, numpy.ndarray) and data.dtype in [ numpy.dtype('int'), numpy.dtype('float') ]: data = numpy.where(numpy.isnan(data), nodata_value, data) ds.GetRasterBand(1).WriteArray(data) ds = None
def run(function_name, in_files, wp=None, in_params=None, out_files=None, mpi_params=None, log_params=None): """ Run TauDEM function. 1. The command will not execute if any input file does not exist. 2. An error will be detected after running the TauDEM command if any output file does not exist; Args: function_name (str): Full path of TauDEM function. in_files (dict, required): Dict of pairs of parameter id (string) and file path (string or list) for input files, e.g.:: {'-z': '/full/path/to/dem.tif'} wp (str, optional): Workspace for outputs. If not specified, the directory of the first input file in ``in_files`` will be used. in_params (dict, optional): Dict of pairs of parameter id (string) and value (or None for a flag parameter without a value) for input parameters, e.g.:: {'-nc': None} {'-thresh': threshold} {'-m': 'ave' 's', '-nc': None} out_files (dict, optional): Dict of pairs of parameter id (string) and file path (string or list) for output files, e.g.:: {'-fel': 'filleddem.tif'} {'-maxS': ['harden.tif', 'maxsimi.tif']} mpi_params (dict, optional): Dict of pairs of parameter id (string) and value or path for MPI setting, e.g.:: {'mpipath':'/soft/bin','hostfile':'/soft/bin/cluster.node','n':4} {'mpipath':'/soft/bin', 'n':4} {'n':4} log_params (dict, optional): Dict of pairs of parameter id (string) and value or path for runtime and log output parameters. e.g.:: {'logfile': '/home/user/log.txt', 'runtimefile': '/home/user/runtime.txt'} Returns: True if TauDEM run successfully, otherwise False. """ # Check input files if in_files is None: TauDEM.error('Input files parameter is required!') if not isinstance(in_files, dict): TauDEM.error('The input files parameter must be a dict!') for (pid, infile) in list(in_files.items()): if infile is None: continue if isinstance(infile, list) or isinstance(infile, tuple): for idx, inf in enumerate(infile): if inf is None: continue inf, wp = TauDEM.check_infile_and_wp(inf, wp) in_files[pid][idx] = inf continue if os.path.exists(infile): infile, wp = TauDEM.check_infile_and_wp(infile, wp) in_files[pid] = os.path.abspath(infile) else: # For more flexible input files extension. # e.g., -inputtags 1 <path/to/tag1.tif> 2 <path/to/tag2.tif> ... # in such unpredictable circumstance, we cannot check the existance of # input files, so the developer will check it in other place. if len(StringClass.split_string(infile, ' ')) > 1: continue else: # the infile still should be a existing file, so check in workspace if wp is None: TauDEM.error('Workspace should not be None!') infile = wp + os.sep + infile if not os.path.exists(infile): TauDEM.error( 'Input files parameter %s: %s is not existed!' % (pid, infile)) in_files[pid] = os.path.abspath(infile) # Make workspace dir if not existed UtilClass.mkdir(wp) # Check the log parameter log_file = None runtime_file = None if log_params is not None: if not isinstance(log_params, dict): TauDEM.error('The log parameter must be a dict!') if 'logfile' in log_params and log_params['logfile'] is not None: log_file = log_params['logfile'] # If log_file is just a file name, then save it in the default workspace. if os.sep not in log_file: log_file = wp + os.sep + log_file log_file = os.path.abspath(log_file) if 'runtimefile' in log_params and log_params[ 'runtimefile'] is not None: runtime_file = log_params['runtimefile'] # If log_file is just a file name, then save it in the default workspace. if os.sep not in runtime_file: runtime_file = wp + os.sep + runtime_file runtime_file = os.path.abspath(runtime_file) # remove out_files to avoid any file IO related error new_out_files = list() if out_files is not None: if not isinstance(out_files, dict): TauDEM.error('The output files parameter must be a dict!') for (pid, out_file) in list(out_files.items()): if out_file is None: continue if isinstance(out_file, list) or isinstance(out_file, tuple): for idx, outf in enumerate(out_file): if outf is None: continue outf = FileClass.get_file_fullpath(outf, wp) FileClass.remove_files(outf) out_files[pid][idx] = outf new_out_files.append(outf) else: out_file = FileClass.get_file_fullpath(out_file, wp) FileClass.remove_files(out_file) out_files[pid] = out_file new_out_files.append(out_file) # concatenate command line commands = list() # MPI header if mpi_params is not None: if not isinstance(mpi_params, dict): TauDEM.error('The MPI settings parameter must be a dict!') if 'mpipath' in mpi_params and mpi_params['mpipath'] is not None: commands.append(mpi_params['mpipath'] + os.sep + 'mpiexec') else: commands.append('mpiexec') if 'hostfile' in mpi_params and mpi_params['hostfile'] is not None \ and not StringClass.string_match(mpi_params['hostfile'], 'none') \ and os.path.isfile(mpi_params['hostfile']): commands.append('-f') commands.append(mpi_params['hostfile']) if 'n' in mpi_params and mpi_params['n'] > 1: commands.append('-n') commands.append(str(mpi_params['n'])) else: # If number of processor is less equal than 1, then do not call mpiexec. commands = [] # append TauDEM function name, which can be full path or just one name commands.append(function_name) # append input files for (pid, infile) in list(in_files.items()): if infile is None: continue if pid[0] != '-': pid = '-' + pid commands.append(pid) if isinstance(infile, list) or isinstance(infile, tuple): commands.append(' '.join(tmpf for tmpf in infile)) else: commands.append(infile) # append input parameters if in_params is not None: if not isinstance(in_params, dict): TauDEM.error('The input parameters must be a dict!') for (pid, v) in list(in_params.items()): if pid[0] != '-': pid = '-' + pid commands.append(pid) # allow for parameter which is an flag without value if v != '' and v is not None: if MathClass.isnumerical(v): commands.append(str(v)) else: commands.append(v) # append output parameters if out_files is not None: for (pid, outfile) in list(out_files.items()): if outfile is None: continue if pid[0] != '-': pid = '-' + pid commands.append(pid) if isinstance(outfile, list) or isinstance(outfile, tuple): commands.append(' '.join(tmpf for tmpf in outfile)) else: commands.append(outfile) # run command runmsg = UtilClass.run_command(commands) TauDEM.log(runmsg, log_file) TauDEM.output_runtime_to_log(function_name, runmsg, runtime_file) # Check out_files, raise RuntimeError if not exist. for of in new_out_files: if not os.path.exists(of): TauDEM.error('%s failed, and the %s was not generated!' % (function_name, of)) return False return True
def run(function_name, in_files, wp=None, in_params=None, out_files=None, mpi_params=None, log_params=None): """ Run TauDEM function. - 1. The command will not execute if any input file does not exist. - 2. An error will be detected after running the TauDEM command if any output file does not exist; Args: function_name (str): Full path of TauDEM function. in_files (dict, required): Dict of pairs of parameter id (string) and file path (string or list) for input files, e.g.:: {'-z': '/full/path/to/dem.tif'} wp (str, optional): Workspace for outputs. If not specified, the directory of the first input file in ``in_files`` will be used. in_params (dict, optional): Dict of pairs of parameter id (string) and value (or None for a flag parameter without a value) for input parameters, e.g.:: {'-nc': None} {'-thresh': threshold} {'-m': 'ave' 's', '-nc': None} out_files (dict, optional): Dict of pairs of parameter id (string) and file path (string or list) for output files, e.g.:: {'-fel': 'filleddem.tif'} {'-maxS': ['harden.tif', 'maxsimi.tif']} mpi_params (dict, optional): Dict of pairs of parameter id (string) and value or path for MPI setting, e.g.:: {'mpipath':'/soft/bin','hostfile':'/soft/bin/cluster.node','n':4} {'mpipath':'/soft/bin', 'n':4} {'n':4} log_params (dict, optional): Dict of pairs of parameter id (string) and value or path for runtime and log output parameters. e.g.:: {'logfile': '/home/user/log.txt', 'runtimefile': '/home/user/runtime.txt'} Returns: True if TauDEM run successfully, otherwise False. """ # Check input files if in_files is None: TauDEM.error('Input files parameter is required!') if not isinstance(in_files, dict): TauDEM.error('The input files parameter must be a dict!') for (pid, infile) in iteritems(in_files): if infile is None: continue if isinstance(infile, list) or isinstance(infile, tuple): for idx, inf in enumerate(infile): if inf is None: continue inf, wp = TauDEM.check_infile_and_wp(inf, wp) in_files[pid][idx] = inf continue if os.path.exists(infile): infile, wp = TauDEM.check_infile_and_wp(infile, wp) in_files[pid] = os.path.abspath(infile) else: # For more flexible input files extension. # e.g., -inputtags 1 <path/to/tag1.tif> 2 <path/to/tag2.tif> ... # in such unpredictable circumstance, we cannot check the existance of # input files, so the developer will check it in other place. if len(StringClass.split_string(infile, ' ')) > 1: continue else: # the infile still should be a existing file, so check in workspace if wp is None: TauDEM.error('Workspace should not be None!') infile = wp + os.sep + infile if not os.path.exists(infile): TauDEM.error('Input files parameter %s: %s is not existed!' % (pid, infile)) in_files[pid] = os.path.abspath(infile) # Make workspace dir if not existed UtilClass.mkdir(wp) # Check the log parameter log_file = None runtime_file = None if log_params is not None: if not isinstance(log_params, dict): TauDEM.error('The log parameter must be a dict!') if 'logfile' in log_params and log_params['logfile'] is not None: log_file = log_params['logfile'] # If log_file is just a file name, then save it in the default workspace. if os.sep not in log_file: log_file = wp + os.sep + log_file log_file = os.path.abspath(log_file) if 'runtimefile' in log_params and log_params['runtimefile'] is not None: runtime_file = log_params['runtimefile'] # If log_file is just a file name, then save it in the default workspace. if os.sep not in runtime_file: runtime_file = wp + os.sep + runtime_file runtime_file = os.path.abspath(runtime_file) # remove out_files to avoid any file IO related error new_out_files = list() if out_files is not None: if not isinstance(out_files, dict): TauDEM.error('The output files parameter must be a dict!') for (pid, out_file) in iteritems(out_files): if out_file is None: continue if isinstance(out_file, list) or isinstance(out_file, tuple): for idx, outf in enumerate(out_file): if outf is None: continue outf = FileClass.get_file_fullpath(outf, wp) FileClass.remove_files(outf) out_files[pid][idx] = outf new_out_files.append(outf) else: out_file = FileClass.get_file_fullpath(out_file, wp) FileClass.remove_files(out_file) out_files[pid] = out_file new_out_files.append(out_file) # concatenate command line commands = list() # MPI header if mpi_params is not None: if not isinstance(mpi_params, dict): TauDEM.error('The MPI settings parameter must be a dict!') if 'mpipath' in mpi_params and mpi_params['mpipath'] is not None: commands.append(mpi_params['mpipath'] + os.sep + 'mpiexec') else: commands.append('mpiexec') if 'hostfile' in mpi_params and mpi_params['hostfile'] is not None \ and not StringClass.string_match(mpi_params['hostfile'], 'none') \ and os.path.isfile(mpi_params['hostfile']): commands.append('-f') commands.append(mpi_params['hostfile']) if 'n' in mpi_params and mpi_params['n'] > 1: commands.append('-n') commands.append(str(mpi_params['n'])) else: # If number of processor is less equal than 1, then do not call mpiexec. commands = [] # append TauDEM function name, which can be full path or just one name commands.append(function_name) # append input files for (pid, infile) in iteritems(in_files): if infile is None: continue if pid[0] != '-': pid = '-' + pid commands.append(pid) if isinstance(infile, list) or isinstance(infile, tuple): commands.append(' '.join(tmpf for tmpf in infile)) else: commands.append(infile) # append input parameters if in_params is not None: if not isinstance(in_params, dict): TauDEM.error('The input parameters must be a dict!') for (pid, v) in iteritems(in_params): if pid[0] != '-': pid = '-' + pid commands.append(pid) # allow for parameter which is an flag without value if v != '' and v is not None: if MathClass.isnumerical(v): commands.append(str(v)) else: commands.append(v) # append output parameters if out_files is not None: for (pid, outfile) in iteritems(out_files): if outfile is None: continue if pid[0] != '-': pid = '-' + pid commands.append(pid) if isinstance(outfile, list) or isinstance(outfile, tuple): commands.append(' '.join(tmpf for tmpf in outfile)) else: commands.append(outfile) # run command runmsg = UtilClass.run_command(commands) TauDEM.log(runmsg, log_file) TauDEM.output_runtime_to_log(function_name, runmsg, runtime_file) # Check out_files, raise RuntimeError if not exist. for of in new_out_files: if not os.path.exists(of): TauDEM.error('%s failed, and the %s was not generated!' % (function_name, of)) return False return True