def set_ldpath(gisbase): """Add GRASS libraries to the dynamic library path And then re-execute the process to take the changes into account """ # choose right path for each platform if (sys.platform.startswith('linux') or 'bsd' in sys.platform or 'solaris' in sys.platform): ldvar = 'LD_LIBRARY_PATH' elif sys.platform.startswith('win'): ldvar = 'PATH' elif sys.platform == 'darwin': ldvar = 'DYLD_LIBRARY_PATH' else: msgr.fatal("Platform not configured: {}".format(sys.platform)) ld_base = os.path.join(gisbase, u"lib") if not os.environ.get(ldvar): # if the path variable is not set msgr.debug("{} not set. Setting and restart".format(ldvar)) os.environ[ldvar] = ld_base reexec() elif ld_base not in os.environ[ldvar]: msgr.debug("{} not in {}. Setting and restart".format(ld_base, ldvar)) # if the variable exists but does not have the path os.environ[ldvar] += os.pathsep + ld_base reexec()
def check_output_files(file_list): """Check if the output files exist """ for map_name in file_list: if file_exists(map_name) and not gscript.overwrite(): msgr.fatal( u"File {} exists and will not be overwritten".format(map_name))
def check_combination(self): """Verifies if the given input times combination is valid. Sets temporal type. """ comb_err_msg = (u"accepted combinations: " u"{d} alone, {s} and {d}, " u"{s} and {e}").format(d='duration', s='start_time', e='end_time') b_dur = (self.raw_duration and not self.raw_start and not self.raw_end) b_start_dur = (self.raw_start and self.raw_duration and not self.raw_end) b_start_end = (self.raw_start and self.raw_end and not self.raw_duration) if not (b_dur or b_start_dur or b_start_end): msgr.fatal(comb_err_msg) # if only duration is given, temporal type is relative if b_dur: self.temporal_type = 'relative' else: self.temporal_type = 'absolute' return self
def raster_list_from_strds(self, strds_name): """Return a list of maps from a given strds for all the simulation duration Each map data is stored as a MapData namedtuple """ assert isinstance(strds_name, str), u"expect a string" # transform simulation start and end time in strds unit strds = tgis.open_stds.open_old_stds(strds_name, 'strds') sim_start, sim_end = self.get_sim_extend_in_stds_unit(strds) # retrieve data from DB where = "start_time <= '{e}' AND end_time >= '{s}'".format( e=str(sim_end), s=str(sim_start)) maplist = strds.get_registered_maps(columns=','.join(self.strds_cols), where=where, order='start_time') # check if every map exist maps_not_found = [m[0] for m in maplist if not self.name_is_map(m[0])] if any(maps_not_found): err_msg = u"STRDS <{}>: Can't find following maps: {}" str_lst = ','.join(maps_not_found) msgr.fatal(err_msg.format(strds_name, str_lst)) # change time data to datetime format if strds.get_temporal_type() == 'relative': rel_unit = strds.get_relative_time_unit().encode('ascii', 'ignore') maplist = [(i[0], self.to_datetime(rel_unit, i[1]), self.to_datetime(rel_unit, i[2])) for i in maplist] return [self.MapData(*i) for i in maplist]
def read(self, map_names): """Read maps names from GIS take as input map_names, a dictionary of maps/STDS names for each entry in map_names: if the name is empty or None, store None if a strds, load all maps in the instance's time extend, store them as a list if a single map, set the start and end time to fit simulation. store it in a list for consistency each map is stored as a MapData namedtuple store result in instance's dictionary """ for k, map_name in map_names.iteritems(): if not map_name: map_list = None continue elif self.name_is_stds(self.format_id(map_name)): strds_id = self.format_id(map_name) if not self.stds_temporal_sanity(strds_id): msgr.fatal(u"{}: inadequate temporal format" u"".format(map_name)) map_list = self.raster_list_from_strds(strds_id) elif self.name_is_map(self.format_id(map_name)): map_list = [ self.MapData(id=self.format_id(map_name), start_time=self.start_time, end_time=self.end_time) ] else: msgr.fatal(u"{} not found!".format(map_name)) self.maps[k] = map_list return self
def __init__(self, start_time, end_time, dtype, mkeys): assert isinstance(start_time, datetime), \ "start_time not a datetime object!" assert isinstance(end_time, datetime), \ "end_time not a datetime object!" assert start_time <= end_time, "start_time > end_time!" self.start_time = start_time self.end_time = end_time self.dtype = dtype self.region = Region() self.xr = self.region.cols self.yr = self.region.rows # Check if region is at least 3x3 if self.xr < 3 or self.yr < 3: msgr.fatal(u"GRASS Region should be at least 3 cells by 3 cells") self.dx = self.region.ewres self.dy = self.region.nsres self.reg_bbox = { 'e': self.region.east, 'w': self.region.west, 'n': self.region.north, 's': self.region.south } self.overwrite = gscript.overwrite() self.mapset = gutils.getenv('MAPSET') self.maps = dict.fromkeys(mkeys) # init temporal module tgis.init() assert os.path.isfile(self.rules_h) assert os.path.isfile(self.rules_v) assert os.path.isfile(self.rules_def)
def check_mandatory(self): """check if mandatory parameters are present """ if not all([self.input_map_names['dem'], self.input_map_names['friction'], self.sim_times.record_step]): msgr.fatal(u"inputs <dem>, <friction> and " u"<record_step> are mandatory")
def reexec(): """Re-execute the software with the same arguments """ args = [sys.executable] + sys.argv try: os.execv(sys.executable, args) except Exception as exc: msgr.fatal(u"Failed to re-execute: {}".format(exc))
def check_grass_params(self): """Check if all grass params are presents if one is given """ grass_any = any(self.grass_params[i] for i in self.grass_mandatory) grass_all = all(self.grass_params[i] for i in self.grass_mandatory) if grass_any and not grass_all: msgr.fatal(u"{} are mutualy inclusive".format(self.grass_mandatory)) return self
def read_param_file(self): """Read the parameter file and populate the relevant dictionaries """ self.out_values = [] # read the file params = ConfigParser(allow_no_value=True) f = params.read(self.config_file) if not f: msgr.fatal(u"File <{}> not found".format(self.config_file)) # populate dictionaries using loops instead of using update() method # in order to not add invalid key for k in self.raw_input_times: if params.has_option('time', k): self.raw_input_times[k] = params.get('time', k) for k in self.sim_param: if params.has_option('options', k): self.sim_param[k] = params.getfloat('options', k) for k in self.grass_params: if params.has_option('grass', k): self.grass_params[k] = params.get('grass', k) # check for deprecated input names if params.has_option('input', "drainage_capacity"): msgr.warning(u"'drainage_capacity' is deprecated. " u"Use 'losses' instead.") self.input_map_names['losses'] = params.get( 'input', "drainage_capacity") # search for valid inputs for k in self.input_map_names: if params.has_option('input', k): self.input_map_names[k] = params.get('input', k) # drainage parameters for k in self.drainage_params: if params.has_option('drainage', k): if k in ['swmm_inp', 'output', 'swmm_gage']: self.drainage_params[k] = params.get('drainage', k) else: self.drainage_params[k] = params.getfloat('drainage', k) # statistic file if params.has_option('statistics', 'stats_file'): self.stats_file = params.get('statistics', 'stats_file') else: self.stats_file = None # output maps if params.has_option('output', 'prefix'): self.out_prefix = params.get('output', 'prefix') if params.has_option('output', 'values'): out_values = params.get('output', 'values').split(',') self.out_values = [e.strip() for e in out_values] # check for deprecated values if 'drainage_cap' in self.out_values and 'losses' not in self.out_values: msgr.warning(u"'drainage_cap' is deprecated. " u"Use 'losses' instead.") self.out_values.append('losses') self.generate_output_name() return self
def __set_models(self): """Instantiate models objects """ # RasterDomain msgr.debug(u"Setting up raster domain...") try: self.rast_domain = RasterDomain(self.dtype, self.gis, self.in_map_names, self.out_map_names) except MemoryError: msgr.fatal(u"Out of memory.") # Infiltration msgr.debug(u"Setting up raster infiltration...") inf_class = { 'constant': infiltration.InfConstantRate, 'green-ampt': infiltration.InfGreenAmpt, 'null': infiltration.InfNull } try: self.infiltration = inf_class[self.inf_model](self.rast_domain, self.dtinf) except KeyError: assert False, u"Unknow infiltration model: {}".format( self.inf_model) # Hydrology msgr.debug(u"Setting up hydrologic model...") self.hydrology = hydrology.Hydrology(self.rast_domain, self.dtinf, self.infiltration) # Surface flows simulation msgr.debug(u"Setting up surface model...") self.surf_sim = SurfaceFlowSimulation(self.rast_domain, self.sim_param) # Instantiate Massbal object if self.stats_file: msgr.debug(u"Setting up mass balance object...") self.massbal = MassBal(self.stats_file, self.rast_domain, self.start_time, self.temporal_type) else: self.massbal = None # Drainage if self.drainage_params['swmm_inp']: msgr.debug(u"Setting up drainage model...") self.drainage = DrainageSimulation(self.rast_domain, self.drainage_params, self.gis, self.sim_param['g']) else: self.drainage = None # reporting object msgr.debug(u"Setting up reporting object...") self.report = Report(self.gis, self.temporal_type, self.sim_param['hmin'], self.massbal, self.rast_domain, self.drainage, self.drainage_params['output']) return self
def __init__(self, start_time, end_time, dtype, mkeys, region_id, raster_mask_id): assert isinstance(start_time, datetime), \ "start_time not a datetime object!" assert isinstance(end_time, datetime), \ "end_time not a datetime object!" assert start_time <= end_time, "start_time > end_time!" self.region_id = region_id self.raster_mask_id = raster_mask_id self.start_time = start_time self.end_time = end_time self.dtype = dtype self.old_mask_name = None # LatLon is not supported if gscript.locn_is_latlong(): msgr.fatal(u"latlong location is not supported. " u"Please use a projected location") # Set region if self.region_id: gscript.use_temp_region() gscript.run_command("g.region", region=region_id) self.region = Region() self.xr = self.region.cols self.yr = self.region.rows # Check if region is at least 3x3 if self.xr < 3 or self.yr < 3: msgr.fatal(u"GRASS Region should be at least 3 cells by 3 cells") self.dx = self.region.ewres self.dy = self.region.nsres self.reg_bbox = { 'e': self.region.east, 'w': self.region.west, 'n': self.region.north, 's': self.region.south } # Set temporary mask if self.raster_mask_id: self.set_temp_mask() self.overwrite = gscript.overwrite() self.mapset = gutils.getenv('MAPSET') self.maps = dict.fromkeys(mkeys) # init temporal module tgis.init() # Create thread and queue for writing raster maps self.raster_lock = Lock() self.raster_writer_queue = Queue(maxsize=15) worker_args = (self.raster_writer_queue, self.raster_lock) self.raster_writer_thread = Thread(name="RasterWriter", target=raster_writer, args=worker_args) self.raster_writer_thread.start()
def check_coherence(self): """Sets end or duration if not given Verifies if end is superior to starts """ if self.start is None: self.start = datetime.min if self.end is None: self.end = self.start + self.duration if self.start >= self.end: msgr.fatal("Simulation duration must be positive") if self.duration is None: self.duration = self.end - self.start
def check_sim_params(self): """Check if the simulations parameters are positives and valid """ for k, v in self.sim_param.items(): if k == 'theta': if not 0 <= v <= 1: msgr.fatal(u"{} value must be between 0 and 1".format(k)) elif k == 'inf_model': continue else: if not v > 0: msgr.fatal(u"{} value must be positive".format(k))
def read_timedelta(self, string): """Try to transform string in timedelta object. If it fail, return an error message If string is None, return None """ if string: try: return self.str_to_timedelta(string) except ValueError: msgr.fatal(self.rel_err_msg.format(string)) else: return None
def read_datetime(self, string): """Try to transform string in datetime object. If it fail, return an error message If string is None, return None """ if string: try: return datetime.strptime(string, self.date_format) except ValueError: msgr.fatal(self.abs_err_msg.format(string)) else: return None
def set_grass_session(self): """Set the GRASS session. """ gisdb = self.conf.grass_params['grassdata'] location = self.conf.grass_params['location'] mapset = self.conf.grass_params['mapset'] if location is None: msgr.fatal(("[grass] section is missing.")) # Check if the given parameters exist and can be accessed error_msg = u"'{}' does not exist or does not have adequate permissions" if not os.access(gisdb, os.R_OK): msgr.fatal(error_msg.format(gisdb)) elif not os.access(os.path.join(gisdb, location), os.R_OK): msgr.fatal(error_msg.format(location)) elif not os.access(os.path.join(gisdb, location, mapset), os.W_OK): msgr.fatal(error_msg.format(mapset)) # Start Session if self.conf.grass_params['grass_bin']: grassbin = self.conf.grass_params['grass_bin'] else: grassbin = None self.grass_session = GrassSession(grassbin=grassbin) self.grass_session.open(gisdb=gisdb, location=location, mapset=mapset, loadlibs=True) return self
def run(self): """prepare the simulation, run it and clean up """ if self.prof: self.prof.start() # If run outside of grass, set it if self.grass_use_file: self.set_grass_session() import itzi.gis as gis msgr.debug('GRASS session set') # return error if output files exist # (should be done once GRASS set up) gis.check_output_files(self.conf.output_map_names.itervalues()) msgr.debug('Output files OK') # stop program if location is latlong if gis.is_latlon(): msgr.fatal(u"latlong location is not supported. " u"Please use a projected location") # set region if self.conf.grass_params['region']: gis.set_temp_region(self.conf.grass_params['region']) # set mask if self.conf.grass_params['mask']: gis.set_temp_mask(self.conf.grass_params['mask']) # Run simulation (SimulationManager needs GRASS, so imported now) from itzi.simulation import SimulationManager sim = SimulationManager(sim_times=self.conf.sim_times, stats_file=self.conf.stats_file, dtype=np.float32, input_maps=self.conf.input_map_names, output_maps=self.conf.output_map_names, sim_param=self.conf.sim_param, drainage_params=self.conf.drainage_params) sim.run() # return to previous region and mask if self.conf.grass_params['region']: gis.del_temp_region() if self.conf.grass_params['mask']: gis.del_temp_mask() # Delete the rcfile if self.grass_use_file: os.remove(self.rcfile) # end profiling and print results if self.prof: self.prof.stop() print(self.prof.output_text(unicode=True, color=True)) return self
def get_gisbase(grassbin): """query GRASS 7 itself for its GISBASE """ startcmd = [grassbin, '--config', 'path'] try: p = subprocess.Popen(startcmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() except OSError as error: msgr.fatal("Cannot find GRASS GIS binary" " '{cmd}' {error}".format(cmd=startcmd[0], error=error)) if p.returncode != 0: msgr.fatal("Error while running GRASS GIS start-up script" " '{cmd}': {error}".format(cmd=' '.join(startcmd), error=stderr)) return stdout.strip().decode(encoding='UTF-8')
def set_grass_session(self): """Inspired by example on GRASS wiki """ grassbin = self.conf.grass_params['grass_bin'] gisdb = self.conf.grass_params['grassdata'] location = self.conf.grass_params['location'] mapset = self.conf.grass_params['mapset'] # check if the given parameters exist and can be accessed error_msg = u"'{}' does not exist or does not have adequate permissions" if not os.access(gisdb, os.R_OK): msgr.fatal(error_msg.format(gisdb)) elif not os.access(os.path.join(gisdb, location), os.R_OK): msgr.fatal(error_msg.format(location)) elif not os.access(os.path.join(gisdb, location, mapset), os.W_OK): msgr.fatal(error_msg.format(mapset)) # query GRASS 7 itself for its GISBASE gisbase = get_gisbase(grassbin) # Set GISBASE environment variable os.environ['GISBASE'] = gisbase # define GRASS Python environment grass_python = os.path.join(gisbase, u"etc", u"python") sys.path.append(grass_python) # launch session import grass.script.setup as gsetup self.rcfile = gsetup.init(gisbase, gisdb, location, mapset) return self
def check_inf_maps(self): """check coherence of input infiltration maps set infiltration model type """ inf_k = 'infiltration' # if at least one Green-Ampt parameters is present ga_any = any(self.input_map_names[i] for i in self.ga_list) # if all Green-Ampt parameters are present ga_all = all(self.input_map_names[i] for i in self.ga_list) # verify parameters if not self.input_map_names[inf_k] and not ga_any: self.sim_param['inf_model'] = 'null' elif self.input_map_names[inf_k] and not ga_any: self.sim_param['inf_model'] = 'constant' elif self.input_map_names[inf_k] and ga_any: msgr.fatal(u"Infiltration model incompatible with user-defined rate") # check if all maps for Green-Ampt are presents elif ga_any and not ga_all: msgr.fatal(u"{} are mutualy inclusive".format(self.ga_list)) elif ga_all and not self.input_map_names[inf_k]: self.sim_param['inf_model'] = 'green-ampt' return self
def step(self): """Step each of the models if needed """ # HYDROLOGY MODEL - compute rainfall rates, losses rates, infiltration rates# #------------------------------------------------------------------------------ if self.sim_time == self.next_ts['hyd']: self.hydrology.solve_dt() # calculate when will happen the next time-step self.next_ts['hyd'] += self.hydrology.dt self.hydrology.step() # update stat array self.rast_domain.populate_stat_array('inf', self.sim_time) self.rast_domain.populate_stat_array('capped_losses', self.sim_time) #SWMM MODEL - Apply linkage flow to SWMM Model and STEP SWMM Model #------------------------------------------------------------------------------ if self.sim_time == self.next_ts['drain'] and self.drainage: self.drainage.apply_drainage_flow() self.drainage.step() self.drainage.solve_dt() self.next_ts['drain'] += self.drainage.dt #compute next 1D time # SURFACE FLOW SIM #------------------------------------------------------------------------------ #calculate fluxes at cell interfaces self.surf_sim.solve_q() # calculate drainage flows and apply it to 2D Model at each 2D timestep if self.drainage: self.drainage.drainage_flow( self.dt.total_seconds()) #compute drainage flow self.rast_domain.update_ext_array() self.rast_domain.populate_stat_array('n_drain', self.sim_time) else: self.rast_domain.update_ext_array() self.surf_sim.apply_boundary_conditions() try: self.surf_sim.update_h() # in case of NaN/NULL cells, raisea NullError self.surf_sim.arr_err = np.isnan(self.rast_domain.get('h')) if np.any(self.surf_sim.arr_err): raise NullError except NullError: self.report.write_error_to_gis(self.surf_sim.arr_err) msgr.fatal(u"{}: Null value detected in simulation, " u"terminating".format(self.sim_time)) self.surf_sim.swap_flow_arrays() # calculate when should happen the next surface time-step self.surf_sim.solve_dt() # increment time when should happen next timestep if self.drainage: self.next_ts['surf'] += min( self.surf_sim.dt, self.drainage.dt2d ) #self.drainage.dt2d is the courant condition based on maximum node head. else: self.next_ts['surf'] += self.surf_sim.dt #REPORTING #------------------------------------------------------------------------------ # send current time-step duration to mass balance object if self.massbal: self.massbal.add_value('tstep', self.dt.total_seconds()) # Reporting # if self.sim_time == self.next_ts['rec']: msgr.verbose(u"{}: Writing output maps...".format(self.sim_time)) self.report.step(self.sim_time) self.next_ts['rec'] += self.record_step # reset statistic maps self.rast_domain.reset_stats(self.sim_time) #GLOBAL STEP CALCULATION #------------------------------------------------------------------------------ # find next step self.nextstep = min(self.next_ts.values()) # force the surface time-step to the lowest time-step self.next_ts['surf'] = self.nextstep self.dt = self.nextstep - self.sim_time # force time-step to be the general time-step self.surf_sim.dt = self.dt return self
def step(self): """Step each of the models if needed """ # hydrology # if self.sim_time == self.next_ts['hyd']: self.hydrology.solve_dt() # calculate when will happen the next time-step self.next_ts['hyd'] += self.hydrology.dt self.hydrology.step() # update stat array self.rast_domain.populate_stat_array('inf', self.sim_time) self.rast_domain.populate_stat_array('capped_losses', self.sim_time) # drainage # if self.sim_time == self.next_ts['drain'] and self.drainage: self.drainage.solve_dt() # calculate when will happen the next time-step self.next_ts['drain'] += self.drainage.dt self.drainage.step() self.drainage.apply_linkage(self.dt.total_seconds()) self.rast_domain.isnew['n_drain'] = True # update stat array self.rast_domain.populate_stat_array('n_drain', self.sim_time) else: self.rast_domain.isnew['n_drain'] = False # surface flow # # update arrays of infiltration, rainfall etc. self.rast_domain.update_ext_array() # force time-step to be the general time-step self.surf_sim.dt = self.dt # surf_sim.step() raise NullError in case of NaN/NULL cell # if this happen, stop simulation and # output a map showing the errors try: self.surf_sim.step() except NullError: self.report.write_error_to_gis(self.surf_sim.arr_err) msgr.fatal(u"{}: Null value detected in simulation, " u"terminating".format(self.sim_time)) # calculate when should happen the next surface time-step self.surf_sim.solve_dt() self.next_ts['surf'] += self.surf_sim.dt # send current time-step duration to mass balance object if self.massbal: self.massbal.add_value('tstep', self.dt.total_seconds()) # Reporting # if self.sim_time >= self.next_ts['rec']: msgr.verbose(u"{}: Writing output maps...".format(self.sim_time)) self.report.step(self.sim_time) self.next_ts['rec'] += self.record_step # reset statistic maps self.rast_domain.reset_stats(self.sim_time) # find next step self.nextstep = min(self.next_ts.values()) # force the surface time-step to the lowest time-step self.next_ts['surf'] = self.nextstep self.dt = self.nextstep - self.sim_time return self
def itzi_run(args): # Check if being run within GRASS session try: import grass.script except ImportError: grass_use_file = True else: grass_use_file = False # set environment variables if args.o: os.environ['GRASS_OVERWRITE'] = '1' else: os.environ['GRASS_OVERWRITE'] = '0' # verbosity if args.q and args.q == 2: os.environ['ITZI_VERBOSE'] = str(VerbosityLevel.SUPER_QUIET) elif args.q == 1: os.environ['ITZI_VERBOSE'] = str(VerbosityLevel.QUIET) elif args.v == 1: os.environ['ITZI_VERBOSE'] = str(VerbosityLevel.VERBOSE) elif args.v and args.v >= 2: os.environ['ITZI_VERBOSE'] = str(VerbosityLevel.DEBUG) else: os.environ['ITZI_VERBOSE'] = str(VerbosityLevel.MESSAGE) # setting GRASS verbosity (especially for maps registration) if args.q and args.q >= 1: # no warnings os.environ['GRASS_VERBOSE'] = '-1' elif args.v and args.v >= 1: # normal os.environ['GRASS_VERBOSE'] = '2' else: # only warnings os.environ['GRASS_VERBOSE'] = '0' # start total time counter total_sim_start = time.time() # dictionary to store computation times times_dict = {} for conf_file in args.config_file: # parsing configuration file conf = ConfigReader(conf_file) grassbin = conf.grass_params['grass_bin'] # if outside from GRASS, set path to shared libraries and restart if grass_use_file and not grassbin: msgr.fatal(u"Please define [grass] section in parameter file") elif grass_use_file: set_ldpath(get_gisbase(grassbin)) file_name = os.path.basename(conf_file) msgr.message(u"Starting simulation of {}...".format(file_name)) # display parameters (if verbose) conf.display_sim_param() # run in a subprocess sim_start = time.time() worker_args = (conf, grass_use_file, args) p = Process(target=sim_runner_worker, args=worker_args) p.start() p.join() if p.exitcode != 0: msgr.warning((u"Execution of {} " u"ended with an error").format(file_name)) # store computational time comp_time = timedelta(seconds=int(time.time() - sim_start)) times_dict[file_name] = comp_time # stop total time counter total_elapsed_time = timedelta(seconds=int(time.time() - total_sim_start)) # display total computation duration msgr.message(u"Simulations complete. Elapsed times:") for f, t in times_dict.items(): msgr.message(u"{}: {}".format(f, t)) msgr.message(u"Total: {}".format(total_elapsed_time)) avg_time_s = int(total_elapsed_time.total_seconds() / len(times_dict)) msgr.message(u"Average: {}".format(timedelta(seconds=avg_time_s)))