def apply_transition(self, ls_ids, current_pop_maps, destination_maps): # Timing of process start_time = time.time() ## Import Rasters # check Index raster name, check it exists index_raster = grass.get_g().get_index_raster(self.index_source) # check list of rasters that comprise stages (in order) pop_raster_list = current_pop_maps self.log.debug("Converting stage rasters to ASCII...") ascii_pop_rasters = [] ascii_out_rasters = [] for i in pop_raster_list: # output to ascii, and also replace null values with 0 data_fn, data_out_fn = grass.get_g().raster_to_ascii(i, null_as_zero=True) ascii_pop_rasters.append(data_fn) ascii_out_rasters.append(data_out_fn) self.log.debug("Converting index file...") ascii_indexes = grass.get_g().index_to_ascii(index_raster) # TODO temp ascii files are currently cleaned up by process rows, # but should probably be removed outside of the function as they # generated above. # apply matrix multiplication self.process_rows(ls_ids, ascii_indexes, ascii_pop_rasters, ascii_out_rasters, destination_maps) # remove temporary ascii files for i in ascii_indexes: os.remove(i) processingTime = time.time() - start_time self.log.debug("Transition matrix application completed. " + "Processing time %f seconds" % processingTime)
def add_completed_raster_map(self,t,ls,file_name,interval=1): mdig_config = config.get_config() # If the command line has specified that the null bitmask # of completed raster maps should be removed: if mdig_config.remove_null: grass.get_g().null_bitmask(file_name,generate="False") # TODO: Check if the filename has already been associated with an analysis ls_node = self.node.xpath('lifestage[@id="%s"]' % ls) if len(ls_node) == 0: ls_node.append(lxml.etree.SubElement(self.node,'lifestage')) ls_node[0].attrib["id"] = ls ls_node[0].text = '\n' maps_node = ls_node[0].xpath('maps') if len(maps_node) == 0: new_maps_node = lxml.etree.SubElement(ls_node[0],'maps') if interval != 1: new_maps_node.attrib['interval']=interval maps_node.append(new_maps_node) correct_m = None for m in maps_node[0]: if string.atoi(m.attrib["time"]) == t: correct_m = m if correct_m is None: # Not found, so add: correct_m=lxml.etree.SubElement(maps_node[0],'map') correct_m.attrib["time"]="%d" % t correct_m.text = file_name
def null_bitmask(self, generate_null=True): """ Create null bitmasks for raster maps""" ls_keys = self.instance.experiment.get_lifestage_ids() for ls_key in ls_keys: maps = self.get_saved_maps(ls_key) for m in maps.values(): grass.get_g().null_bitmask(m, generate=generate_null)
def add_completed_raster_map(self, t, ls, file_name, interval=1): mdig_config = config.get_config() # If the command line has specified that the null bitmask # of completed raster maps should be removed: if mdig_config.remove_null: grass.get_g().null_bitmask(file_name, generate="False") # TODO: Check if the filename has already been associated with an analysis ls_node = self.node.xpath('lifestage[@id="%s"]' % ls) if len(ls_node) == 0: ls_node.append(lxml.etree.SubElement(self.node, 'lifestage')) ls_node[0].attrib["id"] = ls ls_node[0].text = '\n' maps_node = ls_node[0].xpath('maps') if len(maps_node) == 0: new_maps_node = lxml.etree.SubElement(ls_node[0], 'maps') if interval != 1: new_maps_node.attrib['interval'] = interval maps_node.append(new_maps_node) correct_m = None for m in maps_node[0]: if string.atoi(m.attrib["time"]) == t: correct_m = m if correct_m is None: # Not found, so add: correct_m = lxml.etree.SubElement(maps_node[0], 'map') correct_m.attrib["time"] = "%d" % t correct_m.text = file_name
def get_treatment_area(self, replicate, last_area=None): """ Get the map name representing the treatment area, generating it dynamically if necessary. """ if isinstance(self.area, Event): if self.area_filter_output is not None: grass.get_g().remove_map(self.area_filter_output) if last_area: dist_map = last_area else: dist_map = replicate.temp_map_names[self.treatment.area_ls][0] self.area.run(dist_map, self.area_temp, replicate, False) self.area_filter_output = self.area_temp return self.area_filter_output elif isinstance(self.area, GrassMap): replacements = { "POP_MAP": replicate.temp_map_names[self.treatment.area_ls][0], "START_MAP": replicate.initial_maps[ self.treatment.area_ls].get_map_filename() } return self.area.get_map_filename(replacements)
def null_bitmask(self, generate_null=True): """ Create null bitmasks for raster maps""" ls_keys = self.instance.experiment.get_lifestage_ids() for ls_key in ls_keys: maps=self.get_saved_maps(ls_key) for m in maps.values(): grass.get_g().null_bitmask(m,generate=generate_null)
def run(self, in_name, out_name, rep, is_pop): stats_before = grass.get_g().get_univariate_stats({0: in_name}) metrics = super(TreatmentEvent, self).run(in_name, out_name, rep, is_pop) stats_after = grass.get_g().get_univariate_stats({0: out_name}) metrics['AREA_REMOVED'] = stats_before[0].get( 'n', 0) - stats_after[0].get('n', 0) return metrics
def __init__(self, xml_file, model): # Init timing of load process start_time = time.time() # self.m_instance = model_instance self.model = model self.log = logging.getLogger("mdig.popmod") # Do lifestage transitions by individual rather than using # matrix multiplication. (SLOW!) self.by_individual = False # XML parsing self.xml_file = xml_file self.xml_dom = xml.dom.minidom.parse(xml_file) self.index_source = self.xml_to_index() # tm_size, used in defining/parsing 'expressions' list # same as number of lifestages self.tm_size = len(model.get_lifestage_ids()) if model.base_dir: self.parameters = self.xml_to_param(model.base_dir) else: # This should only occur when loading files during model addition # to repository self.parameters = self.xml_to_param(os.path.dirname(self.xml_file)) self.expressions = self.xml_to_expression_list() # determine size of rasters self.range_data = grass.get_g().get_range() # process range_data self.n_rows = int(self.range_data[8][6:]) self.n_cols = int(self.range_data[9][6:]) self.header = str( self.range_data[2] + self.range_data[3] + self.range_data[4] + self.range_data[5] + self.range_data[8] + self.range_data[9] ) # End timing of load process load_time = time.time() - start_time self.log.debug("Transition sources loaded. Load time %f seconds" % load_time) self.log.debug("Transition matrix size set to %i x %i" % (self.tm_size, self.tm_size)) # Get the different indexes available index_values = grass.get_g().raster_value_freq(self.index_source) index_values = [int(x[0]) for x in index_values] self.log.debug("Index values found were: " + str(index_values)) # Create matrix instance self.t_matrix = TVGenerator(self.parameters, self.expressions, self.tm_size, index_values)
def delete_maps(self): """ Deletes all maps created by replicate, this currently DOES NOT update the xml, as it's only used by the instance.remove_rep method which removes the entire replicate xml node. TODO: update xml """ g = grass.get_g() for ls_id in self.instance.experiment.get_lifestage_ids(): ls_saved_maps = [] try: ls_saved_maps = self.get_saved_maps(ls_id) except grass.MapNotFoundException: # If maps are missing, there still might be some found, even # though it's unlikely ls_saved_maps = self.get_saved_maps(ls_id) except grass.SetMapsetException: self.log.warning("Couldn't find mapset '" + \ self.instance.get_mapset() + \ "', so forgetting about it.") finally: for m in ls_saved_maps: # remove map g.remove_map(m, self.instance.get_mapset()) ls_node = self.node.xpath('lifestage[@id="%s"]' % ls_id) if len(ls_node) == 0: continue maps_node = ls_node[0].xpath('maps') if len(maps_node) != 0: ls_node[0].remove(maps_node[0])
def _merge_areas(self, replicate): """ Merge all the TreatmentArea maps based on the combine attribute ("and" or "or" them) """ # Check whether the component Areas change between calls if self.area_temp is not None: # Whether we need to regenerate the merged area map generate = False for a in self.areas: if a.is_dynamic(): # if just one component area is dynamic, we have to regenerate # the merged treatment area. generate = True break if not generate: # We can just return the last area map we generated if it's not dynamic return self.area_temp else: self.area_temp = "x_t___strategy_" + self.strategy.get_name() + \ "_area_t_" + str(self.index) g = grass.get_g() # remove previous map g.remove_map(self.area_temp) component_areas = self._get_component_area_maps(replicate) if len(component_areas) > 1: merge_str = self._get_area_merge_mapcalc_expression( component_areas) g.mapcalc(self.area_temp, merge_str) else: g.copy_map(component_areas[0], self.area_temp) return self.area_temp
def _generate_mask(self, interval, r_id): """ Actually generates a mask for a given interval and region (r_id) """ # Get GRASS interface instance g = grass.get_g() # Generate a random map name mapname = g.generate_map_name("mask") if r_id in self.bins.keys(): bins = self.bins[r_id] elif "__default" in self.bin.keys(): bins = self.bins["__default"] else: self.log.error( "Could not find any phenology bins for generating mask") # Find what bin the interval lies within for d_range, mean in bins.items(): if interval >= d_range[0] and interval <= d_range[1]: g.mapcalc(mapname, "if(%s>=%f,if(%s<=%f,1,0),0)" % (self.p_map_names[ r_id], d_range[0], self.p_map_names[r_id], d_range[1])) grassmap_mask = GrassMap(mapname) grassmap_mask.temporary = True return grassmap_mask self.log.debug( "No appropriate interval range found for interval %d" % interval)
def _generate_mask(self, interval, r_id): """ Actually generates a mask for a given interval and region (r_id) """ # Get GRASS interface instance g = grass.get_g() # Generate a random map name mapname = g.generate_map_name("mask") if r_id in self.bins.keys(): bins = self.bins[r_id] elif "__default" in self.bin.keys(): bins = self.bins["__default"] else: self.log.error("Could not find any phenology bins for generating mask") # Find what bin the interval lies within for d_range, mean in bins.items(): if interval >= d_range[0] and interval <= d_range[1]: g.mapcalc( mapname, "if(%s>=%f,if(%s<=%f,1,0),0)" % (self.p_map_names[r_id], d_range[0], self.p_map_names[r_id], d_range[1]), ) grassmap_mask = GrassMap(mapname) grassmap_mask.temporary = True return grassmap_mask self.log.debug("No appropriate interval range found for interval %d" % interval)
def update_occupancy_envelope(self, ls_list=None, start=None, end=None, force=False): """ Go through and update the occupancy envelopes if necessary Note: ls_list has to be a list or None """ # Set the region in case it hasn't been yet self.set_region() if ls_list == None: ls_list = self.experiment.get_lifestage_ids() if start == None: start = self.experiment.get_period()[0] if end == None: start = self.experiment.get_period()[1] ls = ls_list self.log.debug("Checking whether envelopes are fresh...") missing_envelopes = self.are_envelopes_fresh(ls, start, end, force=force) if not missing_envelopes: return if not self.is_complete(): raise InstanceIncompleteException( "Instance isn't complete, can't create occupancy envelope") for l in ls: maps = [] for r_idx in range(0, len(self.replicates)): r = self.replicates[r_idx] self.log.debug("Getting saved maps for replicate %d", r_idx) saved_maps = r.get_saved_maps(l) if saved_maps: maps.append(saved_maps) else: raise DispersalInstanceException( "Missing maps for replicate %d" % r_idx) for t in missing_envelopes[l]: maps_to_combine = [] for r in maps: if str(t) in r: maps_to_combine.append(r[str(t)]) else: self.log.warning("Missing map for time=" + str(t)) raise DispersalInstanceException( "Missing map for time %s" % str(t)) filename = self.get_map_name_base() filename += "_ls_" + l + "_t_" + repr(t) + "_prob" prob_env = grass.get_g().occupancy_envelope( maps_to_combine, filename) if prob_env is not None: self._add_envelope(prob_env, l, t) for li in self.listeners: if "occupancy_envelope_complete" in dir(li): li.occupancy_envelope_complete(self, l, t)
def _merge_areas(self, replicate): """ Merge all the TreatmentArea maps based on the combine attribute ("and" or "or" them) """ # Check whether the component Areas change between calls if self.area_temp is not None: # Whether we need to regenerate the merged area map generate = False for a in self.areas: if a.is_dynamic(): # if just one component area is dynamic, we have to regenerate # the merged treatment area. generate = True break if not generate: # We can just return the last area map we generated if it's not dynamic return self.area_temp else: self.area_temp = "x_t___strategy_" + self.strategy.get_name() + \ "_area_t_" + str(self.index) g = grass.get_g() # remove previous map g.remove_map(self.area_temp) component_areas = self._get_component_area_maps(replicate) if len(component_areas) > 1: merge_str = self._get_area_merge_mapcalc_expression(component_areas) g.mapcalc(self.area_temp, merge_str) else: g.copy_map(component_areas[0], self.area_temp) return self.area_temp
def _load_saved_maps(self, skip_check=False): self.saved_maps = {} self.map_intervals = {} missing_maps = [] ls_keys = self.instance.experiment.get_lifestage_ids() for ls_key in ls_keys: self.map_intervals[ ls_key] = 0 # If no maps node exist then interval is 0 ls_maps_node = self.node.xpath('lifestage[@id="%s"]/maps' % ls_key) if len(ls_maps_node) == 1: self.saved_maps[ls_key] = {} if "interval" in ls_maps_node[0].attrib: self.map_intervals[ls_key] = int( ls_maps_node.attrib["interval"]) else: self.map_intervals[ls_key] = 1 for m in ls_maps_node[0]: if m.tag == "map": time_step = m.attrib["time"] if not skip_check and \ not grass.get_g().check_map(m.text,self.instance.get_mapset()): missing_maps.append(m.text) else: self.saved_maps[ls_key][time_step] = m.text elif len(ls_maps_node) > 1: raise model.InvalidXMLException, "More than one maps node" if missing_maps: raise MapNotFoundException(missing_maps)
def _load_saved_maps(self, skip_check = False): self.saved_maps = {} self.map_intervals = {} missing_maps=[] ls_keys = self.instance.experiment.get_lifestage_ids() for ls_key in ls_keys: self.map_intervals[ls_key] = 0 # If no maps node exist then interval is 0 ls_maps_node = self.node.xpath('lifestage[@id="%s"]/maps' % ls_key) if len(ls_maps_node) == 1: self.saved_maps[ls_key] = {} if "interval" in ls_maps_node[0].attrib: self.map_intervals[ls_key] = int(ls_maps_node.attrib["interval"]) else: self.map_intervals[ls_key] = 1 for m in ls_maps_node[0]: if m.tag == "map": time_step=m.attrib["time"] if not skip_check and \ not grass.get_g().check_map(m.text,self.instance.get_mapset()): missing_maps.append(m.text) else: self.saved_maps[ls_key][time_step] = m.text elif len(ls_maps_node) > 1: raise model.InvalidXMLException, "More than one maps node" if missing_maps: raise MapNotFoundException(missing_maps)
def delete_maps(self): """ Deletes all maps created by replicate, this currently DOES NOT update the xml, as it's only used by the instance.remove_rep method which removes the entire replicate xml node. TODO: update xml """ g = grass.get_g() for ls_id in self.instance.experiment.get_lifestage_ids(): ls_saved_maps=[] try: ls_saved_maps = self.get_saved_maps(ls_id) except grass.MapNotFoundException: # If maps are missing, there still might be some found, even # though it's unlikely ls_saved_maps = self.get_saved_maps(ls_id) except grass.SetMapsetException: self.log.warning("Couldn't find mapset '" + \ self.instance.get_mapset() + \ "', so forgetting about it.") finally: for m in ls_saved_maps: # remove map g.remove_map(m,self.instance.get_mapset()) ls_node = self.node.xpath('lifestage[@id="%s"]' % ls_id) if len(ls_node) == 0: continue maps_node = ls_node[0].xpath('maps') if len(maps_node) != 0: ls_node[0].remove(maps_node[0])
def __init__(self,xml_node=None,filename=None): # Get the logger object self.log = logging.getLogger("mdig.map") # Set the xml_node if one exists for this map self.xml_node = xml_node # Set the filename is one exists self.filename = filename self.mapset = None # Initialise values self.map_type=None # type of map: raster or vector self.xml_map_type=None # type of map as defined by xml self.value=None # For maps that are just a constant value or for a mapcalc expression self.ready=False # Is the map ready for use? self.refresh=False # Should the map be refreshed any time someone attempts to obtain the filename? self.temporary = True # Should this map be deleted on quit? if self.xml_node is not None: # Read xml settings if this map is based on an xml node self._read_XML() elif self.filename is not None: # If a filename was passed, check it's type and if it exists self.map_type = grass.get_g().check_map(self.filename) self.temporary = False if self.map_type is None: # ... raise an exception if it doesn't raise grass.MapNotFoundException(self.filename) if self.xml_map_type in ["name",None]: # Only maps specified in xml that are not existing maps # are temporary by default self.temporary = False
def get_mdig_dir_path(self): # Get the mdig directory g = grass.get_g() db = g.grass_vars['GISDBASE'] loc = self.experiment.infer_location() mapset = self.get_mapset() d = os.path.join(db,loc,mapset,'mdig') return d
def get_mdig_dir_path(self): # Get the mdig directory g = grass.get_g() db = g.grass_vars['GISDBASE'] loc = self.experiment.infer_location() mapset = self.get_mapset() d = os.path.join(db, loc, mapset, 'mdig') return d
def change_to_web_mapset(): log.debug("Changing the web service mapset") g = grass.get_g(create=False) ms = "mdig_webservice" if not g.check_mapset(ms): g.change_mapset(ms, create=True) mapsets[g.grass_vars["LOCATION_NAME"]] = ms else: g.change_mapset(ms)
def get_map_resources(self): maps = [] if isinstance(self.area, Event): m = self.treatment.strategy.experiment maps.extend(self.area.get_map_resources(m)) elif isinstance(self.area, GrassMap): if self.area.xml_map_type == "name": g = grass.get_g() maps.append((self.area.filename, g.find_mapset(self.area.filename))) return maps
def set_region(self): """ Set up GRASS so that the instance is working in the correct region (and consequently the right mapset/location too) """ current_region = self.experiment.get_region(self.r_id) g = grass.get_g() try: g.change_mapset(self.get_mapset(), self.experiment.infer_location()) g.set_region(current_region) except grass.SetRegionException, e: raise e
def record_maps(self, remove_null=False): # If not active, then there are no temp_map_names to copy if not self.active: return for ls_id in self.instance.experiment.get_lifestage_ids(): new_map = grass.get_g().generate_map_name(ls_id) self.grass_i.copy_map(self.temp_map_names[ls_id][0],new_map,True) self.push_previous_map(ls_id,new_map) if remove_null: # Remove the null bitmask which is uncompressed, so saves space # at the expense of cpu time self.grass_i.null_bitmask(self.get_previous_map(ls_id),generate=False)
def get_map_resources(self): maps = [] if isinstance(self.area, Event): m = self.treatment.strategy.experiment maps.extend(self.area.get_map_resources(m)) elif isinstance(self.area, GrassMap): if self.area.xml_map_type == "name": g = grass.get_g() maps.append( (self.area.filename, g.find_mapset(self.area.filename))) return maps
def get_map_resources(self, model): var_maps = model.get_variable_maps() params = self.get_params() maps = [] for p_key in params: p = params[p_key] if p[0] == "MAP": maps.append(p[1]) elif p[0] == "VAR": maps.extend(var_maps[p[1]]) maps_w_mapset = grass.get_g().find_mapsets(maps) return maps_w_mapset
def get_map_resources(self,model): var_maps = model.get_variable_maps() params = self.get_params() maps = [] for p_key in params: p = params[p_key] if p[0] == "MAP": maps.append(p[1]) elif p[0] == "VAR": maps.extend(var_maps[p[1]]) maps_w_mapset = grass.get_g().find_mapsets(maps) return maps_w_mapset
def record_maps(self, remove_null=False): # If not active, then there are no temp_map_names to copy if not self.active: return for ls_id in self.instance.experiment.get_lifestage_ids(): new_map = grass.get_g().generate_map_name(ls_id) self.grass_i.copy_map(self.temp_map_names[ls_id][0], new_map, True) self.push_previous_map(ls_id, new_map) if remove_null: # Remove the null bitmask which is uncompressed, so saves space # at the expense of cpu time self.grass_i.null_bitmask(self.get_previous_map(ls_id), generate=False)
def mdig_worker_start(work_q, results_q): # Have to replace some of the environment variables, otherwise they get # remembered and will confuse original Web server process g = grass.get_g() g.init_pid_specific_files() g.grass_vars["MAPSET"] = "PERMANENT" g.set_gis_env() worker = MDiGWorker(work_q, results_q) try: worker.run() except KeyboardInterrupt, e: worker.running = False worker.clean_up()
def shutdown_webapp(): if app is None: return log.info("Shutting down the web service...") rt.running = False work_q.put({'action': "SHUTDOWN"}) if mdig_worker_process.pid: mdig_worker_process.join() g = grass.get_g(create=False) # Remove webservice mapsets log.debug("Removing temporary mapsets") global mapsets for loc in mapsets: g.remove_mapset(mapsets[loc], loc, force=True)
def _load_parameter(self): source = self.source dist = self.dist vals = self.vals index = self.index model_dir = self.model_dir try: if source == "map": # TODO create a GRASSInterface command to load map to an array g = grass.get_g() self.map_name = str(vals[0]) if g.check_map(self.map_name) != "raster": raise grass.MapNotFoundException(self.map_name) map_range = g.get_range() n_rows = int(map_range[8][6:]) n_cols = int(map_range[9][6:]) cmd = "r.out.ascii -h input=" + self.map_name p = Popen(cmd, shell=True, stdout=subprocess.PIPE) map_ascii = p.communicate()[0] if map_ascii.find("*") != -1: raise Exception("Null values in parameter map %s not allowed" % self.map_name) self.mat = numpy.matrix(map_ascii) self.mat = self.mat.reshape((n_rows, n_cols)) elif source == "CODA": self.coda = {} prefix = "" if not os.path.exists(index): prefix = model_dir self.coda_index = loadtxt(os.path.join(prefix, index), converters={0: lambda x: 0.0}) for i in range(len(vals)): temp = loadtxt(os.path.join(prefix, vals[i]), converters={0: lambda x: 0.0}) if i == 0: for j in range(len(self.coda_index[:, 0])): self.coda[j + 1] = temp[int(self.coda_index[j, 1]) - 1 : int(self.coda_index[j, 2]), 1] else: for j in range(len(self.coda_index[:, 0])): self.coda[j + 1] = concatenate( (self.coda[j + 1], temp[int(self.coda_index[j, 1] - 1) : int(self.coda_index[j, 2]), 1]) ) elif source == "random": self.str = "(%s.%s(%f,%f))" % (source, dist, vals[0], vals[1]) # elif source == 'zero': # break elif source == "static": self.static = vals[0] self.ready = True except IOError, e: errstr = "%s %s %s %s parameter value source coding not valid" % (source, index, dist, vals) self.log.error(errstr + "\nAre your CODA files okay?") raise e
def get_map_resources(self, model): """ We need the model to get instances and resolve variables """ maps = [] # get initial_maps for r_id in self.initial_maps: im = self.initial_maps[r_id] if not im.temporary: maps.append(im.filename) maps_w_mapset = grass.get_g().find_mapsets(maps) # TODO: get phenology maps! # get maps in events for e in self.events: maps_w_mapset.extend(e.get_map_resources(model)) maps_w_mapset = set(maps_w_mapset) # remove duplicate maps return maps_w_mapset
def get_variable_map(self, var_key, var_val, replicate): """ Get the map that represents a variable that is impacted by affectsVarable, for the specific regions within get_treatment_area. Returns None if this treatment does not affect var_key. """ if not self.affects_var(var_key): return None area_mask_map = self.get_treatment_area_map(replicate) if area_mask_map is None: # This means the treatment is applied globally, no need to return # a map return None altered_value = self.get_altered_variable_value(var_key, var_val) if altered_value is None: altered_value = "null()" orig_value = var_val if orig_value is None: orig_value = "null()" grass.get_g().mapcalc( self.var_temp, "if(" + area_mask_map + "==1," + str(altered_value) + "," + str(orig_value) + ")") return self.var_temp
def get_treatment_area(self, replicate, last_area=None): """ Get the map name representing the treatment area, generating it dynamically if necessary. """ if isinstance(self.area, Event): if self.area_filter_output is not None: grass.get_g().remove_map(self.area_filter_output) if last_area: dist_map = last_area else: dist_map = replicate.temp_map_names[self.treatment.area_ls][0] self.area.run(dist_map, self.area_temp, replicate, False) self.area_filter_output = self.area_temp return self.area_filter_output elif isinstance(self.area, GrassMap): replacements = { "POP_MAP": replicate.temp_map_names[self.treatment.area_ls][0], "START_MAP": replicate.initial_maps[self.treatment.area_ls].get_map_filename() } return self.area.get_map_filename(replacements)
def get_variable_map(self, var_key, var_val, replicate): """ Get the map that represents a variable that is impacted by affectsVarable, for the specific regions within get_treatment_area. Returns None if this treatment does not affect var_key. """ if not self.affects_var(var_key): return None area_mask_map = self.get_treatment_area_map(replicate) if area_mask_map is None: # This means the treatment is applied globally, no need to return # a map return None altered_value = self.get_altered_variable_value(var_key, var_val) if altered_value is None: altered_value = "null()" orig_value = var_val if orig_value is None: orig_value = "null()" grass.get_g().mapcalc(self.var_temp, "if(" + area_mask_map + "==1," + str(altered_value) + "," + str(orig_value) + ")") return self.var_temp
def update_occupancy_envelope(self, ls_list=None, start=None, end=None, force=False): """ Go through and update the occupancy envelopes if necessary Note: ls_list has to be a list or None """ # Set the region in case it hasn't been yet self.set_region() if ls_list == None: ls_list = self.experiment.get_lifestage_ids() if start == None: start = self.experiment.get_period()[0] if end == None: start = self.experiment.get_period()[1] ls = ls_list self.log.debug("Checking whether envelopes are fresh...") missing_envelopes = self.are_envelopes_fresh(ls, start, end, force=force) if not missing_envelopes: return if not self.is_complete(): raise InstanceIncompleteException("Instance isn't complete, can't create occupancy envelope") for l in ls: maps = [] for r_idx in range(0,len(self.replicates)): r = self.replicates[r_idx] self.log.debug("Getting saved maps for replicate %d", r_idx) saved_maps = r.get_saved_maps(l) if saved_maps: maps.append(saved_maps) else: raise DispersalInstanceException("Missing maps for replicate %d" % r_idx) for t in missing_envelopes[l]: maps_to_combine = [] for r in maps: if str(t) in r: maps_to_combine.append(r[str(t)]) else: self.log.warning("Missing map for time=" + str(t)) raise DispersalInstanceException("Missing map for time %s" % str(t)) filename = self.get_map_name_base() filename += "_ls_" + l + "_t_" + repr(t) + "_prob" prob_env = grass.get_g().occupancy_envelope(maps_to_combine,filename) if prob_env is not None: self._add_envelope(prob_env,l,t) for li in self.listeners: if "occupancy_envelope_complete" in dir(li): li.occupancy_envelope_complete(self,l,t)
def are_envelopes_fresh(self, ls, start, end, force=False): previous_envelopes = self.get_occupancy_envelopes() missing_years = {} envelopes_current = self.are_envelopes_newer_than_reps() if not envelopes_current and not force: self.log.warning( "Envelopes are older than some replicates use -p to" " regenerate.") # if there are no envelopes yet or we want to overwrite them if force or previous_envelopes is None: for l in ls: missing_years[l] = [ y for y in self.experiment.map_year_generator( l, [start, end]) ] return missing_years for l in ls: interval = self.experiment.get_map_output_interval(l) if interval < 0: self.log.info( "No raster output defined to create occupancy envelope" " for lifestage " + l) return None missing_years[l] = [] for t in self.experiment.map_year_generator(l, [start, end]): # is map in the model xml? if str(t) not in previous_envelopes[l]: # no, then add year missing_years[l].append(t) else: # yes... then check if map exists self.log.debug("Checking for envelope %s" % previous_envelopes[l][str(t)]) if grass.get_g().check_map( previous_envelopes[l][str(t)]) is None: missing_years[l].append(t) self.log.debug("Missing envelope %s" % previous_envelopes[l][str(t)]) else: self.log.debug("Found envelope %s" % previous_envelopes[l][str(t)]) # delete ls in missing years if it's empty if len(missing_years[l]) == 0: del missing_years[l] return missing_years
def change_mapset(self): """ Change the current mapset to the one associated with this instance """ g = grass.get_g() loc = self.experiment.infer_location() mapset = self.get_mapset() # Create new mapset and link back to experiment's original mapset if g.check_mapset(mapset,loc): g.change_mapset(mapset,loc,in_path=[self.experiment.get_mapset()]) else: g.change_mapset(mapset,loc,True) # create mdig dir in mapset try: mdig_dir = g.create_mdig_subdir(mapset) self.create_mdig_files(mdig_dir) except OSError, e: g.remove_mapset(mapset,force=True) raise e
def __init__(self, node, experiment, instance=None): self.log = logging.getLogger("mdig.strategy") self.grass_i = grass.get_g() self.temp_map_names = {} self.active = False self.treatments = None # We save the experiment instead of the instance # because instances are only temporarily associated with strategies self.experiment = experiment self.instance = instance if node is None: self.node = self.init_strategy(experiment) else: self.node = node
def __init__(self, node, instance, r_index=0): self.instance = instance self.log = logging.getLogger("mdig.replicate") self.grass_i = grass.get_g() self.temp_map_names = {} self.active = False self.current_t = -1 self.initial_maps = {} self.previous_maps = None self.saved_maps = None self.map_intervals = None # used to keep track of index in replicates while loading: self.r_index = r_index # used to calculate time taken to complete self.start_time = None self.metrics = Metric(self) if node is None: if instance is None: raise ValueError( "Can't create Replicate connected to None value as instance." ) self.node = self.instance.experiment.add_replicate( self.instance.node) self.complete = False self.set_seed(self.instance.experiment.next_random_value()) self.instance.replicates.append(self) self.r_index = self.instance.replicates.index(self) else: # if node is provided then create replicate node from xml self.node = node c = config.get_config() if "replicate" not in c or \ "check_complete" not in c["replicate"] or \ c["replicate"]["check_complete"] != "false": self.complete = self.check_complete() else: # If the mdig.conf file has turned off check # then we just assume the replicate is complete self.complete = True self.random = random.Random() self.random.seed(self.get_seed())
def change_mapset(self): """ Change the current mapset to the one associated with this instance """ g = grass.get_g() loc = self.experiment.infer_location() mapset = self.get_mapset() # Create new mapset and link back to experiment's original mapset if g.check_mapset(mapset, loc): g.change_mapset(mapset, loc, in_path=[self.experiment.get_mapset()]) else: g.change_mapset(mapset, loc, True) # create mdig dir in mapset try: mdig_dir = g.create_mdig_subdir(mapset) self.create_mdig_files(mdig_dir) except OSError, e: g.remove_mapset(mapset, force=True) raise e
def __init__(self, node, instance, r_index=0): self.instance = instance self.log = logging.getLogger("mdig.replicate") self.grass_i = grass.get_g() self.temp_map_names={} self.active = False self.current_t = -1 self.initial_maps = {} self.previous_maps = None self.saved_maps = None self.map_intervals = None # used to keep track of index in replicates while loading: self.r_index = r_index # used to calculate time taken to complete self.start_time = None self.metrics = Metric(self) if node is None: if instance is None: raise ValueError("Can't create Replicate connected to None value as instance.") self.node = self.instance.experiment.add_replicate(self.instance.node) self.complete = False self.set_seed(self.instance.experiment.next_random_value()) self.instance.replicates.append(self) self.r_index = self.instance.replicates.index(self) else: # if node is provided then create replicate node from xml self.node = node c = config.get_config() if "replicate" not in c or \ "check_complete" not in c["replicate"] or \ c["replicate"]["check_complete"] != "false": self.complete = self.check_complete() else: # If the mdig.conf file has turned off check # then we just assume the replicate is complete self.complete = True self.random = random.Random() self.random.seed(self.get_seed())
def are_envelopes_fresh(self, ls, start, end, force=False): previous_envelopes = self.get_occupancy_envelopes() missing_years = {} envelopes_current = self.are_envelopes_newer_than_reps() if not envelopes_current and not force: self.log.warning("Envelopes are older than some replicates use -p to" " regenerate.") # if there are no envelopes yet or we want to overwrite them if force or previous_envelopes is None: for l in ls: missing_years[l] = [y for y in self.experiment.map_year_generator(l, [start,end])] return missing_years for l in ls: interval = self.experiment.get_map_output_interval(l) if interval < 0: self.log.info("No raster output defined to create occupancy envelope" " for lifestage " + l) return None missing_years[l] = [] for t in self.experiment.map_year_generator(l, [start,end]): # is map in the model xml? if str(t) not in previous_envelopes[l]: # no, then add year missing_years[l].append(t) else: # yes... then check if map exists self.log.debug("Checking for envelope %s" % previous_envelopes[l][str(t)]) if grass.get_g().check_map(previous_envelopes[l][str(t)]) is None: missing_years[l].append(t) self.log.debug("Missing envelope %s" % previous_envelopes[l][str(t)]) else: self.log.debug("Found envelope %s" % previous_envelopes[l][str(t)]) # delete ls in missing years if it's empty if len(missing_years[l]) == 0: del missing_years[l] return missing_years
def get_map_filename(self, map_replacements=None): """ Retrieve filename for the map. If this is the first time retrieving the map filename, or if the map is set to refresh itself every time it is retrieved, then generate it. @param ls The lifestage to base dynamic maps on. So that if the map is created by mapcalc, POP_MAP will be replaced with the latest map from that lifestage. """ if self.filename is None or self.refresh: g = grass.get_g() # If the map needs to be refreshed and has already been initiated # then destroy the old map... if self.refresh and self.ready: g.destruct_map(self.filename) self.mapset = g.get_mapset() self.filename, self.map_type = g.init_map(self, map_replacements) self.ready = True if self.mapset: return self.filename + "@" + self.mapset else: return self.filename
def clean_up(self): """ Just removes map if it is temporary """ if self.temporary and self.ready: grass.get_g().destruct_map(self.filename)
def run(self, in_name, rep): """ Run the analysis on a replicate. @param in_name: The name of the current map. @param rep: The replicate to run on. @todo: remove in_name as the output and use get_previous_map once it's implemented in replicate. """ #rawCommand = self.get_command() cmd = "" #if rawCommand in analysis.inbuiltCommands: # cmd = analysis.inbuiltCommands[rawCommand].create_cmd_string(in_name,rep) #else p = self._fill_in_map_parameters(rep, self.get_params(), in_name) # put all the parameters and command into a command string cmd = self.create_cmd_string(p) fn = "" # base_cmd has the input map parameter removed for recording # in xml. base_cmd = cmd if self.is_redirected_stdout(): fn = self._make_filename(rep) # if generating a file for each time step then check file # doesn't exist if not self.is_append() and os.path.exists(fn): if not MDiGConfig().overwrite_flag: raise AnalysisOutputFileExists() else: os.remove(fn) # remove input map parameter from base_cmd res = re.search("(input=\w+)", base_cmd) if res is not None: base_cmd = base_cmd.replace(res.groups()[0], "") # if the analysis requires the timestep to be written/appended # to the output file then do so if self.is_interval(): fh = open(fn, 'a') fh.write('%d ' % rep.current_t) fh.close() # add the output filename to the command if self.is_append(): cmd += " >> " else: cmd += " > " # run command! grass.get_g().run_command(cmd + fn) # if a file was generated then add this to the replicate ls_id = self.get_lifestage_id() if self.is_redirected_stdout(): class mock_ac: def __init__(self, base_cmd, fn): self.cmd_string = base_cmd self.output_fn = fn rep.add_analysis_result(ls_id, mock_ac(base_cmd, fn))
# Get existing models in repository models = mdig.repository.get_models() ms = models.keys()[:] ms.sort() m_list = [] for m in ms: try: dm = DispersalModel(models[m], setup=False) desc = dm.get_description() desc = re.sub("[\\s\\t]+", " ", desc) m_list.append((m, desc, dm.infer_location())) except mdig.model.ValidationError, e: log.error(str(e)) env = grass.get_g().get_gis_env() task_order, task_updates = process_tasks() return dict(name=mdig.version_string, version=mdig.version, v_name=mdig.version_name, models=m_list, repo_location=mdig.repository.db, grass_env=env, task_order=task_order, task_updates=task_updates) @route('/models/:model', method='GET') @route('/models/:model', method='POST') @view('model.tpl') @validate(model=validate_model_name)
def run(self, in_name, out_name, rep, is_pop): """ Run the event using in_name as the input map and out_name as the output map. """ template_p = self.get_params(is_pop, None) p = {} # If this event has a fixed input specified if self.fixed_input is not None: in_name = self.fixed_input # Parameter names for input and output maps in_param = self.get_input_name() if in_param is not None: template_p[in_param] = ("IN", in_name) out_param = self.get_output_name() if out_param is not None: template_p[out_param] = ("OUT", out_name) # Some commands report some interesting information to aggregate, # like r.mdig.survival's AREA_EVALUATED report_file = None s_name = rep.instance.strategy s = rep.instance.experiment.get_management_strategy(s_name) # TODO strategies should be pre initialised with instances if s is not None: s.set_instance(rep.instance) for p_name, value in template_p.items(): p_type, p_value = value # print 'name is', p_name, # print 'value is', value if p_type == "VAR": instance_value = rep.instance.get_var(p_value) instance_map = None treatments = [] if s: treatments = s.get_treatments_for_param( p_value, rep.current_t) if treatments: self.log.debug("treatments for variable %s are: %s" % (p_value, repr(treatments))) # TODO support blending of multiple treatments on param # (move below operations from treatment to strategy) assert len( treatments ) == 1, "MDiG does not currently support multiple treatments to a parameter" instance_map = treatments[0].get_variable_map( p_value, instance_value, rep) if instance_map is None: instance_value = treatments[ 0].get_altered_variable_value( p_value, instance_value) assert instance_value is not None if instance_value: p[p_name] = instance_value self.log.debug( "Variable %s has value %s for this instance" % (p_name, instance_value)) elif instance_map: p[p_name] = instance_map self.log.debug("Variable %s is map %s for this instance" % (p_name, instance_map)) else: self.log.debug( "Variable %s has None value for this instance" % p_name) elif p_type == "SEED": p[p_name] = rep.random.randint(-2.14748e+09, 2.14748e+09) elif p_type == "REPORT_FILE": report_file = trm.temp_filename(prefix='mdig_event_report') p[p_name] = report_file elif p_type in ["VALUE", "MAP", "IN", "OUT"]: p[p_name] = p_value elif p_type == "FLAG": p[p_name] = "FLAG" else: raise Exception("Unknown parameter type %s" % p_type) cmd = self.create_cmd_string(p) grass.get_g().remove_map(out_name) grass.get_g().run_command(cmd) metrics = {} if report_file: metrics = self.read_report_file(report_file) return metrics
def clean_up(self): g = grass.get_g() g.clean_up()
def run(self, in_name, out_name, rep, is_pop): """ Run the event using in_name as the input map and out_name as the output map. """ template_p = self.get_params(is_pop,None) p = {} # If this event has a fixed input specified if self.fixed_input is not None: in_name = self.fixed_input # Parameter names for input and output maps in_param = self.get_input_name() if in_param is not None: template_p[in_param] = ("IN", in_name) out_param = self.get_output_name() if out_param is not None: template_p[out_param] = ("OUT", out_name) # Some commands report some interesting information to aggregate, # like r.mdig.survival's AREA_EVALUATED report_file = None s_name = rep.instance.strategy s = rep.instance.experiment.get_management_strategy(s_name) # TODO strategies should be pre initialised with instances if s is not None: s.set_instance(rep.instance) for p_name, value in template_p.items(): p_type, p_value = value # print 'name is', p_name, # print 'value is', value if p_type == "VAR": instance_value = rep.instance.get_var(p_value) instance_map = None treatments = [] if s: treatments = s.get_treatments_for_param(p_value,rep.current_t) if treatments: self.log.debug("treatments for variable %s are: %s" % (p_value, repr(treatments))) # TODO support blending of multiple treatments on param # (move below operations from treatment to strategy) assert len(treatments) == 1, "MDiG does not currently support multiple treatments to a parameter" instance_map = treatments[0].get_variable_map(p_value, instance_value, rep) if instance_map is None: instance_value = treatments[0].get_altered_variable_value(p_value,instance_value) assert instance_value is not None if instance_value: p[p_name]=instance_value self.log.debug("Variable %s has value %s for this instance" % (p_name, instance_value)) elif instance_map: p[p_name]=instance_map self.log.debug("Variable %s is map %s for this instance" % (p_name, instance_map)) else: self.log.debug("Variable %s has None value for this instance" % p_name) elif p_type == "SEED": p[p_name] = rep.random.randint(-2.14748e+09,2.14748e+09) elif p_type == "REPORT_FILE": report_file = trm.temp_filename(prefix='mdig_event_report') p[p_name] = report_file elif p_type in ["VALUE", "MAP", "IN", "OUT"]: p[p_name] = p_value elif p_type == "FLAG": p[p_name] = "FLAG" else: raise Exception("Unknown parameter type %s" % p_type) cmd=self.create_cmd_string(p) grass.get_g().remove_map(out_name) grass.get_g().run_command(cmd) metrics = {} if report_file: metrics = self.read_report_file(report_file) return metrics
def init_phenology_bins(self): """ Initialise the bins with their range and mean values. Bins are dictionaries, with the bin range as a tuple key and the value is the mean value within that bin (which may not be directly in the middle depending on the frequency distribution of values for that bin. """ self.bins = {} n_bins = 10 p_nodes = self.xml_node.xpath("phenology") for nodes in p_nodes: if "region" in nodes.attrib.keys(): # If there is a region specified for the phenology settings self.bins[nodes.attrib["region"]] = {} current_bins = self.bins[nodes.attrib["region"]] else: # otherwise the setting is for all regions if "__default" in self.bins.keys(): self.log.error("Default region phenology map already defined") current_bins = None break else: self.bins["__default"] = {} current_bins = self.bins["__default"] for n in nodes: if n.tag == "delay": # Unsure what delay does, it might be the delay before # new individuals can be processed by the next lifestage self.phenology_delay = int(n.text.strip()) elif n.tag == "value": # If phenology is simply a value then it occurs at the same # time everywhere interval = int(n.text) current_bins[(interval, interval)] = interval elif n.tag == "map": # If it's a map we have to process it to work out the bin # ranges and their means p_mapname = n.text.strip() assert False, "Phenology maps currently broken" # What on earth is region_id? # hiding this to avoid the region_id showing pyflakes. # self.p_map_names[region_id] = GrassMap(p_mapname) sums = [0] * n_bins bin_counts = [0] * n_bins freqs = grass.get_g().raster_value_freq(p_mapname) min_cell = int(freqs[0][0]) max_cell = int(freqs[-1][0]) bin_size = float((max_cell - (min_cell - 1))) / n_bins i = 1 for value, freq in freqs: value = int(value) freq = int(freq) # if we have gone beyond the range of the current bin while value > (min_cell - 1 + (bin_size * i)): i += 1 # add value freq times to the sum sums[i - 1] += freq * value bin_counts[i - 1] += freq for i in range(1, n_bins + 1): bin_range = (min_cell + ((i - 1) * bin_size), min_cell - 1 + ((i) * bin_size)) if bin_counts[i - 1] != 0: current_bins[bin_range] = float(sums[i - 1] / bin_counts[i - 1]) else: current_bins[bin_range] = None # end for nodes in p_nodes self.log.debug("Phenology bins are: %s" % (repr(self.bins)))
def run(self, interval, rep, temp_map_names, strategy=None): grass_i = grass.get_g() # Run through events for this lifestage for e in self.events: mask = "" p_intervals = self.get_phenology_intervals(rep.instance.r_id) if len(p_intervals) > 1: mask = self.get_phenology_mask(interval, rep.instance.r_id) grass_i.make_mask(mask) metrics = e.run(temp_map_names[0], temp_map_names[1], rep, self.populationBased) rep.metrics.add_event_metrics(self, e, metrics, interval) if len(p_intervals) > 1: # Remove mask because we can't access anything outside of it # using mapcalc grass_i.make_mask(None) # Join Maps grass_i.mapcalc( temp_map_names[0], "if(isnull(%s),%s,%s)" % (mask, temp_map_names[0], temp_map_names[1]) ) # Merge value outside of original mask, e.g. long distance jumps if self.populationBased: grass_i.mapcalc( temp_map_names[1], "if(isnull(%s) && isnull(%s),%s,%s+%s)" % (mask, temp_map_names[1], temp_map_names[0], temp_map_names[0], temp_map_names[1]), ) else: grass_i.mapcalc( temp_map_names[1], "if(isnull(%s) && isnull(%s),%s,%s)" % (mask, temp_map_names[1], temp_map_names[0], temp_map_names[1]), ) temp_map_names.reverse() # Get management strategy treatments that affect this lifestage. treatments = [] if strategy is not None: treatments = strategy.get_treatments_for_ls(self.name, rep.current_t) for t in treatments: self.log.debug("Applying treatment %d for strategy %s" % (t.index, strategy.get_name())) t_area = t.get_treatment_area_map(rep) self.log.debug("Treatment area map is %s" % t_area) if "NULL" in grass_i.get_raster_range(t_area).values(): # If mask is empty then don't run treatment, as it does # nothing, and GRASS's mask system is broken. # http://trac.osgeo.org/grass/ticket/1999 continue # Mask so that only treatment area is affected grass_i.make_mask(t_area) e = t.get_event() metrics = e.run(temp_map_names[0], temp_map_names[1], rep, False) rep.metrics.add_event_metrics(self, e, metrics, interval, treatment=t) # Remove mask when done grass_i.make_mask(None) # Now we have to combine the unmasked region from the original map with the # the alteration made by the treatment on the masked area. if t_area is not None: self.tempmap = "wibble_foss" grass_i.mapcalc( self.tempmap, 'if(isnull("%s"),"%s","%s")' % (t_area, temp_map_names[0], temp_map_names[1]) ) grass_i.copy_map(self.tempmap, temp_map_names[1], overwrite=True) temp_map_names.reverse()