def exit_cleanup(): global simulations logger = logging.getLogger("mdig") logger.debug("Cleaning up") # clean up simulations and save changes for exp in simulations: exp.clean_up() exp.save_model() # clean up web server from mdig.webui import shutdown_webapp shutdown_webapp() # clean up GRASS environment if grass.get_g(False) is not None: grass.get_g().clean_up() # save config changes config.get_config().write() from mdig.tempresource import trm trm.cleanup() logger.debug("Finished at %s" % repr(datetime.now().ctime()))
def remove_model(self, model_name, force=False): models = self.get_models() if model_name not in models: raise RepositoryException("The model '" + model_name + "' doesn't exist in the repository.") if not force: # TODO list ALL associated mapsets ans = raw_input("Are you sure you wish to remove model " + model_name + " and it's associate mapset? [y/n] ") if ans.upper() == "Y": force = True else: print "Not removing model " + model_name if force: g = grass.get_g() dm = model.DispersalModel(models[model_name],setup=False) loc = dm.get_location() if loc == None: loc = dm.infer_location() if not os.path.isdir(os.path.join(self.db,loc,"PERMANENT")): raise RepositoryException("Model is in a GIS Location " + loc + " that doesn't exist in " + self.db) g.change_mapset("PERMANENT",loc) # remove ALL associated instance mapsets first (because we need the # model file to tell us which ones are associated) for i in dm.get_instances(): i_mapset = i.get_mapset() if i_mapset != model_name: try: grass.get_g().remove_mapset(i_mapset, loc, force) except OSError, e: self.log.warn("Failed to remove mapset %s@%s" % (i_mapset,loc)) try: grass.get_g().remove_mapset(model_name, loc, force) except OSError, e: self.log.warn("Failed to remove mapset %s@%s" % (model_name,loc))
def export_map(self, map, out_fn, envelope=False): old_region = "ExportActionBackupRegion" g = grass.get_g() g.run_command('g.region --o save=%s' % old_region) g.set_region(raster=map) cmd = 'r.out.gdal input=%s output=%s.tif format=GTiff type=%s createopt="COMPRESS=PACKBITS,INTERLEAVE=PIXEL"' if envelope: if self.float64: cmd = cmd % (map, out_fn, 'Float64') else: cmd = cmd % (map, out_fn, 'Float32') else: cmd = cmd % (map, out_fn, 'UInt16') try: g.run_command(cmd) out_fn += ".tif" except grass.GRASSCommandException, e: # This swaps to 64 bit floats if GRASS complains about # losing precision on export. if "Precision loss" in e.stderr: self.float64 = True out_fn = self.export_map(map, out_fn, envelope) else: raise e
def replicate_update(self, rep, t): g = grass.get_g() fn = None if rep.instance.experiment.interval_modulus(self.interval, t) == 0: fn = self.create_filename(rep) fn += "_" + repr(t) + ".png" self.log.debug("Writing PNG %s" % fn) g.set_output(fn, display=None) g.clear_monitor() current_region = rep.instance.experiment.get_region(rep.instance.r_id) if current_region.getBackgroundMap(): g.paint_map(current_region.getBackgroundMap().get_map_filename()) for l in rep.temp_map_names.keys(): g.paint_map(rep.temp_map_names[l][0]) if self.show_grid: g.paint_grid(5) if self.show_year: g.paint_year(t) self.last_output = t g.close_output() return [None, fn]
def replicate_update(self, rep, t): g = grass.get_g() fn = None if rep.instance.experiment.interval_modulus(self.interval, t) == 0: fn = self.create_filename(rep) fn += "_" + repr(t) + ".png" self.log.debug("Writing PNG %s" % fn) g.set_output(fn, display=None) g.clear_monitor() current_region = rep.instance.experiment.get_region( rep.instance.r_id) if current_region.getBackgroundMap(): g.paint_map( current_region.getBackgroundMap().get_map_filename()) for l in rep.temp_map_names.keys(): g.paint_map(rep.temp_map_names[l][0]) if self.show_grid: g.paint_grid(5) if self.show_year: g.paint_year(t) self.last_output = t g.close_output() return [None, fn]
def export_map(self, map, out_fn, envelope=False): old_region = "ExportActionBackupRegion" g = grass.get_g() g.run_command('g.region --o save=%s' % old_region) g.set_region(raster=map) cmd = 'r.out.gdal input=%s output=%s.tif format=GTiff type=%s createopt="COMPRESS=PACKBITS,INTERLEAVE=PIXEL"' if envelope: if self.float64: cmd = cmd % (map, out_fn, 'Float64') else: cmd = cmd % (map, out_fn, 'Float32') else: cmd = cmd % (map, out_fn, 'UInt16') try: g.run_command(cmd) out_fn += ".tif" except grass.GRASSCommandException, e: # This swaps to 64 bit floats if GRASS complains about # losing precision on export. if "Precision loss" in e.stderr: self.float64 = True out_fn = self.export_map(map,out_fn,envelope) else: raise e
def do_rep(self,i,r): ls = self.options.output_lifestage map_list = [] saved_maps = r.get_saved_maps(ls) model_name = i.experiment.get_name() # Normalise the color scale so that the lgend and range doesn't # keep changing for map to map self.log.info("Normalising colours") the_range = grass.get_g().normalise_map_colors(saved_maps.values()) times = saved_maps.keys() times.sort(key=lambda x: float(x)) output_images = self.options.output_gif or self.options.output_image output_maps = self.options.output_map_pack if output_maps: rep_filenames = r.get_base_filenames(ls, output_dir=self.options.outdir) elif output_images: rep_filenames = r.get_base_filenames(ls, extension='.png', output_dir=self.options.outdir) for t in times: m = saved_maps[t] if output_images: map_list.append(self.create_frame(m, rep_filenames[t], model_name, t, ls, the_range)) self.update_listeners(None, r, ls, t) elif output_maps: map_list.append(self.export_map(m, rep_filenames[t])) self.update_listeners_map_pack(None, r, ls, t) if self.options.output_gif: self.create_gif(map_list, r.get_base_filenames(ls, extension='_anim.gif', single_file=True, output_dir=self.options.outdir)) elif output_maps: zip_fn = r.get_base_filenames(ls, extension='.zip', single_file=True, output_dir=self.options.outdir) self.zip_maps(map_list, zip_fn) return map_list
def check_background_map(self): g = grass.get_g() if self.options.background and not g.check_map( self.options.background): self.log.error("Couldn't find background map %s" % self.options.background) self.options.background = None
def create_frame(self, map_name, output_name, model_name, year, ls, the_range=None): g = grass.get_g() if os.path.isfile(output_name) and not self.options.overwrite_flag: raise OSError("Gif file %s exists, use -o flag to overwrite" % output_name) g.set_output(filename = output_name, \ width=self.options.width, height=self.options.height, display=None) g.run_command("d.erase") os.environ['GRASS_PNG_READ'] = "TRUE" # Check the background map exists if self.options.background: bg = self.options.background.split('@') if len(bg) > 1: map_ok = g.check_map(bg[0], bg[1]) else: map_ok = g.check_map(bg) g.run_command("r.colors color=grey map=" + self.options.background) g.run_command("d.rast " + self.options.background) # This is code for setting the color table of each map manually # hasn't been easily integrated into interface, but easy for hacking # custom map output custom_color = False if custom_color: pcolor= Popen('r.colors map=%s rules=-' % map_name, \ shell=True, stdout=PIPE, stdin=PIPE) rule_string = "0%% %d:%d:%d\n" % (255, 255, 0) rule_string += "100%% %d:%d:%d\n" % (255, 255, 0) rule_string += 'end' output = pcolor.communicate(rule_string)[0] ### # Draw the map g.run_command("d.rast " + map_name + " -o") # Draw the scale g.run_command("d.barscale tcolor=0:0:0 bcolor=none at=2,18 -l -t") # Code to enable/disable the legend do_legend = True if do_legend: if the_range: g.run_command("d.legend -s " + map_name + " range=%f,%f at=5,50,85,90" % the_range) else: g.run_command("d.legend -s " + map_name + " at=5,50,85,90") ### # Show the model name and year g.run_command("d.text at=2,90 size=3 text=" + model_name) g.run_command("d.text text=" + year + " at=2,93") # Save frame g.close_output() return output_name
def add_model(self, model_fn): if not os.path.isfile(model_fn): raise RepositoryException("Model file %s is not a file." % model_fn) g = grass.get_g() c = config.get_config() dm = model.DispersalModel(model_fn, setup=False) loc = dm.get_location() if loc == None: raise RepositoryException( "Model doesn't define GIS Location for simulation") if not os.path.isdir(os.path.join(self.db, loc, "PERMANENT")): raise RepositoryException( "Model defines a GIS Location %s that doesn't exist in %s" % (loc, self.db)) models = self.get_models() if dm.get_name() in models: if c.overwrite_flag: # remove instance mapsets and replace model.xml self.log.info("Remove instances of old version of model %s" % dm.get_name()) dm_old = model.DispersalModel(models[dm.get_name()], setup=False) dm_old.hard_reset() del dm_old else: raise RepositoryException("The model '" + dm.get_name() + \ "' already exists in the repository. Replace with -o.") g.change_mapset("PERMANENT", loc) model_mapset = dm.get_mapset() if not g.check_mapset(dm.get_name()): # Create model mapset self.log.info("Creating mapset for model %s" % model_mapset) if not g.change_mapset(dm.get_name(), loc, True): raise RepositoryException("Couldn't create mapset %s." % model_mapset) self.log.info("Created mapset for model " + dm.get_name()) else: # Use existing mapset which matches model name self.log.warning( "Using existing mapset with same name as model %s" % model_mapset) if not g.change_mapset(dm.get_name(), loc): raise RepositoryException("Couldn't change into mapset %s." % model_mapset) # create mdig dir in mapset try: dest_dir = g.create_mdig_subdir(model_mapset, c.overwrite_flag) except OSError, e: g.remove_mapset(model_mapset, force=True) raise RepositoryException("Error creating mdig dir in mapset. %s" % str(e))
def do_instances(self, mdig_model, instances): _update_probability_envelopes(mdig_model, instances, self.log, self.options) ls = self.options.analysis_lifestage if ls is None: ls = self.experiment.get_lifestage_ids() elif not isinstance(ls, list): ls = [ls] self.files_written = [] self.log.info("Calculating area...") g=grass.get_g() if self.options.combined_analysis: if self.options.analysis_step == "all": for l in ls: for i in instances: self.log.info("Calculating stats for instance %d" % i.get_index()) maps = i.get_occupancy_envelopes()[l] i.change_mapset() g.make_mask(self.options.mask) stats=g.get_univariate_stats(maps) fn = os.path.split(i.get_occ_envelope_img_filenames(ls=l, extension=False,gif=True)[:-5]) g.make_mask(None) fn = os.path.join(fn[0], self.options.analysis_filename_base + fn[1]) self.write_stats_to_file(stats,fn) self.files_written.append(fn) else: # just run on last map raise NotImplementedError("Only supports running all maps current") else: if self.options.analysis_step == "all": for l in ls: for i in instances: i.change_mapset() g.make_mask(self.options.mask) for r in i.replicates: self.log.info("Calculating stats for instance %d, rep %d" % \ (i.get_index(), r.r_index)) maps = r.get_saved_maps(l) stats = g.get_univariate_stats(maps) fn = os.path.split(r.get_base_filenames(l, single_file=True)) fn = os.path.join(fn[0], self.options.analysis_filename_base + fn[1]) self.write_stats_to_file(stats, fn) self.files_written.append(fn) g.make_mask(None) else: # just run on last map raise NotImplementedError("Only supports running all maps current") if len(self.files_written) > 1: self.log.info("Output is in: %s .. etc" % str(self.files_written[0])) else: self.log.info("Output is in: %s" % str(self.files_written[0]))
def main(argv): global simulations # Do a migration of model repository data if len(argv) > 0 and argv[0] == 'migrate': do_migration(argv[1:]) # Otherwise start up normally # set up logger logger, loghandler = setup_logger() # get config mdig_config = config.get_config() # set ansi logging if enabled ansi_bool_string = mdig_config["LOGGING"]["ansi"] if ansi_bool_string == "true": set_log_formatter(loghandler, True) the_action = process_options(argv) # We can't rely on logging before this point as process_options # determines whether to enable debug level messages logging.getLogger("mdig.config").debug("Using MDIG config file " + mdig_config.cf_full_path) signal.signal(signal.SIGINT, exit_catcher) # Load model repository if the_action.repository is not None: mdig_config["GRASS"]["GISDBASE"] = the_action.repository if the_action.location is not None: mdig_config["GRASS"]["LOCATION_NAME"] = the_action.location try: if the_action.init_repository: mdig.repository = modelrepository.ModelRepository() models = mdig.repository.get_models() if the_action.init_grass: # Check for grass environment and set up interface grass.get_g() except grass.EnvironmentException, e: logger.error("Failed to initialize due to environment error: %s" % str(e)) logger.error("Perhaps check your config file? (%s)" % mdig_config.cf_full_path) sys.exit(1)
def remove_model(self, model_name, force=False): models = self.get_models() if model_name not in models: raise RepositoryException("The model '" + model_name + "' doesn't exist in the repository.") if not force: # TODO list ALL associated mapsets ans = raw_input("Are you sure you wish to remove model " + model_name + " and it's associate mapset? [y/n] ") if ans.upper() == "Y": force = True else: print "Not removing model " + model_name if force: g = grass.get_g() dm = model.DispersalModel(models[model_name], setup=False) loc = dm.get_location() if loc == None: loc = dm.infer_location() if not os.path.isdir(os.path.join(self.db, loc, "PERMANENT")): raise RepositoryException("Model is in a GIS Location " + loc + " that doesn't exist in " + self.db) g.change_mapset("PERMANENT", loc) # remove ALL associated instance mapsets first (because we need the # model file to tell us which ones are associated) for i in dm.get_instances(): i_mapset = i.get_mapset() if i_mapset != model_name: try: grass.get_g().remove_mapset(i_mapset, loc, force) except OSError, e: self.log.warn("Failed to remove mapset %s@%s" % (i_mapset, loc)) try: grass.get_g().remove_mapset(model_name, loc, force) except OSError, e: self.log.warn("Failed to remove mapset %s@%s" % (model_name, loc))
def replicate_update(self, rep, t): g = grass.get_g() fn = None if rep.instance.experiment.interval_modulus(self.interval, t) == 0: for l in rep.temp_map_names.keys(): if self.lifestage == l: fn = self.create_filename(rep) fn += "_ls_" + l + "_" + repr(t) self.log.debug("Writing raster %s" % fn) g.copy_map(rep.temp_map_names[l][0], fn, True) self.last_output = t return [self.lifestage, fn]
def create_frame(self, map_name, output_name, model_name, year, ls, the_range = None): g = grass.get_g() if os.path.isfile(output_name) and not self.options.overwrite_flag: raise OSError("Gif file %s exists, use -o flag to overwrite" % output_name) g.set_output(filename = output_name, \ width=self.options.width, height=self.options.height, display=None) g.run_command("d.erase") os.environ['GRASS_PNG_READ']="TRUE" # Check the background map exists if self.options.background: bg = self.options.background.split('@') if len(bg) > 1: map_ok = g.check_map(bg[0], bg[1]) else: map_ok = g.check_map(bg) g.run_command("r.colors color=grey map=" + self.options.background) g.run_command("d.rast " + self.options.background) # This is code for setting the color table of each map manually # hasn't been easily integrated into interface, but easy for hacking # custom map output custom_color = False if custom_color: pcolor= Popen('r.colors map=%s rules=-' % map_name, \ shell=True, stdout=PIPE, stdin=PIPE) rule_string = "0%% %d:%d:%d\n" % (255,255,0) rule_string += "100%% %d:%d:%d\n" % (255,255,0) rule_string += 'end' output = pcolor.communicate(rule_string)[0] ### # Draw the map g.run_command("d.rast " + map_name + " -o") # Draw the scale g.run_command("d.barscale tcolor=0:0:0 bcolor=none at=2,18 -l -t") # Code to enable/disable the legend do_legend = True if do_legend: if the_range: g.run_command("d.legend -s " + map_name + " range=%f,%f at=5,50,85,90" % the_range) else: g.run_command("d.legend -s " + map_name + " at=5,50,85,90") ### # Show the model name and year g.run_command("d.text at=2,90 size=3 text=" + model_name) g.run_command("d.text text=" + year + " at=2,93") # Save frame g.close_output() return output_name
def __init__(self, gisdbase=None): self.log = logging.getLogger("mdig.repos") c = config.get_config() # Model repository is now a part of a GRASS db directory g = grass.get_g() if gisdbase is None: if g.in_grass_shell: self.db = g.grass_vars["GISDBASE"] else: self.db = c["GRASS"]["GISDBASE"] else: if not os.path.isdir(gisdbase): raise OSError("Bad GISDBASE") self.db = gisdbase self.log.info("Using GRASS DB directory" + self.db)
def __init__(self, model_names, options): # 2 for 1-specificity, 3 for area self.roc_xaxis = 3 self.model_names = model_names self.yearly_vectors = {} self.save_to_star = False self.g = grass.get_g() self.log = logging.getLogger("mdig.roc") self.reps = [] self.import_options(options) self.models = {} self.load_models(self.model_names, options.model_tags)
def __init__(self, model_names, options): # 2 for 1-specificity, 3 for area self.roc_xaxis=3 self.model_names = model_names self.yearly_vectors = {} self.save_to_star = False self.g = grass.get_g() self.log = logging.getLogger("mdig.roc") self.reps = [] self.import_options(options) self.models = {} self.load_models(self.model_names,options.model_tags)
def add_model(self, model_fn): if not os.path.isfile(model_fn): raise RepositoryException("Model file %s is not a file."%model_fn) g = grass.get_g() c = config.get_config() dm = model.DispersalModel(model_fn, setup=False) loc = dm.get_location() if loc == None: raise RepositoryException("Model doesn't define GIS Location for simulation") if not os.path.isdir(os.path.join(self.db, loc, "PERMANENT")): raise RepositoryException("Model defines a GIS Location %s that doesn't exist in %s" % (loc, self.db)) models = self.get_models() if dm.get_name() in models: if c.overwrite_flag: # remove instance mapsets and replace model.xml self.log.info("Remove instances of old version of model %s" % dm.get_name()) dm_old = model.DispersalModel(models[dm.get_name()], setup=False) dm_old.hard_reset() del dm_old else: raise RepositoryException("The model '" + dm.get_name() + \ "' already exists in the repository. Replace with -o.") g.change_mapset("PERMANENT",loc) model_mapset = dm.get_mapset() if not g.check_mapset(dm.get_name()): # Create model mapset self.log.info("Creating mapset for model %s" % model_mapset) if not g.change_mapset(dm.get_name(), loc, True): raise RepositoryException("Couldn't create mapset %s." % model_mapset) self.log.info("Created mapset for model " + dm.get_name()) else: # Use existing mapset which matches model name self.log.warning("Using existing mapset with same name as model %s" % model_mapset) if not g.change_mapset(dm.get_name(),loc): raise RepositoryException("Couldn't change into mapset %s." % model_mapset) # create mdig dir in mapset try: dest_dir = g.create_mdig_subdir(model_mapset, c.overwrite_flag) except OSError, e: g.remove_mapset(model_mapset, force=True) raise RepositoryException("Error creating mdig dir in mapset. %s" % str(e))
def do_rep(self, i, r): ls = self.options.output_lifestage map_list = [] saved_maps = r.get_saved_maps(ls) model_name = i.experiment.get_name() # Normalise the color scale so that the lgend and range doesn't # keep changing for map to map self.log.info("Normalising colours") the_range = grass.get_g().normalise_map_colors(saved_maps.values()) times = saved_maps.keys() times.sort(key=lambda x: float(x)) output_images = self.options.output_gif or self.options.output_image output_maps = self.options.output_map_pack if output_maps: rep_filenames = r.get_base_filenames( ls, output_dir=self.options.outdir) elif output_images: rep_filenames = r.get_base_filenames( ls, extension='.png', output_dir=self.options.outdir) for t in times: m = saved_maps[t] if output_images: map_list.append( self.create_frame(m, rep_filenames[t], model_name, t, ls, the_range)) self.update_listeners(None, r, ls, t) elif output_maps: map_list.append(self.export_map(m, rep_filenames[t])) self.update_listeners_map_pack(None, r, ls, t) if self.options.output_gif: self.create_gif( map_list, r.get_base_filenames(ls, extension='_anim.gif', single_file=True, output_dir=self.options.outdir)) elif output_maps: zip_fn = r.get_base_filenames(ls, extension='.zip', single_file=True, output_dir=self.options.outdir) self.zip_maps(map_list, zip_fn) return map_list
def replicate_update(self, rep, t): g = grass.get_g() g.set_output() g.clear_monitor() bm = config.get_config()['OUTPUT']['background_map'] if bm is not None: try: g.paint_map(bm) except grass.GRASSCommandException: pass layer_index = 0 for l in rep.temp_map_names: g.paint_map(rep.temp_map_names[l][0], layer=layer_index) layer_index += 1 g.close_output()
def replicate_update(self, rep, t): g = grass.get_g() g.set_output() g.clear_monitor() bm=config.get_config()['OUTPUT']['background_map'] if bm is not None: try: g.paint_map(bm) except grass.GRASSCommandException: pass layer_index = 0 for l in rep.temp_map_names: g.paint_map(rep.temp_map_names[l][0], layer=layer_index) layer_index += 1 g.close_output()
def do_me(self,mdig_model): for m in self.model_names: mdig.repository.add_model(m) grass.get_g().clean_up()
def check_background_map(self): g = grass.get_g() if self.options.background and not g.check_map(self.options.background): self.log.error("Couldn't find background map %s" % self.options.background) self.options.background = None
def do_me(self, mdig_model): for m in self.model_names: mdig.repository.add_model(m) grass.get_g().clean_up()
def test_init_map(self): xml = "<mapcalc refresh=\"true\">x()</mapcalc>" map_node = string_as_xml_node(xml) self.gmap = GrassMap(xml_node=map_node) grass.get_g().init_map(self.gmap)
def do_instances(self, mdig_model, instances): _update_probability_envelopes(mdig_model, instances, self.log, self.options) ls = self.options.analysis_lifestage if ls is None: ls = self.experiment.get_lifestage_ids() elif not isinstance(ls, list): ls = [ls] self.files_written = [] self.log.info("Calculating area...") g = grass.get_g() if self.options.combined_analysis: if self.options.analysis_step == "all": for l in ls: for i in instances: self.log.info("Calculating stats for instance %d" % i.get_index()) maps = i.get_occupancy_envelopes()[l] i.change_mapset() g.make_mask(self.options.mask) stats = g.get_univariate_stats(maps) fn = os.path.split( i.get_occ_envelope_img_filenames(ls=l, extension=False, gif=True)[:-5]) g.make_mask(None) fn = os.path.join( fn[0], self.options.analysis_filename_base + fn[1]) self.write_stats_to_file(stats, fn) self.files_written.append(fn) else: # just run on last map raise NotImplementedError( "Only supports running all maps current") else: if self.options.analysis_step == "all": for l in ls: for i in instances: i.change_mapset() g.make_mask(self.options.mask) for r in i.replicates: self.log.info("Calculating stats for instance %d, rep %d" % \ (i.get_index(), r.r_index)) maps = r.get_saved_maps(l) stats = g.get_univariate_stats(maps) fn = os.path.split( r.get_base_filenames(l, single_file=True)) fn = os.path.join( fn[0], self.options.analysis_filename_base + fn[1]) self.write_stats_to_file(stats, fn) self.files_written.append(fn) g.make_mask(None) else: # just run on last map raise NotImplementedError( "Only supports running all maps current") if len(self.files_written) > 1: self.log.info("Output is in: %s .. etc" % str(self.files_written[0])) else: self.log.info("Output is in: %s" % str(self.files_written[0]))