def exit_cleanup():
global simulations
logger = logging.getLogger("mdig")
logger.debug("Cleaning up")
# clean up simulations and save changes
for exp in simulations:
exp.clean_up()
exp.save_model()
# clean up web server
from mdig.webui import shutdown_webapp
shutdown_webapp()
# clean up GRASS environment
if grass.get_g(False) is not None:
grass.get_g().clean_up()
# save config changes
config.get_config().write()
from mdig.tempresource import trm
trm.cleanup()
logger.debug("Finished at %s" % repr(datetime.now().ctime()))
def write_stats_to_file(self, stats, fn):
c = config.get_config()
if os.path.isfile(fn):
if c.overwrite_flag:
os.remove(fn)
else:
raise Exception("File %s already exists, -o to overwrite" % fn)
expected=['n', 'area', 'null_cells', 'cells', 'min', 'max', 'range', 'mean',
'mean_of_abs', 'stddev', 'variance', 'coeff_var', 'sum']
f = open(fn,'w')
f.write('time,' + ','.join(expected))
f.write('\r\n')
times = stats.keys()
times.sort()
for t in times:
f.write(str(t))
for stat_name in expected:
if stat_name in stats[t]:
f.write(',%f' % stats[t][stat_name])
else:
# value missing
f.write(',')
f.write('\r\n')
f.close()
def test_add_analysis_result(self,m_shutil,m_remove,m_path,m_get_g):
m_path.basename.return_value = 'test'
m_get_g.return_value.grass_vars = {'GISDBASE':self.gisdb}
# check with no strategy
i = self.m_strategy.get_instances()[0]
analysis_cmd = Mock()
analysis_cmd.cmd_string = 'wibble'
analysis_cmd.output_fn = 'wibble'
# Test with file existing
i.add_analysis_result('all',analysis_cmd)
# Test with overwrite
config.get_config().overwrite_flag = True
i.add_analysis_result('all',analysis_cmd)
# run again to check parsing existing lifestage analysis results
i.add_analysis_result('all',analysis_cmd)
config.get_config().overwrite_flag = False
def test_add_lifestage_model(self,get_g,m_ls):
self.make_grass_mock(get_g.return_value)
# Assume no appropriate files in tmp
c = config.get_config()
m = ModelRepository(self.temp_dir)
m2 = ModelRepository()
a_file = m2.get_models()['lifestage_test']
self.create_mock_location(self.temp_dir)
self.assertEqual(len(m.get_models()), 0)
# add location to model, save as new
dm = DispersalModel(a_file)
dm.set_location('grass_location')
temp_model_fn = os.path.join(os.path.dirname(a_file),"with_location_model.xml")
dm.save_model(filename=temp_model_fn)
get_g.return_value.get_range.return_value = [ 'xxxxxx10' ] * 10
get_g.return_value.raster_value_freq.return_value = [ [1],[2],[3] ]
m_ls.return_value = {}
# and then try to add
m.add_model(temp_model_fn)
self.assertEqual(len(m.get_models()), 1)
# more tests about lifestage resources?
self.remove_mock_location(self.temp_dir)
def empty_repository_test(self,get_g):
self.make_grass_mock(get_g.return_value)
# Assume no appropriate files in tmp
c = config.get_config()
m = ModelRepository(self.temp_dir)
self.assertEqual(len(m.get_models()), 0)
# Test that Repository gets db from shell when it exists
g = get_g.return_value
g.grass_vars["GISDBASE"] = self.temp_dir
g.in_grass_shell = True
m_in_grass = ModelRepository(self.temp_dir)
self.assertEqual(m_in_grass.db, self.temp_dir)
# Test with no specified dir
m_in_grass = ModelRepository()
self.assertEqual(m_in_grass.db, self.temp_dir)
g.in_grass_shell = False
# Test with dir missing
self.assertRaises(OSError,ModelRepository,'invalid/dir')
# Try to add a model from one repository to the empty one
try: self.remove_mock_location(self.temp_dir)
except OSError, e:
stre = str(e)
if 'No such file' not in stre and \
'system cannot find the path specified' not in stre:
raise e
def test_add_lifestage_model(self, get_g, m_ls):
self.make_grass_mock(get_g.return_value)
# Assume no appropriate files in tmp
c = config.get_config()
m = ModelRepository(self.temp_dir)
m2 = ModelRepository()
a_file = m2.get_models()['lifestage_test']
self.create_mock_location(self.temp_dir)
self.assertEqual(len(m.get_models()), 0)
# add location to model, save as new
dm = DispersalModel(a_file)
dm.set_location('grass_location')
temp_model_fn = os.path.join(os.path.dirname(a_file),
"with_location_model.xml")
dm.save_model(filename=temp_model_fn)
get_g.return_value.get_range.return_value = ['xxxxxx10'] * 10
get_g.return_value.raster_value_freq.return_value = [[1], [2], [3]]
m_ls.return_value = {}
# and then try to add
m.add_model(temp_model_fn)
self.assertEqual(len(m.get_models()), 1)
# more tests about lifestage resources?
self.remove_mock_location(self.temp_dir)
def test_run_command_once(self, po_mock):
# expect return value from function of 0
po_mock.return_value.close.return_value = 0
model_fn = self.repo.get_models()['lifestage_test']
model = DispersalModel(model_fn, setup=False)
i = model.get_instances()[0]
ac = AnalysisCommand("test")
# test normal
ac.times = [1990, 1991, 1992]
t_strings = [str(x) for x in ac.times]
maps = ['map1', 'map2', 'map3']
maps = dict(zip(t_strings, maps))
return_val = ac.run_command_once(1990, maps, "testcommand")
self.assertEqual(return_val, 0)
# test with map_name replacements, but too early
self.assertRaises(IndexError, ac.run_command_once, 1991, maps,
"testcommand %1 %2")
# test with map_name replacements
return_val = ac.run_command_once(1991, maps, "testcommand %1")
self.assertEqual(po_mock.call_args[0][0], "testcommand map1")
return_val = ac.run_command_once(1990, maps, "testcommand %0")
self.assertEqual(po_mock.call_args[0][0], "testcommand map1")
return_val = ac.run_command_once(1992, maps, "testcommand %0")
self.assertEqual(po_mock.call_args[0][0], "testcommand map3")
mdig_config = config.get_config()
mdig_config.analysis_print_time = True
self.__do_run_command_once(model, ac, maps)
def write_stats_to_file(self, stats, fn):
c = config.get_config()
if os.path.isfile(fn):
if c.overwrite_flag:
os.remove(fn)
else:
raise Exception("File %s already exists, -o to overwrite" % fn)
expected = [
'n', 'area', 'null_cells', 'cells', 'min', 'max', 'range', 'mean',
'mean_of_abs', 'stddev', 'variance', 'coeff_var', 'sum'
]
f = open(fn, 'w')
f.write('time,' + ','.join(expected))
f.write('\r\n')
times = stats.keys()
times.sort()
for t in times:
f.write(str(t))
for stat_name in expected:
if stat_name in stats[t]:
f.write(',%f' % stats[t][stat_name])
else:
# value missing
f.write(',')
f.write('\r\n')
f.close()
def test_add_analysis_result(self, m_shutil, m_remove, m_path, m_get_g):
m_path.basename.return_value = 'test'
m_get_g.return_value.grass_vars = {'GISDBASE': self.gisdb}
# check with no strategy
i = self.m_strategy.get_instances()[0]
analysis_cmd = Mock()
analysis_cmd.cmd_string = 'wibble'
analysis_cmd.output_fn = 'wibble'
# Test with file existing
i.add_analysis_result('all', analysis_cmd)
# Test with overwrite
config.get_config().overwrite_flag = True
i.add_analysis_result('all', analysis_cmd)
# run again to check parsing existing lifestage analysis results
i.add_analysis_result('all', analysis_cmd)
config.get_config().overwrite_flag = False
def act_on_options(self, options):
super(ROCAction, self).act_on_options(self, options)
config.get_config().overwrite_flag = self.options.overwrite_flag
if self.options.lifestage is None:
self.options.lifestage = "all"
if self.options.sites_vector is None:
self.log.error("No sites vector provided.")
sys.exit(mdig.mdig_exit_codes["cmdline_error"])
if self.options.area_mask is None:
self.log.error("No area mask specified.")
sys.exit(mdig.mdig_exit_codes["cmdline_error"])
if self.options.output_dir is not None:
if not os.path.isdir(options.output_dir):
self.log.info(
"Directory %s doesn't exist, attemping to" + " create\n",
options.output_dir)
utils.make_path(options.output_dir)
else:
self.options.output_dir = "."
if self.options.model_tags is not None:
tags = self.options.model_tags.split(",")
if len(tags) != len(self.model_names):
self.log.error("Number of tags given not the same as number "+\
" of models.")
sys.exit(mdig.mdig_exit_codes["cmdline_error"])
self.options.model_tags = tags
def do_migration(args):
help_str = "Use 'mdig.py migrate old_repo_dir grassdb' to manually migrate a separate mdig/GISDBASE repository"
help_str += "or 'mdig.py migrate repo_dir' to manually migrate an already merged repository"
if len(args) == 1 and args[0] in ['-h', '--help']:
print help_str
sys.exit(0)
if len(args) == 0:
config.MDiGConfig.migration_is_allowed = True
mdig_config = config.get_config()
if not mdig_config.migration_occurred:
print "Nothing to migrate within mdig.conf"
print help_str
elif len(args) == 1:
import mdig.migrate
print "Migrating repository %s" % args[0]
mdig.migrate.migrate_repository(args[0])
elif len(args) == 2:
import mdig.migrate
print "Old repository: %s" % args[0]
print "GRASSDB destination: %s" % args[1]
mdig.migrate.migrate_old_repository(args[0], args[1])
else:
print "Syntax error, just use 'mdig.py migrate <grassdb>'; or"
print "Use 'mdig.py migrate <grassdb>' to manually " + \
"migrate a repository."
print "Use 'mdig.py migrate <old_repo_dir> <grassdb>' to manually " + \
"migrate an old style repository."
sys.exit(0)
def test_run_command_once(self,po_mock):
# expect return value from function of 0
po_mock.return_value.close.return_value = 0
model_fn = self.repo.get_models()['lifestage_test']
model = DispersalModel(model_fn,setup=False)
i = model.get_instances()[0]
ac = AnalysisCommand("test")
# test normal
ac.times=[1990,1991,1992]
t_strings = [str(x) for x in ac.times]
maps = ['map1','map2','map3']
maps = dict(zip(t_strings, maps))
return_val = ac.run_command_once(1990,maps,"testcommand")
self.assertEqual(return_val, 0)
# test with map_name replacements, but too early
self.assertRaises(IndexError,ac.run_command_once, 1991,maps,"testcommand %1 %2")
# test with map_name replacements
return_val = ac.run_command_once(1991,maps,"testcommand %1")
self.assertEqual(po_mock.call_args[0][0], "testcommand map1")
return_val = ac.run_command_once(1990,maps,"testcommand %0")
self.assertEqual(po_mock.call_args[0][0], "testcommand map1")
return_val = ac.run_command_once(1992,maps,"testcommand %0")
self.assertEqual(po_mock.call_args[0][0], "testcommand map3")
mdig_config = config.get_config()
mdig_config.analysis_print_time = True
self.__do_run_command_once(model,ac,maps)
def act_on_options(self,options):
super(ROCAction, self).act_on_options(self, options)
config.get_config().overwrite_flag = self.options.overwrite_flag
if self.options.lifestage is None:
self.options.lifestage = "all"
if self.options.sites_vector is None:
self.log.error("No sites vector provided.")
sys.exit(mdig.mdig_exit_codes["cmdline_error"])
if self.options.area_mask is None:
self.log.error("No area mask specified.")
sys.exit(mdig.mdig_exit_codes["cmdline_error"])
if self.options.output_dir is not None:
if not os.path.isdir(options.output_dir):
self.log.info("Directory %s doesn't exist, attemping to" +
" create\n",options.output_dir)
utils.make_path(options.output_dir)
else:
self.options.output_dir = "."
if self.options.model_tags is not None:
tags = self.options.model_tags.split(",")
if len(tags) != len(self.model_names):
self.log.error("Number of tags given not the same as number "+\
" of models.")
sys.exit(mdig.mdig_exit_codes["cmdline_error"])
self.options.model_tags = tags
def empty_repository_test(self, get_g):
self.make_grass_mock(get_g.return_value)
# Assume no appropriate files in tmp
c = config.get_config()
m = ModelRepository(self.temp_dir)
self.assertEqual(len(m.get_models()), 0)
# Test that Repository gets db from shell when it exists
g = get_g.return_value
g.grass_vars["GISDBASE"] = self.temp_dir
g.in_grass_shell = True
m_in_grass = ModelRepository(self.temp_dir)
self.assertEqual(m_in_grass.db, self.temp_dir)
# Test with no specified dir
m_in_grass = ModelRepository()
self.assertEqual(m_in_grass.db, self.temp_dir)
g.in_grass_shell = False
# Test with dir missing
self.assertRaises(OSError, ModelRepository, 'invalid/dir')
# Try to add a model from one repository to the empty one
try:
self.remove_mock_location(self.temp_dir)
except OSError, e:
stre = str(e)
if 'No such file' not in stre and \
'system cannot find the path specified' not in stre:
raise e
def test_get_output_filename_base(self):
ac = AnalysisCommand("test %1")
mdig_config = config.get_config()
spam = "woooo"
old_base = mdig_config.analysis_filename_base
mdig_config.analysis_filename_base = spam
self.assertTrue(spam in ac.get_output_filename_base())
mdig_config.analysis_filename_base = old_base
def test_get_output_filename_base(self):
ac = AnalysisCommand("test %1")
mdig_config = config.get_config()
spam = "woooo"
old_base = mdig_config.analysis_filename_base
mdig_config.analysis_filename_base = spam
self.assertTrue(spam in ac.get_output_filename_base())
mdig_config.analysis_filename_base = old_base
def parse_options(self,argv):
(self.options, args) = self.parser.parse_args(argv[1:])
if len(args) == 1:
self.repo_dir = args[0]
self.log.debug("Repo dir specified is " + self.repo_dir)
else:
c = config.get_config()
print "Current repository: " + c['GRASS']['GISDBASE']
sys.exit(0)
def parse_options(self, argv):
(self.options, args) = self.parser.parse_args(argv[1:])
if len(args) == 1:
self.repo_dir = args[0]
self.log.debug("Repo dir specified is " + self.repo_dir)
else:
c = config.get_config()
print "Current repository: " + c['GRASS']['GISDBASE']
sys.exit(0)
def add_model(self, model_fn):
if not os.path.isfile(model_fn):
raise RepositoryException("Model file %s is not a file." %
model_fn)
g = grass.get_g()
c = config.get_config()
dm = model.DispersalModel(model_fn, setup=False)
loc = dm.get_location()
if loc == None:
raise RepositoryException(
"Model doesn't define GIS Location for simulation")
if not os.path.isdir(os.path.join(self.db, loc, "PERMANENT")):
raise RepositoryException(
"Model defines a GIS Location %s that doesn't exist in %s" %
(loc, self.db))
models = self.get_models()
if dm.get_name() in models:
if c.overwrite_flag:
# remove instance mapsets and replace model.xml
self.log.info("Remove instances of old version of model %s" %
dm.get_name())
dm_old = model.DispersalModel(models[dm.get_name()],
setup=False)
dm_old.hard_reset()
del dm_old
else:
raise RepositoryException("The model '" + dm.get_name() + \
"' already exists in the repository. Replace with -o.")
g.change_mapset("PERMANENT", loc)
model_mapset = dm.get_mapset()
if not g.check_mapset(dm.get_name()):
# Create model mapset
self.log.info("Creating mapset for model %s" % model_mapset)
if not g.change_mapset(dm.get_name(), loc, True):
raise RepositoryException("Couldn't create mapset %s." %
model_mapset)
self.log.info("Created mapset for model " + dm.get_name())
else:
# Use existing mapset which matches model name
self.log.warning(
"Using existing mapset with same name as model %s" %
model_mapset)
if not g.change_mapset(dm.get_name(), loc):
raise RepositoryException("Couldn't change into mapset %s." %
model_mapset)
# create mdig dir in mapset
try:
dest_dir = g.create_mdig_subdir(model_mapset, c.overwrite_flag)
except OSError, e:
g.remove_mapset(model_mapset, force=True)
raise RepositoryException("Error creating mdig dir in mapset. %s" %
str(e))
def make_grass_mock(self, g):
g.grass_vars = {}
g.in_grass_shell = False
g.check_mapset.return_value = False
g.change_mapset.return_value = True
g.create_mdig_subdir.return_value = os.path.join( \
self.temp_dir,'grass_location/variables/mdig')
c = config.get_config()
gisdb = c['GRASS']['GISDBASE']
g.grass_vars = {"GISDBASE": gisdb}
def make_grass_mock(self,g):
g.grass_vars = {}
g.in_grass_shell = False
g.check_mapset.return_value = False
g.change_mapset.return_value = True
g.create_mdig_subdir.return_value = os.path.join( \
self.temp_dir,'grass_location/variables/mdig')
c = config.get_config()
gisdb = c['GRASS']['GISDBASE']
g.grass_vars = {"GISDBASE": gisdb}
def act_on_options(self,options):
super(AnalysisAction, self).act_on_options(options)
c = config.get_config()
c.analysis_add_to_xml = self.options.analysis_add_to_xml
c.analysis_filename_base = self.options.analysis_filename_base
c.analysis_print_time = self.options.analysis_print_time
c.overwrite_flag = self.options.overwrite_flag
# If only a probability envelope is to be created then don't prompt
# for command
if (not self.options.prob_envelope_only and
self.options.analysis_cmd_file is None):
self.analysis_command = self._get_analysis_command(self.options.analysis_lifestage)
def act_on_options(self, options):
super(RunAction, self).act_on_options(options)
c = config.get_config()
if options.output_dir is not None:
if not os.path.isdir(options.output_dir):
self.log.info("Directory %s doesn't exist, attemping to" +
" create\n",options.output_dir)
c.output_dir = \
utils.make_path(options.output_dir)
else:
c.output_dir = options.output_dir
c.overwrite_flag = self.options.overwrite_flag
c.remove_null = self.options.remove_null
def remove_other_test(self,m_in,get_g):
self.make_grass_mock(get_g.return_value)
# Assume no appropriate files in tmp
c = config.get_config()
m = ModelRepository(self.temp_dir)
# Try to add a model from one repository to the empty one
try: self.remove_mock_location(self.temp_dir)
except OSError, e:
stre = str(e)
if 'No such file' not in stre and \
'system cannot find the path specified' not in stre:
raise e
def remove_other_test(self, m_in, get_g):
self.make_grass_mock(get_g.return_value)
# Assume no appropriate files in tmp
c = config.get_config()
m = ModelRepository(self.temp_dir)
# Try to add a model from one repository to the empty one
try:
self.remove_mock_location(self.temp_dir)
except OSError, e:
stre = str(e)
if 'No such file' not in stre and \
'system cannot find the path specified' not in stre:
raise e
def act_on_options(self, options):
super(AnalysisAction, self).act_on_options(options)
c = config.get_config()
c.analysis_add_to_xml = self.options.analysis_add_to_xml
c.analysis_filename_base = self.options.analysis_filename_base
c.analysis_print_time = self.options.analysis_print_time
c.overwrite_flag = self.options.overwrite_flag
# If only a probability envelope is to be created then don't prompt
# for command
if (not self.options.prob_envelope_only
and self.options.analysis_cmd_file is None):
self.analysis_command = self._get_analysis_command(
self.options.analysis_lifestage)
def __init__(self, gisdbase=None):
self.log = logging.getLogger("mdig.repos")
c = config.get_config()
# Model repository is now a part of a GRASS db directory
g = grass.get_g()
if gisdbase is None:
if g.in_grass_shell:
self.db = g.grass_vars["GISDBASE"]
else:
self.db = c["GRASS"]["GISDBASE"]
else:
if not os.path.isdir(gisdbase):
raise OSError("Bad GISDBASE")
self.db = gisdbase
self.log.info("Using GRASS DB directory" + self.db)
def __init__(self, gisdbase=None):
self.log = logging.getLogger("mdig.repos")
c = config.get_config()
# Model repository is now a part of a GRASS db directory
g = grass.get_g()
if gisdbase is None:
if g.in_grass_shell:
self.db = g.grass_vars["GISDBASE"]
else:
self.db = c["GRASS"]["GISDBASE"]
else:
if not os.path.isdir(gisdbase):
raise OSError("Bad GISDBASE")
self.db = gisdbase
self.log.info("Using GRASS DB directory" + self.db)
def act_on_options(self, options):
Action.act_on_options(self, options)
c = config.get_config()
c.overwrite_flag = self.options.overwrite_flag
if self.options.output_lifestage is None:
self.options.output_lifestage = "all"
if self.options.width is None:
self.options.width = c["OUTPUT"]["output_width"]
if self.options.height is None:
self.options.height = c["OUTPUT"]["output_height"]
if self.options.background is None:
self.options.background = c["OUTPUT"]["background_map"]
if self.options.outdir is not None:
if not os.path.isdir(self.options.outdir):
sys.exit("No such output dir: %s" % self.options.outdir)
def act_on_options(self,options):
Action.act_on_options(self,options)
c = config.get_config()
c.overwrite_flag = self.options.overwrite_flag
if self.options.output_lifestage is None:
self.options.output_lifestage = "all"
if self.options.width is None:
self.options.width = c["OUTPUT"]["output_width"]
if self.options.height is None:
self.options.height = c["OUTPUT"]["output_height"]
if self.options.background is None:
self.options.background = c["OUTPUT"]["background_map"]
if self.options.outdir is not None:
if not os.path.isdir(self.options.outdir):
sys.exit("No such output dir: %s" % self.options.outdir)
def add_model(self, model_fn):
if not os.path.isfile(model_fn):
raise RepositoryException("Model file %s is not a file."%model_fn)
g = grass.get_g()
c = config.get_config()
dm = model.DispersalModel(model_fn, setup=False)
loc = dm.get_location()
if loc == None:
raise RepositoryException("Model doesn't define GIS Location for simulation")
if not os.path.isdir(os.path.join(self.db, loc, "PERMANENT")):
raise RepositoryException("Model defines a GIS Location %s that doesn't exist in %s" %
(loc, self.db))
models = self.get_models()
if dm.get_name() in models:
if c.overwrite_flag:
# remove instance mapsets and replace model.xml
self.log.info("Remove instances of old version of model %s" % dm.get_name())
dm_old = model.DispersalModel(models[dm.get_name()], setup=False)
dm_old.hard_reset()
del dm_old
else:
raise RepositoryException("The model '" + dm.get_name() + \
"' already exists in the repository. Replace with -o.")
g.change_mapset("PERMANENT",loc)
model_mapset = dm.get_mapset()
if not g.check_mapset(dm.get_name()):
# Create model mapset
self.log.info("Creating mapset for model %s" % model_mapset)
if not g.change_mapset(dm.get_name(), loc, True):
raise RepositoryException("Couldn't create mapset %s." % model_mapset)
self.log.info("Created mapset for model " + dm.get_name())
else:
# Use existing mapset which matches model name
self.log.warning("Using existing mapset with same name as model %s"
% model_mapset)
if not g.change_mapset(dm.get_name(),loc):
raise RepositoryException("Couldn't change into mapset %s." % model_mapset)
# create mdig dir in mapset
try:
dest_dir = g.create_mdig_subdir(model_mapset, c.overwrite_flag)
except OSError, e:
g.remove_mapset(model_mapset, force=True)
raise RepositoryException("Error creating mdig dir in mapset. %s" % str(e))
def setup():
# Enabling this makes logging show lots of information on nosetests
#setup_logger()
# Setup config file to use test mdig.conf which refers to test GRASS db
config.MDiGConfig.config_file = os.path.join(os.path.dirname(__file__), 'mdig.conf')
# Find GRASS directory
c = config.get_config()
c['GRASS']['GISBASE'] = config.find_grass_base_dir()
assert c['GRASS']['GISBASE'], "Couldn't find GRASS GISBASE"
# Copy test repository
global test_dir
test_dir = tempfile.mkdtemp(prefix="mdig_test_")
end_part = os.path.split(c['GRASS']['GISDBASE'])[1]
shutil.copytree(c['GRASS']['GISDBASE'], os.path.join(test_dir,end_part))
c['GRASS']['GISDBASE'] = os.path.join(test_dir,end_part)
def do_me(self, mdig_model):
c = config.get_config()
gisdbase, location = os.path.split(os.path.abspath(self.repo_dir))
if not os.path.isdir(gisdbase):
self.log.error("GISDBASE '%s' is not a directory", gisdbase)
sys.exit(1)
if not os.path.isdir(self.repo_dir):
self.log.error("LOCATION '%s' is not a directory", location)
sys.exit(1)
if not os.path.isdir(os.path.join(self.repo_dir,'PERMANENT')):
self.log.error("LOCATION '%s' doesn't have a PERMANENT mapset." + \
" Is this path a proper location within a GRASS database?", location)
sys.exit(1)
c['GRASS']['GISDBASE'] = gisdbase
c['GRASS']['LOCATION_NAME'] = location
c.write()
print "Set repository to: " + c['GRASS']['GISDBASE']
print "Set default location to: " + c['GRASS']['LOCATION_NAME']
def replicate_update(self, rep, t):
g = grass.get_g()
g.set_output()
g.clear_monitor()
bm=config.get_config()['OUTPUT']['background_map']
if bm is not None:
try:
g.paint_map(bm)
except grass.GRASSCommandException:
pass
layer_index = 0
for l in rep.temp_map_names:
g.paint_map(rep.temp_map_names[l][0], layer=layer_index)
layer_index += 1
g.close_output()
def test_init_output_file(self,rm_mock,isfile_mock):
isfile_mock.return_value = True
mdig_config = config.get_config()
mdig_config.overwrite_flag = False
model_fn = self.repo.get_models()['lifestage_test']
model = DispersalModel(model_fn,setup=False)
i = model.get_instances()[0]
ac = AnalysisCommand("test")
self.assertRaises(mdig.OutputFileExistsException, ac.init_output_file, i)
mdig_config.overwrite_flag = True
output_file = ac.init_output_file(i)
self.assertEqual(output_file.index("lifestage_test_region_a_i0"), 0)
# test with rep
isfile_mock.return_value = False
output_file = ac.init_output_file(i, i.replicates[0])
print output_file
self.assertEqual(output_file.index("lifestage_test_region_a_i0_rep_0"), 0)
def do_me(self, mdig_model):
c = config.get_config()
gisdbase, location = os.path.split(os.path.abspath(self.repo_dir))
if not os.path.isdir(gisdbase):
self.log.error("GISDBASE '%s' is not a directory", gisdbase)
sys.exit(1)
if not os.path.isdir(self.repo_dir):
self.log.error("LOCATION '%s' is not a directory", location)
sys.exit(1)
if not os.path.isdir(os.path.join(self.repo_dir, 'PERMANENT')):
self.log.error("LOCATION '%s' doesn't have a PERMANENT mapset." + \
" Is this path a proper location within a GRASS database?", location)
sys.exit(1)
c['GRASS']['GISDBASE'] = gisdbase
c['GRASS']['LOCATION_NAME'] = location
c.write()
print "Set repository to: " + c['GRASS']['GISDBASE']
print "Set default location to: " + c['GRASS']['LOCATION_NAME']
def replicate_update(self, rep, t):
g = grass.get_g()
g.set_output()
g.clear_monitor()
bm = config.get_config()['OUTPUT']['background_map']
if bm is not None:
try:
g.paint_map(bm)
except grass.GRASSCommandException:
pass
layer_index = 0
for l in rep.temp_map_names:
g.paint_map(rep.temp_map_names[l][0], layer=layer_index)
layer_index += 1
g.close_output()
def act_on_options(self, options):
if options.output_level == "debug":
logging.getLogger("mdig").handlers[0].setLevel(logging.DEBUG)
self.log.debug("Debug messages enabled.")
elif options.output_level == "verbose":
self.output_level = "verbose"
logging.getLogger("mdig").handlers[0].setLevel(logging.INFO)
self.log.info("Verbose messages enabled.")
elif options.output_level == "quiet":
self.output_level = "quiet"
logging.getLogger("mdig").handlers[0].setLevel(logging.ERROR)
# Make the verbosity level globally available through the config object
config.get_config().output_level = self.output_level
if options.repository is not None:
self.repository = options.repository
self.log.debug("Repository location manually set to " + options.repository)
if options.location is not None:
self.location = options.location
self.log.debug("GRASS geo location manually set to " + options.location)
def main(argv):
global simulations
# Do a migration of model repository data
if len(argv) > 0 and argv[0] == 'migrate':
do_migration(argv[1:])
# Otherwise start up normally
# set up logger
logger, loghandler = setup_logger()
# get config
mdig_config = config.get_config()
# set ansi logging if enabled
ansi_bool_string = mdig_config["LOGGING"]["ansi"]
if ansi_bool_string == "true":
set_log_formatter(loghandler, True)
the_action = process_options(argv)
# We can't rely on logging before this point as process_options
# determines whether to enable debug level messages
logging.getLogger("mdig.config").debug("Using MDIG config file " +
mdig_config.cf_full_path)
signal.signal(signal.SIGINT, exit_catcher)
# Load model repository
if the_action.repository is not None:
mdig_config["GRASS"]["GISDBASE"] = the_action.repository
if the_action.location is not None:
mdig_config["GRASS"]["LOCATION_NAME"] = the_action.location
try:
if the_action.init_repository:
mdig.repository = modelrepository.ModelRepository()
models = mdig.repository.get_models()
if the_action.init_grass:
# Check for grass environment and set up interface
grass.get_g()
except grass.EnvironmentException, e:
logger.error("Failed to initialize due to environment error: %s" %
str(e))
logger.error("Perhaps check your config file? (%s)" %
mdig_config.cf_full_path)
sys.exit(1)
def setup():
# Enabling this makes logging show lots of information on nosetests
#setup_logger()
# Setup config file to use test mdig.conf which refers to test GRASS db
config.MDiGConfig.config_file = os.path.join(os.path.dirname(__file__),
'mdig.conf')
# Find GRASS directory
c = config.get_config()
c['GRASS']['GISBASE'] = config.find_grass_base_dir()
assert c['GRASS']['GISBASE'], "Couldn't find GRASS GISBASE"
# Copy test repository
global test_dir
test_dir = tempfile.mkdtemp(prefix="mdig_test_")
end_part = os.path.split(c['GRASS']['GISDBASE'])[1]
shutil.copytree(c['GRASS']['GISDBASE'], os.path.join(test_dir, end_part))
c['GRASS']['GISDBASE'] = os.path.join(test_dir, end_part)
def test_init_output_file(self, rm_mock, isfile_mock):
isfile_mock.return_value = True
mdig_config = config.get_config()
mdig_config.overwrite_flag = False
model_fn = self.repo.get_models()['lifestage_test']
model = DispersalModel(model_fn, setup=False)
i = model.get_instances()[0]
ac = AnalysisCommand("test")
self.assertRaises(mdig.OutputFileExistsException, ac.init_output_file,
i)
mdig_config.overwrite_flag = True
output_file = ac.init_output_file(i)
self.assertEqual(output_file.index("lifestage_test_region_a_i0"), 0)
# test with rep
isfile_mock.return_value = False
output_file = ac.init_output_file(i, i.replicates[0])
print output_file
self.assertEqual(output_file.index("lifestage_test_region_a_i0_rep_0"),
0)
def setUp(self):
mdig.repository = self.repo = ModelRepository()
#logging.getLogger('mdig').setLevel(logging.CRITICAL)
models = mdig.repository.get_models()
fn = models['lifestage_test']
self.m_lifestage = DispersalModel(fn)
# Model initialise with management strategy
fn = models['management_area_combine']
self.m_strategy = DispersalModel(fn)
# Model initialise with variables
fn = models['variables']
self.m_variables = DispersalModel(fn)
#logging.getLogger('mdig').setLevel(logging.WARNING)
fn = models['variables_complete']
self.m_variables_complete = DispersalModel(fn)
c = config.get_config()
self.gisdb = c['GRASS']['GISDBASE']
def test_lifestage_model_missing_files(self,get_g):
self.make_grass_mock(get_g.return_value)
# Assume no appropriate files in tmp
c = config.get_config()
m = ModelRepository(self.temp_dir)
m2 = ModelRepository()
a_file = m2.get_models()['lifestage_test']
self.create_mock_location(self.temp_dir)
self.assertEqual(len(m.get_models()), 0)
# add location to model, save as new
dm = DispersalModel(a_file)
dm.set_location('grass_location')
temp_model_fn = os.path.join(self.temp_dir,"with_location_model.xml")
dm.save_model(filename=temp_model_fn)
# and then try to add
self.assertRaises(mdig.modelrepository.RepositoryException,m.add_model,temp_model_fn)
self.assertEqual(len(m.get_models()), 0)
# more tests about lifestage resources?
self.remove_mock_location(self.temp_dir)
def setUp(self):
mdig.repository = self.repo = ModelRepository()
#logging.getLogger('mdig').setLevel(logging.CRITICAL)
models = mdig.repository.get_models()
fn = models['lifestage_test']
self.m_lifestage = DispersalModel(fn)
# Model initialise with management strategy
fn = models['management_area_combine']
self.m_strategy = DispersalModel(fn)
# Model initialise with variables
fn = models['variables']
self.m_variables = DispersalModel(fn)
#logging.getLogger('mdig').setLevel(logging.WARNING)
fn = models['variables_complete']
self.m_variables_complete = DispersalModel(fn)
c = config.get_config()
self.gisdb = c['GRASS']['GISDBASE']
def act_on_options(self, options):
if options.output_level == "debug":
logging.getLogger("mdig").handlers[0].setLevel(logging.DEBUG)
self.log.debug("Debug messages enabled.")
elif options.output_level == "verbose":
self.output_level = "verbose"
logging.getLogger("mdig").handlers[0].setLevel(logging.INFO)
self.log.info("Verbose messages enabled.")
elif options.output_level == "quiet":
self.output_level = "quiet"
logging.getLogger("mdig").handlers[0].setLevel(logging.ERROR)
# Make the verbosity level globally available through the config object
config.get_config().output_level = self.output_level
if options.repository is not None:
self.repository = options.repository
self.log.debug("Repository location manually set to " +
options.repository)
if options.location is not None:
self.location = options.location
self.log.debug("GRASS geo location manually set to " +
options.location)
def process_options(argv):
global logger
mdig_config = config.get_config()
# Remove action keyword first
action_keyword = None
if len(argv) >= 1:
action_keyword = argv[0]
the_action = actions.mdig_actions.get(action_keyword, None)
if the_action is not None:
the_action = the_action()
the_action.parse_options(argv)
else:
if action_keyword != "--help" and \
action_keyword != "-h" and \
action_keyword != "help":
print "Unknown action '%s'" % action_keyword
usage(mdig_config["GRASS"]["GISDBASE"])
sys.exit(mdig.mdig_exit_codes["ok"])
return the_action
def test_lifestage_model_missing_files(self, get_g):
self.make_grass_mock(get_g.return_value)
# Assume no appropriate files in tmp
c = config.get_config()
m = ModelRepository(self.temp_dir)
m2 = ModelRepository()
a_file = m2.get_models()['lifestage_test']
self.create_mock_location(self.temp_dir)
self.assertEqual(len(m.get_models()), 0)
# add location to model, save as new
dm = DispersalModel(a_file)
dm.set_location('grass_location')
temp_model_fn = os.path.join(self.temp_dir, "with_location_model.xml")
dm.save_model(filename=temp_model_fn)
# and then try to add
self.assertRaises(mdig.modelrepository.RepositoryException,
m.add_model, temp_model_fn)
self.assertEqual(len(m.get_models()), 0)
# more tests about lifestage resources?
self.remove_mock_location(self.temp_dir)
instances = mdig_model.get_instances()
else:
# or we convert instance indices to instance objects
instances = []
all_instances = mdig_model.get_instances()
for i in self.options.instances:
try:
instances.append(all_instances[i])
except IndexError, e:
self.log.error("Bad instance index specified")
sys.exit("Bad instance index specified")
except TypeError, e:
self.log.error("Bad instance index specified")
sys.exit("Bad instance index specified")
show_output = config.get_config().output_level == "normal"
for i in instances:
try:
if show_output:
if self.options.output_map_pack:
print "Creating map pack for instance %d" % i.get_index(
)
elif output_images:
print "Creating images for instance %d" % i.get_index()
self.do_instance(i)
except InvalidLifestageException, e:
sys.exit(mdig.mdig_exit_codes["invalid_lifestage"])
except InstanceIncompleteException, e:
sys.exit(mdig.mdig_exit_codes["instance_incomplete"])
except InvalidReplicateException, e:
sys.exit(mdig.mdig_exit_codes["invalid_replicate_index"])
def act_on_options(self, options):
Action.act_on_options(self, options)
config.get_config().overwrite_flag = self.options.overwrite_flag
def test_check_complete_on_creation(self):
config.get_config()['replicate']['check_complete'] = 'true'
i = self.m_variables.get_instances()[0]
r = Replicate(node=None, instance=i)
config.get_config()['replicate']['check_complete'] = 'false'
instances = mdig_model.get_instances()
else:
# or we convert instance indices to instance objects
instances = []
all_instances = mdig_model.get_instances()
for i in self.options.instances:
try:
instances.append(all_instances[i])
except IndexError,e:
self.log.error("Bad instance index specified")
sys.exit("Bad instance index specified")
except TypeError,e:
self.log.error("Bad instance index specified")
sys.exit("Bad instance index specified")
show_output = config.get_config().output_level == "normal"
for i in instances:
try:
if show_output:
if self.options.output_map_pack:
print "Creating map pack for instance %d" % i.get_index()
elif output_images:
print "Creating images for instance %d" % i.get_index()
self.do_instance(i)
except InvalidLifestageException, e:
sys.exit(mdig.mdig_exit_codes["invalid_lifestage"])
except InstanceIncompleteException, e:
sys.exit(mdig.mdig_exit_codes["instance_incomplete"])
except InvalidReplicateException, e:
sys.exit(mdig.mdig_exit_codes["invalid_replicate_index"])
except NoOccupancyEnvelopesException, e:
def act_on_options(self,options):
Action.act_on_options(self,options)
config.get_config().overwrite_flag = self.options.overwrite_flag
def test_check_complete_on_creation(self):
config.get_config()['replicate']['check_complete']='true'
i = self.m_variables.get_instances()[0]
r = Replicate(node=None,instance=i)
config.get_config()['replicate']['check_complete']='false'
def act_on_options(self, options):
super(StatsAction, self).act_on_options(options)
c = config.get_config()
c.analysis_add_to_xml = self.options.analysis_add_to_xml
c.analysis_filename_base = self.options.analysis_filename_base
c.overwrite_flag = self.options.overwrite_flag
def act_on_options(self, options):
super(StatsAction, self).act_on_options(options)
c = config.get_config()
c.analysis_add_to_xml = self.options.analysis_add_to_xml
c.analysis_filename_base = self.options.analysis_filename_base
c.overwrite_flag = self.options.overwrite_flag