def process_map(a_map, config):
"""Creates shapefile from a map. Updates the respective db record.
The minimum/maximum values as well as the shapefile path/URL will be
captured in the output's db record.
:param a_map: :py:class:`geonode.mtapi.models.Output` instance in question
"""
logger.info("> process_map")
logger.info("map = %s" % a_map)
commands = ["%s/bin/map_transformer.py" % settings.OQ_APIAPP_DIR]
commands.append("-k")
commands.append(str(a_map.id))
commands.append("-p")
commands.append(a_map.path)
if config.get("shapefile"):
commands.append("--shapefile")
commands.append("-t")
commands.append("hazard" if a_map.output_type == "hazard_map" else "loss")
code, out, _ = view_utils.run_cmd(commands, ignore_exit_code=True)
if code == 0:
# All went well
results = extract_results(out)
if results is None:
logger.error("Failed to parse map_transformer stdout ('%s')" % out)
else:
if config.get("shapefile"):
a_map.shapefile_path, a_map.min_value, a_map.max_value = \
results
else:
_, a_map.min_value, a_map.max_value = results
a_map.save()
logger.info("< process_map")
def process_map(a_map, config):
"""Creates shapefile from a map. Updates the respective db record.
The minimum/maximum values as well as the shapefile path/URL will be
captured in the output's db record.
:param a_map: :py:class:`geonode.mtapi.models.Output` instance in question
"""
logger.info("> process_map")
logger.info("map = %s" % a_map)
commands = ["%s/bin/map_transformer.py" % settings.OQ_APIAPP_DIR]
commands.append("-k")
commands.append(str(a_map.id))
commands.append("-p")
commands.append(a_map.path)
if config.get("shapefile"):
commands.append("--shapefile")
commands.append("-t")
commands.append("hazard" if a_map.output_type == "hazard_map" else "loss")
code, out, _ = view_utils.run_cmd(commands, ignore_exit_code=True)
if code == 0:
# All went well
results = extract_results(out)
if results is None:
logger.error("Failed to parse map_transformer stdout ('%s')" % out)
else:
if config.get("shapefile"):
a_map.shapefile_path, a_map.min_value, a_map.max_value = \
results
else:
_, a_map.min_value, a_map.max_value = results
a_map.save()
logger.info("< process_map")
def run_engine(job, utc_provider=datetime.datetime.utcnow):
"""Run the OpenQuake engine and wait for it to terminate.
:param job: the :py:class:`geonode.mtapi.models.OqJob` instance in question
:param utc_provider: the function that will be used to obtain an UTC time
stamp. This dependency is injected to facilitate testing.
:returns: a triple (exit code, stdout, stderr) with engine's execution
outcome
"""
logger.info("> run_engine")
cmds = [os.path.join(settings.OQ_ENGINE_DIR, "bin/openquake")]
cmds.append("--config_file")
cmds.append(os.path.join(job.path, "config.gem"))
logger.info("cmds: %s" % cmds)
start_time = utc_provider()
code, out, err = view_utils.run_cmd(cmds, ignore_exit_code=True)
end_time = utc_provider()
duration = end_time - start_time
job.duration = duration.seconds
job.save()
logger.info("code: '%s'" % code)
logger.info("out: '%s'" % out)
if code != 0:
logger.error(err)
logger.info("< run_engine")
return (code, out, err)
def test_run_cmd_with_errors(self):
"""
Invoke a command with errors.
This is handled in a slightly diferent way depending on the platform.
In Linux, the command we will test is considered a 'misuse of shell
built-ins' resulting in a exit code 2, according to this source:
http://tldp.org/LDP/abs/html/exitcodes.html
However, OSX considers this be a general error and returns an exit code
of 1. Also, the actual error message is slightly different.
"""
if sys.platform == "darwin":
expected_msg = (
"['ls', '-AF', '/this/does/not/exist'] terminated with "
"exit code: 1\nls: /this/does/not/exist: No "
"such file or directory\n")
else:
expected_msg = (
"['ls', '-AF', '/this/does/not/exist'] terminated with "
"exit code: 2\nls: cannot access /this/does/not/exist: No "
"such file or directory\n")
try:
code, out, err = run_cmd(["ls", "-AF", "/this/does/not/exist"])
except Exception, e:
self.assertEqual(expected_msg, e.args[0])
def test_run_cmd_with_errors(self):
"""
Invoke a command with errors.
This is handled in a slightly diferent way depending on the platform.
In Linux, the command we will test is considered a 'misuse of shell
built-ins' resulting in a exit code 2, according to this source:
http://tldp.org/LDP/abs/html/exitcodes.html
However, OSX considers this be a general error and returns an exit code
of 1. Also, the actual error message is slightly different.
"""
if sys.platform == "darwin":
expected_msg = (
"['ls', '-AF', '/this/does/not/exist'] terminated with "
"exit code: 1\nls: /this/does/not/exist: No "
"such file or directory\n")
else:
expected_msg = (
"['ls', '-AF', '/this/does/not/exist'] terminated with "
"exit code: 2\nls: cannot access /this/does/not/exist: No "
"such file or directory\n")
try:
code, out, err = run_cmd(["ls", "-AF", "/this/does/not/exist"])
except Exception, e:
self.assertEqual(expected_msg, e.args[0])
def test_run_cmd_with_errors_and_ignore_exit_code(self):
"""Invoke a command with errors but ignore the exit code."""
if sys.platform == "darwin":
expected_code = 1
expected_msg = \
"ls: /this/does/not/exist: No such file or directory\n"
else:
expected_code = 2
expected_msg = ("ls: cannot access /this/does/not/exist: No such "
"file or directory\n")
code, out, err = run_cmd(["ls", "-AF", "/this/does/not/exist"],
ignore_exit_code=True)
self.assertEqual(expected_code, code)
self.assertEqual("", out)
self.assertEqual(expected_msg, err)
def test_run_cmd_with_errors_and_ignore_exit_code(self):
"""Invoke a command with errors but ignore the exit code."""
if sys.platform == "darwin":
expected_code = 1
expected_msg = \
"ls: /this/does/not/exist: No such file or directory\n"
else:
expected_code = 2
expected_msg = (
"ls: cannot access /this/does/not/exist: No such "
"file or directory\n")
code, out, err = run_cmd(
["ls", "-AF", "/this/does/not/exist"], ignore_exit_code=True)
self.assertEqual(expected_code, code)
self.assertEqual("", out)
self.assertEqual(expected_msg, err)
def run_engine(job):
"""Run the OpenQuake engine and wait for it to terminate.
:param job: the :py:class:`geonode.mtapi.models.OqJob` instance in question
:returns: a triple (exit code, stdout, stderr) with engine's execution
outcome
"""
logger.info("> run_engine")
cmds = [os.path.join(settings.OQ_ENGINE_DIR, "bin/openquake")]
cmds.append("--config_file")
cmds.append(os.path.join(job.path, "config.gem"))
logger.info("cmds: %s" % cmds)
code, out, err = view_utils.run_cmd(cmds, ignore_exit_code=True)
logger.info("code: '%s'" % code)
logger.info("out: '%s'" % out)
if code != 0:
logger.error(err)
logger.info("< run_engine")
return (code, out, err)
def register_shapefiles_in_location(location, datastore):
"""Register the shapefiles in the given location with the Geoserver.
:param str location: a server-side file system path.
:param str datastore: one of "<job_id>-hazardmap", "<job_id>-lossmap"
"""
logger.info("> register_shapefiles_in_location")
url = urljoin(
settings.GEOSERVER_BASE_URL,
"rest/workspaces/geonode/datastores/%s/external.shp?configure=all")
url %= datastore
command = ("curl -v -u 'admin:@dm1n' -XPUT -H 'Content-type: text/plain' "
"-d '%s' '%s'" % (urljoin('file://', location), url))
logger.info("location: '%s'" % location)
logger.info("url: '%s'" % url)
logger.info("command: %s" % command)
code, out, err = view_utils.run_cmd(
command, ignore_exit_code=True, shell=True)
logger.info("code: '%s'" % code)
logger.info("out: '%s'" % out)
logger.info("err: '%s'" % err)
logger.info("< register_shapefiles_in_location")
def register_shapefiles_in_location(location, datastore):
"""Register the shapefiles in the given location with the Geoserver.
:param str location: a server-side file system path.
:param str datastore: one of "<job_id>-hazardmap", "<job_id>-lossmap"
"""
logger.info("> register_shapefiles_in_location")
url = urljoin(
settings.GEOSERVER_BASE_URL,
"rest/workspaces/geonode/datastores/%s/external.shp?configure=all")
url %= datastore
command = ("curl -v -u 'admin:@dm1n' -XPUT -H 'Content-type: text/plain' "
"-d '%s' '%s'" % (urljoin('file://', location), url))
logger.info("location: '%s'" % location)
logger.info("url: '%s'" % url)
logger.info("command: %s" % command)
code, out, err = view_utils.run_cmd(
command, ignore_exit_code=True, shell=True)
logger.info("code: '%s'" % code)
logger.info("out: '%s'" % out)
logger.info("err: '%s'" % err)
logger.info("< register_shapefiles_in_location")
def test_run_cmd_with_success(self):
"""Invoke a command without errors."""
code, out, err = run_cmd(["echo", "-n", "Hello world!"])
self.assertEqual(0, code)
self.assertEqual("Hello world!", out)
self.assertEqual("", err)
def test_run_cmd_with_success(self):
"""Invoke a command without errors."""
code, out, err = run_cmd(["echo", "-n", "Hello world!"])
self.assertEqual(0, code)
self.assertEqual("Hello world!", out)
self.assertEqual("", err)
