def test_gmprocess():
eid = 'usb000syza'
tpath = pkg_resources.resource_filename('gmprocess', 'tests')
gmprocess = os.path.abspath(os.path.join(tpath, '..', '..',
'bin', 'gmprocess'))
dpath = os.path.join('data', 'testdata', 'knet', eid)
cfgpath = os.path.join('data',
'testdata',
'knet',
eid,
'config.yml')
knetdir = pkg_resources.resource_filename('gmprocess', dpath)
cfgfile = pkg_resources.resource_filename('gmprocess', cfgpath)
try:
tdir = tempfile.mkdtemp()
fmt = '%s %s -i %s --directory %s -c %s'
tpl = (gmprocess, tdir, eid, knetdir, cfgfile)
cmd = fmt % tpl
res, stdout, stderr = get_command_output(cmd)
assert res
except Exception:
pass
finally:
shutil.rmtree(tdir)
def test_rjb_WC94():
cmd = "progs/RjbMeanVar.py tests/config/test_Rjb_WC94.ini"
rc, so, se = get_command_output(cmd)
r1 = pd.DataFrame.from_csv("tests/data/test_Rjb_WC94_mechA_ar1p7_seis0_20_Ratios.csv", header = 6)
v1 = pd.DataFrame.from_csv("tests/data/test_Rjb_WC94_mechA_ar1p7_seis0_20_Var.csv", header = 6)
r2 = pd.DataFrame.from_csv("TestData/test_Rjb_WC94_mechA_ar1p7_seis0_20_Ratios.csv", header = 6)
v2 = pd.DataFrame.from_csv("TestData/test_Rjb_WC94_mechA_ar1p7_seis0_20_Var.csv", header = 6)
pd.util.testing.assert_frame_equal(r1, r2)
pd.util.testing.assert_frame_equal(v1, v2)
# Clean up
shutil.rmtree('TestData')
def test_rrup_WC94_R_F():
conf = 'fast_rrup_WC94_R_F.ini'
cmd = "progs/RrupMeanVar.py tests/config/%s" %conf
rc, so, se = get_command_output(cmd)
r1 = pd.DataFrame.from_csv("tests/data/test_Rrup_WC94_mechR_ar1p0_seis0_15_Ratios.csv", header = 6)
v1 = pd.DataFrame.from_csv("tests/data/test_Rrup_WC94_mechR_ar1p0_seis0_15_Var.csv", header = 6)
r2 = pd.DataFrame.from_csv("TestData/test_Rrup_WC94_mechR_ar1p0_seis0_15_Ratios.csv", header = 6)
v2 = pd.DataFrame.from_csv("TestData/test_Rrup_WC94_mechR_ar1p0_seis0_15_Var.csv", header = 6)
pd.util.testing.assert_frame_equal(r1, r2)
pd.util.testing.assert_frame_equal(v1, v2)
# Clean up
shutil.rmtree('TestData')
def test_gm2table():
tpath = pkg_resources.resource_filename('gmprocess', 'tests')
gm2table = os.path.abspath(os.path.join(tpath, '..', '..',
'bin', 'gm2table'))
dpath = os.path.join('data', 'testdata', 'knet')
knetdir = pkg_resources.resource_filename('gmprocess', dpath)
tfile = tempfile.mkstemp()[1]
cmd = '%s %s %s -f csv' % (gm2table, knetdir, tfile)
res, stdout, stderr = get_command_output(cmd)
os.remove(tfile)
tfile = tempfile.mkstemp()[1]
cmd = '%s %s %s -f xlsx' % (gm2table, knetdir, tfile)
res, stdout, stderr = get_command_output(cmd)
os.remove(tfile)
cwbdir = os.path.join(knetdir, '..', 'cwb')
lon = 21.69
lat = 24.14
tfile = tempfile.mkstemp()[1]
cmd = '%s %s %s --lat %s --lon %s' % (gm2table, cwbdir, tfile, lat, lon)
res, stdout, stderr = get_command_output(cmd)
os.remove(tfile)
def send(self):
# we can really only support sending of one file and/or one directory, so error out
# if someone has specified more than one of either.
if len(self.files) > 1:
raise ShakeMapException(
'For PDL, you may only send one file at a time.')
if len(self.directories) > 1:
raise ShakeMapException(
'For PDL, you may only send one directory at a time.')
# make sure we have all the required properties
for prop in self.__required_properties:
if prop not in list(self.properties.keys()):
raise ShakeMapException(
'"%s" property must be supplied to send via PDL')
# build pdl command line from properties
self.properties['command'] = 'send'
self.properties['status'] = 'UPDATE'
if self.files:
self.properties['file'] = self.files[0]
else:
self.properties['file'] = ''
if self.directories:
self.properties['directory'] = self.directories[0]
else:
self.properties['directory'] = ''
cmd = self.__pdlcmd
for propkey, propvalue in self.properties.items():
cmd = cmd.replace('[' + propkey.upper() + ']', propvalue)
# call PDL on the command line
retcode, stdout, stderr = get_command_output(cmd)
if not retcode:
fmt = 'Could not send product "%s" due to error "%s"'
tpl = (code, stdout + stderr)
raise ShakeMapException(fmt % tpl)
# return the number of files we just sent
nfiles = 0
if self.properties['file']:
nfiles += 1
if self.properties['directory']:
nfiles += len(os.listdir(self.properties['directory']))
return nfiles
def test_amps2xml():
homedir = os.path.dirname(
os.path.abspath(__file__)) # where is this script?
excelfile = os.path.join(homedir, '..', 'data', 'minimum_mmi.xlsx')
amps2xml = os.path.join(homedir, '..', '..', 'bin', 'amps2xml')
tmpdir = tempfile.mkdtemp()
try:
cmd = '%s foo --debug-dir=%s %s' % (amps2xml, tmpdir, excelfile)
res, stdout, stderr = get_command_output(cmd)
if not res:
raise AssertionError(
'amps2xml command %s failed with errors "%s"' % (cmd, stderr))
print(stdout)
except Exception as e:
raise (e)
finally:
shutil.rmtree(tmpdir)
def test_rrup_WC94_SS_T():
conf = 'fast_WC94_SS_T.ini'
cmd = "bin/run_ps2ff -w rrup tests/config/%s" % conf
rc, so, se = get_command_output(cmd)
r1 = pd.DataFrame.from_csv(
"tests/data/test_Rrup_WC94_mechSS_LW_seis0_15_Ratios.csv", header=6)
v1 = pd.DataFrame.from_csv(
"tests/data/test_Rrup_WC94_mechSS_LW_seis0_15_Var.csv", header=6)
r2 = pd.DataFrame.from_csv(
"TestData/Rrup_WC94_mechSS_LW_seis0_15_Ratios.csv", header=6)
v2 = pd.DataFrame.from_csv("TestData/Rrup_WC94_mechSS_LW_seis0_15_Var.csv",
header=6)
pd.util.testing.assert_frame_equal(r1, r2)
pd.util.testing.assert_frame_equal(v1, v2)
# Clean up
shutil.rmtree('TestData')
def delete(self):
for prop in self.__required_properties:
if prop not in list(self.properties.keys()):
raise ShakeMapException(
'"%s" property must be supplied to send via PDL')
# build pdl command line from properties
self.properties['status'] = 'DELETE'
self.properties['files'] = ''
self.properties['directories'] = ''
cmd = self.__pdlcmd
for propkey, propvalue in self.properties.items():
cmd = cmd.replace('[' + propkey.upper() + ']', propvalue)
retcode, stdout, stderr = get_command_output(cmd)
if not retcode:
fmt = 'Could not delete product "%s" due to error "%s"'
tpl = (code, stdout + stderr)
raise ShakeMapException(fmt % tpl)
def cancel(self, cancel_content=None):
"""Send a delete message out via PDL regarding the product in question.
Args:
cancel_content: String containing cancel message. This is NOT used
in the implementation for this class.
Returns:
Standard output from PDL DELETE command.
"""
# build pdl command line from properties
self._properties['status'] = 'DELETE'
self._properties['file'] = ''
self._properties['directory'] = ''
cmd = self._pdlcmd
prop_nuggets = []
for propkey, propvalue in self._properties.items():
if isinstance(propvalue, float):
prop_nuggets.append('--property-%s=%.4f' %
(propkey, propvalue))
elif isinstance(propvalue, int):
prop_nuggets.append('--property-%s=%i' % (propkey, propvalue))
elif isinstance(propvalue, datetime.datetime):
prop_nuggets.append(
'--property-%s=%s' %
(propkey, propvalue.strftime(DATE_TIME_FMT)[0:23]))
elif isinstance(propvalue, str):
prop_nuggets.append('--property-%s="%s"' %
(propkey, propvalue))
else:
prop_nuggets.append('--property-%s=%s' %
(propkey, str(propvalue)))
cmd = cmd.replace('[PRODUCT_PROPERTIES]', ' '.join(prop_nuggets))
retcode, stdout, stderr = get_command_output(cmd)
if not retcode:
fmt = 'Could not delete product "%s" due to error "%s"'
tpl = (retcode, stdout + stderr)
raise Exception(fmt % tpl)
return stdout
def test_parallel():
gmprocess = pkg_resources.resource_filename(
'gmprocess', os.path.join('..', 'bin', 'gmprocess'))
data_dir = pkg_resources.resource_filename(
'gmprocess', os.path.join('data', 'testdata', 'demo'))
out_dir = 'temp_dir'
try:
cmd = ('%s -o %s --assemble --directory %s -n 2' %
(gmprocess, out_dir, data_dir))
rc, so, se = get_command_output(cmd)
assert rc
except Exception as e:
print(so.decode())
print(se.decode())
raise e
finally:
shutil.rmtree(out_dir)
def test_fetchflat():
tpath = pkg_resources.resource_filename('gmprocess', 'tests')
fetchflat = os.path.abspath(
os.path.join(tpath, '..', '..', 'bin', 'fetchflat'))
dpath = os.path.join('data', 'testdata', 'geonet2')
cfgpath = os.path.join('data', 'testdata', 'geonet2', 'config.yml')
datadir = pkg_resources.resource_filename('gmprocess', dpath)
cfgfile = pkg_resources.resource_filename('gmprocess', cfgpath)
try:
tdir = tempfile.mkdtemp()
fmt = '%s %s --directory %s -c %s'
tpl = (fetchflat, tdir, datadir, cfgfile)
cmd = fmt % tpl
res, stdout, stderr = get_command_output(cmd)
assert res
except Exception:
pass
finally:
shutil.rmtree(tdir)
def sendAction(self,action,eventid):
if action not in ALLOWED_ACTIONS:
fmt = 'Action "%s" not in list of allowed actions: "%s"'
raise Exception(fmt % (action,str(ALLOWED_ACTIONS)))
pdl_cmd = self._pdlcmd.replace('[JAVA]',self._init_params['java'])
pdl_cmd = pdl_cmd.replace('[JARFILE]',self._init_params['jarfile'])
pdl_cmd = pdl_cmd.replace('[privatekey]',self._init_params['privatekey'])
pdl_cmd = pdl_cmd.replace('[configfile]',self._init_params['configfile'])
pdl_cmd = pdl_cmd.replace('[CODE]',eventid)
source,source_code = split_event(eventid)
pdl_cmd = pdl_cmd.replace('[EVENTSOURCE]',source)
pdl_cmd = pdl_cmd.replace('[EVENTSOURCECODE]',source_code)
user = getpass.getuser()
action_time = datetime.datetime.utcnow().strftime(self._date_time_fmt)
pdl_cmd = pdl_cmd.replace('[ACTION]',action)
pdl_cmd = pdl_cmd.replace('[USER]',user)
pdl_cmd = pdl_cmd.replace('[ACTION-TIME]',action_time)
res,stdout,stderr = get_command_output(pdl_cmd)
return (res,stdout,stderr)
def test_gmprocess():
eid = 'usb000syza'
tpath = pkg_resources.resource_filename('gmprocess', 'tests')
gmprocess = os.path.abspath(
os.path.join(tpath, '..', '..', 'bin', 'gmprocess'))
dpath = os.path.join('data', 'testdata', 'knet', eid)
cfgpath = os.path.join('data', 'testdata', 'knet', eid, 'config.yml')
knetdir = pkg_resources.resource_filename('gmprocess', dpath)
cfgfile = pkg_resources.resource_filename('gmprocess', cfgpath)
try:
tdir = tempfile.mkdtemp()
fmt = '%s %s -i %s --directory %s -c %s'
tpl = (gmprocess, tdir, eid, knetdir, cfgfile)
cmd = fmt % tpl
res, stdout, stderr = get_command_output(cmd)
assert res
except Exception:
pass
finally:
shutil.rmtree(tdir)
def test_sm2xml():
# TEST for KNET
homedir = os.path.dirname(os.path.abspath(
__file__)) # where is this script?
indir = os.path.join(homedir, '..', 'data', 'geonet')
sm2xml = os.path.join(homedir, '..', '..', 'bin', 'sm2xml')
eventid = 'us1000778i'
tmpdir = tempfile.mkdtemp()
try:
cmd = '%s %s %s --debug-dir=%s' % (sm2xml, eventid, indir, tmpdir)
res, stdout, stderr = get_command_output(cmd)
if not res:
raise AssertionError(
'sm2xml command %s failed with errors "%s"' % (cmd, stderr))
print(stdout)
# assert target_file.encode() in stdout
# excel, ref = read_excel(target_file.replace('xml', 'xlsx'))
except Exception as e:
raise(e)
finally:
shutil.rmtree(tmpdir)
def test_rjb_WC94():
cmd = "bin/run_ps2ff -w rjb tests/config/test_WC94.ini"
rc, so, se = get_command_output(cmd)
r1 = pd.DataFrame.from_csv(
"tests/data/test_Rjb_WC94_mechA_ar1p7_seis0_20_Ratios.csv",
header=6)
v1 = pd.DataFrame.from_csv(
"tests/data/test_Rjb_WC94_mechA_ar1p7_seis0_20_Var.csv",
header=6)
r2 = pd.DataFrame.from_csv(
"TestData/Rjb_WC94_mechA_ar1p7_seis0_20_Ratios.csv",
header=6)
v2 = pd.DataFrame.from_csv(
"TestData/Rjb_WC94_mechA_ar1p7_seis0_20_Var.csv",
header=6)
pd.util.testing.assert_frame_equal(r1, r2)
pd.util.testing.assert_frame_equal(v1, v2)
# Clean up
shutil.rmtree('TestData')
def test_eventfile():
out_dir = 'temp_dir'
conf_file = pkg_resources.resource_filename(
'gmprocess', os.path.join('data', 'testdata', 'conf_small.yml'))
eventfile = pkg_resources.resource_filename(
'gmprocess', os.path.join('data', 'testdata', 'example_eventfile.txt'))
try:
cmd = ('gmprocess2 -o %s --assemble --textfile %s --config %s' %
(out_dir, eventfile, conf_file))
rc, so, se = get_command_output(cmd)
assert rc
except Exception as e:
print(so.decode())
print(se.decode())
raise e
finally:
shutil.rmtree(out_dir)
def test_single():
cmd = "progs/RrupRjbMeanVar_SingleEvent.py tests/config/test_single.ini"
rc, so, se = get_command_output(cmd)
rjb1 = pd.DataFrame.from_csv("tests/data/Rjb_bytheta_Ratios.csv", header = 6)
vjb1 = pd.DataFrame.from_csv("tests/data/Rjb_bytheta_Var.csv", header = 6)
rjb2 = pd.DataFrame.from_csv("DataSingle/Rjb_bytheta_Ratios.csv", header = 6)
vjb2 = pd.DataFrame.from_csv("DataSingle/Rjb_bytheta_Var.csv", header = 6)
rrup1 = pd.DataFrame.from_csv("tests/data/Rrup_bytheta_Ratios.csv", header = 6)
vrup1 = pd.DataFrame.from_csv("tests/data/Rrup_bytheta_Var.csv", header = 6)
rrup2 = pd.DataFrame.from_csv("DataSingle/Rrup_bytheta_Ratios.csv", header = 6)
vrup2 = pd.DataFrame.from_csv("DataSingle/Rrup_bytheta_Var.csv", header = 6)
pd.util.testing.assert_frame_equal(rjb1, rjb2)
pd.util.testing.assert_frame_equal(vjb1, vjb2)
pd.util.testing.assert_frame_equal(rrup1, rrup2)
pd.util.testing.assert_frame_equal(vrup1, vrup2)
# Clean up
shutil.rmtree('DataSingle')
def cancel(self, cancel_content=None):
"""Send a delete message out via PDL regarding the product in question.
Args:
cancel_content: String containing cancel message. This is NOT used
in the implementation for this class.
Returns:
Standard output from PDL DELETE command.
"""
# build pdl command line from properties
self._properties["file"] = ""
self._properties["directory"] = ""
cmd = self._pdlcmd
# make this a delete status
cmd = cmd.replace("[STATUS]", "DELETE")
# fill out the required properties
cmd = self._replace_required_properties(cmd)
# fill out any files or directories we'll be sending
cmd = self._replace_files(cmd)
# fill in all the product properties
cmd = self._replace_product_properties(cmd)
# fill in all the optional properties
cmd = self._replace_optional_properties(cmd)
retcode, stdout, stderr = get_command_output(cmd)
if not retcode:
ptype = self._properties["type"]
fmt = (
f'Could not delete product "{ptype}" due to error '
f'"{stdout + stderr}"'
)
raise PDLError(fmt)
return stdout
def test_single():
cmd = "bin/run_ps2ff_single_event tests/config/test_single.ini"
rc, so, se = get_command_output(cmd)
rjb1 = pd.DataFrame.from_csv("tests/data/Rjb_bytheta_Ratios.csv", header=6)
vjb1 = pd.DataFrame.from_csv("tests/data/Rjb_bytheta_Var.csv", header=6)
rjb2 = pd.DataFrame.from_csv("DataSingle/Rjb_bytheta_Ratios.csv", header=6)
vjb2 = pd.DataFrame.from_csv("DataSingle/Rjb_bytheta_Var.csv", header=6)
rrup1 = pd.DataFrame.from_csv("tests/data/Rrup_bytheta_Ratios.csv",
header=6)
vrup1 = pd.DataFrame.from_csv("tests/data/Rrup_bytheta_Var.csv", header=6)
rrup2 = pd.DataFrame.from_csv("DataSingle/Rrup_bytheta_Ratios.csv",
header=6)
vrup2 = pd.DataFrame.from_csv("DataSingle/Rrup_bytheta_Var.csv", header=6)
pd.util.testing.assert_frame_equal(rjb1, rjb2)
pd.util.testing.assert_frame_equal(vjb1, vjb2)
pd.util.testing.assert_frame_equal(rrup1, rrup2)
pd.util.testing.assert_frame_equal(vrup1, vrup2)
# Clean up
shutil.rmtree('DataSingle')
def test_gmsetup():
out_dir = 'temp_dir'
if not os.path.exists(out_dir):
os.mkdir(out_dir)
try:
# Create config file:
fname = os.path.join(out_dir, 'test.yml')
cmd = ('gmsetup %s -f %s -e %s'
% (fname,
'Test User',
'*****@*****.**')
)
rc, so, se = get_command_output(cmd)
assert rc
except Exception as e:
print(so.decode())
print(se.decode())
raise e
finally:
shutil.rmtree(out_dir)
def main(args):
#-------------------------------------------------------------
# where should .rst files, Makefile, _build folder be written?
#-------------------------------------------------------------
API_DIR = os.path.join(os.path.expanduser('~'), '__api-doc')
shutil.rmtree(API_DIR, ignore_errors=True)
#-------------------------------------------------------------
# where should the temporary clone of the ground failure gh-pages repo live?
#-------------------------------------------------------------
CLONE_DIR = os.path.join(os.path.expanduser('~'), '__gf-doc')
shutil.rmtree(CLONE_DIR, ignore_errors=True)
#-------------------------------------------------------------
# Some additional useful directories
#-------------------------------------------------------------
REPO_DIR = os.path.dirname(os.path.abspath(__file__))
PACKAGE_DIR = os.path.join(REPO_DIR, 'groundfailure')
#-------------------------------------------------------------
# what is the package called and who are the authors
#-------------------------------------------------------------
PACKAGE = "GroundFailure"
AUTHORS = 'Kate Allstadt, Eric Thompson, Mike Hearne, Katie Biegel'
# find the make command on this system
res, stdout, stderr = get_command_output('which make')
if not res:
print('Could not find the "make" command on your system. Exiting.')
sys.exit(1)
make_cmd = stdout.decode().strip()
#-------------------------------------------------------------
# clone the repository
#-------------------------------------------------------------
if args.post:
sys.stderr.write('Cloning groundfailure gh-pages branch...\n')
if os.path.isdir(CLONE_DIR):
shutil.rmtree(CLONE_DIR)
clonecmd = 'git clone -b gh-pages https://github.com/usgs/'\
'groundfailure.git %s' % CLONE_DIR
res, stdout, stderr = get_command_output(clonecmd)
if not res:
raise Exception('Could not clone gh-pages branch.')
# Delete everything in the repository (except hidden git files)
cmd = 'rm -fr %s/*' % CLONE_DIR
res, stdout, stderr = get_command_output(cmd)
#-------------------------------------------------------------
# run the api doc command; this creates the .rst files
#-------------------------------------------------------------
sys.stderr.write('Building groundfailure API documentation (REST)...\n')
sphinx_cmd = 'sphinx-apidoc -o %s -f -e -l -d 12 -F -H %s -A "%s"'\
' %s' % (API_DIR, PACKAGE, AUTHORS, PACKAGE_DIR)
res, stdout, stderr = get_command_output(sphinx_cmd)
if not res:
raise Exception('Could not build GroundFailure API documentation'
' - error "%s".' % stderr)
#-------------------------------------------------------------
# Edit the conf.py file to include the theme.
#-------------------------------------------------------------
fname = os.path.join(API_DIR, 'conf.py')
f = open(fname, 'at')
f.write("import os\nimport sys\n")
f.write("sys.path.insert(0, os.path.abspath('%s'))\n" % (REPO_DIR))
f.write("sys.path.insert(0, os.path.abspath('.'))\n")
f.write("temp = sys.executable\n")
f.write("EXECPATH='/'.join(temp.split('/')[:-2])\n")
f.write("sys.path.append(os.path.join(os.path.expanduser('~'), EXECPATH, 'lib'))\n")
#-------------------------------------
# RTD theme
#-------------------------------------
f.write("import sphinx_rtd_theme\n")
f.write("html_theme = 'sphinx_rtd_theme'\n")
f.write("html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]\n")
f.write("html_theme_options = {\n")
f.write(" 'collapse_navigation': False,\n")
f.write("}\n")
#-------------------------------------
# Napolean extension? Supports Goggle and Numpy style docstrings, but it
# also has some side effects such as restrictions on what sections are
# allowed and it seems to suppress the [source] link to code; maybe this
# is a configurable option though.
# f.write("extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.napoleon']\n")
# This line is needed to inclue __init__ methods in documentation
f.write("autoclass_content = 'both'\n")
f.write("autodoc_member_order = 'bysource'\n")
f.write("html_show_copyright = False\n")
# f.write("extensions = extensions + [ 'sphinx.ext.autodoc', "\
# "'sphinx.ext.napoleon', 'sphinx.ext.todo' ] \n")
f.write("napoleon_include_special_with_doc = False\n")
f.write("todo_include_todos = True\n")
f.close()
#-------------------------------------------------------------
# Go to the api directory and build the html
#-------------------------------------------------------------
sys.stderr.write('Building groundfailure manual (HTML)...\n')
os.chdir(API_DIR)
res, stdout, stderr = get_command_output('%s html' % make_cmd)
if not res:
raise Exception('Could not build HTML for API documentation. - error "%s"' % stderr)
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
#-------------------------------------------------------------
# Copy the generated content to the gh-pages branch we created
# earlier
#-------------------------------------------------------------
htmldir = os.path.join(API_DIR, '_build', 'html')
if not os.path.isdir(CLONE_DIR):
os.makedirs(CLONE_DIR)
copy_tree(htmldir, CLONE_DIR)
if args.post:
#-------------------------------------------------------------
# Post to gh-pages
#-------------------------------------------------------------
# cd to directory above where html content was pushed
os.chdir(CLONE_DIR)
res, stdout, stderr = get_command_output('touch .nojekyll')
res1, stdout, stderr1 = get_command_output('git add --all')
res2, stdout, stderr2 = get_command_output(
'git commit -am"Pushing to GitHub pages"')
res3, stdout, stderr3 = get_command_output(
'git push -u origin +gh-pages')
if res1 + res2 + res3 < 3:
stderr = stderr1 + stderr2 + stderr3
print('Something bad happened when attempting to add, commit, '
'or push gh-pages content to GitHub - error "%s". Exiting.'
% stderr)
sys.exit(1)
print('You can inspect the GroundFailure API docs by looking '
'here: http://usgs.github.io/groundfailure/index.html')
else:
if not args.clean:
indexpage = os.path.join(CLONE_DIR, 'index.html')
print('GroundFailure documentation index: %s' % indexpage)
def transfer(event_dir,
version,
pdl_conf,
pdl_bin=None,
source="us",
dryrun=False,
status='UPDATE'):
"""
This is to transfer the event's 'pdl_directory' to comcat. PDL must be
installed separately, see https://usgs.github.io/pdl/ for information.
Args:
event_dir (str): File path to location of results for event.
version (int): Version number of ground-failure run.
pdl_conf (str): Path to PDL conf file.
pdl_bin (str): Path to 'ProductClient.jar'. If None it guesses that it
is installed in the user's home directory:
``~/ProductClient/ProductClient.jar``.
source (str): PDL 'source'. This is 'us' for products coming from NEIC,
and is tied to the authentication key but this only matters once
we start sending to production servers rather than dev servers, at
which point we'll need to add a configuration option for the key.
dryrun (bool): If True, the PDL command is constructed and printed but
not executed.
status (str): Status of ground-failure product being sent to comcat.
Default is "UPDATE" but can also be "WARNING" so that the product
page displays the warning banner.
Returns:
dict or str: Dictionary of pdl return code, standard out, and standard
error for dryrun=False; PDL command for dryrun=True.
"""
if pdl_bin is None:
pdl_bin = os.path.join(os.path.expanduser('~'), 'ProductClient',
'ProductClient.jar')
pdl_dir = os.path.join(event_dir, 'pdl_directory')
# Load info.json
with open(os.path.join(pdl_dir, 'info.json')) as f:
info_dict = json.load(f)
# Get some event info for pdl send command
lat = info_dict['Summary']['lat']
lon = info_dict['Summary']['lon']
dep = info_dict['Summary']['depth']
mag = info_dict['Summary']['magnitude']
time_stamp = info_dict['Summary']['time']
code = info_dict['Summary']['code']
eventsourcecode = info_dict['Summary']['code']
eventsource = info_dict['Summary']['net']
shake_version = info_dict['Summary']['shakemap_version']
xmin, xmax, ymin, ymax = info_dict['Summary']['zoom_extent']
pdl_type = 'ground-failure'
# PDL properties
# Get preferred models to extract PDL properties
lqs = info_dict['Liquefaction']
for lq in lqs:
if lq['preferred']:
lq_pref = lq
lss = info_dict['Landslides']
for ls in lss:
if ls['preferred']:
ls_pref = ls
ls_alert = '"--property-landslide-alert=%s" ' % \
ls_pref['alert']
lq_alert = '"--property-liquefaction-alert=%s" ' % \
lq_pref['alert']
lq_haz_alert = '"--property-liquefaction-hazard-alert-color=%s" ' % \
lq_pref['hazard_alert']['color']
ls_haz_alert = '"--property-landslide-hazard-alert-color=%s" ' % \
ls_pref['hazard_alert']['color']
lq_pop_alert = '"--property-liquefaction-population-alert-color=%s" ' % \
lq_pref['population_alert']['color']
ls_pop_alert = '"--property-landslide-population-alert-color=%s" ' % \
ls_pref['population_alert']['color']
lq_haz_alert_value = '"--property-liquefaction-hazard-alert-value=%s" ' % \
lq_pref['hazard_alert']['value']
ls_haz_alert_value = '"--property-landslide-hazard-alert-value=%s" ' % \
ls_pref['hazard_alert']['value']
lq_pop_alert_value = '"--property-liquefaction-population-alert-value=%s" ' % \
lq_pref['population_alert']['value']
ls_pop_alert_value = '"--property-landslide-population-alert-value=%s" ' % \
ls_pref['population_alert']['value']
lq_haz_alert_parameter = '"--property-liquefaction-hazard-alert-parameter=%s" ' % \
lq_pref['hazard_alert']['parameter']
ls_haz_alert_parameter = '"--property-landslide-hazard-alert-parameter=%s" ' % \
ls_pref['hazard_alert']['parameter']
lq_pop_alert_parameter = '"--property-liquefaction-population-alert-parameter=%s" ' % \
lq_pref['population_alert']['parameter']
ls_pop_alert_parameter = '"--property-landslide-population-alert-parameter=%s" ' % \
ls_pref['population_alert']['parameter']
lq_overlay = '"--property-liquefaction-overlay=%s" ' % lq_pref['overlay']
ls_overlay = '"--property-landslide-overlay=%s" ' % ls_pref['overlay']
lq_extent = lq_pref['extent']
ls_extent = ls_pref['extent']
# Liquefaction extent
lq_xmin = '"--property-liquefaction-minimum-longitude=%s" ' % lq_extent[0]
lq_xmax = '"--property-liquefaction-maximum-longitude=%s" ' % lq_extent[1]
lq_ymin = '"--property-liquefaction-minimum-latitude=%s" ' % lq_extent[2]
lq_ymax = '"--property-liquefaction-maximum-latitude=%s" ' % lq_extent[3]
# Landslide extent
ls_xmin = '"--property-landslide-minimum-longitude=%s" ' % ls_extent[0]
ls_xmax = '"--property-landslide-maximum-longitude=%s" ' % ls_extent[1]
ls_ymin = '"--property-landslide-minimum-latitude=%s" ' % ls_extent[2]
ls_ymax = '"--property-landslide-maximum-latitude=%s" ' % ls_extent[3]
# Product extent -- note, for now I'm just setting this to the landslide
# extent, which seems a bit pointless. But by providing the property, it
# gives us the ability to update it later if we can come up with a more
# sensible product extent.
prod_xmin = '"--property-minimum-longitude=%s" ' % xmin
prod_xmax = '"--property-maximum-longitude=%s" ' % xmax
prod_ymin = '"--property-minimum-latitude=%s" ' % ymin
prod_ymax = '"--property-maximum-latitude=%s" ' % ymax
rupt_warn = '"--property-rupture-warning=%s" ' % \
info_dict['Summary']['rupture_warning']
# Check for PDL key:
defaults = os.path.join(os.path.expanduser('~'), '.gfail_defaults')
configs = ConfigObj(defaults)
if 'key' in configs.keys():
pdl_key = configs['key']
else:
pdl_key = None
# Construct PDL command
pdl_cmd = ('java -jar %s ' % pdl_bin +
'--send --configFile=%s ' % pdl_conf + '--source=%s ' % source +
'--eventsource=%s ' % eventsource + '--code=%s ' % code +
'--status=%s ' % status +
'--eventsourcecode=%s ' % eventsourcecode +
'--version=%s ' % version + '--latitude=%s ' % lat +
'--longitude=%s ' % lon + '--magnitude=%s ' % mag +
'--depth=%s ' % dep + '--eventtime=%s ' % time_stamp +
'--type=%s ' % pdl_type + '--directory=%s ' % pdl_dir +
ls_alert + lq_alert + lq_haz_alert + ls_haz_alert +
lq_pop_alert + ls_pop_alert + lq_haz_alert_value +
ls_haz_alert_value + lq_pop_alert_value + ls_pop_alert_value +
lq_haz_alert_parameter + ls_haz_alert_parameter +
lq_pop_alert_parameter + ls_pop_alert_parameter + lq_overlay +
ls_overlay + lq_xmin + lq_xmax + lq_ymin + lq_ymax + ls_xmin +
ls_xmax + ls_ymin + ls_ymax + prod_xmin + prod_xmax +
prod_ymin + prod_ymax + rupt_warn +
'"--property-shakemap-version=%s" ' % shake_version)
if pdl_key is not None:
pdl_cmd = pdl_cmd + " --privateKey=%s " % pdl_key
if not dryrun:
rc, so, se = get_command_output(pdl_cmd)
print('PDL return code: %s ' % rc)
print('PDL standard output:\n%s ' % so)
print('PDL standard error:\n%s ' % se)
return {'rc': rc, 'so': so, 'se': se}
else:
print(pdl_cmd)
return {'rc': True, 'so': b'', 'se': b''}
def main(args):
"""
Generate API docs.
Args:
args: Output of argparse. Currently only holds the verbose flag.
Returns:
Nothing. Function will exit upon success or failure.
"""
verbose = args.verbose
#-------------------------------------------------------------
# Some useful directories
#-------------------------------------------------------------
REPO_DIR = os.path.dirname(os.path.abspath(__file__))
DOCS_DIR = os.path.join(REPO_DIR, 'docs')
API_DIR = os.path.join(REPO_DIR, 'doc_source')
PACKAGE_DIR = os.path.join(REPO_DIR, 'ps2ff')
#-------------------------------------------------------------
# what is the package called and who are the authors
#-------------------------------------------------------------
PACKAGE = "ps2ff"
AUTHORS = 'Eric Thompson, Bruce Worden'
verstr = '1.1'
#-------------------------------------------------------------
# run the api doc command; this creates the .rst files
#-------------------------------------------------------------
sys.stderr.write('Building ps2ff API documentation (REST)...\n')
sphinx_cmd = 'sphinx-apidoc -o %s -f -e -l -M -d 12 -H %s -A "%s"'\
' -V %s %s' % (API_DIR, PACKAGE, AUTHORS, verstr,
PACKAGE_DIR)
res, stdout, stderr = get_command_output(sphinx_cmd)
if not res:
raise Exception('Could not build ps2ff API documentation'
' - error "%s".' % stderr)
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
#--------------------------------------------
# try to clean up some of the excess labeling
#--------------------------------------------
clean_cmd = "sed -i '' -e 's/ module//g' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -i '' -e 's/ package//g' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -i '' -e '/Subpackages/d' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -i '' -e '/-.*-/d' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
#-------------------------------------------------------------
# Build the html
#-------------------------------------------------------------
sys.stderr.write('Building ps2ff pages (HTML)...\n')
res, stdout, stderr = get_command_output(
'sphinx-build -a -E doc_source docs')
if not res:
raise Exception('Could not build HTML for API documentation. - '
'error "%s"' % stderr.decode())
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
pathlib.Path(os.path.join(DOCS_DIR, '.nojekyll')).touch(exist_ok=True)
def create_twopager(pdata, hazinfo, version_dir):
"""
:param pdata:
PagerData object.
:param hazinfo:
HazusInfo object.
:param version_dir:
Path of event version directory.
"""
# ---------------------------------------------------------------------------
# Sort out some paths
# ---------------------------------------------------------------------------
# Location of this module
mod_dir, dummy = os.path.split(__file__)
# losspager package direcotry
losspager_dir = os.path.join(mod_dir, '..')
# Repository root directory
root_dir = os.path.join(losspager_dir, '..')
# Logos directory
logos_dir = os.path.join(losspager_dir, 'logos')
# twopager latex template file
template_file = os.path.join(logos_dir, 'twopager.tex')
# ---------------------------------------------------------------------------
# Read in pager data, Hazus data, and latex template
# ---------------------------------------------------------------------------
json_dir = os.path.join(version_dir, 'json')
pdict = pdata._pagerdict
edict = pdata.getEventInfo()
with open(template_file, 'r') as f:
template = f.read()
# ---------------------------------------------------------------------------
# Fill in template values
# ---------------------------------------------------------------------------
# Sort out origin time
olat = edict['lat']
olon = edict['lon']
otime_utc = edict['time']
date_utc = datetime.strptime(otime_utc, "%Y-%m-%d %H:%M:%S")
date_local = pdata.local_time
DoW = date_local.strftime('%a')
otime_local = date_local.strftime('%H:%M:%S')
otime_local = DoW + ' ' + otime_local
template = template.replace("[ORIGTIME]", otime_utc)
template = template.replace("[LOCALTIME]", otime_local)
# Some paths
template = template.replace("[VERSIONFOLDER]", version_dir)
template = template.replace("[HOMEDIR]", root_dir)
# Magnitude location string under USGS logo
magloc = 'M %.1f, %s' % (edict['mag'], texify(edict['location']))
template = template.replace("[MAGLOC]", magloc)
# Pager version
ver = "Version " + str(pdict['pager']['version_number'])
template = template.replace("[VERSION]", ver)
# Epicenter location
lat = edict['lat']
lon = edict['lon']
dep = edict['depth']
if lat > 0:
hlat = "N"
else:
hlat = "S"
if lon > 0:
hlon = "E"
else:
hlon = "W"
template = template.replace("[LAT]", '%.4f' % abs(lat))
template = template.replace("[LON]", '%.4f' % abs(lon))
template = template.replace("[HEMILAT]", hlat)
template = template.replace("[HEMILON]", hlon)
template = template.replace("[DEPTH]", '%.1f' % dep)
# Tsunami warning? --- need to fix to be a function of tsunamic flag
if edict['tsunami']:
template = template.replace(
"[TSUNAMI]", "FOR TSUNAMI INFORMATION, SEE: tsunami.gov")
else:
template = template.replace("[TSUNAMI]", "")
# Elapsed time
if pdata.isScenario():
elapse = ''
else:
elapse = "Created: " + \
pdict['pager']['elapsed_time'] + " after earthquake"
template = template.replace("[ELAPSED]", elapse)
# Summary alert color
template = template.replace("[SUMMARYCOLOR]",
pdata.summary_alert.capitalize())
template = template.replace("[ALERTFILL]", pdata.summary_alert)
# Summary comment
template = template.replace("[IMPACT1]",
texify(pdict['comments']['impact1']))
template = template.replace("[IMPACT2]",
texify(pdict['comments']['impact2']))
# Hazus arrow color and relative position
hazdel = (hazinfo.hazloss) / LOSS_CONV
if hazdel < 0.1:
hazdelval = 0.1
elif hazdel > 1000000:
hazdelval = 1000000
else:
hazdelval = hazdel
arrowloc = (((6 - log10(hazdelval)) * 0.83) - 0.07)
# distance (in cm) to the left from right end of the econ histogram
template = template.replace("[ARROWSHIFT]", '%.2f' % arrowloc)
shift = arrowloc + 1.75
# value is ARROWSHIFT plus 1.75
# white box around the arrow and text to "open" the lines between values
template = template.replace("[BOXSHIFT]", '%.2f' % shift)
# color of the Hazus econ loss value using PAGER color scale
template = template.replace("[HAZUS_SUMMARY]", hazinfo.summary_color)
# MMI color pal
pal = ColorPalette.fromPreset('mmi')
# get all of the tag tables
(green_tag_table, yellow_tag_table,
red_tag_table) = hazinfo.createTaggingTables()
# Building Tags by occupancy
template = template.replace('[GREEN_TAG_TABLE]', green_tag_table)
template = template.replace('[YELLOW_TAG_TABLE]', yellow_tag_table)
template = template.replace('[RED_TAG_TABLE]', red_tag_table)
# Direct economic losses table
econ_losses_table = hazinfo.createEconTable()
template = template.replace('[DEL_TABLE]', econ_losses_table)
# Non-fatal injuries table
injuries_table = hazinfo.createInjuryTable()
template = template.replace('[NFI_TABLE]', injuries_table)
# Shelter needs table
shelter_table = hazinfo.createShelterTable()
template = template.replace('[SHELTER_TABLE]', shelter_table)
# Earthquake Debris table
debris_table = hazinfo.createDebrisTable()
template = template.replace('[DEBRIS_TABLE]', debris_table)
eventid = edict['eventid']
# query ComCat for information about this event
# fill in the url, if we can find it
try:
ccinfo = ComCatInfo(eventid)
eventid, allids = ccinfo.getAssociatedIds()
event_url = ccinfo.getURL() + '#pager'
except:
event_url = DEFAULT_PAGER_URL
eventid = "Event ID: " + eventid
template = template.replace("[EVENTID]", texify(eventid))
template = template.replace("[EVENTURL]", texify(event_url))
template = template.replace("[HAZUSURL]", texify(DEFAULT_FEMA_URL))
# Write latex file
tex_output = os.path.join(version_dir, 'twopager.tex')
with open(tex_output, 'w') as f:
f.write(template)
pdf_output = os.path.join(version_dir, 'twopager.pdf')
stderr = ''
try:
cwd = os.getcwd()
os.chdir(version_dir)
cmd = '%s -interaction nonstopmode --output-directory %s %s' % (
LATEX_TO_PDF_BIN, version_dir, tex_output)
logging.info('Running %s...' % cmd)
res, stdout, stderr = get_command_output(cmd)
os.chdir(cwd)
if not res:
if os.path.isfile(pdf_output):
msg = 'pdflatex created output file with non-zero exit code.'
return (pdf_output, msg)
return (None, stderr)
else:
if os.path.isfile(pdf_output):
return (pdf_output, stderr)
else:
pass
except Exception as e:
pass
finally:
os.chdir(cwd)
return (None, stderr)
def main(args):
#-------------------------------------------------------------
# Some additional useful directories
#-------------------------------------------------------------
REPO_DIR = os.path.dirname(os.path.abspath(__file__))
PACKAGE_DIR = os.path.join(REPO_DIR, 'libcomcat')
DOC_SRC_DIR = os.path.join(REPO_DIR, 'doc_source')
API_DIR = os.path.join(DOC_SRC_DIR, 'apidoc')
DOCS_DIR = os.path.join(REPO_DIR, 'docs')
#-------------------------------------------------------------
# get the human-friendly version of the libcomcat version
#-------------------------------------------------------------
verstr = '0.5'
#-------------------------------------------------------------
# what is the package called and who are the authors
#-------------------------------------------------------------
PACKAGE = "libcomcat 0.5 API"
AUTHORS = 'Mike Hearne'
#-------------------------------------------------------------
# run the api doc command; this creates the .rst files
#-------------------------------------------------------------
sys.stderr.write('Building libcomcat API documentation (REST)...\n')
sphinx_cmd = 'sphinx-apidoc -o %s -f -e -d 12 -H "%s" -A "%s"'\
' -V %s -T %s' % (API_DIR, PACKAGE, AUTHORS, verstr,
PACKAGE_DIR)
res, stdout, stderr = get_command_output(sphinx_cmd)
if not res:
raise Exception('Could not build libcomcat API documentation'
' - error "%s".' % stderr.decode())
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
#--------------------------------------------
# try to clean up some of the excess labeling
#--------------------------------------------
clean_cmd = "sed -e 's/ module//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e 's/ package//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/Subpackages/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/-.*-/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
#-------------------------------------------------------------
# Go to the api directory and build the html
#-------------------------------------------------------------
sys.stderr.write('Building HTML...\n')
res, stdout, stderr = get_command_output('sphinx-build -a -E %s %s'
% (DOC_SRC_DIR, DOCS_DIR))
if not res:
raise Exception('Could not build HTML. - '
'error "%s"' % stderr.decode())
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
pathlib.Path(os.path.join(DOCS_DIR, '.nojekyll')).touch(exist_ok=True)
def main(args):
# -------------------------------------------------------------
# Some additional useful directories
# -------------------------------------------------------------
REPO_DIR = os.path.dirname(os.path.abspath(__file__))
SHAKEMAP_PACKAGE_DIR = os.path.join(REPO_DIR, 'shakemap')
SHAKELIB_PACKAGE_DIR = os.path.join(REPO_DIR, 'shakelib')
DOC_SRC_DIR = os.path.join(REPO_DIR, 'doc_source')
SHAKEMAP_API_DIR = os.path.join(DOC_SRC_DIR, 'apidoc')
SHAKELIB_API_DIR = os.path.join(DOC_SRC_DIR, 'shakelib')
DOCS_DIR = os.path.join(REPO_DIR, 'docs')
# -------------------------------------------------------------
# get the human-friendly version of the ShakeMap version
# -------------------------------------------------------------
verstr = '4.0a'
# -------------------------------------------------------------
# what is the package called and who are the authors
# -------------------------------------------------------------
SHAKEMAP_PACKAGE = "ShakeMap 4.0a API"
SHAKELIB_PACKAGE = "ShakeLib API"
AUTHORS = 'Bruce Worden, Eric Thompson, Mike Hearne, David Wald'
# -------------------------------------------------------------
# run the api doc command; this creates the .rst files
# -------------------------------------------------------------
# First clear out the apidoc and shakelib directory
for f in os.listdir(SHAKEMAP_API_DIR):
fpath = os.path.join(SHAKEMAP_API_DIR, f)
if os.path.isfile(fpath):
os.unlink(fpath)
for f in os.listdir(SHAKELIB_API_DIR):
fpath = os.path.join(SHAKELIB_API_DIR, f)
if os.path.isfile(fpath):
os.unlink(fpath)
sys.stderr.write('Building shakemap API documentation (REST)...\n')
sphinx_cmd = 'sphinx-apidoc -o %s -f -e -d 12 -H "%s" -A "%s"'\
' -V %s -T %s' % (SHAKEMAP_API_DIR, SHAKEMAP_PACKAGE,
AUTHORS, verstr,
SHAKEMAP_PACKAGE_DIR)
res, stdout, stderr = get_command_output(sphinx_cmd)
if not res:
raise Exception('Could not build ShakeMap API documentation'
' - error "%s".' % stderr.decode())
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
sys.stderr.write('Building shakelib API documentation (REST)...\n')
sphinx_cmd = 'sphinx-apidoc -o %s -f -e -d 12 -H "%s" -A "%s"'\
' -V %s -T %s shakelib/rupture/gc2.py' % \
(SHAKELIB_API_DIR, SHAKELIB_PACKAGE, AUTHORS, verstr,
SHAKELIB_PACKAGE_DIR)
res, stdout, stderr = get_command_output(sphinx_cmd)
if not res:
raise Exception('Could not build ShakeLib API documentation'
' - error "%s".' % stderr.decode())
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
# --------------------------------------------
# try to clean up some of the excess labeling
# --------------------------------------------
clean_cmd = "sed -e 's/ module//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKEMAP_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e 's/ package//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKEMAP_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/Subpackages/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKEMAP_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/-.*-/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKEMAP_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e 's/ module//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKELIB_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e 's/ package//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKELIB_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/Subpackages/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKELIB_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/-.*-/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKELIB_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
# -------------------------------------------------------------
# Go to the api directory and build the html
# -------------------------------------------------------------
sys.stderr.write('Building shakemap manual (HTML)...\n')
res, stdout, stderr = get_command_output('sphinx-build -a -E %s %s'
% (DOC_SRC_DIR, DOCS_DIR))
if not res:
raise Exception('Could not build HTML. - '
'error "%s"' % stderr.decode())
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
pathlib.Path(os.path.join(DOCS_DIR, '.nojekyll')).touch(exist_ok=True)
def test_viewdb(tmpdir):
# Make a copy of current defaults
default_file = os.path.join(os.path.expanduser("~"), ".gfail_defaults")
if os.path.exists(default_file):
shutil.copy(default_file, default_file + '_bak')
try:
# Point to test database file
pathcmd = 'gfail --set-default-paths -db %s' % \
os.path.join(datadir, 'testevents.db')
rc, so, se = get_command_output(pathcmd)
# Run some examples
runcmd1 = "viewdb -r -c -n 10 --all -s"
rc1, so1, se1 = get_command_output(runcmd1)
runcmd2 = "viewdb -r -c -n 10 -p --LShazmin yellow"
rc2, so2, se2 = get_command_output(runcmd2)
runcmd3 = "viewdb -e us1000h3p4 us1000h5el -s" # --timeplots"
rc3, so3, se3 = get_command_output(runcmd3)
runcmd4 = "viewdb -p -c --minmag 7.5 --color"
rc4, so4, se4 = get_command_output(runcmd4)
runcmd5 = "viewdb -s --summaryplotfile %s" % \
os.path.join(tmpdir.name, 'test.png')
rc5, so5, se5 = get_command_output(runcmd5)
runcmd6 = "viewdb -s --csvfile %s" % \
os.path.join(tmpdir.name, 'test.csv')
rc6, so6, se6 = get_command_output(runcmd6)
# Test that everything ran
np.testing.assert_equal(True, rc1,
'%s did not run successfully' % runcmd1)
np.testing.assert_equal(True, rc2,
'%s did not run successfully' % runcmd2)
np.testing.assert_equal(True, rc3,
'%s did not run successfully' % runcmd3)
np.testing.assert_equal(True, rc4,
'%s did not run successfully' % runcmd4)
np.testing.assert_equal(True, rc5,
'%s did not run successfully' % runcmd5)
# Make sure figure was created
np.testing.assert_equal(
True, os.path.isfile(os.path.join(tmpdir.name,
'test_overall.png')))
# Make sure csv file was created
np.testing.assert_equal(
True, os.path.isfile(os.path.join(tmpdir.name, 'test.csv')))
except Exception as e:
print(e)
# Put defaults back
if os.path.exists(default_file + '_bak'):
shutil.copy(default_file + '_bak', default_file)
# Remove backup and tempfile
if os.path.exists(default_file + '_bak'):
os.remove(default_file + '_bak')
def test_zhu2015_web(tmpdir):
shakegrid = os.path.join(datadir, 'loma_prieta', 'grid.xml')
pathcmd = """
gfail --set-default-paths \
-d %s/loma_prieta/model_inputs \
-o [TMPOUT] \
-c %s/defaultconfigfiles/models \
-m %s/defaultconfigfiles/mapconfig.ini \
-md %s/loma_prieta/mapping_inputs
""" % (datadir, upone, upone, datadir)
# Make a copy of current defaults
default_file = os.path.join(os.path.expanduser("~"), ".gfail_defaults")
if os.path.exists(default_file):
shutil.copy(default_file, default_file + '_bak')
try:
try:
p = os.path.join(str(tmpdir.name), "sub")
except:
p = os.path.join(str(tmpdir), "sub")
if not os.path.exists(p):
os.makedirs(p)
else:
shutil.rmtree(p)
os.makedirs(p)
# Clear paths
rc, so, se = get_command_output('gfail -reset')
np.testing.assert_equal(True, rc, 'gfail reset failed')
# Modify paths
pathcmd = pathcmd.replace('[TMPOUT]', p)
rc, so, se = get_command_output(pathcmd)
np.testing.assert_equal(True, rc, 'gfail path modification failed')
with open(default_file, "a") as f:
f.write("popfile = %s" %
os.path.join(datadir, 'loma_prieta/lspop2016_lp.flt'))
# Run model
conf = os.path.join(datadir, 'test_conf')
runcmd = "gfail %s %s -w --hdf5 -ext" % (conf, shakegrid)
rc, so, se = get_command_output(runcmd)
np.testing.assert_equal(True, rc, se.decode())
# event_dir = os.path.join(p, '19891018000415')
# # Make png
# cmd = 'create_png -e %s' % event_dir
# rc1, so1, se1 = get_command_output(cmd)
# np.testing.assert_equal(True, rc1, se1.decode())
# # Make info
# cmd = 'create_info -e %s' % event_dir
# rc2, so2, se2 = get_command_output(cmd)
# np.testing.assert_equal(True, rc2, se2.decode())
# Make PDL directory
# pdldir = os.path.join(p, '19891018000415')
pdldir = p
pdl.prepare_pdl_directory(pdldir)
# Transfer dry run
pdl_out = pdl.transfer(pdldir, 1, 'None', dryrun=True)
except Exception as e: # So that defaults are put back even if something goes wrong
print(e)
# Put defaults back
if os.path.exists(default_file + '_bak'):
shutil.copy(default_file + '_bak', default_file)
# remove backup
os.remove(default_file + '_bak')
# Then do test
assert pdl_out['rc'] is True
# Remove tempfile
shutil.rmtree(p)
def build_report_latex(sc, directory, origin, config=None):
"""
Build latex summary report.
Args:
st (StreamCollection):
StreamCollection of data.
directory (str):
Directory for saving report.
origin (ScalarEvent):
ScalarEvent object.
config (dict):
Config dictionary.
Returns:
tuple:
- Name of pdf or latex report file created.
- boolean indicating whether PDF creation was successful.
"""
# Need to get config to know where the plots are located
if config is None:
config = get_config()
# Check if directory exists, and if not, create it.
if not os.path.exists(directory):
os.makedirs(directory)
# Initialize report string with PREAMBLE
report = PREAMBLE
timestr = origin.time.strftime('%m/%d/%Y %H:%M:%S')
# Does the map exist?
map_file = os.path.join(directory, 'stations_map.png')
if os.path.isfile(map_file):
TB = TITLEBLOCK.replace('[MAPPATH]', 'stations_map.png')
TB = TB.replace('[VERSION]', gmprocess.__version__)
moveout_file = os.path.join(directory, 'moveout_plot.png')
if os.path.isfile(moveout_file):
TB = TB.replace('[MOVEOUTPATH]', 'moveout_plot.png')
report += TB
# Loop over each StationStream and append it's page to the report
# do not include more than three.
for st in sc:
plot_path = os.path.join('plots',
origin.id + '_' + st.get_id() + '.png')
SB = STREAMBLOCK.replace('[PLOTPATH]', plot_path)
SB = SB.replace(
'[EVENT]',
'M %s - %s - %s' % (origin.magnitude, origin.id, timestr))
SB = SB.replace('[STATION]', st.get_id())
report += SB
prov_latex = get_prov_latex(st)
report += prov_latex
report += '\n'
if st[0].hasParameter('signal_split'):
pick_method = st[0].getParameter('signal_split')['picker_type']
report += 'Pick Method: %s\n\n' % str_for_latex(pick_method)
if 'nnet_qa' in st.getStreamParamKeys():
score_lq = st.getStreamParam('nnet_qa')['score_LQ']
score_hq = st.getStreamParam('nnet_qa')['score_HQ']
report += ('Neural Network LQ score: %s\n\n' %
str_for_latex(str(score_lq)))
report += ('Neural Network HQ score: %s\n\n' %
str_for_latex(str(score_hq)))
if not st.passed:
for tr in st:
if tr.hasParameter('failure'):
report += (
'Failure reason: %s\n\n' %
str_for_latex(tr.getParameter('failure')['reason']))
break
report += '\\newpage\n\n'
# Finish the latex file
report += POSTAMBLE
res = False
# Do not save report if running tests
if 'CALLED_FROM_PYTEST' not in os.environ:
# Set working directory to be the event subdirectory
current_directory = os.getcwd()
os.chdir(directory)
# File name relative to current location
file_name = ('report_%s.tex' % (origin.id))
# File name for printing out later relative base directory
latex_file = os.path.join(directory, file_name)
with open(file_name, 'w') as f:
f.write(report)
# Can we find pdflatex?
try:
pdflatex_bin = which('pdflatex')
pdflatex_options = '-interaction=nonstopmode -halt-on-error'
cmd = '%s %s %s' % (pdflatex_bin, pdflatex_options, file_name)
res, stdout, stderr = get_command_output(cmd)
report_file = latex_file
if res:
base, ext = os.path.splitext(file_name)
pdf_file = base + '.pdf'
if os.path.isfile(pdf_file):
report_file = pdf_file
auxfiles = glob.glob(base + '*')
auxfiles.remove(pdf_file)
for auxfile in auxfiles:
os.remove(auxfile)
else:
res = False
else:
print('pdflatex output:')
print(stdout.decode())
print(stderr.decode())
except Exception:
report_file = ''
pass
finally:
os.chdir(current_directory)
else:
report_file = 'not run'
# make report file an absolute path
report_file = os.path.join(directory, report_file)
return (report_file, res)
def quickcut(filename,
gdict,
tempname=None,
extrasamp=5.,
method='bilinear',
precise=True,
cleanup=True,
verbose=False,
override=False):
"""
Use gdal to trim a large global file down quickly so mapio can read it
efficiently. (Cannot read Shakemap.xml files, must save as .bil filrst)
Args:
filename (str): File path to original input file (raster).
gdict (geodict): Geodictionary to cut around and align with.
tempname (str): File path to desired location of clipped part of
filename.
extrasamp (int): Number of extra cells to cut around each edge of
geodict to have resampling buffer for future steps.
method (str): If resampling is necessary, method to use.
precise (bool): If true, will resample to the gdict as closely as
possible, if False it will just roughly cut around the area of
interest without changing resolution
cleanup (bool): if True, delete tempname after reading it back in
verbose (bool): if True, prints more details
override (bool): if True, if filename extent is not fully contained by
gdict, read in the entire file (only used for ShakeMaps)
Returns: New grid2D layer
Note: This function uses the subprocess approach because ``gdal.Translate``
doesn't hang on the command until the file is created which causes
problems in the next steps.
"""
if gdict.xmax < gdict.xmin:
raise Exception('quickcut: your geodict xmax is smaller than xmin')
try:
filegdict = GDALGrid.getFileGeoDict(filename)
except:
try:
filegdict = GMTGrid.getFileGeoDict(filename)
except:
raise Exception('Cannot get geodict for %s' % filename)
if tempname is None:
tempdir = tempfile.mkdtemp()
tempname = os.path.join(tempdir, 'junk.tif')
deltemp = True
else:
tempdir = None
deltemp = False
# if os.path.exists(tempname):
# os.remove(tempname)
# print('Temporary file already there, removing file')
filegdict = filegdict[0]
# Get the right methods for mapio (method) and gdal (method2)
if method == 'linear':
method2 = 'bilinear'
if method == 'nearest':
method2 = 'near'
if method == 'bilinear':
method = 'linear'
method2 = 'bilinear'
if method == 'near':
method = 'nearest'
method2 = 'near'
else:
method2 = method
if filegdict != gdict:
# First cut without resampling
tempgdict = GeoDict.createDictFromBox(gdict.xmin,
gdict.xmax,
gdict.ymin,
gdict.ymax,
filegdict.dx,
filegdict.dy,
inside=True)
try:
egdict = filegdict.getBoundsWithin(tempgdict)
ulx = egdict.xmin - extrasamp * egdict.dx
uly = egdict.ymax + extrasamp * egdict.dy
lrx = egdict.xmax + (extrasamp + 1) * egdict.dx
lry = egdict.ymin - (extrasamp + 1) * egdict.dy
cmd = 'gdal_translate -a_srs EPSG:4326 -of GTiff -projwin %1.8f \
%1.8f %1.8f %1.8f -r %s %s %s' % (ulx, uly, lrx, lry, method2,
filename, tempname)
except Exception as e:
if override:
# When ShakeMap is being loaded, sometimes they won't align
# right because it's already cut to the area, so just load
# the whole file
cmd = 'gdal_translate -a_srs EPSG:4326 -of GTiff -r %s %s %s' \
% (method2, filename, tempname)
else:
raise Exception('Failed to cut layer: %s' % e)
rc, so, se = get_command_output(cmd)
if not rc:
raise Exception(se.decode())
else:
if verbose:
print(so.decode())
newgrid2d = GDALGrid.load(tempname)
if precise:
# Resample to exact geodictionary
newgrid2d = newgrid2d.interpolate2(gdict, method=method)
if cleanup:
os.remove(tempname)
if deltemp:
shutil.rmtree(tempdir)
else:
ftype = GMTGrid.getFileType(filename)
if ftype != 'unknown':
newgrid2d = GMTGrid.load(filename)
elif filename.endswith('.xml'):
newgrid2d = ShakeGrid.load(filename)
else:
newgrid2d = GDALGrid.load(filename)
return newgrid2d
def run_gfail(args):
"""Runs ground failure.
Args:
args: dictionary or argument parser Namespace output by bin/gfail
program.
Returns:
list: Names of created files.
"""
# TODO: ADD CONFIG VALIDATION STEP THAT MAKES SURE ALL THE FILES EXIST
filenames = []
# If args is a dictionary, convert to a Namespace
if isinstance(args, dict):
args = Namespace(**args)
if args.set_default_paths:
set_default_paths(args)
print('default paths set, continuing...\n')
if args.list_default_paths:
list_default_paths()
return
if args.reset_default_paths:
reset_default_paths()
return
if args.make_webpage:
# Turn on GIS and HDF5 flags
gis = True
hdf5 = True
kmz = True
else:
gis = args.gis
hdf5 = args.hdf5
kmz = args.kmz
# Figure out what models will be run
if args.shakefile is not None: # user intends to actually run some models
shakefile = args.shakefile
# make output location for things
if args.output_filepath is None:
outdir = os.getcwd()
else:
outdir = args.output_filepath
if hdf5 or gis or kmz:
if not os.path.exists(outdir):
os.makedirs(outdir)
# download if is url
# cleanup = False
if not os.path.isfile(shakefile):
if isURL(shakefile):
# getGridURL returns a named temporary file object
shakefile = getGridURL(shakefile)
# cleanup = True # Be sure to delete it after
else:
raise NameError('Could not find "%s" as a file or a valid url'
% shakefile)
eventid = getHeaderData(shakefile)[0]['event_id']
# Get entire path so won't break if running gfail with relative path
shakefile = os.path.abspath(shakefile)
if args.extract_contents:
outfolder = outdir
else: # Nest in a folder named by eventid
outfolder = os.path.join(outdir, eventid)
if not os.path.exists(outfolder):
os.makedirs(outfolder)
# Copy shake grid into output directory
# --- this is base on advice from Mike that when running in production
# the shake grids are not archived and so if we need/want to have
# the exact grid used for the calculation later if there's every a
# question about how the calculation was done, the safest thing is
# to store a copy of it here.
shake_copy = os.path.join(outfolder, "grid.xml")
shutil.copyfile(shakefile, shake_copy)
if args.uncertfile is not None:
uncertfile = os.path.abspath(args.uncertfile)
unc_copy = os.path.join(outfolder, "uncertainty.xml")
shutil.copyfile(uncertfile, unc_copy)
else:
uncertfile = None
# Write shakefile to a file for use later
shakename = os.path.join(outfolder, "shakefile.txt")
shake_file = open(shakename, "wt")
shake_file.write(shake_copy)
shake_file.close()
filenames.append(shakename)
# Check that shakemap bounds do not cross 180/-180 line
if args.set_bounds is None:
sd = ShakeGrid.getFileGeoDict(shakefile)
if sd.xmin > sd.xmax:
print('\nShakeMap crosses 180/-180 line, setting bounds so '
'only side with more land area is run')
if sd.xmax + 180. > 180 - sd.xmin:
set_bounds = '%s, %s, %s, %s' % (
sd.ymin, sd.ymax, -180., sd.xmax)
else:
set_bounds = '%s, %s, %s, %s' % (sd.ymin, sd.ymax, sd.xmin,
180.)
print('Bounds applied: %s' % set_bounds)
else:
set_bounds = args.set_bounds
else:
set_bounds = args.set_bounds
config = args.config
if args.config_filepath is not None:
# only add config_filepath if full filepath not given and file
# ext is .ini
if (not os.path.isabs(config) and
os.path.splitext(config)[-1] == '.ini'):
config = os.path.join(args.config_filepath, config)
if os.path.splitext(config)[-1] == '.ini':
temp = ConfigObj(config)
if len(temp) == 0:
raise Exception(
'Could not find specified .ini file: %s' % config)
if args.data_path is not None:
temp = correct_config_filepaths(args.data_path, temp)
configs = [temp]
conffail = []
else:
# input is a list of config files
f = open(config, 'r')
configlist = f.readlines()
configs = []
conffail = []
for conf in configlist:
conf = conf.strip()
if not os.path.isabs(conf):
# only add config_filepath if full filepath not given
conf = os.path.join(args.config_filepath, conf)
try:
temp = ConfigObj(conf)
if temp:
if args.data_path is not None:
temp = correct_config_filepaths(
args.data_path, temp)
configs.append(temp)
else:
conffail.append(conf)
except BaseException:
conffail.append(conf)
print('\nRunning the following models:')
for conf in configs:
print('\t%s' % conf.keys()[0])
if len(conffail) > 0:
print('Could not find or read in the following config files:\n')
for conf in conffail:
print('\t%s' % conf)
print('\nContinuing...\n')
if set_bounds is not None:
if 'zoom' in set_bounds:
temp = set_bounds.split(',')
print('Using %s threshold of %1.1f to cut model bounds'
% (temp[1].strip(), float(temp[2].strip())))
bounds = get_bounds(shakefile, temp[1].strip(),
float(temp[2].strip()))
else:
temp = eval(set_bounds)
latmin = temp[0]
latmax = temp[1]
lonmin = temp[2]
lonmax = temp[3]
bounds = {'xmin': lonmin, 'xmax': lonmax,
'ymin': latmin, 'ymax': latmax}
print('Applying bounds of lonmin %1.2f, lonmax %1.2f, '
'latmin %1.2f, latmax %1.2f'
% (bounds['xmin'], bounds['xmax'],
bounds['ymin'], bounds['ymax']))
else:
bounds = None
if args.make_webpage:
results = []
# pre-read in ocean trimming file polygons so only do this step once
if args.trimfile is not None:
if not os.path.exists(args.trimfile):
print('trimfile defined does not exist: %s\n'
'Ocean will not be trimmed.' % args.trimfile)
trimfile = None
elif os.path.splitext(args.trimfile)[1] != '.shp':
print('trimfile must be a shapefile, '
'ocean will not be trimmed')
trimfile = None
else:
trimfile = args.trimfile
else:
trimfile = None
# Get finite fault ready, if exists
ffault = None
point = True
if args.finite_fault is not None:
point = False
try:
if os.path.splitext(args.finite_fault)[-1] == '.txt':
ffault = text_to_json(args.finite_fault)
elif os.path.splitext(args.finite_fault)[-1] == '.json':
ffault = args.finite_fault
else:
print('Could not read in finite fault, will '
'try to download from comcat')
ffault = None
except BaseException:
print('Could not read in finite fault, will try to '
'download from comcat')
ffault = None
if ffault is None:
# Try to get finite fault file, if it exists
try:
returned_ev = get_event_comcat(shakefile)
if returned_ev is not None:
testjd, detail, temp = returned_ev
evinfo = testjd['input']['event_information']
if 'faultfiles' in evinfo:
ffilename = evinfo['faultfiles']
if len(ffilename) > 0:
# Download the file
with tempfile.NamedTemporaryFile(
delete=False, mode='w') as f:
temp.getContent(ffilename, filename=f.name)
ffault = text_to_json(f.name)
os.remove(f.name)
point = False
else:
point = True
else:
print('Unable to determine source type, unknown if finite'
' fault or point source')
ffault = None
point = False
except Exception as e:
print(e)
print('Unable to determine source type, unknown if finite'
' fault or point source')
ffault = None
point = False
# Loop over config files
for conf in configs:
modelname = conf.keys()[0]
print('\nNow running %s:' % modelname)
notcov, newbnds = check_input_extents(
conf, shakefile=shakefile,
bounds=bounds
)
if len(notcov) > 0:
print('\nThe following input layers do not cover'
' the area of interest:\n\t%s' % '\n\t'.join(notcov))
if newbnds is None:
print('\nCannnot make bounds that work. '
'Skipping to next model\n')
continue
else:
pnt = '%s, %s, %s, %s' % (
newbnds['xmin'], newbnds['xmax'],
newbnds['ymin'], newbnds['ymax'])
print('Running model for new bounds that are fully covered'
' by input layer: %s' % pnt)
bounds2 = newbnds
else:
bounds2 = bounds
modelfunc = conf[modelname]['funcname']
if modelfunc == 'LogisticModel':
lm = LM.LogisticModel(shakefile, conf,
uncertfile=uncertfile,
saveinputs=args.save_inputs,
bounds=bounds2,
trimfile=trimfile)
maplayers = lm.calculate()
elif modelfunc == 'godt2008':
maplayers = godt2008(shakefile, conf,
uncertfile=uncertfile,
saveinputs=args.save_inputs,
bounds=bounds2,
trimfile=trimfile)
else:
print('Unknown model function specified in config for %s '
'model, skipping to next config' % modelfunc)
continue
# time1 = datetime.datetime.utcnow().strftime('%d%b%Y_%H%M')
# filename = ('%s_%s_%s' % (eventid, modelname, time1))
if args.appendname is not None:
filename = ('%s_%s_%s' % (eventid, modelname, args.appendname))
else:
filename = ('%s_%s' % (eventid, modelname))
if hdf5:
filenameh = filename + '.hdf5'
if os.path.exists(filenameh):
os.remove(filenameh)
savelayers(maplayers, os.path.join(outfolder, filenameh))
filenames.append(filenameh)
if gis or kmz:
for key in maplayers:
# Rename 'std' key to 'beta_sigma'
if key == 'std':
key_label = 'beta_sigma'
else:
key_label = key
if gis:
filen = os.path.join(outfolder, '%s_%s.bil'
% (filename, key_label))
fileh = os.path.join(outfolder, '%s_%s.hdr'
% (filename, key_label))
fileg = os.path.join(outfolder, '%s_%s.tif'
% (filename, key_label))
GDALGrid.copyFromGrid(
maplayers[key]['grid']).save(filen)
cflags = '-co COMPRESS=DEFLATE -co predictor=2'
srs = '-a_srs EPSG:4326'
cmd = 'gdal_translate %s %s -of GTiff %s %s' % (
srs, cflags, filen, fileg)
rc, so, se = get_command_output(cmd)
# Delete bil file and its header
os.remove(filen)
os.remove(fileh)
filenames.append(fileg)
if kmz and (not key.startswith('quantile') and not key.startswith('std')) :
plotorder, logscale, lims, colormaps, maskthresh = \
parseConfigLayers(maplayers, conf, keys=['model'])
maxprob = np.nanmax(maplayers[key]['grid'].getData())
if key == 'model':
qdict = {
k: maplayers[k] for k in maplayers.keys()
if k.startswith('quantile')
}
else:
qdict = None
if maskthresh is None:
maskthresh = [0.]
if maxprob >= maskthresh[0]:
filen = os.path.join(outfolder, '%s_%s.kmz'
% (filename, key_label))
filek = create_kmz(maplayers[key], filen,
mask=maskthresh[0],
levels=lims[0],
qdict=qdict)
filenames.append(filek)
else:
print('No unmasked pixels present, skipping kmz '
'file creation')
if args.make_webpage:
# Compile into list of results for later
results.append(maplayers)
# # Make binary output for ShakeCast
# filef = os.path.join(outfolder, '%s_model.flt'
# % filename)
# # And get name of header
# filefh = os.path.join(outfolder, '%s_model.hdr'
# % filename)
# # Make file
# write_floats(filef, maplayers['model']['grid'])
# filenames.append(filef)
# filenames.append(filefh)
eventid = getHeaderData(shakefile)[0]['event_id']
if not hasattr(args, 'eventsource'):
args.eventsource = 'us'
if not hasattr(args, 'eventsourcecode'):
args.eventsourcecode = eventid
if args.make_webpage:
if len(results) == 0:
raise Exception('No models were run. Cannot make webpages.')
outputs = hazdev(
results, configs,
shakefile, outfolder=outfolder,
pop_file=args.popfile,
pager_alert=args.property_alertlevel,
eventsource=args.eventsource,
eventsourcecode=args.eventsourcecode,
point=point, gf_version=args.gf_version,
pdlcall=args.pdlcall)
filenames = filenames + outputs
# # create transparent png file
# outputs = create_png(outdir)
# filenames = filenames + outputs
#
# # create info file
# infofile = create_info(outdir)
# filenames = filenames + infofile
print('\nFiles created:\n')
for filen in filenames:
print('%s' % filen)
return filenames
def main(args):
# -------------------------------------------------------------
# Some additional useful directories
# -------------------------------------------------------------
REPO_DIR = os.path.dirname(os.path.abspath(__file__))
SHAKEMAP_PACKAGE_DIR = os.path.join(REPO_DIR, 'shakemap')
SHAKELIB_PACKAGE_DIR = os.path.join(REPO_DIR, 'shakelib')
DOC_SRC_DIR = os.path.join(REPO_DIR, 'doc_source')
SHAKEMAP_API_DIR = os.path.join(DOC_SRC_DIR, 'apidoc')
SHAKELIB_API_DIR = os.path.join(DOC_SRC_DIR, 'shakelib')
DOCS_DIR = os.path.join(REPO_DIR, 'docs')
# -------------------------------------------------------------
# get the human-friendly version of the ShakeMap version
# -------------------------------------------------------------
verstr = '4.0a'
# -------------------------------------------------------------
# what is the package called and who are the authors
# -------------------------------------------------------------
SHAKEMAP_PACKAGE = "ShakeMap 4.0a API"
SHAKELIB_PACKAGE = "ShakeLib API"
AUTHORS = 'Bruce Worden, Eric Thompson, Mike Hearne'
# -------------------------------------------------------------
# run the api doc command; this creates the .rst files
# -------------------------------------------------------------
# First clear out the apidoc and shakelib directory
for f in os.listdir(SHAKEMAP_API_DIR):
fpath = os.path.join(SHAKEMAP_API_DIR, f)
if os.path.isfile(fpath):
os.unlink(fpath)
for f in os.listdir(SHAKELIB_API_DIR):
fpath = os.path.join(SHAKELIB_API_DIR, f)
if os.path.isfile(fpath):
os.unlink(fpath)
sys.stderr.write('Building shakemap API documentation (REST)...\n')
sphinx_cmd = 'sphinx-apidoc -o %s -f -e -d 12 -H "%s" -A "%s"'\
' -V %s -T %s' % (SHAKEMAP_API_DIR, SHAKEMAP_PACKAGE,
AUTHORS, verstr,
SHAKEMAP_PACKAGE_DIR)
res, stdout, stderr = get_command_output(sphinx_cmd)
if not res:
raise Exception('Could not build ShakeMap API documentation'
' - error "%s".' % stderr.decode())
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
sys.stderr.write('Building shakelib API documentation (REST)...\n')
sphinx_cmd = 'sphinx-apidoc -o %s -f -e -d 12 -H "%s" -A "%s"'\
' -V %s -T %s shakelib/rupture/gc2\.py' % \
(SHAKELIB_API_DIR, SHAKELIB_PACKAGE, AUTHORS, verstr,
SHAKELIB_PACKAGE_DIR)
res, stdout, stderr = get_command_output(sphinx_cmd)
if not res:
raise Exception('Could not build ShakeLib API documentation'
' - error "%s".' % stderr.decode())
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
# --------------------------------------------
# try to clean up some of the excess labeling
# --------------------------------------------
clean_cmd = "sed -e 's/ module//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKEMAP_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e 's/ package//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKEMAP_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/Subpackages/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKEMAP_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/-.*-/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKEMAP_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e 's/ module//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKELIB_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e 's/ package//g' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKELIB_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/Subpackages/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKELIB_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -e '/-.*-/d' -i '' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % SHAKELIB_API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
# -------------------------------------------------------------
# Go to the api directory and build the html
# -------------------------------------------------------------
sys.stderr.write('Building shakemap manual (HTML)...\n')
res, stdout, stderr = get_command_output('sphinx-build -a -E %s %s'
% (DOC_SRC_DIR, DOCS_DIR))
if not res:
raise Exception('Could not build HTML. - '
'error "%s"' % stderr.decode())
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
pathlib.Path(os.path.join(DOCS_DIR, '.nojekyll')).touch(exist_ok=True)
def test_charlevoix_0(tmpdir):
# make a temporary directory; read in rupture file
p = os.path.join(str(tmpdir), "sub")
if not os.path.exists(p):
os.makedirs(p)
old_shakedir = set_shakehome(p)
v = os.path.join(shakedir, 'tests/data/CharlevoixVs30.grd')
old_vs30file = set_vs30file(v)
old_gmpe = set_gmpe('stable_continental_nshmp2014_rlme')
jsonfile = os.path.join(
shakedir, 'rupture_sets/BSSC2014/bssc2014_ceus.json')
# directory holding test and target data for this event
testinput = os.path.join(p, 'data/charlevoix_0_m7p_se/input')
targetinput = os.path.join(
shakedir, 'tests/output/charlevoix_0_m7p_se/input')
#---------------------------------------------------------------------------
# First test mkinputdir
#---------------------------------------------------------------------------
# Run mkinputdir
cmd = 'mkinputdir -f %s -i 0 ' % jsonfile
rc, so, se = get_command_output(cmd)
if se != b'':
print(so.decode())
print(se.decode())
# Check output files
# Note: Not checking event.xml because the timestamp breaks cmp comparison.
# Would need to parse and to tag comaprisons. Not worth it.
target = os.path.join(targetinput, 'charlevoix_0_m7p_se_for-map_fault.txt')
test = os.path.join(testinput, 'charlevoix_0_m7p_se_for-map_fault.txt')
assert filecmp.cmp(test, target) is True
#---------------------------------------------------------------------------
# Test mkscenariogrids
#---------------------------------------------------------------------------
datadir = os.path.join(p, 'data')
cmd = 'mkscenariogrids -e charlevoix_0_m7p_se -r 0.1 '
rc, so, se = get_command_output(cmd)
# Check output files
target = os.path.join(targetinput, 'mi_estimates.grd')
test = os.path.join(testinput, 'mi_estimates.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'mi_sd.grd')
test = os.path.join(testinput, 'mi_sd.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'pga_estimates.grd')
test = os.path.join(testinput, 'pga_estimates.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'pga_sd.grd')
test = os.path.join(testinput, 'pga_sd.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'pgv_estimates.grd')
test = os.path.join(testinput, 'pgv_estimates.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'pgv_sd.grd')
test = os.path.join(testinput, 'pgv_sd.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'psa03_estimates.grd')
test = os.path.join(testinput, 'psa03_estimates.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'psa03_sd.grd')
test = os.path.join(testinput, 'psa03_sd.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'psa10_estimates.grd')
test = os.path.join(testinput, 'psa10_estimates.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'psa10_sd.grd')
test = os.path.join(testinput, 'psa10_sd.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'psa30_estimates.grd')
test = os.path.join(testinput, 'psa30_estimates.grd')
grdcmp(test, target)
target = os.path.join(targetinput, 'psa30_sd.grd')
test = os.path.join(testinput, 'psa30_sd.grd')
grdcmp(test, target)
# Clean up
set_shakehome(old_shakedir)
set_vs30file(old_vs30file)
set_gmpe(old_gmpe)
shutil.rmtree(p)
def tes_main():
cmd = 'gfail -i -pd -pn -pi %s %s' % (configfile, shakefile)
retcode, stdout, stderr = get_command_output(cmd)
temp = stdout.decode('utf-8')
print(temp)
def test_zhu2015(tmpdir):
shakegrid = os.path.join(datadir, 'loma_prieta', 'grid.xml')
pathcmd = """
gfail --set-default-paths \
-d %s/loma_prieta/model_inputs \
-o [TMPOUT] \
-c %s/defaultconfigfiles/models \
-m %s/defaultconfigfiles/mapconfig.ini \
-md %s/loma_prieta/mapping_inputs
""" % (datadir, upone, upone, datadir)
trimfile = '%s/loma_prieta/mapping_inputs/ne_10m_ocean/ne_10m_ocean.shp' \
% datadir
# Make a copy of current defaults
default_file = os.path.join(os.path.expanduser("~"), ".gfail_defaults")
if os.path.exists(default_file):
shutil.copy(default_file, default_file + '_bak')
try:
try:
p = os.path.join(str(tmpdir.name), "sub")
except:
p = os.path.join(str(tmpdir), "sub")
if not os.path.exists(p):
os.makedirs(p)
# Clear paths
rc, so, se = get_command_output('gfail -reset')
# Modify paths
pathcmd = pathcmd.replace('[TMPOUT]', p)
rc1, so1, se1 = get_command_output(pathcmd)
with open(default_file, "a") as f:
f.write("popfile = %s" %
os.path.join(datadir, 'loma_prieta/lspop2016_lp.flt'))
# List paths
rc3, so3, se3 = get_command_output('gfail --list-default-paths')
# Run model with bounds
runcmd = "gfail %s/test_conf %s -b 'zoom, pga, 2' --hdf5 -tr %s -ext"\
% (datadir, shakegrid, trimfile)
rc4, so4, se4 = get_command_output(runcmd)
# Run model
runcmd = "gfail %s/test_conf %s --gis -pn -pi -pd --hdf5 -ext" \
% (datadir, shakegrid)
rc2, so2, se2 = get_command_output(runcmd)
# Read in the testing data
test_file = os.path.join(p, '19891018000415_zhu_2015_model.tif')
test_grid = GDALGrid.load(test_file)
test_data = test_grid.getData()
# Read in target file
target_file = os.path.join(datadir, 'loma_prieta', 'targets',
'19891018000415_zhu_2015_model.tif')
if changetarget:
# To change target data:
test_grid.save(test_file)
cmd = 'gdal_translate -a_srs EPSG:4326 -of GTiff %s %s' % (
test_file, target_file)
rc, so, se = get_command_output(cmd)
target_grid = GDALGrid.load(target_file)
target_data = target_grid.getData()
except Exception as e:
print(e)
# Put defaults back
if os.path.exists(default_file + '_bak'):
shutil.copy(default_file + '_bak', default_file)
# Remove backup and tempfile
if os.path.exists(default_file + '_bak'):
os.remove(default_file + '_bak')
shutil.rmtree(p)
# Test that everything ran
np.testing.assert_equal(True, rc, 'gfail reset failed')
np.testing.assert_equal(True, rc1, 'gfail path modification failed')
np.testing.assert_equal(True, rc2, se2.decode())
np.testing.assert_equal(True, rc3, 'gfail list-default-paths failed')
np.testing.assert_equal(True, rc4, se4.decode())
# Then do test of values
np.testing.assert_allclose(target_data, test_data, rtol=1e-3)
def run_one_old_shakemap(eventid, topo=True, genex=True):
"""
Convenience method for running old (v 3.5) shakemap with new estimates. This
allows for us to generate all the products with the old code since the new
code cannot do this yet, but use the new code for computing the ground
motions.
Args:
eventid (srt): Specifies the id of the event to process.
topo (bool): Include topography shading?
genex (bool): Should genex be run?
Returns:
dictionary: Each entry is the log file for the different ShakeMap3.5
calls.
"""
config = ConfigObj(os.path.join(os.path.expanduser('~'), 'scenarios.conf'))
shakehome = config['system']['shakehome']
log = {}
shakebin = os.path.join(shakehome, 'bin')
datadir = os.path.join(shakehome, 'data')
# Read event.xml
eventdir = os.path.join(datadir, eventid)
inputdir = os.path.join(eventdir, 'input')
xml_file = os.path.join(inputdir, 'event.xml')
# Read in event.xml
event = read_event_file(xml_file)
# Read in gmpe set name
gmpefile = open(os.path.join(inputdir, "gmpe_set_name.txt"), "r")
set_name = gmpefile.read()
gmpefile.close()
# Add scenario-specific fields:
eventtree = ET.parse(xml_file)
eventroot = eventtree.getroot()
for eq in eventroot.iter('earthquake'):
description = eq.attrib['description']
directivity = eq.attrib['directivity']
if 'reference' in eq.attrib.keys():
reference = eq.attrib['reference']
else:
reference = ''
event['description'] = description
event['directivity'] = directivity
event['reference'] = reference
grd = os.path.join(inputdir, 'pgv_estimates.grd')
gdict = GMTGrid.getFileGeoDict(grd)[0]
# Tolerance is a bit hacky but necessary to prevent GMT
# from barfing becasue it thinks that the estimates files
# do not cover the desired area sampled by grind's call
# with grdsample.
tol = gdict.dx
W = gdict.xmin + tol
E = gdict.xmax - tol
S = gdict.ymin + tol
N = gdict.ymax - tol
# Put into grind.conf (W S E N)
confdir = os.path.join(eventdir, 'config')
if os.path.isdir(confdir) == False:
os.mkdir(confdir)
# need to copy default grind.conf
default_grind_conf = os.path.join(shakehome, 'config', 'grind.conf')
grind_conf = os.path.join(confdir, 'grind.conf')
shutil.copyfile(default_grind_conf, grind_conf)
# Set strictbound and resolution to match estiamtes.grd files
with open(grind_conf, 'a') as f:
f.write('x_grid_interval : %.16f\n' % gdict.dx)
f.write('y_grid_interval : %.16f\n' % gdict.dy)
f.write('strictbound : %.9f %.9f %.9f %.9f\n' % (W, S, E, N))
# Grind
callgrind = os.path.join(shakebin, 'grind') + \
' -event ' + eventid + ' -psa'
rc, so, se = get_command_output(callgrind)
log['grind'] = {'rc': rc, 'so': so, 'se': se}
# Add GMPE set name to info.json
cmd = os.path.join(shakebin, 'edit_info') + ' -event ' + eventid + \
' -tag gmpe_reference' + ' -value ' + set_name
rc, so, se = get_command_output(cmd)
log['edit_info'] = {'rc': rc, 'so': so, 'se': se}
# Tag
calltag = os.path.join(shakebin, 'tag') + \
' -event ' + eventid + ' -name \"' + event['locstring'] + ' - ' + \
event['description'] + '\"'
rc, so, se = get_command_output(calltag)
log['tag'] = {'rc': rc, 'so': so, 'se': se}
# Copy rock_grid.xml from input to output directory
rg_scr = os.path.join(inputdir, 'rock_grid.xml')
rg_dst = os.path.join(eventdir, 'output', 'rock_grid.xml')
cmd = shutil.copy(rg_scr, rg_dst)
# Mapping
if topo is True:
topostr = '-itopo'
else:
topostr = ''
callmapping = os.path.join(shakebin, 'mapping') + ' -event ' + \
eventid + ' -timestamp -nohinges ' + topostr
rc, so, se = get_command_output(callmapping)
log['mapping'] = {'rc': rc, 'so': so, 'se': se}
# Genex
if genex is True:
callgenex = os.path.join(shakebin, 'genex') + ' -event ' + \
eventid + ' -metadata -zip -verbose -shape shape -shape hazus'
rc, so, se = get_command_output(callgenex)
log['genex'] = {'rc': rc, 'so': so, 'se': se}
return log
def trim_ocean(grid2D, mask, all_touched=True, crop=False):
"""Use the mask (a shapefile) to trim offshore areas
Args:
grid2D: MapIO grid2D object of results that need trimming
mask: list of shapely polygon features already loaded in or string of
file extension of shapefile to use for clipping
all_touched (bool): if True, won't mask cells that touch any part of
polygon edge
crop (bool): crop boundaries of raster to new masked area
Returns:
grid2D file with ocean masked
"""
gdict = grid2D.getGeoDict()
tempdir = tempfile.mkdtemp()
# Get shapes ready
if type(mask) == str:
with fiona.open(mask, 'r') as shapefile:
bbox = (gdict.xmin, gdict.ymin, gdict.xmax, gdict.ymax)
hits = list(shapefile.items(bbox=bbox))
features = [feature[1]["geometry"] for feature in hits]
# hits = list(shapefile)
# features = [feature["geometry"] for feature in hits]
elif type(mask) == list:
features = mask
else:
raise Exception('mask is neither a link to a shapefile or a list of \
shapely shapes, cannot proceed')
if len(features) == 0:
print('No coastlines in ShakeMap area')
return grid2D
tempfilen = os.path.join(tempdir, 'temp.bil')
tempfile1 = os.path.join(tempdir, 'temp.tif')
tempfile2 = os.path.join(tempdir, 'temp2.tif')
GDALGrid.copyFromGrid(grid2D).save(tempfilen)
cmd = 'gdal_translate -a_srs EPSG:4326 -of GTiff %s %s' % \
(tempfilen, tempfile1)
rc, so, se = get_command_output(cmd)
if rc:
with rasterio.open(tempfile1, 'r') as src_raster:
out_image, out_transform = rasterio.mask.mask(
src_raster, features, all_touched=all_touched, crop=crop)
out_meta = src_raster.meta.copy()
out_meta.update({
"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform
})
with rasterio.open(tempfile2, "w", **out_meta) as dest:
dest.write(out_image)
newgrid = GDALGrid.load(tempfile2)
else:
print(se)
raise Exception('ocean trimming failed')
shutil.rmtree(tempdir)
return newgrid
def build_report_latex(
st_list, directory, origin, prefix="", config=None, gmprocess_version="unknown"
):
"""
Build latex summary report.
Args:
st_list (list):
List of streams.
directory (str):
Directory for saving report.
origin (ScalarEvent):
ScalarEvent object.
prefix (str):
String to prepend to report file name.
config (dict):
Config dictionary.
gmprocess_version:
gmprocess version.
Returns:
tuple:
- Name of pdf or latex report file created.
- boolean indicating whether PDF creation was successful.
"""
# Need to get config to know where the plots are located
if config is None:
config = get_config()
# Check if directory exists, and if not, create it.
if not os.path.exists(directory):
os.makedirs(directory)
# Initialize report string with PREAMBLE
report = PREAMBLE
timestr = origin.time.strftime("%m/%d/%Y %H:%M:%S")
# Does the map exist?
map_file = os.path.join(directory, "stations_map.png")
if os.path.isfile(map_file):
TB = TITLEBLOCK.replace("[MAPPATH]", "stations_map.png")
TB = TB.replace("[VERSION]", gmprocess_version)
moveout_file = os.path.join(directory, "moveout_plot.png")
if os.path.isfile(moveout_file):
TB = TB.replace("[MOVEOUT_PAGE]", moveout_page_tex)
TB = TB.replace("[MOVEOUTPATH]", "moveout_plot.png")
else:
TB = TB.replace("[MOVEOUT_PAGE]", "")
report += TB
# Loop over each StationStream and append it's page to the report
# do not include more than three.
# sort list of streams:
st_list.sort(key=lambda x: x.id)
for st in st_list:
# Do NOT use os.path.join() here becuase even on windows, latex needs the path
# to use linux-style forward slashs.
streamid = st.get_id()
plot_path = f"plots/{origin.id}_{streamid}.png"
SB = STREAMBLOCK.replace("[PLOTPATH]", plot_path)
SB = SB.replace(
"[EVENT]",
f"M {origin.magnitude} - {str_for_latex(origin.id)} - {timestr}",
)
SB = SB.replace("[STATION]", st.get_id())
report += SB
prov_latex = get_prov_latex(st)
report += prov_latex
report += "\n"
if st[0].hasParameter("signal_split"):
pick_method = st[0].getParameter("signal_split")["picker_type"]
report += f"Pick Method: {str_for_latex(pick_method)}\n\n"
if "nnet_qa" in st.getStreamParamKeys():
score_lq = st.getStreamParam("nnet_qa")["score_LQ"]
score_hq = st.getStreamParam("nnet_qa")["score_HQ"]
report += f"Neural Network LQ score: {str_for_latex(str(score_lq))}\n\n"
report += f"Neural Network HQ score: {str_for_latex(str(score_hq))}\n\n"
if not st.passed:
for tr in st:
if tr.hasParameter("failure"):
report += "Failure reason: %s\n\n" % str_for_latex(
tr.getParameter("failure")["reason"]
)
break
report += "\\newpage\n\n"
# Finish the latex file
report += POSTAMBLE
res = False
# Do not save report if running tests
if "CALLED_FROM_PYTEST" not in os.environ:
# Set working directory to be the event subdirectory
current_directory = os.getcwd()
os.chdir(directory)
# File name relative to current location
file_name = f"{prefix}_report_{origin.id}.tex"
# File name for printing out later relative base directory
latex_file = os.path.join(directory, file_name)
with open(file_name, "w", encoding="utf-8") as f:
f.write(report)
# Can we find pdflatex?
try:
pdflatex_bin = which("pdflatex")
if os.name == "nt":
# seems that windows needs two dashes for the program options
flag = "--"
else:
flag = "-"
pdflatex_options = f"{flag}interaction=nonstopmode {flag}halt-on-error"
cmd = f"{pdflatex_bin} {pdflatex_options} {file_name}"
res, stdout, stderr = get_command_output(cmd)
report_file = latex_file
if res:
base, ext = os.path.splitext(file_name)
pdf_file = base + ".pdf"
if os.path.isfile(pdf_file):
report_file = pdf_file
auxfiles = glob.glob(base + "*")
auxfiles.remove(pdf_file)
for auxfile in auxfiles:
os.remove(auxfile)
else:
res = False
else:
print("pdflatex output:")
print(stdout.decode())
print(stderr.decode())
except BaseException:
report_file = ""
pass
finally:
os.chdir(current_directory)
else:
report_file = "not run"
# make report file an absolute path
report_file = os.path.join(directory, report_file)
return (report_file, res)
def create_onepager(pdata, version_dir, debug=False):
"""
:param pdata:
PagerData object.
:param version_dir:
Path of event version directory.
:param debug:
bool for whether or not to add textpos boxes to onepager.
"""
#---------------------------------------------------------------------------
# Sort out some paths
#---------------------------------------------------------------------------
# Locaiton of this module
mod_dir, dummy = os.path.split(__file__)
# losspager package direcotry
losspager_dir = os.path.join(mod_dir, '..')
# Repository root directory
root_dir = os.path.join(losspager_dir, '..')
# Data directory
data_dir = os.path.join(losspager_dir, 'data')
# Onepager latex template file
template_file = os.path.join(data_dir, 'onepager2.tex')
#---------------------------------------------------------------------------
# Read in pager data and latex template
#---------------------------------------------------------------------------
json_dir = os.path.join(version_dir, 'json')
pdict = pdata._pagerdict
edict = pdata.getEventInfo()
with open(template_file, 'r') as f:
template = f.read()
#---------------------------------------------------------------------------
# Fill in template values
#---------------------------------------------------------------------------
# Sort out origin time
olat = edict['lat']
olon = edict['lon']
otime_utc = edict['time']
date_utc = datetime.strptime(otime_utc, "%Y-%m-%d %H:%M:%S")
date_local = pdata.local_time
DoW = date_local.strftime('%a')
otime_local = date_local.strftime('%H:%M:%S')
otime_local = DoW + ' ' + otime_local
template = template.replace("[ORIGTIME]", otime_utc)
template = template.replace("[LOCALTIME]", otime_local)
# Some paths
template = template.replace("[VERSIONFOLDER]", version_dir)
template = template.replace("[HOMEDIR]", root_dir)
# Magnitude location string under USGS logo
magloc = 'M %.1f, %s' % (edict['mag'], texify(edict['location']))
template = template.replace("[MAGLOC]", magloc)
# Pager version
ver = "Version " + str(pdict['pager']['version_number'])
template = template.replace("[VERSION]", ver)
template = template.replace("[VERSIONX]", "2.5")
# Epicenter location
lat = edict['lat']
lon = edict['lon']
dep = edict['depth']
if lat > 0:
hlat = "N"
else:
hlat = "S"
if lon > 0:
hlon = "E"
else:
hlon = "W"
template = template.replace("[LAT]", '%.4f' % abs(lat))
template = template.replace("[LON]", '%.4f' % abs(lon))
template = template.replace("[HEMILAT]", hlat)
template = template.replace("[HEMILON]", hlon)
template = template.replace("[DEPTH]", '%.1f' % dep)
# Tsunami warning? --- need to fix to be a function of tsunamic flag
if edict['tsunami']:
template = template.replace(
"[TSUNAMI]", "FOR TSUNAMI INFORMATION, SEE: tsunami.gov")
else:
template = template.replace("[TSUNAMI]", "")
if pdata.isScenario():
elapse = ''
else:
elapse = "Created: " + pdict['pager'][
'elapsed_time'] + " after earthquake"
template = template.replace("[ELAPSED]", elapse)
template = template.replace("[IMPACT1]",
texify(pdict['comments']['impact1']))
template = template.replace("[IMPACT2]",
texify(pdict['comments']['impact2']))
template = template.replace("[STRUCTCOMMENT]",
texify(pdict['comments']['struct_comment']))
# Summary alert color
template = template.replace("[SUMMARYCOLOR]",
pdata.summary_alert.capitalize())
template = template.replace("[ALERTFILL]", pdata.summary_alert)
# fill in exposure values
max_border_mmi = pdata._pagerdict['population_exposure'][
'maximum_border_mmi']
explist = pdata.getTotalExposure()
pophold = 0
for mmi in range(1, 11):
iexp = mmi - 1
if mmi == 2:
pophold += explist[iexp]
continue
elif mmi == 3:
pop = explist[iexp] + pophold
macro = '[MMI2-3]'
else:
pop = explist[iexp]
macro = '[MMI%i]' % mmi
if pop < 1000:
pop = round_to_nearest(pop, round_value=1000)
if max_border_mmi > mmi and mmi <= 4:
if pop == 0:
popstr = '--*'
else:
if pop < 1000:
pop = round_to_nearest(pop, round_value=1000)
popstr = pop_round_short(pop) + '*'
else:
popstr = pop_round_short(pop)
template = template.replace(macro, popstr)
# MMI color pal
pal = ColorPalette.fromPreset('mmi')
# Historical table
htab = pdata.getHistoricalTable()
if htab[0] is None:
# use pdata.getHistoricalComment()
htex = pdata.getHistoricalComment()
else:
# build latex table
htex = """
\\begin{tabularx}{7.25cm}{lrc*{1}{>{\\centering\\arraybackslash}X}*{1}{>{\\raggedleft\\arraybackslash}X}}
\hline
\\textbf{Date} &\\textbf{Dist.}&\\textbf{Mag.}&\\textbf{Max} &\\textbf{Shaking}\\\\
\\textbf{(UTC)}&\\textbf{(km)} & &\\textbf{MMI(\#)}&\\textbf{Deaths} \\\\
\hline
[TABLEDATA]
\hline
\multicolumn{5}{p{7.2cm}}{\\small [COMMENT]}
\end{tabularx}"""
comment = pdata._pagerdict['comments']['secondary_comment']
htex = htex.replace("[COMMENT]", texify(comment))
tabledata = ""
nrows = len(htab)
for i in range(nrows):
date = htab[i]['Time'].split()[0]
dist = str(int(htab[i]['Distance']))
mag = str(htab[i]['Magnitude'])
mmi = dec_to_roman(np.round(htab[i]['MaxMMI'], 0))
col = pal.getDataColor(htab[i]['MaxMMI'])
texcol = "%s,%s,%s" % (col[0], col[1], col[2])
nmmi = pop_round_short(htab[i]['NumMaxMMI'])
mmicell = '%s(%s)' % (mmi, nmmi)
shakedeath = htab[i]['ShakingDeaths']
if np.isnan(shakedeath):
death = "--"
else:
death = pop_round_short(shakedeath)
row = '%s & %s & %s & \cellcolor[rgb]{%s} %s & %s \\\\ '\
'\n' %(date, dist, mag, texcol, mmicell, death)
tabledata = tabledata + row
htex = htex.replace("[TABLEDATA]", tabledata)
template = template.replace("[HISTORICAL_BLOCK]", htex)
# City table
ctex = """
\\begin{tabularx}{7.25cm}{lXr}
\hline
\\textbf{MMI} & \\textbf{City} & \\textbf{Population} \\\\
\hline
[TABLEDATA]
\hline
\end{tabularx}"""
ctab = pdata.getCityTable()
nrows = len(ctab.index)
tabledata = ""
for i in range(nrows):
mmi = dec_to_roman(np.round(ctab['mmi'].iloc[i], 0))
city = ctab['name'].iloc[i]
if ctab['pop'].iloc[i] == 0:
pop = '$<$1k'
else:
if ctab['pop'].iloc[i] < 1000:
popnum = round_to_nearest(ctab['pop'].iloc[i],
round_value=1000)
else:
popnum = ctab['pop'].iloc[i]
pop = pop_round_short(popnum)
col = pal.getDataColor(ctab['mmi'].iloc[i])
texcol = "%s,%s,%s" % (col[0], col[1], col[2])
if ctab['on_map'].iloc[i] == 1:
if ctab['pop'].iloc[i] == 0:
pop = '\\boldmath$<$\\textbf{1k}'
row = '\\rowcolor[rgb]{%s}\\textbf{%s} & \\textbf{%s} & '\
'%s\\\\ \n' %(texcol, mmi, city, pop)
else:
row = '\\rowcolor[rgb]{%s}\\textbf{%s} & \\textbf{%s} & '\
'\\textbf{%s}\\\\ \n' %(texcol, mmi, city, pop)
else:
row = '\\rowcolor[rgb]{%s}%s & %s & '\
'%s\\\\ \n' %(texcol, mmi, city, pop)
tabledata = tabledata + row
ctex = ctex.replace("[TABLEDATA]", tabledata)
template = template.replace("[CITYTABLE]", ctex)
eventid = edict['eventid']
# query ComCat for information about this event
# fill in the url, if we can find it
try:
ccinfo = ComCatInfo(eventid)
eventid, allids = ccinfo.getAssociatedIds()
event_url = ccinfo.getURL() + '#pager'
except:
event_url = DEFAULT_PAGER_URL
eventid = "Event ID: " + eventid
template = template.replace("[EVENTID]", texify(eventid))
template = template.replace("[EVENTURL]", texify(event_url))
# Write latex file
tex_output = os.path.join(version_dir, 'onepager.tex')
with open(tex_output, 'w') as f:
f.write(template)
pdf_output = os.path.join(version_dir, 'onepager.pdf')
stderr = ''
try:
cwd = os.getcwd()
os.chdir(version_dir)
cmd = '%s -interaction nonstopmode --output-directory %s %s' % (
LATEX_TO_PDF_BIN, version_dir, tex_output)
print('Running %s...' % cmd)
res, stdout, stderr = get_command_output(cmd)
os.chdir(cwd)
if not res:
return (None, stderr)
else:
if os.path.isfile(pdf_output):
return (pdf_output, stderr)
else:
pass
except Exception as e:
pass
finally:
os.chdir(cwd)
return (None, stderr)
def main(args):
#-------------------------------------------------------------
# where should .rst files, Makefile, _build folder be written?
#-------------------------------------------------------------
API_DIR = os.path.join(os.path.expanduser('~'), '__api-doc')
shutil.rmtree(API_DIR, ignore_errors=True)
#-------------------------------------------------------------
# where should the temporary clone of the shakemap gh-pages repo live?
#-------------------------------------------------------------
TOP_DIR = os.path.join(os.path.expanduser('~'), '__shake-doc')
CLONE_DIR = os.path.join(TOP_DIR, 'html')
shutil.rmtree(TOP_DIR, ignore_errors=True)
#-------------------------------------------------------------
# Some additional useful directories
#-------------------------------------------------------------
REPO_DIR = os.path.dirname(os.path.abspath(__file__))
DOC_DIR = os.path.join(REPO_DIR, 'doc')
REST_DIR = os.path.join(REPO_DIR, 'rest')
PACKAGE_DIR = os.path.join(REPO_DIR, 'shakemap')
#-------------------------------------------------------------
# get the human-friendly version of the ShakeMap version
#-------------------------------------------------------------
res, verstr, stderr = get_command_output('git describe --tags')
verstr = verstr.decode().strip()
if not len(verstr):
verstr = DEFAULT_TAG
else:
if verstr[0] == 'v':
verstr = verstr[1:]
spl = re.findall(r"[\w']+", verstr)
verstr = "%s.%s.%s" % (spl[0], spl[1], spl[2])
#-------------------------------------------------------------
# what is the package called and who are the authors
#-------------------------------------------------------------
PACKAGE = "shakemap"
AUTHORS = 'Bruce Worden, Eric Thompson, Mike Hearne'
# find the make command on this system
res, stdout, stderr = get_command_output('which make')
if not res:
print('Could not find the "make" command on your system. Exiting.')
sys.exit(1)
make_cmd = stdout.decode().strip()
#-------------------------------------------------------------
# clone the repository
#-------------------------------------------------------------
if args.post:
sys.stderr.write('Cloning shakemap gh-pages branch...\n')
if os.path.isdir(CLONE_DIR):
shutil.rmtree(CLONE_DIR)
clonecmd = 'git clone -b gh-pages https://github.com/usgs/'\
'shakemap.git %s' % CLONE_DIR
res, stdout, stderr = get_command_output(clonecmd)
if not res:
raise Exception('Could not clone gh-pages branch.')
# Delete everything in the repository (except hidden git files)
cmd = 'rm -fr %s' %CLONE_DIR
#-------------------------------------------------------------
# run the api doc command; this creates the .rst files
#-------------------------------------------------------------
sys.stderr.write('Building shakemap API documentation (REST)...\n')
sphinx_cmd = 'sphinx-apidoc -o %s -f -e -l -d 12 -F -H %s -A "%s"'\
' -V %s %s' % (API_DIR, PACKAGE, AUTHORS, verstr,
PACKAGE_DIR)
res, stdout, stderr = get_command_output(sphinx_cmd)
if not res:
raise Exception('Could not build ShakeMap API documentation'
' - error "%s".' % stderr)
#-------------------------------------------------------------
# change index.rst to api_index.rst
#-------------------------------------------------------------
move_cmd = 'mv %s/index.rst %s/api_index.rst' % (API_DIR, API_DIR)
res, stdout, stderr = get_command_output(move_cmd)
# Change name of API documentation in api_index.rst
cmd = "sed -i -e 's/Welcome to shakemap.*/ShakeMap 4.0 API/g' "\
"%s/api_index.rst" % API_DIR
res, stdout, stderr = get_command_output(cmd)
#--------------------------------------------
# try to clean up some of the excess labeling
#--------------------------------------------
clean_cmd = "sed -i -e 's/ module//g' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -i -e 's/ package//g' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -i -e '/Subpackages/d' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
clean_cmd = "sed -i -e '/-.*-/d' `find %s/*.rst -type f "\
"-maxdepth 0 -print`" % API_DIR
res, stdout, stderr = get_command_output(clean_cmd)
#-------------------------------------------------------------
# run the make command to build the shakemap manual (pdf version)
#-------------------------------------------------------------
if not args.nopdf:
sys.stderr.write('Building shakemap manual (PDF)...\n')
os.chdir(DOC_DIR)
# Need to run HTML to create __shakedoc/html/_static
manualcmd = '%s latexpdf' % make_cmd
res, stdout, stderr = get_command_output(manualcmd)
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
if not res:
raise Exception('Could not build the PDF version of the ShakeMap '
'manual - error "%s".' % stderr)
# Move PDF to API_DIR/_static
spdf = '%s/latex/ShakeMapManual.pdf' %TOP_DIR
dpdf = '%s/_static/ShakeMapManual.pdf' %API_DIR
shutil.copy2(spdf, dpdf)
#-------------------------------------------------------------
# Edit the conf.py file to include the theme.
#-------------------------------------------------------------
fname = os.path.join(API_DIR, 'conf.py')
f = open(fname, 'at')
f.write("sys.path.insert(0, os.path.abspath('%s'))\n" % (REPO_DIR))
#-------------------------------------
# Built in theme:
#-------------------------------------
# f.write("html_theme = 'haiku'\n")
#-------------------------------------
#-------------------------------------
# RTD theme
#-------------------------------------
f.write("import sphinx_rtd_theme\n")
f.write("html_theme = 'sphinx_rtd_theme'\n")
f.write("html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]\n")
f.write("html_theme_options = {\n")
f.write(" 'collapse_navigation': False,\n")
f.write("}\n")
#-------------------------------------
#-------------------------------------
# Bootstrap theme
#-------------------------------------
# f.write("import sphinx_bootstrap_theme\n")
# f.write("html_theme = 'bootstrap'\n")
# f.write("html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()\n")
# f.write("html_theme_options = {\n")
# f.write(" 'bootswatch_theme': \"Sandstone\",\n")
# f.write(" 'navbar_site_name': \"API Documentation\",\n")
# f.write(" 'navbar_sidebarrel': False,\n")
# f.write(" 'navbar_pagenav': False,\n")
# f.write(" 'navbar_links': [\n")
# f.write(" (\"Manual\", \"../shake_index.html\", 1),\n")
# f.write(" ],\n")
# f.write("}\n")
#-------------------------------------
# Napolean extension? Supports Goggle and Numpy style docstrings, but it
# also has some side effects such as restrictions on what sections are
# allowed and it seems to suppress the [source] link to code; maybe this
# is a configurable option though.
# f.write("extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.napoleon']\n")
# This line is needed to inclue __init__ methods in documentation
f.write("autoclass_content = 'both'\n")
f.write("autodoc_member_order = 'bysource'\n")
f.write("html_show_copyright = False\n")
# f.write("extensions = extensions + [ 'sphinx.ext.autodoc', "\
# "'sphinx.ext.napoleon', 'sphinx.ext.todo' ] \n")
f.write("napoleon_include_special_with_doc = False\n")
f.write("todo_include_todos = True\n")
f.close()
#-------------------------------------------------------------
# Copy the manual REST files to the API directory
#-------------------------------------------------------------
# Get rid of the automatic conf.py so our preferred one is used
os.remove('%s/conf.py' % (API_DIR))
# put aside Makefile so it doesn't get overwritten
oldmake = '%s/Makefile' %API_DIR
tmpmake = '%s/Makefile_save' %API_DIR
os.rename(oldmake, tmpmake)
# move files into API directory; this should raise exceptions if any
# files will get overwritten.
copy_tree(DOC_DIR, API_DIR, update=1)
# put Makefile back
os.rename(tmpmake, oldmake)
# Move index.rst to manual_index.rst
ind1 = '%s/index.rst' %API_DIR
ind2 = '%s/manual_index.rst' %API_DIR
os.rename(ind1, ind2)
# Copy index.rst from rest/ directory into build directory
restind = '%s/index.rst' %REST_DIR
apiind = '%s/index.rst' %API_DIR
shutil.copy2(restind, apiind)
#-------------------------------------------------------------
# Go to the api directory and build the html
#-------------------------------------------------------------
sys.stderr.write('Building shakemap manual (HTML)...\n')
os.chdir(API_DIR)
res, stdout, stderr = get_command_output('%s html' % make_cmd)
if not res:
raise Exception('Could not build HTML for API documentation. - '
'error "%s"' % stderr)
if args.verbose:
print(stdout.decode('utf-8'))
print(stderr.decode('utf-8'))
#-------------------------------------------------------------
# Copy the generated content to the gh-pages branch we created
# earlier
#-------------------------------------------------------------
htmldir = os.path.join(API_DIR, '_build', 'html')
if not os.path.isdir(CLONE_DIR):
os.makedirs(CLONE_DIR)
copy_tree(htmldir, CLONE_DIR)
if args.post:
#-------------------------------------------------------------
# Post to gh-pages
#-------------------------------------------------------------
# cd to directory above where html content was pushed
os.chdir(CLONE_DIR)
res, stdout, stderr = get_command_output('touch .nojekyll')
res1, stdout, stderr1 = get_command_output('git add --all')
res2, stdout, stderr2 = get_command_output(
'git commit -am"Pushing version %s to GitHub pages"' % verstr)
res3, stdout, stderr3 = get_command_output(
'git push -u origin +gh-pages')
if res1 + res2 + res3 < 3:
stderr = stderr1 + stderr2 + stderr3
print('Something bad happened when attempting to add, commit, '
'or push gh-pages content to GitHub - error "%s". Exiting.'
% stderr)
sys.exit(1)
print('You can inspect the ShakeMap manual and API docs by looking '
'here: http://usgs.github.io/shakemap/index.html')
else:
if not args.clean:
indexpage = os.path.join(CLONE_DIR, 'index.html')
print('ShakeMap documentation index: %s' % indexpage)
