def validate_config(mydict, install_path):
"""Recursively validate select.conf.
Args:
mydict (dict): Full or partial config dictionary.
install_path (str):
"""
for key in mydict:
if isinstance(mydict[key], dict):
validate_config(mydict[key], install_path)
continue
if key == 'horizontal_buffer' or key == 'vertical_buffer':
mydict[key] = config.cfg_float(mydict[key])
elif key == 'gmpe':
mydict[key] = config.gmpe_list(mydict[key], 1)
elif key == 'min_depth' or key == 'max_depth':
mydict[key] = config.cfg_float_list(mydict[key])
elif key == 'layer_dir':
mydict[key] = path_macro_sub(mydict[key], ip=install_path)
elif key in ('x1', 'x2', 'p1', 'p2', 'p_kagan_default',
'default_slab_depth'):
mydict[key] = float(mydict[key])
else:
raise ValidateError('Invalid entry in config: "%s"' % (key))
return
def execute(self):
"""
Create KML files.
Raises:
NotADirectoryError: When the event data directory does not exist.
FileNotFoundError: When the the shake_result HDF file does not
exist.
"""
install_path, data_path = get_config_paths()
datadir = os.path.join(data_path, self._eventid, 'current', 'products')
if not os.path.isdir(datadir):
raise NotADirectoryError('%s is not a valid directory.' % datadir)
datafile = os.path.join(datadir, 'shake_result.hdf')
if not os.path.isfile(datafile):
raise FileNotFoundError('%s does not exist.' % datafile)
# Open the ShakeMapOutputContainer and extract the data
container = ShakeMapOutputContainer.load(datafile)
if container.getDataType() != 'grid':
raise NotImplementedError('kml module can only contour '
'gridded data, not sets of points')
# find the low res ocean vector dataset
product_config_file = os.path.join(install_path, 'config',
'products.conf')
pconfig = configobj.ConfigObj(product_config_file)
oceanfile = pconfig['products']['mapping']['layers']['lowres_oceans']
oceanfile = path_macro_sub(oceanfile, install_path, data_path)
# call create_kmz function
create_kmz(container, datadir, oceanfile, self.logger)
def execute(self):
"""
Raises:
NotADirectoryError: When the event data directory does not exist.
FileNotFoundError: When the the shake_result HDF file does not
exist.
"""
install_path, data_path = get_config_paths()
datadir = os.path.join(data_path, self._eventid, 'current', 'products')
if not os.path.isdir(datadir):
raise NotADirectoryError('%s is not a valid directory.' % datadir)
datafile = os.path.join(datadir, 'shake_result.hdf')
if not os.path.isfile(datafile):
raise FileNotFoundError('%s does not exist.' % datafile)
# Open the ShakeMapOutputContainer and extract the data
container = ShakeMapOutputContainer.load(datafile)
if container.getDataType() != 'grid':
raise NotImplementedError('mapping module can only operate on '
'gridded data, not sets of points')
# get the path to the products.conf file, load the config
config_file = os.path.join(install_path, 'config', 'products.conf')
config = ConfigObj(config_file)
# create contour files
self.logger.info('Mapping...')
# get all of the pieces needed for the mapmaker
layerdict = {}
layers = config['products']['mapping']['layers']
layerdict['coast'] = path_macro_sub(
layers['coasts'], install_path, data_path)
layerdict['ocean'] = path_macro_sub(
layers['oceans'], install_path, data_path)
layerdict['lake'] = path_macro_sub(
layers['lakes'], install_path, data_path)
layerdict['country'] = path_macro_sub(
layers['countries'], install_path, data_path)
layerdict['state'] = path_macro_sub(
layers['states'], install_path, data_path)
topofile = path_macro_sub(
layers['topography'], install_path, data_path)
cities = path_macro_sub(layers['cities'], install_path, data_path)
mapmaker = MapMaker(container, topofile, layerdict, cities,
self.logger)
self.logger.info('Drawing intensity map...')
intensity_map = mapmaker.drawIntensityMap(datadir)
self.logger.info('Created intensity map %s' % intensity_map)
for imt in config['products']['mapping']['imts']:
self.logger.info('Drawing %s contour map...' % imt)
contour_file = mapmaker.drawContourMap(imt, datadir)
self.logger.info('Created contour map %s' % contour_file)
def test_create_kmz():
tempdir = tempfile.mkdtemp()
try:
homedir = os.path.dirname(os.path.abspath(__file__))
cfile = os.path.join(homedir, '..', '..', 'data', 'containers',
'northridge', 'shake_result.hdf')
container = ShakeMapOutputContainer.load(cfile)
install_path, data_path = get_config_paths()
global_data_path = os.path.join(os.path.expanduser('~'),
'shakemap_data')
product_config_file = os.path.join(install_path, 'config',
'products.conf')
pconfig = configobj.ConfigObj(product_config_file)
oceanfile = pconfig['products']['mapping']['layers']['lowres_oceans']
oceanfile = path_macro_sub(oceanfile, install_path, data_path,
global_data_path)
logger = logging.getLogger(__name__)
kmzfile = create_kmz(container, tempdir, oceanfile, logger)
myzip = zipfile.ZipFile(kmzfile, mode='r')
kmlstr = myzip.read('shakemap.kml').decode('utf-8')
root = minidom.parseString(kmlstr)
document = root.getElementsByTagName('Document')[0]
folders = document.getElementsByTagName('Folder')
names = []
nstations = 0
nmmi = 0
for folder in folders:
name = folder.getElementsByTagName('name')[0].firstChild.data
names.append(name)
if name == 'Instrumented Stations':
nstations = len(folder.getElementsByTagName('Placemark'))
elif name == 'Macroseismic Stations':
nmmi = len(folder.getElementsByTagName('Placemark'))
assert sorted(names) == [
'Contours', 'Instrumented Stations', 'Macroseismic Stations'
]
assert nstations == 185
assert nmmi == 977
myzip.close()
except Exception as e:
print(str(e))
assert 1 == 2
finally:
shutil.rmtree(tempdir)
def execute(self):
"""
Assemble ShakeMap input data and write and ShakeMapInputContainer named
shake_data.hdf in the event's 'current' directory.
Raises:
NotADirectoryError: When the event data directory does not
exist.
FileNotFoundError: When the the event's event.xml file does
not exist.
RuntimeError: When there are problems parsing the configuration.
ValidateError: When there are configuration items missing or mis-
configured.
"""
install_path, data_path = get_config_paths()
datadir = os.path.join(data_path, self._eventid, 'current')
if not os.path.isdir(datadir):
raise NotADirectoryError('%s is not a valid directory.' % datadir)
eventxml = os.path.join(datadir, 'event.xml')
self.logger.debug('Looking for event.xml file...')
if not os.path.isfile(eventxml):
raise FileNotFoundError('%s does not exist.' % eventxml)
# Prompt for a comment string if none is provided on the command line
if self.comment is None:
if sys.stdout.isatty():
self.comment = input(
'Please enter a comment for this version.\n'
'comment: ')
else:
self.comment = ''
# find any source.txt or moment.xml files
momentfile = os.path.join(datadir, 'moment.xml')
sourcefile = os.path.join(datadir, 'source.txt')
if not os.path.isfile(sourcefile):
sourcefile = None
if not os.path.isfile(momentfile):
momentfile = None
#
# Clear away results from previous runs
#
products_path = os.path.join(datadir, 'products')
if os.path.isdir(products_path):
shutil.rmtree(products_path, ignore_errors=True)
#
# Look for global configs in install_path/config
#
spec_file = get_configspec()
validator = get_custom_validator()
self.logger.debug('Looking for configuration files...')
modules = ConfigObj(
os.path.join(install_path, 'config', 'modules.conf'),
configspec=spec_file)
gmpe_sets = ConfigObj(
os.path.join(install_path, 'config', 'gmpe_sets.conf'),
configspec=spec_file)
global_config = ConfigObj(
os.path.join(install_path, 'config', 'model.conf'),
configspec=spec_file)
#
# this is the event specific model.conf (may not be present)
# prefer model.conf to model_zc.conf
#
event_config_file = os.path.join(datadir, 'model.conf')
event_config_zc_file = os.path.join(datadir, 'model_zc.conf')
if os.path.isfile(event_config_file):
event_config = ConfigObj(event_config_file,
configspec=spec_file)
elif os.path.isfile(event_config_zc_file):
event_config = ConfigObj(event_config_zc_file,
configspec=spec_file)
else:
event_config = ConfigObj()
#
# start merging event_config
#
global_config.merge(event_config)
global_config.merge(modules)
global_config.merge(gmpe_sets)
results = global_config.validate(validator)
if not isinstance(results, bool) or not results:
config_error(global_config, results)
check_config(global_config, self.logger)
#
# The vs30 file may have macros in it
#
vs30file = global_config['data']['vs30file']
global_data_path = os.path.join(os.path.expanduser('~'),
'shakemap_data')
if vs30file:
vs30file = path_macro_sub(vs30file, ip=install_path,
dp=data_path, gp=global_data_path,
ei=self._eventid)
if not os.path.isfile(vs30file):
raise FileNotFoundError("vs30 file '%s' is not a valid file" %
vs30file)
global_config['data']['vs30file'] = vs30file
#
# If there is a prediction_location->file file, then we need
# to expand any macros
#
if 'file' in global_config['interp']['prediction_location']:
loc_file = global_config['interp']['prediction_location']['file']
if loc_file and loc_file != 'None': # 'None' is a string here
loc_file = path_macro_sub(loc_file, ip=install_path,
dp=data_path, gp=global_data_path,
ei=self._eventid)
if not os.path.isfile(loc_file):
raise FileNotFoundError("prediction file '%s' is not "
"a valid file" % loc_file)
global_config['interp']['prediction_location']['file'] = \
loc_file
config = global_config.dict()
self.logger.debug('Looking for data files...')
datafiles = glob.glob(os.path.join(datadir, '*_dat.xml'))
if os.path.isfile(os.path.join(datadir, 'stationlist.xml')):
datafiles.append(os.path.join(datadir, 'stationlist.xml'))
self.logger.debug('Looking for rupture files...')
# look for geojson versions of rupture files
rupturefile = os.path.join(datadir, 'rupture.json')
if not os.path.isfile(rupturefile):
# failing any of those, look for text file versions
rupturefiles = glob.glob(os.path.join(datadir, '*_fault.txt'))
rupturefile = None
if len(rupturefiles):
rupturefile = rupturefiles[0]
#
# Sort out the version history. Get the most recent backup file and
# extract the existing history. Then add a new line for this run.
#
timestamp = datetime.datetime.utcnow().strftime('%FT%TZ')
originator = config['system']['source_network']
backup_dirs = sorted(
glob.glob(os.path.join(datadir, '..', 'backup*')),
reverse=True)
if len(backup_dirs):
#
# Backup files exist so find the latest one and extract its
# history, then add a new line that increments the version
#
bu_file = os.path.join(backup_dirs[0], 'shake_data.hdf')
bu_ic = ShakeMapInputContainer.load(bu_file)
history = bu_ic.getVersionHistory()
bu_ic.close()
version = int(
backup_dirs[0].replace(
os.path.join(datadir, '..', 'backup'), ''))
version += 1
new_line = [timestamp, originator, version, self.comment]
history['history'].append(new_line)
elif os.path.isfile(os.path.join(datadir, 'shake_data.hdf')):
#
# No backups are available, but there is an existing shake_data
# file. Extract its history and update the timestamp and
# source network (but leave the version alone).
# If there is no history, just start a new one with version 1
#
bu_file = os.path.join(datadir, 'shake_data.hdf')
bu_ic = ShakeMapInputContainer.load(bu_file)
history = bu_ic.getVersionHistory()
bu_ic.close()
if 'history' in history:
new_line = [timestamp, originator, history['history'][-1][2],
self.comment]
history['history'][-1] = new_line
else:
history = {'history': []}
new_line = [timestamp, originator, 1, self.comment]
history['history'].append(new_line)
else:
#
# No backup and no existing file. Make this version 1
#
history = {'history': []}
new_line = [timestamp, originator, 1, self.comment]
history['history'].append(new_line)
hdf_file = os.path.join(datadir, 'shake_data.hdf')
self.logger.debug('Creating input container...')
shake_data = ShakeMapInputContainer.createFromInput(
hdf_file,
config,
eventxml,
history,
rupturefile=rupturefile,
sourcefile=sourcefile,
momentfile=momentfile,
datafiles=datafiles)
self.logger.debug('Created HDF5 input container in %s' %
shake_data.getFileName())
ah = AmplitudeHandler(install_path, data_path)
event = ah.getEvent(self._eventid)
if event is None:
origin = shake_data.getRuptureObject().getOrigin()
event = {'id': self._eventid,
'netid': origin.netid,
'network': origin.network,
'time': origin.time.strftime(queue.TIMEFMT),
'lat': origin.lat,
'lon': origin.lon,
'depth': origin.depth,
'mag': origin.mag,
'locstring': origin.locstring}
ah.insertEvent(event)
shake_data.close()
def execute(self):
"""
Augment a ShakeMap input data file with local configs, data, rupture,
etc. The version history will only be incremented if the originator
differs from the originator in the previous line of the history.
Raises:
NotADirectoryError: When the event data directory does not
exist.
FileNotFoundError: When the the event's event.xml file does
not exist.
RuntimeError: When there are problems parsing the configuration.
ValidateError: When there are configuration items missing or mis-
configured.
"""
install_path, data_path = get_config_paths()
datadir = os.path.join(data_path, self._eventid, 'current')
if not os.path.isdir(datadir):
raise NotADirectoryError('%s is not a valid directory.' % datadir)
hdf_file = os.path.join(datadir, 'shake_data.hdf')
if not os.path.isfile(hdf_file):
raise FileNotFoundError('%s does not exist. Use assemble.' %
hdf_file)
shake_data = ShakeMapInputContainer.load(hdf_file)
# Prompt for a comment string if none is provided on the command line
if self.comment is None:
if sys.stdout.isatty():
self.comment = input(
'Please enter a comment for this version.\n'
'(Start with "+" if you wish to append to the\n'
'existing comment; "+" by itself will preserve\n'
'existing comments.)\n'
'comment: ')
else:
self.comment = ''
#
# Clear away results from previous runs
#
products_path = os.path.join(datadir, 'products')
if os.path.isdir(products_path):
shutil.rmtree(products_path, ignore_errors=True)
#
# Get the config from the HDF file and merge in the local configs
#
spec_file = get_configspec()
validator = get_custom_validator()
shake_config = shake_data.getConfig()
shake_config = ConfigObj(shake_config, configspec=spec_file)
#
# This is a weird hack to get around a bug/feature of ConfigObj
# that results in the validation failing if max_workers is already
# an integer.
#
if 'max_workers' in shake_config['system']:
shake_config['system']['max_workers'] = \
str(shake_config['system']['max_workers'])
modules_file = os.path.join(install_path, 'config', 'modules.conf')
if os.path.isfile(modules_file):
self.logger.debug('Found a modules file.')
modules = ConfigObj(modules_file, configspec=spec_file)
shake_config.merge(modules)
gmpe_file = os.path.join(install_path, 'config', 'gmpe_sets.conf')
if os.path.isfile(gmpe_file):
self.logger.debug('Found a gmpe file.')
gmpe_sets = ConfigObj(gmpe_file, configspec=spec_file)
shake_config.merge(gmpe_sets)
config_file = os.path.join(install_path, 'config', 'model.conf')
if os.path.isfile(config_file):
self.logger.debug('Found a global config file.')
global_config = ConfigObj(config_file, configspec=spec_file)
shake_config.merge(global_config)
# extent conf (may not be present)
extent_config = os.path.join(install_path, 'config', 'extent.conf')
if os.path.isfile(extent_config):
extent_config = ConfigObj(extent_config, configspec=spec_file)
else:
extent_config = ConfigObj()
shake_config.merge(extent_config)
#
# this is the event specific model.conf (may not be present)
# prefer model.conf to model_zc.conf
#
event_config_file = os.path.join(datadir, 'model.conf')
event_config_zc_file = os.path.join(datadir, 'model_zc.conf')
if os.path.isfile(event_config_file):
self.logger.debug('Found an event specific model.conf file.')
event_config = ConfigObj(event_config_file, configspec=spec_file)
shake_config.merge(event_config)
elif os.path.isfile(event_config_zc_file):
self.logger.debug('Found an event specific model_zc file.')
event_config = ConfigObj(event_config_zc_file,
configspec=spec_file)
shake_config.merge(event_config)
#
# Validate the resulting config
#
results = shake_config.validate(validator)
if not results or isinstance(results, dict):
config_error(shake_config, results)
check_config(shake_config, self.logger)
#
# The vs30 file may have macros in it
#
vs30file = shake_config['data']['vs30file']
global_data_path = os.path.join(os.path.expanduser('~'),
'shakemap_data')
if vs30file:
vs30file = path_macro_sub(vs30file,
ip=install_path,
dp=data_path,
gp=global_data_path,
ei=self._eventid)
if not os.path.isfile(vs30file):
raise FileNotFoundError('vs30 file "%s" is not a '
'valid file' % vs30file)
shake_config['data']['vs30file'] = vs30file
#
# If there is a prediction_location->file file, then we need
# to expand any macros
#
if 'file' in shake_config['interp']['prediction_location']:
loc_file = shake_config['interp']['prediction_location']['file']
if loc_file and loc_file != 'None': # 'None' is a string here
loc_file = path_macro_sub(loc_file,
ip=install_path,
dp=data_path,
gp=global_data_path,
ei=self._eventid)
if not os.path.isfile(loc_file):
raise FileNotFoundError('prediction file "%s" is not a '
'valid file' % loc_file)
shake_config['interp']['prediction_location']['file'] = \
loc_file
#
# Put the updated config back into shake_data.hdf`
#
config = shake_config.dict()
shake_data.setConfig(config)
#
# Look for additional data files and update the stationlist if found
#
datafiles = glob.glob(os.path.join(datadir, '*_dat.xml'))
if os.path.isfile(os.path.join(datadir, 'stationlist.xml')):
datafiles.append(os.path.join(datadir, 'stationlist.xml'))
if datafiles:
self.logger.debug('Found additional data files...')
shake_data.addStationData(datafiles)
#
# Look for a rupture file and replace the existing one if found
#
rupturefile = os.path.join(datadir, 'rupture.json')
eventxml = os.path.join(datadir, 'event.xml')
if not os.path.isfile(eventxml):
eventxml = None
if not os.path.isfile(rupturefile):
faultfiles = glob.glob(os.path.join(datadir, '*_fault.txt'))
if len(faultfiles):
rupturefile = faultfiles[0]
else:
rupturefile = None
if (rupturefile and os.path.isfile(rupturefile)) \
or eventxml is not None:
self.logger.debug('Updating rupture/origin information.')
shake_data.updateRupture(eventxml=eventxml,
rupturefile=rupturefile)
#
# Sort out the version history. We're working with an existing
# HDF file, so: if we are the originator, just update the timestamp,
# otherwise add a new line.
#
timestamp = datetime.datetime.utcnow().strftime('%FT%TZ')
originator = config['system']['source_network']
history = shake_data.getVersionHistory()
if history['history'][-1][1] == originator:
history['history'][-1][0] = timestamp
if self.comment.startswith('+'):
if self.comment.replace('+', '') != '':
history['history'][-1][3] += self.comment.replace('+', ' ')
else:
history['history'][-1][3] = self.comment
else:
version = int(history['history'][-1][2]) + 1
if self.comment.startswith('+'):
new_line = [
timestamp, originator, version,
self.comment.replace('+', '')
]
else:
new_line = [timestamp, originator, version, self.comment]
history['history'].append(new_line)
shake_data.setVersionHistory(history)
shake_data.close()