def execute(self): """ Output the version history of an event. Raises: NotADirectoryError: When the event data directory does not exist. """ _, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current') backups = glob.glob(os.path.join(data_path, self._eventid, 'backup*')) backups.sort(reverse=True) if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) # First try the current results file... datafile = os.path.join(datadir, 'products', 'shake_result.hdf') if os.path.isfile(datafile): # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) try: metadata = container.getMetadata() except LookupError: print("\nNo version history available for this event.\n") return history = (metadata['processing']['shakemap_versions'] ['map_data_history']) final = False if len(backups) > 0: last_ver = int(backups[0][-4:]) last_hist = history[-1][2] if last_ver == last_hist: final = True print_history(history, final=final) return # Nope. Are there any backup files? if len(backups) == 0: print("\nNo version history available for this event.\n") return # There should be a results file in the backup directory... datafile = os.path.join(data_path, self._eventid, backups[0], 'products', 'shake_result.hdf') if os.path.isfile(datafile): # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) try: metadata = container.getMetadata() except LookupError: print("\nNo version history available for this event.\n") return history = (metadata['processing']['shakemap_versions'] ['map_data_history']) print_history(history, final=True) return print("\nNo version history available for this event.\n") return
def execute(self): """ Write rupture.json file. Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) # create ShakeMap rupture file for fformat in ALLOWED_FORMATS: if fformat == 'json': self.logger.info('Writing rupture.json file...') rupture_dict = container.getRuptureDict() rupture_file = os.path.join(datadir, 'rupture.json') f = open(rupture_file, 'w') json.dump(rupture_dict, f) f.close() container.close() self.contents.addFile('ruptureJSON', 'Fault Rupture', 'JSON Representation of Fault Rupture.', 'rupture.json', 'application/json')
def dummy_test_transfer(): homedir = os.path.dirname(os.path.abspath(__file__)) cfile = os.path.join(homedir, '..', '..', 'data', 'containers', 'northridge', 'shake_result.hdf') products_dir = os.path.join(homedir, '..', '..', 'data', 'eventdata', 'northridge', 'current') pdl_dir = os.path.join(homedir, '..', '..', 'data', 'eventdata', 'northridge', 'current', 'pdl') eventid = 'ci3144585' container = ShakeMapOutputContainer.load(cfile) try: tdir = tempfile.mkdtemp() remote_dir = os.path.join(tdir, eventid) config = {'copy': {'local': {'remote_directory': tdir}}} # transfermod = TransferModule(eventid) _transfer(config, container.getMetadata(), pdl_dir, products_dir) nfiles = len(os.listdir(remote_dir)) nsrcfiles = len(os.listdir(products_dir)) assert nfiles == nsrcfiles except Exception as e: print('Exception: %s' % str(e)) finally: if os.path.isdir(tdir): shutil.rmtree(tdir)
def test_macros(): homedir = os.path.dirname(os.path.abspath(__file__)) shakedir = os.path.abspath(os.path.join(homedir, '..', '..', '..')) out_file = os.path.join(shakedir, 'tests', 'data', 'containers', 'northridge', 'shake_result.hdf') container = ShakeMapOutputContainer.load(out_file) info = container.getMetadata() macros = get_macros(info) test_dict = { 'LON': '-118.5357', 'LAT': '34.213', 'DATE': 'Jan 17, 1994', 'DEP': '18.0', 'MAG': '6.7', 'LOC': 'Northridge', 'NETID': 'ci', 'DATETIME': '1994-01-17T12:30:55.000000Z', 'EVENTID': 'northridge', 'VERSION': '1', 'PRODUCT_CODE': 'northridge', 'TIME': '12:30:55' } assert list(sorted(macros.keys())) == list(sorted(test_dict.keys())) for key, mvalue in macros.items(): tvalue = test_dict[key] print('Testing key %s: %s vs %s.' % (key, mvalue, tvalue)) assert mvalue == tvalue
def execute(self): """ Create KML files. Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) if container.getDataType() != 'grid': raise NotImplementedError('kml module can only contour ' 'gridded data, not sets of points') # call create_kmz function create_kmz(container, datadir, self.logger, self.contents) container.close()
def test_masking(): install_path, data_path = get_config_paths() event_path = os.path.join(data_path, 'masking_test', 'current') set_files( event_path, { 'event.xml': 'event.xml', 'model.conf': 'model.conf', 'au_continental_shelf.geojson': 'au_continental_shelf.geojson', }) assemble = AssembleModule('masking_test', comment='Test comment.') assemble.execute() model = ModelModule('masking_test') model.execute() clear_files(event_path) hdf_file = os.path.join(event_path, 'products', 'shake_result.hdf') oc = ShakeMapOutputContainer.load(hdf_file) sa3 = oc.getIMTGrids('SA(3.0)', 'GREATER_OF_TWO_HORIZONTAL')['mean'] removed = np.isnan(sa3).astype(int) assert (removed[240, 240] == 1) assert (removed[260, 240] == 0) np.testing.assert_equal( removed[::100, ::100], [[1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 0, 1], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]]) oc.close()
def test_directivity(): # # Turned on directivity in model.conf # install_path, data_path = get_config_paths() event_path = os.path.join(data_path, 'directivity_test', 'current') set_files( event_path, { 'event.xml': 'event.xml', 'model.conf': 'model.conf', 'dir_fault.txt': 'dir_fault.txt' }) assemble = AssembleModule('directivity_test', comment='Test comment.') assemble.execute() model = ModelModule('directivity_test') model.execute() clear_files(event_path) hdf_file = os.path.join(event_path, 'products', 'shake_result.hdf') oc = ShakeMapOutputContainer.load(hdf_file) sa3 = np.exp( oc.getIMTGrids('SA(3.0)', 'GREATER_OF_TWO_HORIZONTAL')['mean']) # np.testing.assert_allclose(np.max(sa3), 1.15864273) np.testing.assert_allclose(np.max(sa3), 1.1567265149442174) # np.testing.assert_allclose(np.min(sa3), 0.9278920) np.testing.assert_allclose(np.min(sa3), 0.88508818541678) oc.close()
def test_mapmaker_intensity(): homedir = os.path.dirname( os.path.abspath(__file__)) # where is this script? shakedir = os.path.abspath(os.path.join(homedir, '..', '..', '..')) out_file = os.path.join(shakedir, 'tests', 'data', 'containers', 'northridge', 'shake_result.hdf') container = ShakeMapOutputContainer.load(out_file) topofile = os.path.join(homedir, '..', '..', 'data', 'install', 'data', 'mapping', 'CA_topo.grd') info = container.getMetadata() xmin = info['output']['map_information']['min']['longitude'] xmax = info['output']['map_information']['max']['longitude'] ymin = info['output']['map_information']['min']['latitude'] ymax = info['output']['map_information']['max']['latitude'] xmin = float(xmin) - 0.1 xmax = float(xmax) + 0.1 ymin = float(ymin) - 0.1 ymax = float(ymax) + 0.1 dy = float(info['output']['map_information']['grid_spacing']['latitude']) dx = float(info['output']['map_information']['grid_spacing']['longitude']) sampledict = GeoDict.createDictFromBox(xmin, xmax, ymin, ymax, dx, dy) topogrid = GMTGrid.load(topofile, samplegeodict=sampledict, resample=False) outpath = mkdtemp() model_config = container.getConfig() comp = container.getComponents('MMI')[0] textfile = os.path.join(get_data_path(), 'mapping', 'map_strings.en') text_dict = get_text_strings(textfile) cities = Cities.fromDefault() d = { 'imtype': 'MMI', 'topogrid': topogrid, 'allcities': cities, 'states_provinces': None, 'countries': None, 'oceans': None, 'lakes': None, 'roads': None, 'faults': None, 'datadir': outpath, 'operator': 'NEIC', 'filter_size': 10, 'info': info, 'component': comp, 'imtdict': container.getIMTGrids('MMI', comp), 'ruptdict': copy.deepcopy(container.getRuptureDict()), 'stationdict': container.getStationDict(), 'config': model_config, 'tdict': text_dict } try: fig1, fig2 = draw_map(d) except Exception: assert 1 == 2 finally: shutil.rmtree(outpath)
def execute(self): """Write stationlist.json file. Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) # create ShakeMap station data file for fformat in ALLOWED_FORMATS: if fformat == 'json': self.logger.debug('Writing rupture.json file...') station_dict = container.getStationDict() station_file = os.path.join(datadir, 'stationlist.json') f = open(station_file, 'w') json.dump(station_dict, f) f.close() container.close()
def execute(self): """ Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) if container.getDataType() != 'points': raise NotImplementedError('xtestplot module can only operate on ' 'sets of points, not gridded data') datadict = {} imtlist = container.getIMTs('GREATER_OF_TWO_HORIZONTAL') for myimt in imtlist: datadict[myimt] = container.getIMTArrays( myimt, 'GREATER_OF_TWO_HORIZONTAL') container.close() # # Make plots # for myimt in imtlist: data = datadict[myimt] fig, axa = plt.subplots(2, sharex=True, figsize=(10, 8)) plt.subplots_adjust(hspace=0.1) axa[0].plot(data['lons'], data['mean'], color='k', label='mean') axa[0].plot(data['lons'], data['mean'] + data['std'], '--b', label='mean +/- stddev') axa[0].plot(data['lons'], data['mean'] - data['std'], '--b') axa[1].plot(data['lons'], data['std'], '-.r', label='stddev') plt.xlabel('Longitude') axa[0].set_ylabel('Mean ln(%s) (g)' % myimt) axa[1].set_ylabel('Stddev ln(%s) (g)' % myimt) axa[0].legend(loc='best') axa[1].legend(loc='best') axa[0].set_title(self._eventid) axa[0].grid() axa[1].grid() axa[1].set_ylim(bottom=0) fileimt = oq_to_file(myimt) pfile = os.path.join(datadir, self._eventid + '_' + fileimt + '.pdf') plt.savefig(pfile) pfile = os.path.join(datadir, self._eventid + '_' + fileimt + '.png') plt.savefig(pfile) plt.close()
def test_create_kmz(): tempdir = tempfile.mkdtemp() try: homedir = os.path.dirname(os.path.abspath(__file__)) cfile = os.path.join(homedir, '..', '..', 'data', 'containers', 'northridge', 'shake_result.hdf') container = ShakeMapOutputContainer.load(cfile) install_path, data_path = get_config_paths() product_config_file = os.path.join(install_path, 'config', 'products.conf') spec_file = get_configspec('products') validator = get_custom_validator() pconfig = configobj.ConfigObj(product_config_file, configspec=spec_file) results = pconfig.validate(validator) if not isinstance(results, bool) or not results: config_error(pconfig, results) oceanfile = pconfig['products']['mapping']['layers']['lowres_oceans'] logger = logging.getLogger(__name__) kmzfile = create_kmz(container, tempdir, oceanfile, logger) myzip = zipfile.ZipFile(kmzfile, mode='r') kmlstr = myzip.read('shakemap.kml').decode('utf-8') root = minidom.parseString(kmlstr) document = root.getElementsByTagName('Document')[0] folders = document.getElementsByTagName('Folder') names = [] nstations = 0 nmmi = 0 for folder in folders: name = folder.getElementsByTagName('name')[0].firstChild.data names.append(name) if name == 'Instrumented Stations': nstations = len(folder.getElementsByTagName('Placemark')) elif name == 'Macroseismic Stations': nmmi = len(folder.getElementsByTagName('Placemark')) assert sorted(names) == [ 'Contours', 'Instrumented Stations', 'MMI 4 Polygons', 'MMI 5 Polygons', 'MMI 6 Polygons', 'MMI 7 Polygons', 'MMI 8 Polygons', 'MMI 8.5 Polygons', 'MMI Contours', 'MMI Labels', 'MMI Polygons', 'Macroseismic Stations', 'PGA Contours', 'PGV Contours', 'SA(0.3) Contours', 'SA(1.0) Contours', 'SA(3.0) Contours' ] assert nstations == 185 assert nmmi == 547 myzip.close() except Exception as e: print(str(e)) assert 1 == 2 finally: shutil.rmtree(tempdir)
def execute(self): install_path, data_path = get_config_paths() self.datadir = os.path.join(data_path, self._eventid, 'current') if not os.path.isdir(self.datadir): raise NotADirectoryError('%s is not a valid directory.' % self.datadir) # look for the presence of a NO_TRANSFER file in the datadir. notransfer = os.path.join(self.datadir, NO_TRANSFER) if os.path.isfile(notransfer): self.logger.info('Event has a %s file blocking transfer.' % NO_TRANSFER) return # get the path to the transfer.conf spec file configspec = os.path.join(get_data_path(), 'transferspec.conf') # look for an event specific transfer.conf file transfer_conf = os.path.join(self.datadir, 'transfer.conf') if not os.path.isfile(transfer_conf): # if not there, use the system one transfer_conf = os.path.join(install_path, 'config', 'transfer.conf') if not os.path.isfile(transfer_conf): raise FileNotFoundError('%s does not exist.' % transfer_conf) # get the config information for transfer self.config = ConfigObj(transfer_conf, configspec=configspec) results = self.config.validate(Validator()) if not isinstance(results, bool) or not results: config_error(self.config, results) # get the output container with all the things in it products_dir = os.path.join(self.datadir, 'products') datafile = os.path.join(products_dir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) # extract the info.json object from the container self.info = container.getMetadata() container.close() # check for the presence of a .saved file. If found, do nothing. # Otherwise, create the backup directory. save_file = os.path.join(self.datadir, SAVE_FILE) if not os.path.isfile(save_file): logging.info('Making backup directory...') self._make_backup(data_path) with open(save_file, 'wt') as f: tnow = datetime.utcnow().strftime(constants.TIMEFMT) f.write('Saved %s by %s\n' % (tnow, self.command_name))
def run_event(evid): installpath, datapath = get_config_paths() assemble = AssembleModule(evid, comment='Test comment.') assemble.execute() model = ModelModule(evid) model.execute() res_file = os.path.join(datapath, evid, 'current', 'products', 'shake_result.hdf') oc = ShakeMapOutputContainer.load(res_file) imts = oc.getIMTs() imt = imts[0].split('/')[1] comps = oc.getComponents(imt) imtdict = oc.getIMTArrays(imt, comps[0]) return imtdict
def execute(self): """ Create shape files. Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) if container.getDataType() != 'grid': raise NotImplementedError('shape module can only contour ' 'gridded data, not sets of points') config_file = os.path.join(install_path, 'config', 'products.conf') spec_file = get_configspec('products') validator = get_custom_validator() config = ConfigObj(config_file, configspec=spec_file) results = config.validate(validator) if not isinstance(results, bool) or not results: config_error(config, results) max_workers = config['products']['mapping']['max_workers'] method = config['products']['shape']['method'] create_polygons(container, datadir, self.logger, max_workers, method=method) container.close() self.contents.addFile('shakemap_shapefiles', 'ShakeMap Shape Files', 'Shape Files.', 'shape.zip', 'application/zip')
def test_create_kmz(): tempdir = tempfile.mkdtemp() try: homedir = os.path.dirname(os.path.abspath(__file__)) cfile = os.path.join(homedir, '..', '..', 'data', 'containers', 'northridge', 'shake_result.hdf') container = ShakeMapOutputContainer.load(cfile) install_path, data_path = get_config_paths() logger = logging.getLogger(__name__) contents = Contents(None, None, 'northridge') kmzfile = create_kmz(container, tempdir, logger, contents) myzip = zipfile.ZipFile(kmzfile, mode='r') kmlstr = myzip.read('shakemap.kml').decode('utf-8') root = minidom.parseString(kmlstr) document = root.getElementsByTagName('Document')[0] folders = document.getElementsByTagName('Folder') names = [] nstations = 0 nmmi = 0 for folder in folders: name = folder.getElementsByTagName('name')[0].firstChild.data names.append(name) if name == 'Instrumented Stations': nstations = len(folder.getElementsByTagName('Placemark')) elif name == 'Macroseismic Stations': nmmi = len(folder.getElementsByTagName('Placemark')) assert sorted(names) == [ 'Contours', 'Instrumented Stations', 'MMI 4 Polygons', 'MMI 5 Polygons', 'MMI 6 Polygons', 'MMI 7 Polygons', 'MMI 8 Polygons', 'MMI 8.5 Polygons', 'MMI Contours', 'MMI Labels', 'MMI Polygons', 'Macroseismic Stations', 'PGA Contours', 'PGV Contours', 'SA(0.3) Contours', 'SA(1.0) Contours', 'SA(3.0) Contours' ] assert nstations == 185 assert nmmi == 547 myzip.close() except Exception as e: print(str(e)) assert 1 == 2 finally: shutil.rmtree(tempdir)
def test_mapmaker_contour(): homedir = os.path.dirname(os.path.abspath( __file__)) # where is this script? shakedir = os.path.abspath(os.path.join(homedir, '..', '..', '..')) out_file = os.path.join(shakedir, 'tests', 'data', 'containers', 'northridge', 'shake_result.hdf') container = ShakeMapOutputContainer.load(out_file) topofile = os.path.join(homedir, '..', '..', 'data', 'install', 'data', 'mapping', 'CA_topo.grd') info = container.getMetadata() xmin = info['output']['map_information']['min']['longitude'] xmax = info['output']['map_information']['max']['longitude'] ymin = info['output']['map_information']['min']['latitude'] ymax = info['output']['map_information']['max']['latitude'] xmin = float(xmin) - 0.1 xmax = float(xmax) + 0.1 ymin = float(ymin) - 0.1 ymax = float(ymax) + 0.1 dy = float(info['output']['map_information'] ['grid_spacing']['latitude']) dx = float(info['output']['map_information'] ['grid_spacing']['longitude']) sampledict = GeoDict.createDictFromBox(xmin, xmax, ymin, ymax, dx, dy) topogrid = GMTGrid.load(topofile, samplegeodict=sampledict, resample=False) oceanfile = os.path.join(homedir, '..', '..', 'data', 'install', 'data', 'mapping', 'northridge_ocean.json') outpath = mkdtemp() filter_size = 10 try: pdf, png = draw_contour(container, 'PGA', topogrid, oceanfile, outpath, 'NEIC', filter_size) print(pdf) except Exception: assert 1 == 2 finally: shutil.rmtree(outpath)
def execute(self): """ Write info.json metadata file. Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) # Create ShakeMap metadata file self.logger.debug('Writing info.json file...') info = container.getMetadata() # Clean up strec results to be valid json if 'strec' in info: for k, v in info['strec'].items(): if isinstance(v, float): if not np.isfinite(v): info['strec'][k] = None infostring = json.dumps(info, allow_nan=False) info_file = os.path.join(datadir, 'info.json') f = open(info_file, 'wt') f.write(infostring) f.close() container.close() cap = 'ShakeMap processing parameters and map summary information.' self.contents.addFile('supplementalInformation', 'Supplemental Information', cap, 'info.json', 'application/json')
def execute(self): """ Create contour files for all configured IMT values. Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) # get the path to the products.conf file, load the config config_file = os.path.join(install_path, 'config', 'products.conf') spec_file = get_configspec('products') validator = get_custom_validator() config = ConfigObj(config_file, configspec=spec_file) results = config.validate(validator) if not isinstance(results, bool) or not results: config_error(config, results) if container.getDataType() != 'grid': raise NotImplementedError('contour module can only contour ' 'gridded data, not sets of points') # get the filter size from the products.conf filter_size = config['products']['contour']['filter_size'] # create contour files self.logger.debug('Contouring to files...') contour_to_files(container, datadir, self.logger, self.contents, filter_size) container.close()
def dummy_scp_test(remote_host, remote_directory, private_key): homedir = os.path.dirname(os.path.abspath(__file__)) cfile = os.path.join(homedir, '..', '..', 'data', 'containers', 'northridge', 'shake_result.hdf') products_dir = os.path.join(homedir, '..', '..', 'data', 'eventdata', 'northridge', 'current') pdl_dir = os.path.join(homedir, '..', '..', 'data', 'eventdata', 'northridge', 'current', 'pdl') container = ShakeMapOutputContainer.load(cfile) config = { 'ssh': { 'dest1': { 'remote_host': remote_host, 'remote_directory': remote_directory, 'private_key': private_key } } } # transfermod = TransferModule(eventid) _transfer(config, container.getMetadata(), pdl_dir, products_dir)
def execute(self): """ Create KML files. Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) if container.getDataType() != 'grid': raise NotImplementedError('kml module can only contour ' 'gridded data, not sets of points') # find the low res ocean vector dataset product_config_file = os.path.join(install_path, 'config', 'products.conf') spec_file = get_configspec('products') validator = get_custom_validator() pconfig = configobj.ConfigObj(product_config_file, configspec=spec_file) results = pconfig.validate(validator) if not isinstance(results, bool) or not results: config_error(pconfig, results) oceanfile = pconfig['products']['mapping']['layers']['lowres_oceans'] # call create_kmz function create_kmz(container, datadir, oceanfile, self.logger) container.close()
def dummy_pdl_test(java, jarfile, keyfile, configfile): homedir = os.path.dirname(os.path.abspath(__file__)) cfile = os.path.join(homedir, '..', '..', 'data', 'containers', 'northridge', 'shake_result.hdf') products_dir = os.path.join(homedir, '..', '..', 'data', 'eventdata', 'northridge', 'current') pdl_dir = os.path.join(homedir, '..', '..', 'data', 'eventdata', 'northridge', 'current', 'pdl') container = ShakeMapOutputContainer.load(cfile) config = { 'pdl': { 'dest1': { 'java': java, 'jarfile': jarfile, 'privatekey': keyfile, 'configfile': configfile, 'source': 'us' } } } # transfermod = TransferModule(eventid) _transfer(config, container.getMetadata(), pdl_dir, products_dir)
def execute(self): """ Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ _, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) if container.getDataType() != 'grid': raise NotImplementedError('xtestimage module can only operate on ' 'gridded data not sets of points') datadict = {} imtlist = container.getIMTs('GREATER_OF_TWO_HORIZONTAL') for myimt in imtlist: datadict[myimt] = container.getIMTGrids( myimt, 'GREATER_OF_TWO_HORIZONTAL') container.close() # # Make plots # for myimt in imtlist: if myimt == 'MMI': yunits = '(MMI)' elif myimt == 'PGV': yunits = '[ln(cm/s)]' else: yunits = '[ln(g)]' fileimt = oq_to_file(myimt) # # Do the ground motion plots # data = datadict[myimt] grddata = data['mean'] metadata = data['mean_metadata'] fig = plt.figure(figsize=(10, 10)) gs = plt.GridSpec(4, 4, hspace=0.2, wspace=0.1) ax0 = fig.add_subplot(gs[:-1, 1:]) plt.title(self._eventid + ': ' + myimt + ' mean') im1 = ax0.imshow(grddata, extent=(metadata['xmin'], metadata['xmax'], metadata['ymin'], metadata['ymax'])) cbax = fig.add_axes([0.915, .34, .02, .5]) plt.colorbar(im1, ax=ax0, cax=cbax) ycut = fig.add_subplot(gs[:-1, 0], sharey=ax0) xcut = fig.add_subplot(gs[-1, 1:], sharex=ax0) rows, cols = grddata.shape midrow = int(rows / 2) midcol = int(cols / 2) xvals = np.linspace(metadata['xmin'], metadata['xmax'], cols) yvals = np.linspace(metadata['ymin'], metadata['ymax'], rows) ycut.plot(grddata[:, midcol], yvals) xcut.plot(xvals, grddata[midrow, :]) ycut.set(xlabel=myimt + ' ' + yunits, ylabel='Latitude') xcut.set(xlabel='Longitude', ylabel=myimt + ' ' + yunits) ycut.set_ylim((metadata['ymin'], metadata['ymax'])) xcut.set_xlim((metadata['xmin'], metadata['xmax'])) ax0.label_outer() pfile = os.path.join(datadir, self._eventid + '_' + fileimt + '.pdf') plt.savefig(pfile, bbox_inches='tight') plt.close() # # Do the stddev plots # grddata = data['std'] fig = plt.figure(figsize=(10, 10)) gs = plt.GridSpec(4, 4, hspace=0.2, wspace=0.1) ax0 = fig.add_subplot(gs[:-1, 1:]) plt.title(self._eventid + ': ' + myimt + ' stddev') im1 = ax0.imshow(grddata, extent=(metadata['xmin'], metadata['xmax'], metadata['ymin'], metadata['ymax'])) cbax = fig.add_axes([0.915, .34, .02, .5]) plt.colorbar(im1, ax=ax0, cax=cbax) ycut = fig.add_subplot(gs[:-1, 0], sharey=ax0) xcut = fig.add_subplot(gs[-1, 1:], sharex=ax0) rows, cols = grddata.shape midrow = int(rows / 2) midcol = int(cols / 2) xvals = np.linspace(metadata['xmin'], metadata['xmax'], cols) yvals = np.linspace(metadata['ymin'], metadata['ymax'], rows) ycut.plot(grddata[:, midcol], yvals) xcut.plot(xvals, grddata[midrow, :]) ycut.set(xlabel='stddev ' + yunits, ylabel='Latitude') xcut.set(xlabel='Longitude', ylabel='stddev ' + yunits) xcut.set_xlim((metadata['xmin'], metadata['xmax'])) xcut.set_ylim(bottom=0, top=np.max(grddata[midrow, :]) * 1.1) ycut.set_xlim(left=0, right=np.max(grddata[:, midcol] * 1.1)) ycut.set_ylim((metadata['ymin'], metadata['ymax'])) ax0.label_outer() pfile = os.path.join(datadir, self._eventid + '_' + fileimt + '_sd.pdf') plt.savefig(pfile, bbox_inches='tight') plt.close()
def test_output_arrays(): f, datafile = tempfile.mkstemp() os.close(f) try: container = ShakeMapOutputContainer.create(datafile) # # Test that no data type is set # assert container.getDataType() is None # # Make some array data and metadata # mean = np.random.rand(100) std = np.random.rand(100) lats = np.random.rand(100) lons = np.random.rand(100) ids = np.array([randomword(4).encode('ascii') for x in range(100)]) metadata = {'units': '%g', 'digits': 4} # # Put the data in the container # container.setIMTArrays('PGA', lons, lats, ids, mean, metadata, std, metadata, 'Larger') # # Now extract it and compare it to what we put in there # dout = container.getIMTArrays('PGA', 'Larger') assert all(dout['lons'] == lons) assert all(dout['lats'] == lats) assert all(dout['ids'] == ids) assert all(dout['mean'] == mean) assert all(dout['std'] == std) # # Check the data type # assert container.getDataType() == 'points' # # Try raising some exceptions # # Shouldn't be able to find this IMT with pytest.raises(LookupError): container.getIMTArrays('JUNK', 'Larger') # Shapes of inputs not the same with pytest.raises(ValueError): empty = np.array([]) container.setIMTArrays('PGV', empty, lats, ids, mean, metadata, std, metadata, 'Larger') # IMT already exists with pytest.raises(LookupError): container.setIMTArrays('PGA', lons, lats, ids, mean, metadata, std, metadata, 'Larger') # Trying to set a grid in a file with points with pytest.raises(TypeError): container.setIMTGrids('PGV', mean, metadata, std, metadata, 'Larger') # Trying to get a grid in a file with points with pytest.raises(TypeError): container.getIMTGrids('PGA', 'Larger') container.close() except Exception as e: if os.path.isfile(datafile): os.remove(datafile) raise (e) finally: if os.path.isfile(datafile): os.remove(datafile)
def test_output_container(): nrows = 400 ncols = 230 # create MMI mean data for maximum component mean_mmi_maximum_data = np.random.rand(nrows, ncols) mean_mmi_maximum_metadata = { 'xmin': -118.5, 'xmax': -114.5, 'ymin': 32.1, 'ymax': 36.7, 'dx': 0.01, 'dy': 0.02, 'nx': 400, 'ny': 230, 'name': 'Gandalf', 'color': 'white', 'powers': 'magic' } # create MMI std data for maximum component std_mmi_maximum_data = mean_mmi_maximum_data / 10 std_mmi_maximum_metadata = { 'xmin': -118.5, 'xmax': -114.5, 'ymin': 32.1, 'ymax': 36.7, 'dx': 0.01, 'dy': 0.02, 'nx': 400, 'ny': 230, 'name': 'Legolas', 'color': 'green', 'powers': 'good hair' } # create MMI mean data for rotd50 component mean_mmi_rotd50_data = np.random.rand(nrows, ncols) mean_mmi_rotd50_metadata = { 'xmin': -118.5, 'xmax': -114.5, 'ymin': 32.1, 'ymax': 36.7, 'dx': 0.01, 'dy': 0.02, 'nx': 400, 'ny': 230, 'name': 'Gimli', 'color': 'brown', 'powers': 'axing' } # create MMI std data for rotd50 component std_mmi_rotd50_data = mean_mmi_rotd50_data / 10 std_mmi_rotd50_metadata = { 'xmin': -118.5, 'xmax': -114.5, 'ymin': 32.1, 'ymax': 36.7, 'dx': 0.01, 'dy': 0.02, 'nx': 400, 'ny': 230, 'name': 'Aragorn', 'color': 'white', 'powers': 'scruffiness' } # create PGA mean data for maximum component mean_pga_maximum_data = np.random.rand(nrows, ncols) mean_pga_maximum_metadata = { 'xmin': -118.5, 'xmax': -114.5, 'ymin': 32.1, 'ymax': 36.7, 'dx': 0.01, 'dy': 0.02, 'nx': 400, 'ny': 230, 'name': 'Pippin', 'color': 'purple', 'powers': 'rashness' } # create PGA std data for maximum component std_pga_maximum_data = mean_pga_maximum_data / 10 std_pga_maximum_metadata = { 'xmin': -118.5, 'xmax': -114.5, 'ymin': 32.1, 'ymax': 36.7, 'dx': 0.01, 'dy': 0.02, 'nx': 400, 'ny': 230, 'name': 'Merry', 'color': 'grey', 'powers': 'hunger' } f, datafile = tempfile.mkstemp() os.close(f) try: container = ShakeMapOutputContainer.create(datafile) # LookupError raised if trying to dropIMTs if there are none with pytest.raises(LookupError): container.dropIMT('mmi') # Add imts container.setIMTGrids('mmi', mean_mmi_maximum_data, mean_mmi_maximum_metadata, std_mmi_maximum_data, std_mmi_maximum_metadata, component='maximum') container.setIMTGrids('mmi', mean_mmi_rotd50_data, mean_mmi_rotd50_metadata, std_mmi_rotd50_data, std_mmi_rotd50_metadata, component='rotd50') container.setIMTGrids('pga', mean_pga_maximum_data, mean_pga_maximum_metadata, std_pga_maximum_data, std_pga_maximum_metadata, component='maximum') # get the maximum MMI imt data mmi_max_dict = container.getIMTGrids('mmi', component='maximum') np.testing.assert_array_equal(mmi_max_dict['mean'], mean_mmi_maximum_data) np.testing.assert_array_equal(mmi_max_dict['std'], std_mmi_maximum_data) assert mmi_max_dict['mean_metadata'] == mean_mmi_maximum_metadata assert mmi_max_dict['std_metadata'] == std_mmi_maximum_metadata # get the rotd50 MMI imt data mmi_rot_dict = container.getIMTGrids('mmi', component='rotd50') np.testing.assert_array_equal(mmi_rot_dict['mean'], mean_mmi_rotd50_data) np.testing.assert_array_equal(mmi_rot_dict['std'], std_mmi_rotd50_data) assert mmi_rot_dict['mean_metadata'] == mean_mmi_rotd50_metadata assert mmi_rot_dict['std_metadata'] == std_mmi_rotd50_metadata # get list of all imts imts = container.getIMTs() # get list of maximum imts max_imts = container.getIMTs(component='maximum') assert sorted(max_imts) == ['mmi', 'pga'] # get list of components for mmi mmi_comps = container.getComponents('mmi') assert sorted(mmi_comps) == ['maximum', 'rotd50'] # Test dropIMT imts = container.getIMTs('maximum') assert imts == ['mmi', 'pga'] container.dropIMT('mmi') imts = container.getIMTs('maximum') assert imts == ['pga'] container.close() except Exception as e: raise (e) finally: os.remove(datafile)
def execute(self): """Create high, medium, and low resolution coverage of the mapped parameters. Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) # get all of the grid layers and the geodict if container.getDataType() != 'grid': raise NotImplementedError('coverage module can only function on ' 'gridded data, not sets of points') imtlist = container.getIMTs() for imtype in imtlist: component, imtype = imtype.split('/') fileimt = oq_to_file(imtype) oqimt = imt.from_string(imtype) imtdict = container.getIMTGrids(imtype, component) grid_data = imtdict['mean'] metadata = imtdict['mean_metadata'] if imtype == 'MMI': description = 'Modified Mercalli Intensity', property_id = "https://earthquake.usgs.gov/learn/topics/mercalli.php", # noqa decimals = 1 elif imtype == 'PGA': description = 'Peak Ground Acceleration', units = 'natural logarithm of "g"' symbol = 'ln(g)' decimals = 2 elif imtype == 'PGV': description = 'Peak Ground Velocity', units = 'natural logarithm of centimeters per second' symbol = 'ln(cm/s)' decimals = 2 elif imtype.startswith('SA'): description = str(oqimt.period) + \ '-second Spectral Acceleration', units = 'natural logarithm of "g"' symbol = 'ln(g)' decimals = 2 else: raise TypeError("Unknown IMT in coverage module") for i in range(3): if i == 0: resolution = 'high' fgrid = grid_data decimation = 1 elif i == 1: resolution = 'medium' fgrid = gaussian_filter(grid_data, sigma=1) decimation = 2 elif i == 2: resolution = 'low' fgrid = gaussian_filter(grid_data, sigma=2) decimation = 4 rgrid = fgrid[::decimation, ::decimation] ny, nx = rgrid.shape rnd_grd = np.flipud(np.around(rgrid, decimals=decimals)).flatten() if imtype == 'MMI': rnd_grd = np.clip(rnd_grd, 1.0, 10.0) xstart = metadata["xmin"] xstop = metadata["xmin"] + \ (nx - 1) * decimation * metadata["dx"] ystart = metadata["ymin"] ystop = metadata["ymin"] + \ (ny - 1) * decimation * metadata["dy"] coverage = { "type": "Coverage", "domain": { "type": "Domain", "domainType": "Grid", "axes": { "x": { "start": xstart, "stop": xstop, "num": nx }, "y": { "start": ystart, "stop": ystop, "num": ny } }, "referencing": [{ "coordinates": ["x", "y"], "system": { "type": "GeographicCRS", "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84" # noqa } }] }, "parameters": { imtype: { "type": "Parameter", "description": { "en": description }, "observedProperty": { "id": property_id, "label": { "en": imtype } }, } }, "ranges": { imtype: { "type": "NdArray", "dataType": "float", "axisNames": ["y", "x"], "shape": [ny, nx], "values": rnd_grd.tolist() } } } if imtype == 'MMI': coverage["parameters"]["MMI"]["preferredPalette"] = { "colors": [ "rgb(255, 255, 255)", "rgb(255, 255, 255)", "rgb(191, 204, 255)", "rgb(160, 230, 255)", "rgb(128, 255, 255)", "rgb(122, 255, 147)", "rgb(255, 255, 0)", "rgb(255, 200, 0)", "rgb(255, 145, 0)", "rgb(255, 0, 0)", "rgb(200, 0, 0)" ], "extent": [0, 10], "interpolation": "linear" } else: coverage["parameters"][imtype]["unit"] = { 'label': { "en": units }, "symbol": { 'value': symbol, 'type': "http://www.opengis.net/def/uom/UCUM/" } } if component == 'GREATER_OF_TWO_HORIZONTAL': fname = 'coverage_%s_%s_res.covjson' % (fileimt, resolution) else: fname = 'coverage_%s_%s_%s_res.covjson' % ( fileimt, resolution, component) filepath = os.path.join(datadir, fname) with open(filepath, 'w') as outfile: json.dump(coverage, outfile, separators=(',', ':')) self.contents.addFile( imtype + "_" + resolution + '_res_coverage', resolution + '-res ' + imtype.upper() + ' Coverage', 'Coverage of ' + resolution + ' resolution ' + imtype, fname, 'application/json') container.close()
def execute(self): """ Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) if container.getDataType() != 'grid': raise NotImplementedError('mapping module can only operate on ' 'gridded data, not sets of points') # get the path to the products.conf file, load the config config_file = os.path.join(install_path, 'config', 'products.conf') spec_file = get_configspec('products') validator = get_custom_validator() config = ConfigObj(config_file, configspec=spec_file) results = config.validate(validator) check_extra_values(config, self.logger) if not isinstance(results, bool) or not results: config_error(config, results) # create contour files self.logger.debug('Mapping...') # get the filter size from the products.conf filter_size = config['products']['contour']['filter_size'] # get the operator setting from config operator = config['products']['mapping']['operator'] # get all of the pieces needed for the mapping functions layers = config['products']['mapping']['layers'] if 'topography' in layers and layers['topography'] != '': topofile = layers['topography'] else: topofile = None if 'roads' in layers and layers['roads'] != '': roadfile = layers['roads'] else: roadfile = None if 'faults' in layers and layers['faults'] != '': faultfile = layers['faults'] else: faultfile = None # Get the number of parallel workers max_workers = config['products']['mapping']['max_workers'] # Reading HDF5 files currently takes a long time, due to poor # programming in MapIO. To save us some time until that issue is # resolved, we'll coarsely subset the topo grid once here and pass # it into both mapping functions # get the bounds of the map info = container.getMetadata() xmin = info['output']['map_information']['min']['longitude'] xmax = info['output']['map_information']['max']['longitude'] ymin = info['output']['map_information']['min']['latitude'] ymax = info['output']['map_information']['max']['latitude'] dy = float( info['output']['map_information']['grid_spacing']['latitude']) dx = float( info['output']['map_information']['grid_spacing']['longitude']) padx = 5 * dx pady = 5 * dy sxmin = float(xmin) - padx sxmax = float(xmax) + padx symin = float(ymin) - pady symax = float(ymax) + pady sampledict = GeoDict.createDictFromBox(sxmin, sxmax, symin, symax, dx, dy) if topofile: topogrid = read(topofile, samplegeodict=sampledict, resample=False) else: tdata = np.full([sampledict.ny, sampledict.nx], 0.0) topogrid = Grid2D(data=tdata, geodict=sampledict) model_config = container.getConfig() imtlist = container.getIMTs() textfile = os.path.join( get_data_path(), 'mapping', 'map_strings.' + config['products']['mapping']['language']) text_dict = get_text_strings(textfile) if config['products']['mapping']['fontfamily'] != '': matplotlib.rcParams['font.family'] = \ config['products']['mapping']['fontfamily'] matplotlib.rcParams['axes.unicode_minus'] = False allcities = Cities.fromDefault() states_provs = None countries = None oceans = None lakes = None extent = (float(xmin), float(ymin), float(xmax), float(ymax)) if 'CALLED_FROM_PYTEST' not in os.environ: states_provs = cfeature.NaturalEarthFeature( category='cultural', name='admin_1_states_provinces_lines', scale='10m', facecolor='none') states_provs = list(states_provs.intersecting_geometries(extent)) if len(states_provs) > 300: states_provs = None else: states_provs = cfeature.NaturalEarthFeature( category='cultural', name='admin_1_states_provinces_lines', scale='10m', facecolor='none') countries = cfeature.NaturalEarthFeature(category='cultural', name='admin_0_countries', scale='10m', facecolor='none') oceans = cfeature.NaturalEarthFeature(category='physical', name='ocean', scale='10m', facecolor=WATERCOLOR) lakes = cfeature.NaturalEarthFeature(category='physical', name='lakes', scale='10m', facecolor=WATERCOLOR) if faultfile is not None: faults = ShapelyFeature(Reader(faultfile).geometries(), ccrs.PlateCarree(), facecolor='none') else: faults = None if roadfile is not None: roads = ShapelyFeature(Reader(roadfile).geometries(), ccrs.PlateCarree(), facecolor='none') if len(list(roads.intersecting_geometries(extent))) > 200: roads = None else: roads = ShapelyFeature(Reader(roadfile).geometries(), ccrs.PlateCarree(), facecolor='none') else: roads = None alist = [] for imtype in imtlist: component, imtype = imtype.split('/') comp = container.getComponents(imtype)[0] d = { 'imtype': imtype, 'topogrid': topogrid, 'allcities': allcities, 'states_provinces': states_provs, 'countries': countries, 'oceans': oceans, 'lakes': lakes, 'roads': roads, 'faults': faults, 'datadir': datadir, 'operator': operator, 'filter_size': filter_size, 'info': info, 'component': comp, 'imtdict': container.getIMTGrids(imtype, comp), 'ruptdict': copy.deepcopy(container.getRuptureDict()), 'stationdict': container.getStationDict(), 'config': model_config, 'tdict': text_dict } alist.append(d) if imtype == 'MMI': g = copy.deepcopy(d) g['imtype'] = 'thumbnail' alist.append(g) h = copy.deepcopy(d) h['imtype'] = 'overlay' alist.append(h) self.contents.addFile('intensityMap', 'Intensity Map', 'Map of macroseismic intensity.', 'intensity.jpg', 'image/jpeg') self.contents.addFile('intensityMap', 'Intensity Map', 'Map of macroseismic intensity.', 'intensity.pdf', 'application/pdf') self.contents.addFile('intensityThumbnail', 'Intensity Thumbnail', 'Thumbnail of intensity map.', 'pin-thumbnail.png', 'image/png') self.contents.addFile( 'intensityOverlay', 'Intensity Overlay and World File', 'Macroseismic intensity rendered as a ' 'PNG overlay and associated world file', 'intensity_overlay.png', 'image/png') self.contents.addFile( 'intensityOverlay', 'Intensity Overlay and World File', 'Macroseismic intensity rendered as a ' 'PNG overlay and associated world file', 'intensity_overlay.pngw', 'text/plain') else: fileimt = oq_to_file(imtype) self.contents.addFile(fileimt + 'Map', fileimt.upper() + ' Map', 'Map of ' + imtype + '.', fileimt + '.jpg', 'image/jpeg') self.contents.addFile(fileimt + 'Map', fileimt.upper() + ' Map', 'Map of ' + imtype + '.', fileimt + '.pdf', 'application/pdf') if max_workers > 0: with cf.ProcessPoolExecutor(max_workers=max_workers) as ex: results = ex.map(make_map, alist) list(results) else: for adict in alist: make_map(adict) container.close()
def execute(self): """Create grid.xml and uncertainty.xml files. Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ logger = logging.getLogger(__name__) install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) # get all of the grid layers and the geodict if container.getDataType() != 'grid': raise NotImplementedError('gridxml module can only function on ' 'gridded data, not sets of points') gridnames = container.getIMTs(COMPONENT) xml_types = ['grid', 'uncertainty'] for xml_type in xml_types: layers = OrderedDict() field_keys = OrderedDict() for gridname in gridnames: imt_field = _oq_to_gridxml(gridname) imtdict = container.getIMTGrids(gridname, COMPONENT) if xml_type == 'grid': grid_data = imtdict['mean'] metadata = imtdict['mean_metadata'] elif xml_type == 'uncertainty': grid_data = imtdict['std'] metadata = imtdict['std_metadata'] units = metadata['units'] digits = metadata['digits'] # convert from HDF units to legacy grid.xml units if xml_type == 'grid': if units == 'ln(cm/s)': grid_data = np.exp(grid_data) units = 'cm/s' elif units == 'ln(g)': grid_data = np.exp(grid_data) * 100 units = '%g' else: pass if xml_type == 'grid': layers[imt_field] = grid_data field_keys[imt_field] = (units, digits) else: layers['STD' + imt_field] = grid_data field_keys['STD' + imt_field] = (units, digits) if xml_type == 'grid': grid_data, _ = container.getArray([], 'vs30') units = 'm/s' digits = metadata['digits'] layers['SVEL'] = grid_data field_keys['SVEL'] = (units, digits) geodict = GeoDict(metadata) config = container.getConfig() # event dictionary info = container.getMetadata() event_info = info['input']['event_information'] event_dict = {} event_dict['event_id'] = event_info['event_id'] event_dict['magnitude'] = float(event_info['magnitude']) event_dict['depth'] = float(event_info['depth']) event_dict['lat'] = float(event_info['latitude']) event_dict['lon'] = float(event_info['longitude']) try: event_dict['event_timestamp'] = datetime.strptime( event_info['origin_time'], constants.TIMEFMT) except ValueError: event_dict['event_timestamp'] = datetime.strptime( event_info['origin_time'], constants.ALT_TIMEFMT) event_dict['event_description'] = event_info['location'] event_dict['event_network'] = \ info['input']['event_information']['eventsource'] event_dict['intensity_observations'] =\ info['input']['event_information']['intensity_observations'] event_dict['seismic_stations'] =\ info['input']['event_information']['seismic_stations'] if info['input']['event_information']['fault_ref'] == 'Origin': event_dict['point_source'] = 'True' else: event_dict['point_source'] = 'False' # shake dictionary shake_dict = {} shake_dict['event_id'] = event_dict['event_id'] shake_dict['shakemap_id'] = event_dict['event_id'] shake_dict['shakemap_version'] = \ info['processing']['shakemap_versions']['map_version'] shake_dict['code_version'] = shakemap.__version__ ptime = info['processing']['shakemap_versions']['process_time'] try: shake_dict['process_timestamp'] = datetime.strptime( ptime, constants.TIMEFMT) except ValueError: shake_dict['process_timestamp'] = datetime.strptime( ptime, constants.ALT_TIMEFMT) shake_dict['shakemap_originator'] = \ config['system']['source_network'] shake_dict['map_status'] = config['system']['map_status'] shake_dict['shakemap_event_type'] = 'ACTUAL' if event_dict['event_id'].endswith('_se'): shake_dict['shakemap_event_type'] = 'SCENARIO' shake_grid = ShakeGrid(layers, geodict, event_dict, shake_dict, {}, field_keys=field_keys) fname = os.path.join(datadir, '%s.xml' % xml_type) logger.debug('Saving IMT grids to %s' % fname) shake_grid.save(fname) # TODO - set grid version number self.contents.addFile('xmlGrids', 'XML Grid', 'XML grid of ground motions', 'grid.xml', 'text/xml') self.contents.addFile('uncertaintyGrid', 'Uncertainty Grid', 'XML grid of uncertainties', 'uncertainty.xml', 'text/xml') container.close()
def execute(self): """ Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() event_paths = glob.glob(os.path.join(data_path, "%s*" % self._eventid)) datalist = [] sigmas = [] for path in event_paths: datadir = os.path.join(path, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) if container.getDataType() != 'points': raise NotImplementedError('xtestplot_multi module can only ' 'operate on sets of points, not ' 'gridded data') stas = container.getStationDict() ampd = stas['features'][0]['properties']['channels'][0][ 'amplitudes'][0] if 'ln_sigma' in ampd: sigmas.append(ampd['ln_sigma']) else: sigmas.append(0) datadict = {} imtlist = container.getIMTs('GREATER_OF_TWO_HORIZONTAL') for myimt in imtlist: datadict[myimt] = container.getIMTArrays( myimt, 'GREATER_OF_TWO_HORIZONTAL') datalist.append(datadict) container.close() # # Make plots # colors = ['k', 'b', 'g', 'r', 'c', 'm'] for myimt in imtlist: fig, axa = plt.subplots(2, sharex=True, figsize=(10, 8)) plt.subplots_adjust(hspace=0.1) for ix, dd in enumerate(datalist): data = dd[myimt] axa[0].plot(data['lons'], data['mean'], color=colors[ix], label=r'$\sigma_\epsilon = %.2f$' % sigmas[ix]) axa[1].plot(data['lons'], data['std'], '-.', color=colors[ix], label=r'$\sigma_\epsilon = %.2f$' % sigmas[ix]) plt.xlabel('Longitude') axa[0].set_ylabel('Mean ln(%s) (g)' % myimt) axa[1].set_ylabel('Stddev ln(%s) (g)' % myimt) axa[0].legend(loc='best') axa[1].legend(loc='best') axa[0].set_title(self._eventid) axa[0].grid() axa[1].grid() axa[1].set_ylim(bottom=0) fileimt = oq_to_file(myimt) pfile = os.path.join(event_paths[0], 'current', 'products', self._eventid + '_' + fileimt + '.pdf') plt.savefig(pfile) pfile = os.path.join(event_paths[0], 'current', 'products', self._eventid + '_' + fileimt + '.png') plt.savefig(pfile) plt.close()
def execute(self): """ Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data oc = ShakeMapOutputContainer.load(datafile) if oc.getDataType() != 'grid': raise NotImplementedError('plotregr module can only operate on ' 'gridded data not sets of points') # get the path to the products.conf file, load the config config_file = os.path.join(install_path, 'config', 'products.conf') spec_file = get_configspec('products') validator = get_custom_validator() config = ConfigObj(config_file, configspec=spec_file) results = config.validate(validator) if not isinstance(results, bool) or not results: config_error(config, results) # If mapping runs in parallel, then we want this module too, as well. # Otherwise we get weird errors from matplotlib max_workers = config['products']['mapping']['max_workers'] # # Cheating here a bit by assuming that the IMTs are the same # as the regression IMTs # rockgrid = {} soilgrid = {} rocksd = {} soilsd = {} imtlist = oc.getIMTs('GREATER_OF_TWO_HORIZONTAL') for myimt in imtlist: rockgrid[myimt], _ = oc.getArray(['attenuation', 'rock', myimt], 'mean') soilgrid[myimt], _ = oc.getArray(['attenuation', 'soil', myimt], 'mean') rocksd[myimt], _ = oc.getArray(['attenuation', 'rock', myimt], 'std') soilsd[myimt], _ = oc.getArray(['attenuation', 'soil', myimt], 'std') distances, _ = oc.getArray(['attenuation', 'distances'], 'rrup') stations = oc.getStationDict() # # Make plots # alist = [] for myimt in imtlist: a = { 'myimt': myimt, 'rockgrid': rockgrid, 'soilgrid': soilgrid, 'rocksd': rocksd, 'soilsd': soilsd, 'stations': stations, 'distances': distances, 'eventid': self._eventid, 'datadir': datadir } alist.append(a) if myimt == 'MMI': self.contents.addFile( 'miRegr', 'Intensity Regression', 'Regression plot of macroseismic ' 'intensity.', 'mmi_regr.png', 'image/png') elif myimt == 'PGA': self.contents.addFile( 'pgaRegr', 'PGA Regression', 'Regression plot of peak ' 'ground acceleration (%g).', 'pga_regr.png', 'image/png') elif myimt == 'PGV': self.contents.addFile( 'pgvRegr', 'PGV Regression', 'Regression plot of peak ground ' 'velocity (cm/s).', 'pgv_regr.png', 'image/png') else: oqimt = imt.from_string(myimt) period = str(oqimt.period) filebase = oq_to_file(myimt) psacap = 'Regression plot of ' + period + ' sec 5% damped ' \ 'pseudo-spectral acceleration(%g).' self.contents.addFile(filebase + 'Regr', 'PSA ' + period + ' sec Regression', psacap, filebase + '_regr.png', 'image/png') if max_workers > 0: with cf.ProcessPoolExecutor(max_workers=max_workers) as ex: results = ex.map(make_plots, alist) list(results) else: for adict in alist: make_plots(adict) # # Make attenuation_curves.json # jdict = {'eventid': self._eventid} jdict['gmpe'] = {} for site in ['soil', 'rock']: jdict['gmpe'][site] = {} for myimt in imtlist: jdict['gmpe'][site][myimt] = {} jdict['gmpe'][site][myimt]['mean'] = oc.getArray( ['attenuation', site, myimt], 'mean')[0].round(decimals=5).tolist() jdict['gmpe'][site][myimt]['stddev'] = oc.getArray( ['attenuation', site, myimt], 'std')[0].round(decimals=5).tolist() jdict['distances'] = {} for dtype in ['repi', 'rhypo', 'rjb', 'rrup']: jdict['distances'][dtype] = oc.getArray( ['attenuation', 'distances'], dtype)[0].round(decimals=5).tolist() jdict['mean_bias'] = {} info = oc.getMetadata() for myimt in imtlist: jdict['mean_bias'][myimt] = info['output']['ground_motions'][ myimt]['bias'] jstring = json.dumps(jdict, allow_nan=False) jfile = os.path.join(datadir, 'attenuation_curves.json') f = open(jfile, 'wt') f.write(jstring) f.close() oc.close() cap = "Nominal attenuation curves" self.contents.addFile('attenuationCurves', 'Attenuation Curves', cap, 'attenuation_curves.json', 'application/json')
def test_products(): installpath, datapath = get_config_paths() # # Use a real event for checks of products against the contents of # the output container # evid = 'integration_test_0001' try: # # Make sure an output file exists # assemble = AssembleModule(evid, comment='Test comment.') assemble.execute() del assemble model = ModelModule(evid) model.execute() del model res_file = os.path.join(datapath, evid, 'current', 'products', 'shake_result.hdf') oc = ShakeMapOutputContainer.load(res_file) # # The history module just outputs some info to the operator, so # here we just run it to make sure it doesn't crash anything. # Actual testing should be done via bug reports/feature requests # from users. # history = HistoryModule(evid) history.execute() del history # # Test the creation of products -- currently not checking results # for validity or consistency, but probably should # # # TODO: The stationlist.json should be validated, but we need a # function that will read it and convert it to something # we can test against. # check_failures(evid, datapath, StationModule) mod = StationModule(evid) mod.execute() mod.writeContents() check_failures(evid, datapath, MappingModule) mod = MappingModule(evid) mod.execute() mod.writeContents() check_failures(evid, datapath, PlotRegr) # # PlotRegr gets tested in the model tests for event 72282711 # # mod = PlotRegr(evid) # mod.execute() # mod.writeContents() check_failures(evid, datapath, KMLModule) mod = KMLModule(evid) mod.execute() mod.writeContents() del mod # This just exercises the ShapeModule code without actually # checking for valid results. mod = ShapeModule(evid) mod.execute() mod.writeContents() del mod # # These check that the results are consistent with the output # container # do_rupture(evid, datapath, oc) # do_info(evid, datapath, oc) do_raster(evid, datapath, oc) do_gridxml(evid, datapath, oc) oc.close() # # Checks contours against saved versions; if something # changes, will need to update the files in # data/integration_test_0001 # # TODO: (7/28/2021)I am disabling this test for a little while until # the float32 HDF version of the impactutils containers are in # wide distribution. At that point, I'll update the data files # and the test results. # do_contour(evid, datapath) # do_contour_command_line(evid, datapath) finally: pass data_file = os.path.join(datapath, evid, 'current', 'shake_data.hdf') if os.path.isfile(data_file): os.remove(data_file)