def test_extra_large_source(self):
oq = readinput.get_oqparam('job.ini', case_21)
with mock.patch('logging.error') as error, datastore.hdf5new() as h5:
with mock.patch('openquake.hazardlib.geo.utils.MAX_EXTENT', 80):
readinput.get_composite_source_model(oq, h5)
os.remove(h5.filename)
self.assertEqual(
error.call_args[0][0], 'source SFLT2: too large: 84 km')
def test_extra_large_source(self):
oq = readinput.get_oqparam('job.ini', case_21)
mon = performance.Monitor('csm', datastore.hdf5new())
with mock.patch('logging.error') as error:
with mock.patch('openquake.hazardlib.geo.utils.MAX_EXTENT', 80):
readinput.get_composite_source_model(oq, mon)
mon.hdf5.close()
os.remove(mon.hdf5.path)
self.assertEqual(error.call_args[0][0],
'source SFLT2: too large: 84 km')
def create_parent_hdf5(N, num_gmfs, sites, cinfo, oq_param):
parent_hdf5 = f = hdf5new()
calc_id, datadir = extract_calc_id_datadir(parent_hdf5.path)
logs.dbcmd('import_job', calc_id, 'event_based', 'eb_test_hdf5',
getpass.getuser(), 'complete', None, datadir)
create_gmdata(f, num_gmfs, oq_param.imtls)
create_events(f, num_gmfs)
f['csm_info'] = cinfo
f['sitecol'] = sites
f['oqparam'] = oq_param
return f, calc_id
def main(dirname):
dname = pathlib.Path(dirname)
with hdf5new() as hdf5: # create a new datastore
monitor = Monitor('count', hdf5) # create a new monitor
iterargs = ((open(dname/fname, encoding='utf-8').read(),)
for fname in os.listdir(dname)
if fname.endswith('.rst')) # read the docs
c = collections.Counter() # intially empty counter
for counter in Starmap(count, iterargs, monitor):
c += counter
print(c) # total counts
print('Performance info stored in', hdf5)
def prepare_site_model(exposure_xml,
vs30_csv,
grid_spacing=0,
site_param_distance=5,
output='sites.csv'):
"""
Prepare a site_model.csv file from an exposure xml file, a vs30 csv file
and a grid spacing which can be 0 (meaning no grid). Sites far away from
the vs30 records are discarded and you can see them with the command
`oq plot_assets`. It is up to you decide if you need to fix your exposure
or if it is right to ignore the discarded sites.
"""
logging.basicConfig(level=logging.INFO)
hdf5 = datastore.hdf5new()
with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon:
mesh, assets_by_site = Exposure.read(
exposure_xml, check_dupl=False).get_mesh_assets_by_site()
mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, req_site_params={'vs30'})
if grid_spacing:
grid = mesh.get_convex_hull().dilate(grid_spacing).discretize(
grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params={'vs30'})
logging.info(
'Reducing exposure grid with %d locations to %d sites'
' with assets', len(haz_sitecol), len(assets_by_site))
haz_sitecol, assets_by, _discarded = assoc(assets_by_site,
haz_sitecol,
grid_spacing * SQRT2,
'filter')
haz_sitecol.make_complete()
else:
haz_sitecol = assetcol
vs30orig = read_vs30(vs30_csv.split(','))
logging.info('Associating %d hazard sites to %d site parameters',
len(haz_sitecol), len(vs30orig))
sitecol, vs30, discarded = assoc(
vs30orig, haz_sitecol, grid_spacing * SQRT2 or site_param_distance,
'filter')
sitecol.array['vs30'] = vs30['vs30']
mon.hdf5['sitecol'] = sitecol
if discarded:
mon.hdf5['discarded'] = numpy.array(discarded)
sids = numpy.arange(len(vs30), dtype=numpy.uint32)
sites = compose_arrays(sids, vs30, 'site_id')
write_csv(output, sites)
if discarded:
logging.info('Discarded %d sites with assets [use oq plot_assets]',
len(discarded))
logging.info('Saved %d rows in %s' % (len(sitecol), output))
logging.info(mon)
return sitecol
def prepare_site_model(exposure_xml,
sites_csv,
vs30_csv,
z1pt0,
z2pt5,
vs30measured,
grid_spacing=0,
assoc_distance=5,
output='site_model.csv'):
"""
Prepare a site_model.csv file from exposure xml files/site csv files,
vs30 csv files and a grid spacing which can be 0 (meaning no grid).
For each site the closest vs30 parameter is used. The command can also
generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured
which may be needed by your hazard model, depending on the required GSIMs.
"""
hdf5 = datastore.hdf5new()
req_site_params = {'vs30'}
fields = ['lon', 'lat', 'vs30']
if z1pt0:
req_site_params.add('z1pt0')
fields.append('z1pt0')
if z2pt5:
req_site_params.add('z2pt5')
fields.append('z2pt5')
if vs30measured:
req_site_params.add('vs30measured')
fields.append('vs30measured')
with performance.Monitor(measuremem=True) as mon:
if exposure_xml:
mesh, assets_by_site = Exposure.read(
exposure_xml, check_dupl=False).get_mesh_assets_by_site()
hdf5['assetcol'] = assetcol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, req_site_params=req_site_params)
if grid_spacing:
grid = mesh.get_convex_hull().dilate(grid_spacing).discretize(
grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
logging.info(
'Associating exposure grid with %d locations to %d '
'exposure sites', len(haz_sitecol), len(assets_by_site))
haz_sitecol, assets_by, discarded = assoc(
assets_by_site, haz_sitecol, grid_spacing * SQRT2,
'filter')
if len(discarded):
logging.info(
'Discarded %d sites with assets '
'[use oq plot_assets]', len(discarded))
hdf5['discarded'] = numpy.array(discarded)
haz_sitecol.make_complete()
else:
haz_sitecol = assetcol
discarded = []
elif sites_csv:
lons, lats = [], []
for fname in sites_csv:
check_fname(fname, 'sites_csv', output)
with open(fname) as csv:
for line in csv:
if line.startswith('lon,lat'): # possible header
continue
lon, lat = line.split(',')[:2]
lons.append(valid.longitude(lon))
lats.append(valid.latitude(lat))
haz_sitecol = site.SiteCollection.from_points(
lons, lats, req_site_params=req_site_params)
if grid_spacing:
grid = haz_sitecol.mesh.get_convex_hull().dilate(
grid_spacing).discretize(grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
else:
raise RuntimeError('Missing exposures or missing sites')
vs30orig = read_vs30(vs30_csv, output)
logging.info('Associating %d hazard sites to %d site parameters',
len(haz_sitecol), len(vs30orig))
vs30 = haz_sitecol.assoc(vs30orig,
assoc_distance,
ignore={'z1pt0', 'z2pt5'})
if z1pt0:
haz_sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30'])
if z2pt5:
haz_sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30'])
hdf5['sitecol'] = haz_sitecol
write_csv(output, haz_sitecol.array[fields])
logging.info('Saved %d rows in %s' % (len(haz_sitecol), output))
logging.info(mon)
return haz_sitecol
def prepare_site_model(exposure_xml, vs30_csv,
z1pt0, z2pt5, vs30measured, grid_spacing=0,
site_param_distance=5, output='sites.csv'):
"""
Prepare a sites.csv file from an exposure xml file, a vs30 csv file
and a grid spacing which can be 0 (meaning no grid). For each asset site
or grid site the closest vs30 parameter is used. The command can also
generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured
which may be needed by your hazard model, depending on the required GSIMs.
"""
logging.basicConfig(level=logging.INFO)
hdf5 = datastore.hdf5new()
req_site_params = {'vs30'}
fields = ['lon', 'lat', 'vs30']
if z1pt0:
req_site_params.add('z1pt0')
fields.append('z1pt0')
if z2pt5:
req_site_params.add('z2pt5')
fields.append('z2pt5')
if vs30measured:
req_site_params.add('vs30measured')
fields.append('vs30measured')
with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon:
mesh, assets_by_site = Exposure.read(
exposure_xml, check_dupl=False).get_mesh_assets_by_site()
mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, req_site_params=req_site_params)
if grid_spacing:
grid = mesh.get_convex_hull().dilate(
grid_spacing).discretize(grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
logging.info(
'Associating exposure grid with %d locations to %d '
'exposure sites', len(haz_sitecol), len(assets_by_site))
haz_sitecol, assets_by, _discarded = assoc(
assets_by_site, haz_sitecol, grid_spacing * SQRT2, 'filter')
haz_sitecol.make_complete()
else:
haz_sitecol = assetcol
vs30orig = read_vs30(vs30_csv)
logging.info('Associating %d hazard sites to %d site parameters',
len(haz_sitecol), len(vs30orig))
sitecol, vs30, discarded = assoc(
vs30orig, haz_sitecol, site_param_distance, 'warn')
sitecol.array['vs30'] = vs30['vs30']
if z1pt0:
sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30'])
if z2pt5:
sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30'])
if vs30measured:
sitecol.array['vs30measured'] = False # it is inferred
mon.hdf5['sitecol'] = sitecol
if discarded:
mon.hdf5['discarded'] = numpy.array(discarded)
write_csv(output, sitecol.array[fields])
if discarded:
logging.info('Discarded %d sites with assets [use oq plot_assets]',
len(discarded))
logging.info('Saved %d rows in %s' % (len(sitecol), output))
logging.info(mon)
return sitecol
def prepare_site_model(exposure_xml, sites_csv, vs30_csv,
z1pt0, z2pt5, vs30measured, grid_spacing=0,
assoc_distance=5, output='site_model.csv'):
"""
Prepare a site_model.csv file from exposure xml files/site csv files,
vs30 csv files and a grid spacing which can be 0 (meaning no grid).
For each site the closest vs30 parameter is used. The command can also
generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured
which may be needed by your hazard model, depending on the required GSIMs.
"""
hdf5 = datastore.hdf5new()
req_site_params = {'vs30'}
fields = ['lon', 'lat', 'vs30']
if z1pt0:
req_site_params.add('z1pt0')
fields.append('z1pt0')
if z2pt5:
req_site_params.add('z2pt5')
fields.append('z2pt5')
if vs30measured:
req_site_params.add('vs30measured')
fields.append('vs30measured')
with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon:
if exposure_xml:
mesh, assets_by_site = Exposure.read(
exposure_xml, check_dupl=False).get_mesh_assets_by_site()
mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, req_site_params=req_site_params)
if grid_spacing:
grid = mesh.get_convex_hull().dilate(
grid_spacing).discretize(grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
logging.info(
'Associating exposure grid with %d locations to %d '
'exposure sites', len(haz_sitecol), len(assets_by_site))
haz_sitecol, assets_by, discarded = assoc(
assets_by_site, haz_sitecol,
grid_spacing * SQRT2, 'filter')
if len(discarded):
logging.info('Discarded %d sites with assets '
'[use oq plot_assets]', len(discarded))
mon.hdf5['discarded'] = numpy.array(discarded)
haz_sitecol.make_complete()
else:
haz_sitecol = assetcol
discarded = []
elif sites_csv:
lons, lats = [], []
for fname in sites_csv:
with open(fname) as csv:
for line in csv:
if line.startswith('lon,lat'): # possible header
continue
lon, lat = line.split(',')[:2]
lons.append(valid.longitude(lon))
lats.append(valid.latitude(lat))
haz_sitecol = site.SiteCollection.from_points(
lons, lats, req_site_params=req_site_params)
if grid_spacing:
grid = mesh.get_convex_hull().dilate(
grid_spacing).discretize(grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
else:
raise RuntimeError('Missing exposures or missing sites')
vs30orig = read_vs30(vs30_csv)
logging.info('Associating %d hazard sites to %d site parameters',
len(haz_sitecol), len(vs30orig))
sitecol, vs30, _ = assoc(
vs30orig, haz_sitecol, assoc_distance, 'warn')
sitecol.array['vs30'] = vs30['vs30']
if z1pt0:
sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30'])
if z2pt5:
sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30'])
if vs30measured:
sitecol.array['vs30measured'] = False # it is inferred
mon.hdf5['sitecol'] = sitecol
write_csv(output, sitecol.array[fields])
logging.info('Saved %d rows in %s' % (len(sitecol), output))
logging.info(mon)
return sitecol
