def zip_exposure(exposure_xml, archive_zip='', log=logging.info):
"""
Zip an exposure.xml file with all its .csv subfiles (if any)
"""
archive_zip = archive_zip or exposure_xml[:-4] + '.zip'
if os.path.exists(archive_zip):
sys.exit('%s exists already' % archive_zip)
[exp] = Exposure.read_headers([exposure_xml])
files = [exposure_xml] + exp.datafiles
general.zipfiles(files, archive_zip, log=log, cleanup=True)
return archive_zip
def zip_exposure(exposure_xml, archive_zip='', log=logging.info):
"""
Zip an exposure.xml file with all its .csv subfiles (if any)
"""
archive_zip = archive_zip or exposure_xml[:-4] + '.zip'
if os.path.exists(archive_zip):
sys.exit('%s exists already' % archive_zip)
[exp] = Exposure.read_headers([exposure_xml])
files = [exposure_xml] + exp.datafiles
general.zipfiles(files, archive_zip, log=log, cleanup=True)
return archive_zip
def prepare_site_model(exposure_xml,
vs30_csv,
grid_spacing=0,
site_param_distance=5,
output='sites.csv'):
"""
Prepare a site_model.csv file from an exposure xml file, a vs30 csv file
and a grid spacing which can be 0 (meaning no grid). Sites far away from
the vs30 records are discarded and you can see them with the command
`oq plot_assets`. It is up to you decide if you need to fix your exposure
or if it is right to ignore the discarded sites.
"""
logging.basicConfig(level=logging.INFO)
hdf5 = datastore.hdf5new()
with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon:
mesh, assets_by_site = Exposure.read(
exposure_xml, check_dupl=False).get_mesh_assets_by_site()
mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, req_site_params={'vs30'})
if grid_spacing:
grid = mesh.get_convex_hull().dilate(grid_spacing).discretize(
grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params={'vs30'})
logging.info(
'Reducing exposure grid with %d locations to %d sites'
' with assets', len(haz_sitecol), len(assets_by_site))
haz_sitecol, assets_by, _discarded = assoc(assets_by_site,
haz_sitecol,
grid_spacing * SQRT2,
'filter')
haz_sitecol.make_complete()
else:
haz_sitecol = assetcol
vs30orig = read_vs30(vs30_csv.split(','))
logging.info('Associating %d hazard sites to %d site parameters',
len(haz_sitecol), len(vs30orig))
sitecol, vs30, discarded = assoc(
vs30orig, haz_sitecol, grid_spacing * SQRT2 or site_param_distance,
'filter')
sitecol.array['vs30'] = vs30['vs30']
mon.hdf5['sitecol'] = sitecol
if discarded:
mon.hdf5['discarded'] = numpy.array(discarded)
sids = numpy.arange(len(vs30), dtype=numpy.uint32)
sites = compose_arrays(sids, vs30, 'site_id')
write_csv(output, sites)
if discarded:
logging.info('Discarded %d sites with assets [use oq plot_assets]',
len(discarded))
logging.info('Saved %d rows in %s' % (len(sitecol), output))
logging.info(mon)
return sitecol
def check_input(job_ini_or_zip_or_nrmls):
for job_ini_or_zip_or_nrml in job_ini_or_zip_or_nrmls:
if job_ini_or_zip_or_nrml.endswith('.xml'):
try:
node = nrml.to_python(job_ini_or_zip_or_nrml)
if node.tag.endswith('exposureModel'):
err = Exposure.check(job_ini_or_zip_or_nrml)
if err:
logging.warning(err)
else:
logging.info('Checked %s', job_ini_or_zip_or_nrml)
except Exception as exc:
sys.exit(exc)
else:
oq = readinput.get_oqparam(job_ini_or_zip_or_nrml)
base.calculators(oq, logs.init()).read_inputs()
def main(job_ini_or_zip_or_nrmls):
"""
Check the validity of job.ini files, job.zip files and .xml files.
NB: `oq check_input job_haz.ini job_risk.ini` is special-cased so
that the risk files are checked before the hazard files.
"""
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
all_inis = all(f.endswith('.ini') for f in job_ini_or_zip_or_nrmls)
if all_inis: # the typical case is job_haz.ini + job_risk.ini
dic = {}
for ini in job_ini_or_zip_or_nrmls:
for key, val in readinput.get_params(ini).items():
if key == 'inputs' and key in dic:
dic[key].update(val)
else: # the last wins
dic[key] = val
with logs.init('job', dic) as log:
logging.info('Running oq check_input %s',
' '.join(job_ini_or_zip_or_nrmls))
calc = base.calculators(log.get_oqparam(), log.calc_id)
base.BaseCalculator.gzip_inputs = lambda self: None # disable
with mock.patch.dict(os.environ, {'OQ_CHECK_INPUT': '1'}):
calc.read_inputs()
return
for job_ini_or_zip_or_nrml in job_ini_or_zip_or_nrmls:
if job_ini_or_zip_or_nrml.endswith('.xml'):
try:
node = nrml.to_python(job_ini_or_zip_or_nrml)
if node.tag.endswith('exposureModel'):
err = Exposure.check(job_ini_or_zip_or_nrml)
if err:
logging.warning(err)
else:
logging.info('Checked %s', job_ini_or_zip_or_nrml)
except Exception as exc:
sys.exit(exc)
else: # .zip
with logs.init('job', job_ini_or_zip_or_nrml) as log:
path = os.path.abspath(job_ini_or_zip_or_nrml)
logging.info('Running oq check_input %s', path)
calc = base.calculators(log.get_oqparam(), log.calc_id)
base.BaseCalculator.gzip_inputs = lambda self: None # disable
with mock.patch.dict(os.environ, {'OQ_CHECK_INPUT': '1'}):
calc.read_inputs()
def main(job_ini_or_zip_or_nrmls):
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
for job_ini_or_zip_or_nrml in job_ini_or_zip_or_nrmls:
if job_ini_or_zip_or_nrml.endswith('.xml'):
try:
node = nrml.to_python(job_ini_or_zip_or_nrml)
if node.tag.endswith('exposureModel'):
err = Exposure.check(job_ini_or_zip_or_nrml)
if err:
logging.warning(err)
else:
logging.info('Checked %s', job_ini_or_zip_or_nrml)
except Exception as exc:
sys.exit(exc)
else:
with logs.init('calc', job_ini_or_zip_or_nrml) as log:
calc = base.calculators(log.get_oqparam(), log.calc_id)
base.BaseCalculator.gzip_inputs = lambda self: None # disable
with mock.patch.dict(os.environ, {'OQ_CHECK_INPUT': '1'}):
calc.read_inputs()
def main(vs30_csv,
z1pt0=False,
z2pt5=False,
vs30measured=False,
*,
exposure_xml=None,
sites_csv=None,
grid_spacing: float = 0,
assoc_distance: float = 5,
output='site_model.csv'):
"""
Prepare a site_model.csv file from exposure xml files/site csv files,
vs30 csv files and a grid spacing which can be 0 (meaning no grid).
For each site the closest vs30 parameter is used. The command can also
generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured
which may be needed by your hazard model, depending on the required GSIMs.
"""
hdf5 = datastore.hdf5new()
req_site_params = {'vs30'}
fields = ['lon', 'lat', 'vs30']
if z1pt0:
req_site_params.add('z1pt0')
fields.append('z1pt0')
if z2pt5:
req_site_params.add('z2pt5')
fields.append('z2pt5')
if vs30measured:
req_site_params.add('vs30measured')
fields.append('vs30measured')
with performance.Monitor(measuremem=True) as mon:
if exposure_xml:
mesh, assets_by_site = Exposure.read(
exposure_xml, check_dupl=False).get_mesh_assets_by_site()
hdf5['assetcol'] = assetcol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, req_site_params=req_site_params)
if grid_spacing:
grid = mesh.get_convex_hull().dilate(grid_spacing).discretize(
grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
logging.info(
'Associating exposure grid with %d locations to %d '
'exposure sites', len(haz_sitecol), len(assets_by_site))
haz_sitecol, assets_by, discarded = assoc(
assets_by_site, haz_sitecol, grid_spacing * SQRT2,
'filter')
if len(discarded):
logging.info(
'Discarded %d sites with assets '
'[use oq plot_assets]', len(discarded))
hdf5['discarded'] = numpy.array(discarded)
haz_sitecol.make_complete()
else:
haz_sitecol = assetcol
discarded = []
elif sites_csv:
lons, lats = [], []
for fname in sites_csv:
check_fname(fname, 'sites_csv', output)
with read(fname) as csv:
for line in csv:
if line.startswith('lon,lat'): # possible header
continue
lon, lat = line.split(',')[:2]
lons.append(valid.longitude(lon))
lats.append(valid.latitude(lat))
haz_sitecol = site.SiteCollection.from_points(
lons, lats, req_site_params=req_site_params)
if grid_spacing:
grid = haz_sitecol.mesh.get_convex_hull().dilate(
grid_spacing).discretize(grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
else:
raise RuntimeError('Missing exposures or missing sites')
vs30 = associate(haz_sitecol, vs30_csv, assoc_distance)
if z1pt0:
haz_sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30'])
if z2pt5:
haz_sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30'])
hdf5['sitecol'] = haz_sitecol
writers.write_csv(output, haz_sitecol.array[fields])
logging.info('Saved %d rows in %s' % (len(haz_sitecol), output))
logging.info(mon)
return haz_sitecol
def prepare_site_model(exposure_xml, vs30_csv,
z1pt0, z2pt5, vs30measured, grid_spacing=0,
site_param_distance=5, output='sites.csv'):
"""
Prepare a sites.csv file from an exposure xml file, a vs30 csv file
and a grid spacing which can be 0 (meaning no grid). For each asset site
or grid site the closest vs30 parameter is used. The command can also
generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured
which may be needed by your hazard model, depending on the required GSIMs.
"""
logging.basicConfig(level=logging.INFO)
hdf5 = datastore.hdf5new()
req_site_params = {'vs30'}
fields = ['lon', 'lat', 'vs30']
if z1pt0:
req_site_params.add('z1pt0')
fields.append('z1pt0')
if z2pt5:
req_site_params.add('z2pt5')
fields.append('z2pt5')
if vs30measured:
req_site_params.add('vs30measured')
fields.append('vs30measured')
with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon:
mesh, assets_by_site = Exposure.read(
exposure_xml, check_dupl=False).get_mesh_assets_by_site()
mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, req_site_params=req_site_params)
if grid_spacing:
grid = mesh.get_convex_hull().dilate(
grid_spacing).discretize(grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
logging.info(
'Associating exposure grid with %d locations to %d '
'exposure sites', len(haz_sitecol), len(assets_by_site))
haz_sitecol, assets_by, _discarded = assoc(
assets_by_site, haz_sitecol, grid_spacing * SQRT2, 'filter')
haz_sitecol.make_complete()
else:
haz_sitecol = assetcol
vs30orig = read_vs30(vs30_csv)
logging.info('Associating %d hazard sites to %d site parameters',
len(haz_sitecol), len(vs30orig))
sitecol, vs30, discarded = assoc(
vs30orig, haz_sitecol, site_param_distance, 'warn')
sitecol.array['vs30'] = vs30['vs30']
if z1pt0:
sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30'])
if z2pt5:
sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30'])
if vs30measured:
sitecol.array['vs30measured'] = False # it is inferred
mon.hdf5['sitecol'] = sitecol
if discarded:
mon.hdf5['discarded'] = numpy.array(discarded)
write_csv(output, sitecol.array[fields])
if discarded:
logging.info('Discarded %d sites with assets [use oq plot_assets]',
len(discarded))
logging.info('Saved %d rows in %s' % (len(sitecol), output))
logging.info(mon)
return sitecol
def prepare_site_model(exposure_xml, sites_csv, vs30_csv,
z1pt0, z2pt5, vs30measured, grid_spacing=0,
assoc_distance=5, output='site_model.csv'):
"""
Prepare a site_model.csv file from exposure xml files/site csv files,
vs30 csv files and a grid spacing which can be 0 (meaning no grid).
For each site the closest vs30 parameter is used. The command can also
generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured
which may be needed by your hazard model, depending on the required GSIMs.
"""
hdf5 = datastore.hdf5new()
req_site_params = {'vs30'}
fields = ['lon', 'lat', 'vs30']
if z1pt0:
req_site_params.add('z1pt0')
fields.append('z1pt0')
if z2pt5:
req_site_params.add('z2pt5')
fields.append('z2pt5')
if vs30measured:
req_site_params.add('vs30measured')
fields.append('vs30measured')
with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon:
if exposure_xml:
mesh, assets_by_site = Exposure.read(
exposure_xml, check_dupl=False).get_mesh_assets_by_site()
mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, req_site_params=req_site_params)
if grid_spacing:
grid = mesh.get_convex_hull().dilate(
grid_spacing).discretize(grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
logging.info(
'Associating exposure grid with %d locations to %d '
'exposure sites', len(haz_sitecol), len(assets_by_site))
haz_sitecol, assets_by, discarded = assoc(
assets_by_site, haz_sitecol,
grid_spacing * SQRT2, 'filter')
if len(discarded):
logging.info('Discarded %d sites with assets '
'[use oq plot_assets]', len(discarded))
mon.hdf5['discarded'] = numpy.array(discarded)
haz_sitecol.make_complete()
else:
haz_sitecol = assetcol
discarded = []
elif sites_csv:
lons, lats = [], []
for fname in sites_csv:
with open(fname) as csv:
for line in csv:
if line.startswith('lon,lat'): # possible header
continue
lon, lat = line.split(',')[:2]
lons.append(valid.longitude(lon))
lats.append(valid.latitude(lat))
haz_sitecol = site.SiteCollection.from_points(
lons, lats, req_site_params=req_site_params)
if grid_spacing:
grid = mesh.get_convex_hull().dilate(
grid_spacing).discretize(grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
else:
raise RuntimeError('Missing exposures or missing sites')
vs30orig = read_vs30(vs30_csv)
logging.info('Associating %d hazard sites to %d site parameters',
len(haz_sitecol), len(vs30orig))
sitecol, vs30, _ = assoc(
vs30orig, haz_sitecol, assoc_distance, 'warn')
sitecol.array['vs30'] = vs30['vs30']
if z1pt0:
sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30'])
if z2pt5:
sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30'])
if vs30measured:
sitecol.array['vs30measured'] = False # it is inferred
mon.hdf5['sitecol'] = sitecol
write_csv(output, sitecol.array[fields])
logging.info('Saved %d rows in %s' % (len(sitecol), output))
logging.info(mon)
return sitecol