def zip_exposure(exposure_xml, archive_zip='', log=logging.info): """ Zip an exposure.xml file with all its .csv subfiles (if any) """ archive_zip = archive_zip or exposure_xml[:-4] + '.zip' if os.path.exists(archive_zip): sys.exit('%s exists already' % archive_zip) [exp] = Exposure.read_headers([exposure_xml]) files = [exposure_xml] + exp.datafiles general.zipfiles(files, archive_zip, log=log, cleanup=True) return archive_zip
def prepare_site_model(exposure_xml, vs30_csv, grid_spacing=0, site_param_distance=5, output='sites.csv'): """ Prepare a site_model.csv file from an exposure xml file, a vs30 csv file and a grid spacing which can be 0 (meaning no grid). Sites far away from the vs30 records are discarded and you can see them with the command `oq plot_assets`. It is up to you decide if you need to fix your exposure or if it is right to ignore the discarded sites. """ logging.basicConfig(level=logging.INFO) hdf5 = datastore.hdf5new() with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon: mesh, assets_by_site = Exposure.read( exposure_xml, check_dupl=False).get_mesh_assets_by_site() mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points( mesh.lons, mesh.lats, req_site_params={'vs30'}) if grid_spacing: grid = mesh.get_convex_hull().dilate(grid_spacing).discretize( grid_spacing) haz_sitecol = site.SiteCollection.from_points( grid.lons, grid.lats, req_site_params={'vs30'}) logging.info( 'Reducing exposure grid with %d locations to %d sites' ' with assets', len(haz_sitecol), len(assets_by_site)) haz_sitecol, assets_by, _discarded = assoc(assets_by_site, haz_sitecol, grid_spacing * SQRT2, 'filter') haz_sitecol.make_complete() else: haz_sitecol = assetcol vs30orig = read_vs30(vs30_csv.split(',')) logging.info('Associating %d hazard sites to %d site parameters', len(haz_sitecol), len(vs30orig)) sitecol, vs30, discarded = assoc( vs30orig, haz_sitecol, grid_spacing * SQRT2 or site_param_distance, 'filter') sitecol.array['vs30'] = vs30['vs30'] mon.hdf5['sitecol'] = sitecol if discarded: mon.hdf5['discarded'] = numpy.array(discarded) sids = numpy.arange(len(vs30), dtype=numpy.uint32) sites = compose_arrays(sids, vs30, 'site_id') write_csv(output, sites) if discarded: logging.info('Discarded %d sites with assets [use oq plot_assets]', len(discarded)) logging.info('Saved %d rows in %s' % (len(sitecol), output)) logging.info(mon) return sitecol
def check_input(job_ini_or_zip_or_nrmls): for job_ini_or_zip_or_nrml in job_ini_or_zip_or_nrmls: if job_ini_or_zip_or_nrml.endswith('.xml'): try: node = nrml.to_python(job_ini_or_zip_or_nrml) if node.tag.endswith('exposureModel'): err = Exposure.check(job_ini_or_zip_or_nrml) if err: logging.warning(err) else: logging.info('Checked %s', job_ini_or_zip_or_nrml) except Exception as exc: sys.exit(exc) else: oq = readinput.get_oqparam(job_ini_or_zip_or_nrml) base.calculators(oq, logs.init()).read_inputs()
def main(job_ini_or_zip_or_nrmls): """ Check the validity of job.ini files, job.zip files and .xml files. NB: `oq check_input job_haz.ini job_risk.ini` is special-cased so that the risk files are checked before the hazard files. """ if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'): os.environ['OQ_DISTRIBUTE'] = 'processpool' all_inis = all(f.endswith('.ini') for f in job_ini_or_zip_or_nrmls) if all_inis: # the typical case is job_haz.ini + job_risk.ini dic = {} for ini in job_ini_or_zip_or_nrmls: for key, val in readinput.get_params(ini).items(): if key == 'inputs' and key in dic: dic[key].update(val) else: # the last wins dic[key] = val with logs.init('job', dic) as log: logging.info('Running oq check_input %s', ' '.join(job_ini_or_zip_or_nrmls)) calc = base.calculators(log.get_oqparam(), log.calc_id) base.BaseCalculator.gzip_inputs = lambda self: None # disable with mock.patch.dict(os.environ, {'OQ_CHECK_INPUT': '1'}): calc.read_inputs() return for job_ini_or_zip_or_nrml in job_ini_or_zip_or_nrmls: if job_ini_or_zip_or_nrml.endswith('.xml'): try: node = nrml.to_python(job_ini_or_zip_or_nrml) if node.tag.endswith('exposureModel'): err = Exposure.check(job_ini_or_zip_or_nrml) if err: logging.warning(err) else: logging.info('Checked %s', job_ini_or_zip_or_nrml) except Exception as exc: sys.exit(exc) else: # .zip with logs.init('job', job_ini_or_zip_or_nrml) as log: path = os.path.abspath(job_ini_or_zip_or_nrml) logging.info('Running oq check_input %s', path) calc = base.calculators(log.get_oqparam(), log.calc_id) base.BaseCalculator.gzip_inputs = lambda self: None # disable with mock.patch.dict(os.environ, {'OQ_CHECK_INPUT': '1'}): calc.read_inputs()
def main(job_ini_or_zip_or_nrmls): if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'): os.environ['OQ_DISTRIBUTE'] = 'processpool' for job_ini_or_zip_or_nrml in job_ini_or_zip_or_nrmls: if job_ini_or_zip_or_nrml.endswith('.xml'): try: node = nrml.to_python(job_ini_or_zip_or_nrml) if node.tag.endswith('exposureModel'): err = Exposure.check(job_ini_or_zip_or_nrml) if err: logging.warning(err) else: logging.info('Checked %s', job_ini_or_zip_or_nrml) except Exception as exc: sys.exit(exc) else: with logs.init('calc', job_ini_or_zip_or_nrml) as log: calc = base.calculators(log.get_oqparam(), log.calc_id) base.BaseCalculator.gzip_inputs = lambda self: None # disable with mock.patch.dict(os.environ, {'OQ_CHECK_INPUT': '1'}): calc.read_inputs()
def main(vs30_csv, z1pt0=False, z2pt5=False, vs30measured=False, *, exposure_xml=None, sites_csv=None, grid_spacing: float = 0, assoc_distance: float = 5, output='site_model.csv'): """ Prepare a site_model.csv file from exposure xml files/site csv files, vs30 csv files and a grid spacing which can be 0 (meaning no grid). For each site the closest vs30 parameter is used. The command can also generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured which may be needed by your hazard model, depending on the required GSIMs. """ hdf5 = datastore.hdf5new() req_site_params = {'vs30'} fields = ['lon', 'lat', 'vs30'] if z1pt0: req_site_params.add('z1pt0') fields.append('z1pt0') if z2pt5: req_site_params.add('z2pt5') fields.append('z2pt5') if vs30measured: req_site_params.add('vs30measured') fields.append('vs30measured') with performance.Monitor(measuremem=True) as mon: if exposure_xml: mesh, assets_by_site = Exposure.read( exposure_xml, check_dupl=False).get_mesh_assets_by_site() hdf5['assetcol'] = assetcol = site.SiteCollection.from_points( mesh.lons, mesh.lats, req_site_params=req_site_params) if grid_spacing: grid = mesh.get_convex_hull().dilate(grid_spacing).discretize( grid_spacing) haz_sitecol = site.SiteCollection.from_points( grid.lons, grid.lats, req_site_params=req_site_params) logging.info( 'Associating exposure grid with %d locations to %d ' 'exposure sites', len(haz_sitecol), len(assets_by_site)) haz_sitecol, assets_by, discarded = assoc( assets_by_site, haz_sitecol, grid_spacing * SQRT2, 'filter') if len(discarded): logging.info( 'Discarded %d sites with assets ' '[use oq plot_assets]', len(discarded)) hdf5['discarded'] = numpy.array(discarded) haz_sitecol.make_complete() else: haz_sitecol = assetcol discarded = [] elif sites_csv: lons, lats = [], [] for fname in sites_csv: check_fname(fname, 'sites_csv', output) with read(fname) as csv: for line in csv: if line.startswith('lon,lat'): # possible header continue lon, lat = line.split(',')[:2] lons.append(valid.longitude(lon)) lats.append(valid.latitude(lat)) haz_sitecol = site.SiteCollection.from_points( lons, lats, req_site_params=req_site_params) if grid_spacing: grid = haz_sitecol.mesh.get_convex_hull().dilate( grid_spacing).discretize(grid_spacing) haz_sitecol = site.SiteCollection.from_points( grid.lons, grid.lats, req_site_params=req_site_params) else: raise RuntimeError('Missing exposures or missing sites') vs30 = associate(haz_sitecol, vs30_csv, assoc_distance) if z1pt0: haz_sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30']) if z2pt5: haz_sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30']) hdf5['sitecol'] = haz_sitecol writers.write_csv(output, haz_sitecol.array[fields]) logging.info('Saved %d rows in %s' % (len(haz_sitecol), output)) logging.info(mon) return haz_sitecol
def prepare_site_model(exposure_xml, vs30_csv, z1pt0, z2pt5, vs30measured, grid_spacing=0, site_param_distance=5, output='sites.csv'): """ Prepare a sites.csv file from an exposure xml file, a vs30 csv file and a grid spacing which can be 0 (meaning no grid). For each asset site or grid site the closest vs30 parameter is used. The command can also generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured which may be needed by your hazard model, depending on the required GSIMs. """ logging.basicConfig(level=logging.INFO) hdf5 = datastore.hdf5new() req_site_params = {'vs30'} fields = ['lon', 'lat', 'vs30'] if z1pt0: req_site_params.add('z1pt0') fields.append('z1pt0') if z2pt5: req_site_params.add('z2pt5') fields.append('z2pt5') if vs30measured: req_site_params.add('vs30measured') fields.append('vs30measured') with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon: mesh, assets_by_site = Exposure.read( exposure_xml, check_dupl=False).get_mesh_assets_by_site() mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points( mesh.lons, mesh.lats, req_site_params=req_site_params) if grid_spacing: grid = mesh.get_convex_hull().dilate( grid_spacing).discretize(grid_spacing) haz_sitecol = site.SiteCollection.from_points( grid.lons, grid.lats, req_site_params=req_site_params) logging.info( 'Associating exposure grid with %d locations to %d ' 'exposure sites', len(haz_sitecol), len(assets_by_site)) haz_sitecol, assets_by, _discarded = assoc( assets_by_site, haz_sitecol, grid_spacing * SQRT2, 'filter') haz_sitecol.make_complete() else: haz_sitecol = assetcol vs30orig = read_vs30(vs30_csv) logging.info('Associating %d hazard sites to %d site parameters', len(haz_sitecol), len(vs30orig)) sitecol, vs30, discarded = assoc( vs30orig, haz_sitecol, site_param_distance, 'warn') sitecol.array['vs30'] = vs30['vs30'] if z1pt0: sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30']) if z2pt5: sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30']) if vs30measured: sitecol.array['vs30measured'] = False # it is inferred mon.hdf5['sitecol'] = sitecol if discarded: mon.hdf5['discarded'] = numpy.array(discarded) write_csv(output, sitecol.array[fields]) if discarded: logging.info('Discarded %d sites with assets [use oq plot_assets]', len(discarded)) logging.info('Saved %d rows in %s' % (len(sitecol), output)) logging.info(mon) return sitecol
def prepare_site_model(exposure_xml, sites_csv, vs30_csv, z1pt0, z2pt5, vs30measured, grid_spacing=0, assoc_distance=5, output='site_model.csv'): """ Prepare a site_model.csv file from exposure xml files/site csv files, vs30 csv files and a grid spacing which can be 0 (meaning no grid). For each site the closest vs30 parameter is used. The command can also generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured which may be needed by your hazard model, depending on the required GSIMs. """ hdf5 = datastore.hdf5new() req_site_params = {'vs30'} fields = ['lon', 'lat', 'vs30'] if z1pt0: req_site_params.add('z1pt0') fields.append('z1pt0') if z2pt5: req_site_params.add('z2pt5') fields.append('z2pt5') if vs30measured: req_site_params.add('vs30measured') fields.append('vs30measured') with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon: if exposure_xml: mesh, assets_by_site = Exposure.read( exposure_xml, check_dupl=False).get_mesh_assets_by_site() mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points( mesh.lons, mesh.lats, req_site_params=req_site_params) if grid_spacing: grid = mesh.get_convex_hull().dilate( grid_spacing).discretize(grid_spacing) haz_sitecol = site.SiteCollection.from_points( grid.lons, grid.lats, req_site_params=req_site_params) logging.info( 'Associating exposure grid with %d locations to %d ' 'exposure sites', len(haz_sitecol), len(assets_by_site)) haz_sitecol, assets_by, discarded = assoc( assets_by_site, haz_sitecol, grid_spacing * SQRT2, 'filter') if len(discarded): logging.info('Discarded %d sites with assets ' '[use oq plot_assets]', len(discarded)) mon.hdf5['discarded'] = numpy.array(discarded) haz_sitecol.make_complete() else: haz_sitecol = assetcol discarded = [] elif sites_csv: lons, lats = [], [] for fname in sites_csv: with open(fname) as csv: for line in csv: if line.startswith('lon,lat'): # possible header continue lon, lat = line.split(',')[:2] lons.append(valid.longitude(lon)) lats.append(valid.latitude(lat)) haz_sitecol = site.SiteCollection.from_points( lons, lats, req_site_params=req_site_params) if grid_spacing: grid = mesh.get_convex_hull().dilate( grid_spacing).discretize(grid_spacing) haz_sitecol = site.SiteCollection.from_points( grid.lons, grid.lats, req_site_params=req_site_params) else: raise RuntimeError('Missing exposures or missing sites') vs30orig = read_vs30(vs30_csv) logging.info('Associating %d hazard sites to %d site parameters', len(haz_sitecol), len(vs30orig)) sitecol, vs30, _ = assoc( vs30orig, haz_sitecol, assoc_distance, 'warn') sitecol.array['vs30'] = vs30['vs30'] if z1pt0: sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30']) if z2pt5: sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30']) if vs30measured: sitecol.array['vs30measured'] = False # it is inferred mon.hdf5['sitecol'] = sitecol write_csv(output, sitecol.array[fields]) logging.info('Saved %d rows in %s' % (len(sitecol), output)) logging.info(mon) return sitecol