def export_gmf_xml(key, output, target): """ Export the GMF Collection specified by ``output`` to the ``target``. :param output: :class:`openquake.engine.db.models.Output` with an `output_type` of `gmf`. :param target: The same ``target`` as :func:`export`. :returns: The same return value as defined by :func:`export`. """ gmf = models.Gmf.objects.get(output=output.id) haz_calc = output.oq_job if output.output_type == 'gmf': lt_rlz = gmf.lt_realization sm_lt_path = core.LT_PATH_JOIN_TOKEN.join(lt_rlz.sm_lt_path) gsim_lt_path = core.LT_PATH_JOIN_TOKEN.join(lt_rlz.gsim_lt_path) else: # gmf_scenario sm_lt_path = '' gsim_lt_path = '' dest = _get_result_export_dest(haz_calc.id, target, output.gmf) writer = hazard_writers.EventBasedGMFXMLWriter(dest, sm_lt_path, gsim_lt_path) with floatformat('%12.8E'): writer.serialize(gmf) return dest
def export_gmf_xml(key, export_dir, fname, sitecol, ruptures, gmfs, rlz, investigation_time): """ :param key: output_type and export_type :param export_dir: the directory where to export :param fname: name of the exported file :param sitecol: the full site collection :param ruptures: an ordered list of ruptures :param gmfs: a matrix of ground motion fields of shape (R, N) :param rlz: a realization object :param investigation_time: investigation time (None for scenario) """ dest = os.path.join(export_dir, fname) if hasattr(rlz, 'gsim_rlz'): # event based smltpath = '_'.join(rlz.sm_lt_path) gsimpath = rlz.gsim_rlz.uid else: # scenario smltpath = '' gsimpath = rlz.uid writer = hazard_writers.EventBasedGMFXMLWriter( dest, sm_lt_path=smltpath, gsim_lt_path=gsimpath) with floatformat('%12.8E'): writer.serialize( GmfCollection(sitecol, ruptures, gmfs, investigation_time)) return {key: [dest]}
def test_case_9(self): with writers.floatformat('%10.6E'): out = self.run_calc(case_9.__file__, 'job.ini', exports='xml') f1, f2 = out['gmf_data', 'xml'] self.assertEqualFiles('LinLee2008SSlab_gmf.xml', f1) self.assertEqualFiles('YoungsEtAl1997SSlab_gmf.xml', f2) out = self.run_calc(case_9.__file__, 'job.ini', exports='txt,csv,hdf5') f1, f2 = out['gmf_data', 'txt'] self.assertEqualFiles('LinLee2008SSlab_gmf.txt', f1) self.assertEqualFiles('YoungsEtAl1997SSlab_gmf.txt', f2) f1, f2 = out['gmf_data', 'csv'] self.assertEqualFiles('gmf-LinLee2008SSlab-PGA.csv', f1) self.assertEqualFiles('gmf-YoungsEtAl1997SSlab-PGA.csv', f2) # test the HDF5 export [fname] = out['gmf_data', 'hdf5'] with h5py.File(fname) as f: self.assertEqual(len(f), 2) # there are only two datasets data1 = f['LinLee2008SSlab()'] data2 = f['YoungsEtAl1997SSlab()'] self.assertEqual( data1.dtype.names, ('lon', 'lat', 'PGA-000', 'PGA-001', 'PGA-002', 'PGA-003', 'PGA-004', 'PGA-005', 'PGA-006', 'PGA-007', 'PGA-008', 'PGA-009')) self.assertEqual(data1.shape, (3,)) self.assertEqual(data1.dtype.names, data2.dtype.names) self.assertEqual(data1.shape, data2.shape)
def test_case_9(self): with writers.floatformat('%10.6E'): out = self.run_calc(case_9.__file__, 'job.ini', exports='xml') f1, f2 = out['gmf_data', 'xml'] self.assertEqualFiles('LinLee2008SSlab_gmf.xml', f1) self.assertEqualFiles('YoungsEtAl1997SSlab_gmf.xml', f2) out = self.run_calc(case_9.__file__, 'job.ini', exports='txt,csv,npz') f1, f2 = out['gmf_data', 'txt'] self.assertEqualFiles('LinLee2008SSlab_gmf.txt', f1) self.assertEqualFiles('YoungsEtAl1997SSlab_gmf.txt', f2) f1, f2 = out['gmf_data', 'csv'] self.assertEqualFiles('gmf-LinLee2008SSlab-PGA.csv', f1) self.assertEqualFiles('gmf-YoungsEtAl1997SSlab-PGA.csv', f2) # test the .npz export [fname] = out['gmf_data', 'npz'] with numpy.load(fname) as f: self.assertEqual(len(f.keys()), 2) # there are only two datasets data1 = f['LinLee2008SSlab()'] data2 = f['YoungsEtAl1997SSlab()'] self.assertEqual(data1.dtype.names, ('lon', 'lat', 'PGA')) self.assertEqual(data1.shape, (3,)) self.assertEqual(data1['PGA'].shape, (3, 10)) self.assertEqual(data1.dtype.names, data2.dtype.names) self.assertEqual(data1.shape, data2.shape)
def export_hazard_curves_xml(key, dest, sitecol, curves_by_imt, imtls, investigation_time): """ Export the curves of the given realization into XML. :param key: output_type and export_type :param dest: name of the exported file :param sitecol: site collection :param curves_by_imt: dictionary with the curves keyed by IMT :param imtls: dictionary with the intensity measure types and levels :param investigation_time: investigation time in years """ mdata = [] hcurves = [] for imt_str, imls in sorted(imtls.items()): hcurves.append([ HazardCurve(site.location, poes) for site, poes in zip(sitecol, curves_by_imt[imt_str]) ]) imt = from_string(imt_str) mdata.append({ 'quantile_value': None, 'statistics': None, 'smlt_path': '', 'gsimlt_path': '', 'investigation_time': investigation_time, 'imt': imt[0], 'sa_period': imt[1], 'sa_damping': imt[2], 'imls': imls, }) writer = hazard_writers.MultiHazardCurveXMLWriter(dest, mdata) with floatformat('%12.8E'): writer.serialize(hcurves) return {dest: dest}
def export_gmf_xml(key, output, target): """ Export the GMF Collection specified by ``output`` to the ``target``. :param output: :class:`openquake.engine.db.models.Output` with an `output_type` of `gmf`. :param target: The same ``target`` as :func:`export`. :returns: The same return value as defined by :func:`export`. """ gmf = models.Gmf.objects.get(output=output.id) haz_calc = output.oq_job if output.output_type == 'gmf': lt_rlz = gmf.lt_realization sm_lt_path = core.LT_PATH_JOIN_TOKEN.join(lt_rlz.sm_lt_path) gsim_lt_path = core.LT_PATH_JOIN_TOKEN.join(lt_rlz.gsim_lt_path) else: # gmf_scenario sm_lt_path = '' gsim_lt_path = '' dest = _get_result_export_dest(haz_calc.id, target, output.gmf) writer = hazard_writers.EventBasedGMFXMLWriter( dest, sm_lt_path, gsim_lt_path) with floatformat('%12.8E'): writer.serialize(gmf) return dest
def export_hazard_curves_xml(key, dest, sitecol, curves_by_imt, imtls, investigation_time): """ Export the curves of the given realization into XML. :param key: output_type and export_type :param dest: name of the exported file :param sitecol: site collection :param curves_by_imt: dictionary with the curves keyed by IMT :param imtls: dictionary with the intensity measure types and levels :param investigation_time: investigation time in years """ mdata = [] hcurves = [] for imt_str, imls in sorted(imtls.items()): hcurves.append( [HazardCurve(site.location, poes) for site, poes in zip(sitecol, curves_by_imt[imt_str])]) imt = from_string(imt_str) mdata.append({ 'quantile_value': None, 'statistics': None, 'smlt_path': '', 'gsimlt_path': '', 'investigation_time': investigation_time, 'imt': imt[0], 'sa_period': imt[1], 'sa_damping': imt[2], 'imls': imls, }) writer = hazard_writers.MultiHazardCurveXMLWriter(dest, mdata) with floatformat('%12.8E'): writer.serialize(hcurves) return {dest: dest}
def serialize(self, data): """ :param data: A sequence of data where each datum has the following attributes: * matrix: N-dimensional numpy array containing the disaggregation histogram. * dim_labels: A list of strings which label the dimensions of a given histogram. For example, for a Magnitude-Distance-Epsilon histogram, we would expect `dim_labels` to be ``['Mag', 'Dist', 'Eps']``. * poe: The disaggregation Probability of Exceedance level for which these results were produced. * iml: Intensity measure level, interpolated from the source hazard curve at the given ``poe``. """ with nrml.NRMLFile(self.dest, 'w') as fh, floatformat('%.6E'): root = et.Element('nrml') diss_matrices = et.SubElement(root, 'disaggMatrices') _set_metadata(diss_matrices, self.metadata, _ATTR_MAP) transform = lambda val: ', '.join(map(scientificformat, val)) _set_metadata(diss_matrices, self.metadata, self.BIN_EDGE_ATTR_MAP, transform=transform) for result in data: diss_matrix = et.SubElement(diss_matrices, 'disaggMatrix') # Check that we have bin edges defined for each dimension label # (mag, dist, lon, lat, eps, TRT) for label in result.dim_labels: bin_edge_attr = self.DIM_LABEL_TO_BIN_EDGE_MAP.get(label) assert self.metadata.get(bin_edge_attr) is not None, ( "Writer is missing '%s' metadata" % bin_edge_attr) result_type = ','.join(result.dim_labels) diss_matrix.set('type', result_type) dims = ','.join(str(x) for x in result.matrix.shape) diss_matrix.set('dims', dims) diss_matrix.set('poE', scientificformat(result.poe)) diss_matrix.set('iml', scientificformat(result.iml)) for idxs, value in numpy.ndenumerate(result.matrix): prob = et.SubElement(diss_matrix, 'prob') index = ','.join([str(x) for x in idxs]) prob.set('index', index) prob.set('value', scientificformat(value)) nrml.write(list(root), fh)
def serialize(self, data): """ :param data: A sequence of data where each datum has the following attributes: * matrix: N-dimensional numpy array containing the disaggregation histogram. * dim_labels: A list of strings which label the dimensions of a given histogram. For example, for a Magnitude-Distance-Epsilon histogram, we would expect `dim_labels` to be ``['Mag', 'Dist', 'Eps']``. * poe: The disaggregation Probability of Exceedance level for which these results were produced. * iml: Intensity measure level, interpolated from the source hazard curve at the given ``poe``. """ with open(self.dest, 'wb') as fh, floatformat('%.6E'): root = et.Element('nrml') diss_matrices = et.SubElement(root, 'disaggMatrices') _set_metadata(diss_matrices, self.metadata, _ATTR_MAP) transform = lambda val: ', '.join(map(scientificformat, val)) _set_metadata(diss_matrices, self.metadata, self.BIN_EDGE_ATTR_MAP, transform=transform) for result in data: diss_matrix = et.SubElement(diss_matrices, 'disaggMatrix') # Check that we have bin edges defined for each dimension label # (mag, dist, lon, lat, eps, TRT) for label in result.dim_labels: bin_edge_attr = self.DIM_LABEL_TO_BIN_EDGE_MAP.get(label) assert self.metadata.get(bin_edge_attr) is not None, ( "Writer is missing '%s' metadata" % bin_edge_attr ) result_type = ','.join(result.dim_labels) diss_matrix.set('type', result_type) dims = ','.join(str(x) for x in result.matrix.shape) diss_matrix.set('dims', dims) diss_matrix.set('poE', scientificformat(result.poe)) diss_matrix.set('iml', scientificformat(result.iml)) for idxs, value in numpy.ndenumerate(result.matrix): prob = et.SubElement(diss_matrix, 'prob') index = ','.join([str(x) for x in idxs]) prob.set('index', index) prob.set('value', scientificformat(value)) nrml.write(list(root), fh)
def test_case_9(self): with writers.floatformat('%10.6E'): out = self.run_calc(case_9.__file__, 'job.ini', exports='xml') f1, f2 = out['gmfs', 'xml'] self.assertEqualFiles('LinLee2008SSlab_gmf.xml', f1) self.assertEqualFiles('YoungsEtAl1997SSlab_gmf.xml', f2) out = self.run_calc(case_9.__file__, 'job.ini', exports='csv') f1, f2 = out['gmfs', 'csv'] self.assertEqualFiles('LinLee2008SSlab_gmf.csv', f1) self.assertEqualFiles('YoungsEtAl1997SSlab_gmf.csv', f2)
def to_nrml(self, key, data, fname=None, fmt='%11.7E'): """ :param key: `dmg_dist_per_asset|dmg_dist_per_taxonomy|dmg_dist_total|collapse_map` :param data: sequence of rows to serialize :fname: the path name of the output file; if None, build a name :returns: path name of the saved file """ fname = fname or writetmp() node = getattr(self, key + '_node')(data) with open(fname, 'w') as out, writers.floatformat(fmt): nrml.write([node], out) return fname
def write(nodes, output=sys.stdout, fmt="%8.4E"): """ Convert nodes into a NRML file. output must be a file object open in write mode. If you want to perform a consistency check, open it in read-write mode, then it will be read after creation and validated. :params nodes: an iterable over Node objects :params output: a file-like object in write or read-write mode """ root = Node("nrml", nodes=nodes) with writers.floatformat(fmt): node_to_xml(root, output, {NRML05: "", GML_NAMESPACE: "gml:"}) if hasattr(output, "mode") and "+" in output.mode: # read-write mode output.seek(0) read(output) # validate the written file
def write(nodes, output=sys.stdout, fmt='%8.4E'): """ Convert nodes into a NRML file. output must be a file object open in write mode. If you want to perform a consistency check, open it in read-write mode, then it will be read after creation and validated. :params nodes: an iterable over Node objects :params output: a file-like object in write or read-write mode """ root = Node('nrml', nodes=nodes) with writers.floatformat(fmt): node_to_xml(root, output, {NRML05: '', GML_NAMESPACE: 'gml:'}) if hasattr(output, 'mode') and '+' in output.mode: # read-write mode output.seek(0) read(output) # validate the written file
def test_case_1(self): # ROUNDING ERROR WARNING (MS): numbers such as 2.5 and 2.4999999999 # are extremely close (up to 4E-11) however they must be rounded to # a single digit to compare equal in their string representation; for # this reason the precision here has to be reduced a lot, even it the # numbers are very close. It comes down to the known fact that # comparing the XMLs is not a good idea; suboptimal choises # sometimes have to be made, since we want this test to # to run both on Ubuntu 12.04 and Ubuntu 14.04. # Incidentally, when the approach of comparing the XML was taken, # the idea of supporting at the same time different versions of the # libraries was out of question, so it made a lot of sense to check # the XMLs, since the numbers had to be exactly identical. with writers.floatformat('%5.1E'): out = self.run_calc(case_1.__file__, 'job.ini', exports='xml') raise unittest.SkipTest # because of the rounding errors self.assertEqualFiles('expected.xml', out['gmf_data', 'xml'][0])
def export_hazard_map_csv(key, output, target): """ General hazard map export code. """ file_ext = key[1] data = [] for hazard_map in models.HazardMap.objects.filter(output=output): data.extend(zip(hazard_map.lons, hazard_map.lats, hazard_map.imls)) haz_calc = output.oq_job dest = _get_result_export_dest(haz_calc.id, target, hazard_map, file_ext=file_ext) with open(dest, 'w') as f: writer = csv.writer(f, delimiter=' ') with floatformat('%12.8E'): for row in sorted(data): writer.writerow(map(scientificformat, row)) return dest
def export_hazard_curve_csv(key, output, target): """ Save a hazard curve (of a given IMT) as a .csv file in the format (lon lat poe1 ... poeN), where the fields are space separated. """ data = [] for hc in models.HazardCurve.objects.filter(output=output.id): x_y_poes = models.HazardCurveData.objects.all_curves_simple( filter_args=dict(hazard_curve=hc.id)) data.extend(x_y_poes) haz_calc_id = output.oq_job.id dest = _get_result_export_dest(haz_calc_id, target, hc, file_ext='csv') with open(dest, 'wb') as f: writer = csv.writer(f, delimiter=' ') with floatformat('%11.7E'): for x, y, poes in sorted(data): writer.writerow(map(scientificformat, [x, y] + poes)) return dest
def write_source_model(dest, sources, name=None): """ Writes a source model to XML. :param str dest: Destination path :param list sources: Source model as list of instance of the :class:`openquake.hazardlib.source.base.BaseSeismicSource` :param str name: Name of the source model (if missing, extracted from the filename) """ name = name or os.path.splitext(os.path.basename(dest))[0] nodes = map(obj_to_node, sorted(sources, key=lambda src: src.source_id)) source_model = LiteralNode("sourceModel", {"name": name}, nodes=nodes) with open(dest, 'w') as f, writers.floatformat('%s'): nrml.write([source_model], f) return dest
def write(nodes, output=sys.stdout, fmt='%10.7E', gml=True): """ Convert nodes into a NRML file. output must be a file object open in write mode. If you want to perform a consistency check, open it in read-write mode, then it will be read after creation and validated. :params nodes: an iterable over Node objects :params output: a file-like object in write or read-write mode """ root = Node('nrml', nodes=nodes) namespaces = {NRML05: ''} if gml: namespaces[GML_NAMESPACE] = 'gml:' with writers.floatformat(fmt): node_to_xml(root, output, namespaces) if hasattr(output, 'mode') and '+' in output.mode: # read-write mode output.seek(0) read(output) # validate the written file
def write(nodes, output=sys.stdout, fmt='%.7E', gml=True, xmlns=None): """ Convert nodes into a NRML file. output must be a file object open in write mode. If you want to perform a consistency check, open it in read-write mode, then it will be read after creation and validated. :params nodes: an iterable over Node objects :params output: a file-like object in write or read-write mode :param fmt: format used for writing the floats (default '%.7E') :param gml: add the http://www.opengis.net/gml namespace :param xmlns: NRML namespace like http://openquake.org/xmlns/nrml/0.4 """ root = Node('nrml', nodes=nodes) namespaces = {xmlns or NRML05: ''} if gml: namespaces[GML_NAMESPACE] = 'gml:' with writers.floatformat(fmt): node_to_xml(root, output, namespaces) if hasattr(output, 'mode') and '+' in output.mode: # read-write mode output.seek(0) read(output) # validate the written file