def test_read_written(self): dtype = numpy.dtype([('a', float), ('b', float, 2)]) written = numpy.array([(.1, [.2, .3]), (.3, [.4, .5])], dtype) dest = tempfile.NamedTemporaryFile().name write_csv(dest, written) read = read_composite_array(dest) numpy.testing.assert_equal(read, written)
def get_gmfs(oqparam): """ :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :returns: sitecol, eids, gmf array of shape (G, N, E, I) """ I = len(oqparam.imtls) fname = oqparam.inputs['gmfs'] if fname.endswith('.csv'): array = writers.read_composite_array(fname) R = len(numpy.unique(array['rlzi'])) # the array has the structure rlzi, sid, eid, gmv_PGA, gmv_... dtlist = [(name, array.dtype[name]) for name in array.dtype.names[:3]] num_gmv = len(array.dtype.names[3:]) assert num_gmv == I, (num_gmv, I) dtlist.append(('gmv', (F32, num_gmv))) eids = numpy.unique(array['eid']) assert len(eids) == oqparam.number_of_ground_motion_fields, ( len(eids), oqparam.number_of_ground_motion_fields) eidx = {eid: e for e, eid in enumerate(eids)} sids = numpy.unique(array['sid']) gmfs = numpy.zeros((R, len(sids), len(eids), I), F32) for row in array.view(dtlist): gmfs[row['rlzi'], row['sid'], eidx[row['eid']]] = row['gmv'] elif fname.endswith('.xml'): eids, gmfs_by_imt = get_scenario_from_nrml(oqparam, fname) N, E = gmfs_by_imt.shape gmfs = numpy.zeros((1, N, E, I), F32) for imti, imtstr in enumerate(oqparam.imtls): gmfs[0, :, :, imti] = gmfs_by_imt[imtstr] else: raise NotImplemented('Reading from %s' % fname) return eids, gmfs
def test_read_ok(self): got = read_composite_array(fname) expected = numpy.array([(b'a0', 81.2985, 0.0, 29.1098, 0.0, 542.1328), (b'a1', 83.0823, 0.0, 27.9006, 0.0, 254.7904), (b'a2', 85.7477, 0.0, 27.9015, 0.0, 653.871), (b'a3', 85.7477, 0.0, 27.9015, 0.0, 806.2714)], got.dtype) numpy.testing.assert_equal(got, expected)
def test_read_ok(self): got = read_composite_array(fname) expected = numpy.array( [(b'a0', 81.2985, 0.0, 29.1098, 0.0, 542.1328), (b'a1', 83.0823, 0.0, 27.9006, 0.0, 254.7904), (b'a2', 85.7477, 0.0, 27.9015, 0.0, 653.871), (b'a3', 85.7477, 0.0, 27.9015, 0.0, 806.2714)], got.dtype) numpy.testing.assert_equal(got, expected)
def _get_gmfs(oqparam): M = len(oqparam.imtls) assert M, ('oqparam.imtls is empty, did you call ' 'oqparam.set_risk_imtls(get_risk_models(oqparam))?') fname = oqparam.inputs['gmfs'] if fname.endswith('.csv'): array = writers.read_composite_array(fname).array R = len(numpy.unique(array['rlzi'])) if R > 1: raise InvalidFile('%s: found %d realizations, currently only one ' 'realization is supported' % (fname, R)) # the array has the structure rlzi, sid, eid, gmv_PGA, gmv_... dtlist = [(name, array.dtype[name]) for name in array.dtype.names[:3]] required_imts = list(oqparam.imtls) imts = [name[4:] for name in array.dtype.names[3:]] if imts != required_imts: raise ValueError('Required %s, but %s contains %s' % (required_imts, fname, imts)) dtlist.append(('gmv', (F32, M))) eids = numpy.unique(array['eid']) E = len(eids) found_eids = set(eids) expected_eids = set(range(E)) # expected incremental eids missing_eids = expected_eids - found_eids if missing_eids: raise InvalidFile('Missing eids in the gmfs.csv file: %s' % missing_eids) assert expected_eids == found_eids, (expected_eids, found_eids) eidx = {eid: e for e, eid in enumerate(eids)} sitecol = get_site_collection(oqparam) expected_sids = set(sitecol.sids) found_sids = set(numpy.unique(array['sid'])) missing_sids = found_sids - expected_sids if missing_sids: raise InvalidFile( 'Found site IDs missing in the sites.csv file: %s' % missing_sids) N = len(sitecol) gmfs = numpy.zeros((R, N, E, M), F32) counter = collections.Counter() for row in array.view(dtlist): key = row['rlzi'], row['sid'], eidx[row['eid']] gmfs[key] = row['gmv'] counter[key] += 1 dupl = [key for key in counter if counter[key] > 1] if dupl: raise InvalidFile('Duplicated (rlzi, sid, eid) in the GMFs file: ' '%s' % dupl) elif fname.endswith('.xml'): eids, gmfs_by_imt = get_scenario_from_nrml(oqparam, fname) N, E = gmfs_by_imt.shape gmfs = numpy.zeros((1, N, E, M), F32) for imti, imtstr in enumerate(oqparam.imtls): gmfs[0, :, :, imti] = gmfs_by_imt[imtstr] else: raise NotImplemented('Reading from %s' % fname) return eids, gmfs
def get_gmfs(oqparam): """ :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :returns: sitecol, eids, gmf array of shape (R, N, E, M) """ M = len(oqparam.imtls) fname = oqparam.inputs['gmfs'] if fname.endswith('.csv'): array = writers.read_composite_array(fname) R = len(numpy.unique(array['rlzi'])) # the array has the structure rlzi, sid, eid, gmv_PGA, gmv_... dtlist = [(name, array.dtype[name]) for name in array.dtype.names[:3]] num_gmv = len(array.dtype.names[3:]) assert num_gmv == M, (num_gmv, M) dtlist.append(('gmv', (F32, num_gmv))) eids = numpy.unique(array['eid']) E = len(eids) found_eids = set(eids) expected_eids = set(range(E)) # expected incremental eids missing_eids = expected_eids - found_eids if missing_eids: raise InvalidFile('Missing eids in the gmfs.csv file: %s' % missing_eids) assert expected_eids == found_eids, (expected_eids, found_eids) eidx = {eid: e for e, eid in enumerate(eids)} sitecol = get_site_collection(oqparam) expected_sids = set(sitecol.sids) found_sids = set(numpy.unique(array['sid'])) missing_sids = found_sids - expected_sids if missing_sids: raise InvalidFile( 'Found site IDs missing in the sites.csv file: %s' % missing_sids) N = len(sitecol) gmfs = numpy.zeros((R, N, E, M), F32) counter = collections.Counter() for row in array.view(dtlist): key = row['rlzi'], row['sid'], eidx[row['eid']] gmfs[key] = row['gmv'] counter[key] += 1 dupl = [key for key in counter if counter[key] > 1] if dupl: raise InvalidFile('Duplicated (rlzi, sid, eid) in the GMFs file: ' '%s' % dupl) elif fname.endswith('.xml'): eids, gmfs_by_imt = get_scenario_from_nrml(oqparam, fname) N, E = gmfs_by_imt.shape gmfs = numpy.zeros((1, N, E, M), F32) for imti, imtstr in enumerate(oqparam.imtls): gmfs[0, :, :, imti] = gmfs_by_imt[imtstr] else: raise NotImplemented('Reading from %s' % fname) return eids, gmfs
def test_assetcol(self): expected = writetmp('''\ idx:uint32,lon,lat,site_id:uint32,taxonomy:uint32:,number,area,occupants:float64:,structural:float64:,deductible-structural:float64:,insurance_limit-structural:float64: 0,8.12985001E+01,2.91098003E+01,0,1,3.00000000E+00,1.00000000E+01,1.00000000E+01,1.00000000E+02,2.50000000E+01,1.00000000E+02 1,8.30822983E+01,2.79006004E+01,1,0,5.00000000E+02,1.00000000E+01,2.00000000E+01,4.00000000E-01,1.00000000E-01,2.00000000E-01 ''') assetcol = riskinput.AssetCollection(self.assets_by_site, None, None) numpy.testing.assert_equal( assetcol.array, writers.read_composite_array(expected)) # pickleability pickle.loads(pickle.dumps(assetcol))
def import_gmfs(dstore, fname, sids): """ Import in the datastore a ground motion field CSV file. :param dstore: the datastore :param fname: the CSV file :param sids: the site IDs (complete) :returns: event_ids, num_rlzs """ array = writers.read_composite_array(fname).array # has header rlzi, sid, eid, gmv_PGA, ... imts = [name[4:] for name in array.dtype.names[3:]] n_imts = len(imts) gmf_data_dt = numpy.dtype([('rlzi', U16), ('sid', U32), ('eid', U64), ('gmv', (F32, (n_imts, )))]) # store the events eids = numpy.unique(array['eid']) eids.sort() events = numpy.zeros(len(eids), readinput.stored_event_dt) events['eid'] = eids dstore['events'] = events # store the GMFs dic = general.group_array(array.view(gmf_data_dt), 'sid') lst = [] offset = 0 for sid in sids: n = len(dic.get(sid, [])) lst.append((offset, offset + n)) if n: offset += n dstore.extend('gmf_data/data', dic[sid]) dstore['gmf_data/indices'] = numpy.array(lst, U32) dstore['gmf_data/imts'] = ' '.join(imts) # FIXME: if there is no data for the maximum realization # the inferred number of realizations will be wrong num_rlzs = array['rlzi'].max() + 1 # compute gmdata dic = general.group_array(array.view(gmf_data_dt), 'rlzi') gmdata = {r: numpy.zeros(n_imts + 1, F32) for r in range(num_rlzs)} for r in dic: gmv = dic[r]['gmv'] rec = gmdata[r] # (imt1, ..., imtM, nevents) rec[:-1] += gmv.sum(axis=0) rec[-1] += len(gmv) return eids, num_rlzs, gmdata
def import_gmfs(dstore, fname, sids): """ Import in the datastore a ground motion field CSV file. :param dstore: the datastore :param fname: the CSV file :param sids: the site IDs (complete) :returns: event_ids, num_rlzs """ array = writers.read_composite_array(fname).array # has header rlzi, sid, eid, gmv_PGA, ... imts = [name[4:] for name in array.dtype.names[3:]] n_imts = len(imts) gmf_data_dt = numpy.dtype( [('rlzi', U16), ('sid', U32), ('eid', U64), ('gmv', (F32, (n_imts,)))]) # store the events eids = numpy.unique(array['eid']) eids.sort() E = len(eids) eid2idx = dict(zip(eids, range(E))) events = numpy.zeros(E, rupture.events_dt) events['id'] = eids dstore['events'] = events # store the GMFs dic = general.group_array(array.view(gmf_data_dt), 'sid') lst = [] offset = 0 for sid in sids: n = len(dic.get(sid, [])) lst.append((offset, offset + n)) if n: offset += n gmvs = dic[sid] gmvs['eid'] = get_idxs(gmvs, eid2idx) gmvs['rlzi'] = 0 # effectively there is only 1 realization dstore.extend('gmf_data/data', gmvs) dstore['gmf_data/indices'] = numpy.array(lst, U32) dstore['gmf_data/imts'] = ' '.join(imts) sig_eps_dt = [('eid', U64), ('sig', (F32, n_imts)), ('eps', (F32, n_imts))] dstore['gmf_data/sigma_epsilon'] = numpy.zeros(0, sig_eps_dt) dstore['weights'] = numpy.ones(1) return eids
def import_gmfs(dstore, fname, sids): """ Import in the datastore a ground motion field CSV file. :param dstore: the datastore :param fname: the CSV file :param sids: the site IDs (complete) :returns: event_ids, num_rlzs """ array = writers.read_composite_array(fname).array # has header rlzi, sid, eid, gmv_PGA, ... imts = [name[4:] for name in array.dtype.names[3:]] n_imts = len(imts) gmf_data_dt = numpy.dtype( [('rlzi', U16), ('sid', U32), ('eid', U64), ('gmv', (F32, (n_imts,)))]) # store the events eids = numpy.unique(array['eid']) eids.sort() E = len(eids) eid2idx = dict(zip(eids, range(E))) events = numpy.zeros(E, rupture.events_dt) events['eid'] = eids dstore['events'] = events # store the GMFs dic = general.group_array(array.view(gmf_data_dt), 'sid') lst = [] offset = 0 for sid in sids: n = len(dic.get(sid, [])) lst.append((offset, offset + n)) if n: offset += n gmvs = dic[sid] gmvs['eid'] = get_idxs(gmvs, eid2idx) gmvs['rlzi'] = 0 # effectively there is only 1 realization dstore.extend('gmf_data/data', gmvs) dstore['gmf_data/indices'] = numpy.array(lst, U32) dstore['gmf_data/imts'] = ' '.join(imts) return eids
def test_read_err2(self): with self.assertRaises(InvalidFile) as ctx: read_composite_array(wrong2) self.assertIn("Could not cast '0.0 0.0' in file", str(ctx.exception))
def test_read_err1(self): with self.assertRaises(InvalidFile) as ctx: read_composite_array(wrong1) self.assertIn('expected 6 columns, found 5 in file', str(ctx.exception))