コード例 #1
0
ファイル: readers_test.py プロジェクト: amirj700/oq-risklib
 def test_read_written(self):
     dtype = numpy.dtype([('a', float), ('b', float, 2)])
     written = numpy.array([(.1, [.2, .3]), (.3, [.4, .5])], dtype)
     dest = tempfile.NamedTemporaryFile().name
     write_csv(dest, written)
     read = read_composite_array(dest)
     numpy.testing.assert_equal(read, written)
コード例 #2
0
 def test_read_written(self):
     dtype = numpy.dtype([('a', float), ('b', float, 2)])
     written = numpy.array([(.1, [.2, .3]), (.3, [.4, .5])], dtype)
     dest = tempfile.NamedTemporaryFile().name
     write_csv(dest, written)
     read = read_composite_array(dest)
     numpy.testing.assert_equal(read, written)
コード例 #3
0
def get_gmfs(oqparam):
    """
    :param oqparam:
        an :class:`openquake.commonlib.oqvalidation.OqParam` instance
    :returns:
        sitecol, eids, gmf array of shape (G, N, E, I)
    """
    I = len(oqparam.imtls)
    fname = oqparam.inputs['gmfs']
    if fname.endswith('.csv'):
        array = writers.read_composite_array(fname)
        R = len(numpy.unique(array['rlzi']))
        # the array has the structure rlzi, sid, eid, gmv_PGA, gmv_...
        dtlist = [(name, array.dtype[name]) for name in array.dtype.names[:3]]
        num_gmv = len(array.dtype.names[3:])
        assert num_gmv == I, (num_gmv, I)
        dtlist.append(('gmv', (F32, num_gmv)))
        eids = numpy.unique(array['eid'])
        assert len(eids) == oqparam.number_of_ground_motion_fields, (
            len(eids), oqparam.number_of_ground_motion_fields)
        eidx = {eid: e for e, eid in enumerate(eids)}
        sids = numpy.unique(array['sid'])
        gmfs = numpy.zeros((R, len(sids), len(eids), I), F32)
        for row in array.view(dtlist):
            gmfs[row['rlzi'], row['sid'], eidx[row['eid']]] = row['gmv']
    elif fname.endswith('.xml'):
        eids, gmfs_by_imt = get_scenario_from_nrml(oqparam, fname)
        N, E = gmfs_by_imt.shape
        gmfs = numpy.zeros((1, N, E, I), F32)
        for imti, imtstr in enumerate(oqparam.imtls):
            gmfs[0, :, :, imti] = gmfs_by_imt[imtstr]
    else:
        raise NotImplemented('Reading from %s' % fname)
    return eids, gmfs
コード例 #4
0
 def test_read_ok(self):
     got = read_composite_array(fname)
     expected = numpy.array([(b'a0', 81.2985, 0.0, 29.1098, 0.0, 542.1328),
                             (b'a1', 83.0823, 0.0, 27.9006, 0.0, 254.7904),
                             (b'a2', 85.7477, 0.0, 27.9015, 0.0, 653.871),
                             (b'a3', 85.7477, 0.0, 27.9015, 0.0, 806.2714)],
                            got.dtype)
     numpy.testing.assert_equal(got, expected)
コード例 #5
0
ファイル: readers_test.py プロジェクト: amirj700/oq-risklib
 def test_read_ok(self):
     got = read_composite_array(fname)
     expected = numpy.array(
         [(b'a0', 81.2985, 0.0, 29.1098, 0.0, 542.1328),
          (b'a1', 83.0823, 0.0, 27.9006, 0.0, 254.7904),
          (b'a2', 85.7477, 0.0, 27.9015, 0.0, 653.871),
          (b'a3', 85.7477, 0.0, 27.9015, 0.0, 806.2714)],
         got.dtype)
     numpy.testing.assert_equal(got, expected)
コード例 #6
0
ファイル: readinput.py プロジェクト: cbworden/oq-engine
def _get_gmfs(oqparam):
    M = len(oqparam.imtls)
    assert M, ('oqparam.imtls is empty, did you call '
               'oqparam.set_risk_imtls(get_risk_models(oqparam))?')
    fname = oqparam.inputs['gmfs']
    if fname.endswith('.csv'):
        array = writers.read_composite_array(fname).array
        R = len(numpy.unique(array['rlzi']))
        if R > 1:
            raise InvalidFile('%s: found %d realizations, currently only one '
                              'realization is supported' % (fname, R))
        # the array has the structure rlzi, sid, eid, gmv_PGA, gmv_...
        dtlist = [(name, array.dtype[name]) for name in array.dtype.names[:3]]
        required_imts = list(oqparam.imtls)
        imts = [name[4:] for name in array.dtype.names[3:]]
        if imts != required_imts:
            raise ValueError('Required %s, but %s contains %s' %
                             (required_imts, fname, imts))
        dtlist.append(('gmv', (F32, M)))
        eids = numpy.unique(array['eid'])
        E = len(eids)
        found_eids = set(eids)
        expected_eids = set(range(E))  # expected incremental eids
        missing_eids = expected_eids - found_eids
        if missing_eids:
            raise InvalidFile('Missing eids in the gmfs.csv file: %s' %
                              missing_eids)
        assert expected_eids == found_eids, (expected_eids, found_eids)
        eidx = {eid: e for e, eid in enumerate(eids)}
        sitecol = get_site_collection(oqparam)
        expected_sids = set(sitecol.sids)
        found_sids = set(numpy.unique(array['sid']))
        missing_sids = found_sids - expected_sids
        if missing_sids:
            raise InvalidFile(
                'Found site IDs missing in the sites.csv file: %s' %
                missing_sids)
        N = len(sitecol)
        gmfs = numpy.zeros((R, N, E, M), F32)
        counter = collections.Counter()
        for row in array.view(dtlist):
            key = row['rlzi'], row['sid'], eidx[row['eid']]
            gmfs[key] = row['gmv']
            counter[key] += 1
        dupl = [key for key in counter if counter[key] > 1]
        if dupl:
            raise InvalidFile('Duplicated (rlzi, sid, eid) in the GMFs file: '
                              '%s' % dupl)
    elif fname.endswith('.xml'):
        eids, gmfs_by_imt = get_scenario_from_nrml(oqparam, fname)
        N, E = gmfs_by_imt.shape
        gmfs = numpy.zeros((1, N, E, M), F32)
        for imti, imtstr in enumerate(oqparam.imtls):
            gmfs[0, :, :, imti] = gmfs_by_imt[imtstr]
    else:
        raise NotImplemented('Reading from %s' % fname)
    return eids, gmfs
コード例 #7
0
def get_gmfs(oqparam):
    """
    :param oqparam:
        an :class:`openquake.commonlib.oqvalidation.OqParam` instance
    :returns:
        sitecol, eids, gmf array of shape (R, N, E, M)
    """
    M = len(oqparam.imtls)
    fname = oqparam.inputs['gmfs']
    if fname.endswith('.csv'):
        array = writers.read_composite_array(fname)
        R = len(numpy.unique(array['rlzi']))
        # the array has the structure rlzi, sid, eid, gmv_PGA, gmv_...
        dtlist = [(name, array.dtype[name]) for name in array.dtype.names[:3]]
        num_gmv = len(array.dtype.names[3:])
        assert num_gmv == M, (num_gmv, M)
        dtlist.append(('gmv', (F32, num_gmv)))
        eids = numpy.unique(array['eid'])
        E = len(eids)
        found_eids = set(eids)
        expected_eids = set(range(E))  # expected incremental eids
        missing_eids = expected_eids - found_eids
        if missing_eids:
            raise InvalidFile('Missing eids in the gmfs.csv file: %s'
                              % missing_eids)
        assert expected_eids == found_eids, (expected_eids, found_eids)
        eidx = {eid: e for e, eid in enumerate(eids)}
        sitecol = get_site_collection(oqparam)
        expected_sids = set(sitecol.sids)
        found_sids = set(numpy.unique(array['sid']))
        missing_sids = found_sids - expected_sids
        if missing_sids:
            raise InvalidFile(
                'Found site IDs missing in the sites.csv file: %s' %
                missing_sids)
        N = len(sitecol)
        gmfs = numpy.zeros((R, N, E, M), F32)
        counter = collections.Counter()
        for row in array.view(dtlist):
            key = row['rlzi'], row['sid'], eidx[row['eid']]
            gmfs[key] = row['gmv']
            counter[key] += 1
        dupl = [key for key in counter if counter[key] > 1]
        if dupl:
            raise InvalidFile('Duplicated (rlzi, sid, eid) in the GMFs file: '
                              '%s' % dupl)
    elif fname.endswith('.xml'):
        eids, gmfs_by_imt = get_scenario_from_nrml(oqparam, fname)
        N, E = gmfs_by_imt.shape
        gmfs = numpy.zeros((1, N, E, M), F32)
        for imti, imtstr in enumerate(oqparam.imtls):
            gmfs[0, :, :, imti] = gmfs_by_imt[imtstr]
    else:
        raise NotImplemented('Reading from %s' % fname)
    return eids, gmfs
コード例 #8
0
    def test_assetcol(self):
        expected = writetmp('''\
idx:uint32,lon,lat,site_id:uint32,taxonomy:uint32:,number,area,occupants:float64:,structural:float64:,deductible-structural:float64:,insurance_limit-structural:float64:
0,8.12985001E+01,2.91098003E+01,0,1,3.00000000E+00,1.00000000E+01,1.00000000E+01,1.00000000E+02,2.50000000E+01,1.00000000E+02
1,8.30822983E+01,2.79006004E+01,1,0,5.00000000E+02,1.00000000E+01,2.00000000E+01,4.00000000E-01,1.00000000E-01,2.00000000E-01
''')
        assetcol = riskinput.AssetCollection(self.assets_by_site, None, None)
        numpy.testing.assert_equal(
            assetcol.array, writers.read_composite_array(expected))

        # pickleability
        pickle.loads(pickle.dumps(assetcol))
コード例 #9
0
ファイル: base.py プロジェクト: glorykim99/oq-engine
def import_gmfs(dstore, fname, sids):
    """
    Import in the datastore a ground motion field CSV file.

    :param dstore: the datastore
    :param fname: the CSV file
    :param sids: the site IDs (complete)
    :returns: event_ids, num_rlzs
    """
    array = writers.read_composite_array(fname).array
    # has header rlzi, sid, eid, gmv_PGA, ...
    imts = [name[4:] for name in array.dtype.names[3:]]
    n_imts = len(imts)
    gmf_data_dt = numpy.dtype([('rlzi', U16), ('sid', U32), ('eid', U64),
                               ('gmv', (F32, (n_imts, )))])
    # store the events
    eids = numpy.unique(array['eid'])
    eids.sort()
    events = numpy.zeros(len(eids), readinput.stored_event_dt)
    events['eid'] = eids
    dstore['events'] = events
    # store the GMFs
    dic = general.group_array(array.view(gmf_data_dt), 'sid')
    lst = []
    offset = 0
    for sid in sids:
        n = len(dic.get(sid, []))
        lst.append((offset, offset + n))
        if n:
            offset += n
            dstore.extend('gmf_data/data', dic[sid])
    dstore['gmf_data/indices'] = numpy.array(lst, U32)
    dstore['gmf_data/imts'] = ' '.join(imts)

    # FIXME: if there is no data for the maximum realization
    # the inferred number of realizations will be wrong
    num_rlzs = array['rlzi'].max() + 1

    # compute gmdata
    dic = general.group_array(array.view(gmf_data_dt), 'rlzi')
    gmdata = {r: numpy.zeros(n_imts + 1, F32) for r in range(num_rlzs)}
    for r in dic:
        gmv = dic[r]['gmv']
        rec = gmdata[r]  # (imt1, ..., imtM, nevents)
        rec[:-1] += gmv.sum(axis=0)
        rec[-1] += len(gmv)
    return eids, num_rlzs, gmdata
コード例 #10
0
ファイル: base.py プロジェクト: gem/oq-engine
def import_gmfs(dstore, fname, sids):
    """
    Import in the datastore a ground motion field CSV file.

    :param dstore: the datastore
    :param fname: the CSV file
    :param sids: the site IDs (complete)
    :returns: event_ids, num_rlzs
    """
    array = writers.read_composite_array(fname).array
    # has header rlzi, sid, eid, gmv_PGA, ...
    imts = [name[4:] for name in array.dtype.names[3:]]
    n_imts = len(imts)
    gmf_data_dt = numpy.dtype(
        [('rlzi', U16), ('sid', U32), ('eid', U64), ('gmv', (F32, (n_imts,)))])
    # store the events
    eids = numpy.unique(array['eid'])
    eids.sort()
    E = len(eids)
    eid2idx = dict(zip(eids, range(E)))
    events = numpy.zeros(E, rupture.events_dt)
    events['id'] = eids
    dstore['events'] = events
    # store the GMFs
    dic = general.group_array(array.view(gmf_data_dt), 'sid')
    lst = []
    offset = 0
    for sid in sids:
        n = len(dic.get(sid, []))
        lst.append((offset, offset + n))
        if n:
            offset += n
            gmvs = dic[sid]
            gmvs['eid'] = get_idxs(gmvs, eid2idx)
            gmvs['rlzi'] = 0   # effectively there is only 1 realization
            dstore.extend('gmf_data/data', gmvs)
    dstore['gmf_data/indices'] = numpy.array(lst, U32)
    dstore['gmf_data/imts'] = ' '.join(imts)
    sig_eps_dt = [('eid', U64), ('sig', (F32, n_imts)), ('eps', (F32, n_imts))]
    dstore['gmf_data/sigma_epsilon'] = numpy.zeros(0, sig_eps_dt)
    dstore['weights'] = numpy.ones(1)
    return eids
コード例 #11
0
def import_gmfs(dstore, fname, sids):
    """
    Import in the datastore a ground motion field CSV file.

    :param dstore: the datastore
    :param fname: the CSV file
    :param sids: the site IDs (complete)
    :returns: event_ids, num_rlzs
    """
    array = writers.read_composite_array(fname).array
    # has header rlzi, sid, eid, gmv_PGA, ...
    imts = [name[4:] for name in array.dtype.names[3:]]
    n_imts = len(imts)
    gmf_data_dt = numpy.dtype(
        [('rlzi', U16), ('sid', U32), ('eid', U64), ('gmv', (F32, (n_imts,)))])
    # store the events
    eids = numpy.unique(array['eid'])
    eids.sort()
    E = len(eids)
    eid2idx = dict(zip(eids, range(E)))
    events = numpy.zeros(E, rupture.events_dt)
    events['eid'] = eids
    dstore['events'] = events
    # store the GMFs
    dic = general.group_array(array.view(gmf_data_dt), 'sid')
    lst = []
    offset = 0
    for sid in sids:
        n = len(dic.get(sid, []))
        lst.append((offset, offset + n))
        if n:
            offset += n
            gmvs = dic[sid]
            gmvs['eid'] = get_idxs(gmvs, eid2idx)
            gmvs['rlzi'] = 0   # effectively there is only 1 realization
            dstore.extend('gmf_data/data', gmvs)
    dstore['gmf_data/indices'] = numpy.array(lst, U32)
    dstore['gmf_data/imts'] = ' '.join(imts)

    return eids
コード例 #12
0
 def test_read_err2(self):
     with self.assertRaises(InvalidFile) as ctx:
         read_composite_array(wrong2)
     self.assertIn("Could not cast '0.0 0.0' in file", str(ctx.exception))
コード例 #13
0
 def test_read_err1(self):
     with self.assertRaises(InvalidFile) as ctx:
         read_composite_array(wrong1)
     self.assertIn('expected 6 columns, found 5 in file',
                   str(ctx.exception))
コード例 #14
0
ファイル: readers_test.py プロジェクト: amirj700/oq-risklib
 def test_read_err2(self):
     with self.assertRaises(InvalidFile) as ctx:
         read_composite_array(wrong2)
     self.assertIn("Could not cast '0.0 0.0' in file",
                   str(ctx.exception))
コード例 #15
0
ファイル: readers_test.py プロジェクト: amirj700/oq-risklib
 def test_read_err1(self):
     with self.assertRaises(InvalidFile) as ctx:
         read_composite_array(wrong1)
     self.assertIn('expected 6 columns, found 5 in file',
                   str(ctx.exception))