def test_write_block(self): """ Test that writing to a user block does not destroy the file """ name = self.mktemp() f = File(name, 'w', userblock_size=512) f.create_group("Foobar") f.close() pyfile = open(name, 'r+b') try: pyfile.write(b'X'*512) finally: pyfile.close() f = h5py.File(name, 'r') try: assert "Foobar" in f finally: f.close() pyfile = open(name, 'rb') try: self.assertEqual(pyfile.read(512), b'X'*512) finally: pyfile.close()
class BaseDataset(TestCase): """ data is a 3-dimensional dataset with dimensions [z, y, x] The z dimension is labeled. It does not have any attached scales. The y dimension is not labeled. It has one attached scale. The x dimension is labeled. It has two attached scales. data2 is a 3-dimensional dataset with no associated dimension scales. """ def setUp(self): self.f = File(self.mktemp(), 'w') self.f['data'] = np.ones((4, 3, 2), 'f') self.f['data2'] = np.ones((4, 3, 2), 'f') self.f['x1'] = np.ones((2), 'f') h5py.h5ds.set_scale(self.f['x1'].id) h5py.h5ds.attach_scale(self.f['data'].id, self.f['x1'].id, 2) self.f['x2'] = np.ones((2), 'f') h5py.h5ds.set_scale(self.f['x2'].id, b'x2 name') h5py.h5ds.attach_scale(self.f['data'].id, self.f['x2'].id, 2) self.f['y1'] = np.ones((3), 'f') h5py.h5ds.set_scale(self.f['y1'].id, b'y1 name') h5py.h5ds.attach_scale(self.f['data'].id, self.f['y1'].id, 1) self.f['z1'] = np.ones((4), 'f') h5py.h5ds.set_label(self.f['data'].id, 0, b'z') h5py.h5ds.set_label(self.f['data'].id, 2, b'x') def tearDown(self): if self.f: self.f.close()
def test_blocksize(self): """ Core driver supports variable block size """ fname = self.mktemp() fid = File(fname, 'w', driver='core', block_size=1024, backing_store=False) self.assertTrue(fid) fid.close()
def test_issue_212(self): """ Issue 212 Fails with: AttributeError: 'SharedConfig' object has no attribute 'lapl' """ def closer(x): def w(): try: if x: x.close() except IOError: pass return w orig_name = self.mktemp() new_name = self.mktemp() f = File(orig_name, 'w') self.addCleanup(closer(f)) f.create_group('a') f.close() g = File(new_name, 'w') self.addCleanup(closer(g)) g['link'] = ExternalLink(orig_name, '/') # note root group g.close() h = File(new_name, 'r') self.addCleanup(closer(h)) self.assertIsInstance(h['link']['a'], Group)
class BasH5(object): """ This class is deprecated. .. deprecated:: 0.3.0 Use `BasH5Reader` instead. """ def __init__(self, filename, readType='Raw'): self._h5f = File(filename, 'r') self.rgnTable = RegionTable(self._h5f) self.baseCallsDG = None if readType == 'Raw': self.baseCallsDG = BaseCallsDataGroup(self._h5f, '/PulseData/BaseCalls') elif readType == 'CCS': self.baseCallsDG = CCSBaseCallsDataGroup(self._h5f, '/PulseData/ConsensusBaseCalls') self.rbaseCallsDG = BaseCallsDataGroup(self._h5f, '/PulseData/BaseCalls') def __del__(self): self._h5f.close() def getZMWs(self): for hn in self.baseCallsDG.holeNumber: yield hn def getSequencingZMWs(self): for hn in self.getZMWs(): if self.baseCallsDG.getStatusStringForZMW(hn) == 'SEQUENCING' and self.baseCallsDG.getBaseCallLenForZMW(hn): yield hn
def test_hdf5_writer_data(tmpdir): filename = tmpdir.join('test1.hdf5').strpath data = Data(x=np.arange(6).reshape(2, 3), y=(np.arange(6) * 2).reshape(2, 3)) hdf5_writer(filename, data) from h5py import File f = File(filename) assert len(f) == 2 np.testing.assert_equal(f['x'].value, data['x']) np.testing.assert_equal(f['y'].value, data['y']) f.close() # Only write out some components filename = tmpdir.join('test2.hdf5').strpath hdf5_writer(filename, data, components=[data.id['x']]) f = File(filename) assert len(f) == 1 np.testing.assert_equal(f['x'].value, data['x']) f.close()
def __init__(self, fname): from h5py import File from numpy import array, log10 import json h5f = File(fname, "r") log = sorted(json.loads(h5f["measure"].value).values(), key=lambda e: e["Status"]["Iteration"]) runargs = json.loads(h5f["runargs"].value) self.N = runargs["N"] self.zeta = runargs.get("zeta", 1.0) self.time = array([entry["Status" ]["CurrentTime"] for entry in log]) self.mean_T = array([entry["mean_T" ] for entry in log]) self.max_T = array([entry["max_T" ] for entry in log]) self.mean_Ms = array([entry["mean_Ms"] for entry in log]) self.max_Ms = array([entry["max_Ms" ] for entry in log]) self.mean_Ma = array([entry["mean_Ma"] for entry in log]) self.min_Ma = array([entry["min_Ma" ] for entry in log]) self.kin = array([entry["energies"]["kinetic" ] for entry in log]) self.tie = array([entry["energies"]["internal"] for entry in log]) self.mag = array([entry["energies"]["magnetic"] for entry in log]) self.tot = array([entry["energies"]["total" ] for entry in log]) self.mean_gamma = array([entry["mean_velocity"][0] for entry in log]) self.max_gamma = array([entry["max_lorentz_factor"] for entry in log]) self.runargs = runargs h5f.close()
def orderByQmodulus(filename,outfile=None): """ Sassena does not enforce any ordering of the structure factors. Here we order by increasing value of modulus of Q-vectors. """ from h5py import File import numpy f=File(filename,'r') overwrite=False if not outfile: outfile=tempfile() # temporaty output file overwrite=True g=File(outfile,'w') ds_q = numpy.array(f["qvectors"]) # shape==(nvectors,3) moduli=numpy.square(ds_q).sum(axis=1) # moduli-squared of the Q-vectors rank=numpy.argsort(moduli) # rank from smallest to greatest for dset in ('qvectors', 'fqt', 'fq', 'fq0', 'fq2'): if dset in f.keys(): x=numpy.array(f[dset]) if not outfile: del f[dset] f[dset]=x[rank] else: g[dset]=x[rank] for key,val in f.attrs.items(): g.attrs[key]=val g.close() f.close() if overwrite: os.system('/bin/mv %s %s'%(outfile,filename)) return None
class TestSimpleSlicing(TestCase): """ Feature: Simple NumPy-style slices (start:stop:step) are supported. """ def setUp(self): self.f = File(self.mktemp(), 'w') self.arr = np.arange(10) self.dset = self.f.create_dataset('x', data=self.arr) def tearDown(self): if self.f: self.f.close() def test_negative_stop(self): """ Negative stop indexes work as they do in NumPy """ self.assertArrayEqual(self.dset[2:-2], self.arr[2:-2]) def test_write(self): """Assigning to a 1D slice of a 2D dataset """ dset = self.f.create_dataset('x2', (10, 2)) x = np.zeros((10, 1)) dset[:, 0] = x[:, 0] with self.assertRaises(TypeError): dset[:, 1] = x
def testRetrieveData(self): try: from h5py import File except ImportError: print('module h5py not found') return device_database = PropertyTree() device_database.put_string('type', 'SeriesRC') device_database.put_double('series_resistance', R) device_database.put_double('capacitance', C) device = EnergyStorageDevice(device_database, MPI.COMM_WORLD) eis_database = setup_expertiment() eis_database.put_int('steps_per_decade', 1) eis_database.put_int('steps_per_cycle', 64) eis_database.put_int('cycles', 2) eis_database.put_int('ignore_cycles', 1) fout = File('trash.hdf5', 'w') spectrum_data = measure_impedance_spectrum(device, eis_database, fout) fout.close() fin = File('trash.hdf5', 'r') retrieved_data = retrieve_impedance_spectrum(fin) fin.close() print(spectrum_data['impedance']-retrieved_data['impedance']) print(retrieved_data) self.assertEqual(linalg.norm(spectrum_data['frequency'] - retrieved_data['frequency'], inf), 0.0) # not sure why we don't get equality for the impedance self.assertLess(linalg.norm(spectrum_data['impedance'] - retrieved_data['impedance'], inf), 1e-10)
def dump( histogram, filename = None, pathinfile = '/', mode = 'c', fs = None, compression = 'lzf'): '''dump( histogram, hdf_filename, path_in_hdf_file, mode ) -> save histogram into a hdf file. histogram: The histogram to be written hdf_filename: The hdf filename in which the histogram will be saved path_in_hdf_file: The path inside the hdf file where the histogram is located. mode: The mode to be used to write to the hdf file. 'c': create new hdf file. If hdf file of the same name exists, this command will fail. 'w': write to existing hdf file. If the path_in_hdf_file already exists in the hdf file, this command will fail. compression: The compression ratio. If it is 0, no compression will be done. The valid values are integers from 0 to 9 (inclusive). ''' from Renderer import Renderer #g = graphFromHDF5File( filename, pathinfile, fs = fs ) pathinfile = pathinfile.split( '/' ) p = pathinfile + [histogram.name()] p = '/'.join( p ) if not p.startswith('/'): p = '/' + p writeCodes = {'c':'w','w':'a'} if fs is None: from h5py import File fs = File(filename, writeCodes[mode]) Renderer(fs, compression).render(histogram) fs.close() else: Renderer(fs, compression).render(histogram)
class BaseDataset(TestCase): def setUp(self): self.f = File(self.mktemp(), 'w') def tearDown(self): if self.f: self.f.close()
def test_iter_zero(self): """ Iteration works properly for the case with no group members """ hfile = File(self.mktemp(), 'w') try: lst = [x for x in hfile] self.assertEqual(lst, []) finally: hfile.close()
def test_core(self): """ Core driver is supported (no backing store) """ fname = self.mktemp() fid = File(fname, 'w', driver='core', backing_store=False) self.assertTrue(fid) self.assertEqual(fid.driver, 'core') fid.close() self.assertFalse(os.path.exists(fname))
def test_create_exclusive(self): """ Mode 'w-' opens file in exclusive mode """ fname = self.mktemp() fid = File(fname, 'w-') self.assertTrue(fid) fid.close() with self.assertRaises(IOError): File(fname, 'w-')
def test_mode(self): """ Retrieved File objects have a meaningful mode attribute """ hfile = File(self.mktemp(),'w') try: grp = hfile.create_group('foo') self.assertEqual(grp.file.mode, hfile.mode) finally: hfile.close()
def hasVersion(filename): """Check filename as sassena version""" from h5py import File f = File(filename,'r') value=False if 'sassena_version' in f.attrs.keys(): value=True f.close() return value
def ReadFiniteRadiusWaveform(n, filename, WaveformName, ChMass, InitialAdmEnergy, YLMRegex, LModes, DataType, Ws) : """ This is just a worker function defined for ReadFiniteRadiusData, below, reading a single waveform from an h5 file of many waveforms. You probably don't need to call this directly. """ from scipy.integrate import cumtrapz as integrate from numpy import setdiff1d, empty, delete, sqrt, log, array from h5py import File import GWFrames try : f = File(filename, 'r') except IOError : print("ReadFiniteRadiusWaveform could not open the file '{0}'".format(filename)) raise try : W = f[WaveformName] NTimes_Input = W['AverageLapse.dat'].shape[0] T = W['AverageLapse.dat'][:,0] Indices = MonotonicIndices(T) T = T[Indices] Radii = array(W['ArealRadius.dat'])[Indices,1] AverageLapse = array(W['AverageLapse.dat'])[Indices,1] CoordRadius = W['CoordRadius.dat'][0,1] YLMdata = [DataSet for DataSet in list(W) for m in [YLMRegex.search(DataSet)] if (m and int(m.group('L')) in LModes)] YLMdata = sorted(YLMdata, key=lambda DataSet : [int(YLMRegex.search(DataSet).group('L')), int(YLMRegex.search(DataSet).group('M'))]) LM = sorted([[int(m.group('L')), int(m.group('M'))] for DataSet in YLMdata for m in [YLMRegex.search(DataSet)] if m]) NModes = len(LM) # Lapse is given by 1/sqrt(-g^{00}), where g is the full 4-metric T[1:] = integrate(AverageLapse/sqrt(((-2.0*InitialAdmEnergy)/Radii) + 1.0), T) + T[0] T -= (Radii + (2.0*InitialAdmEnergy)*log((Radii/(2.0*InitialAdmEnergy))-1.0)) Ws[n].SetTime(T/ChMass) # WRONG!!!: # Radii /= ChMass NTimes = Ws[n].NTimes() # Ws[n].SetFrame is not done, because we assume the inertial frame Ws[n].SetFrameType(GWFrames.Inertial) # Assumption! (but this should be safe) Ws[n].SetDataType(DataType) Ws[n].SetRIsScaledOut(True) # Assumption! (but it should be safe) Ws[n].SetMIsScaledOut(True) # We have made this true Ws[n].SetLM(LM) Data = empty((NModes, NTimes), dtype='complex') if(DataType == GWFrames.h) : UnitScaleFactor = 1.0 / ChMass elif(DataType == GWFrames.hdot) : UnitScaleFactor = 1.0 elif(DataType == GWFrames.Psi4) : UnitScaleFactor = ChMass else : raise ValueError('DataType "{0}" is unknown.'.format(DataType)) RadiusRatio = Radii / CoordRadius for m,DataSet in enumerate(YLMdata) : modedata = array(W[DataSet]) Data[m,:] = (modedata[Indices,1] + 1j*modedata[Indices,2]) * RadiusRatio * UnitScaleFactor Ws[n].SetData(Data) finally : f.close() return Radii/ChMass
def test_filename(self): """ .filename behaves properly for string data """ fname = self.mktemp() fid = File(fname, 'w') try: self.assertEqual(fid.filename, fname) self.assertIsInstance(fid.filename, six.text_type) finally: fid.close()
def test_property(self): """ File object can be retrieved from subgroup """ fname = self.mktemp() hfile = File(fname, 'w') try: hfile2 = hfile['/'].file self.assertEqual(hfile, hfile2) finally: hfile.close()
def test_close_multiple_mpio_driver(self): """ MPIO driver and options """ from mpi4py import MPI fname = self.mktemp() f = File(fname, 'w', driver='mpio', comm=MPI.COMM_WORLD) f.create_group("test") f.close() f.close()
def test_backing(self): """ Core driver saves to file when backing store used """ fname = self.mktemp() fid = File(fname, 'w', driver='core', backing_store=True) fid.create_group('foo') fid.close() fid = File(fname, 'r') assert 'foo' in fid fid.close()
def h5_writer(data, h5_path): from h5py import File from os.path import exists if exists(h5_path): remove(h5_path) f = File(h5_path, 'w') f.create_dataset('default', data=data, compression='gzip', chunks=True, shuffle=True) f.close()
def test_create(self): """ Mode 'w' opens file in overwrite mode """ fname = self.mktemp() fid = File(fname, 'w') self.assertTrue(fid) fid.create_group('foo') fid.close() fid = File(fname, 'w') self.assertNotIn('foo', fid) fid.close()
def weightsToHDF(w, name): f=File(name+".h5","w") weights=f.create_group("Weights") for i in range(len(w[:-1])): weights.create_dataset("Hidden "+str(i+1),data=w[i]) weights.create_dataset("Output",data=w[-1]) f.close()
def test_unicode(self): """ Unicode filenames can be used, and retrieved properly via .filename """ fname = self.mktemp(prefix = six.unichr(0x201a)) fid = File(fname, 'w') try: self.assertEqual(fid.filename, fname) self.assertIsInstance(fid.filename, six.text_type) finally: fid.close()
def hdf5_writer(filename, data, components=None): """ Write a dataset or a subset to a FITS file. Parameters ---------- data : `~glue.core.data.Data` or `~glue.core.subset.Subset` The data or subset to export components : `list` or `None` The components to export. Set this to `None` to export all components. """ if isinstance(data, Subset): mask = data.to_mask() data = data.data else: mask = None from h5py import File f = File(filename, 'w') for cid in data.visible_components: if components is not None and cid not in components: continue comp = data.get_component(cid) if comp.categorical: if comp.labels.dtype.kind == 'U': values = np.char.encode(comp.labels, encoding='ascii', errors='replace') else: values = comp.labels.copy() else: values = comp.data.copy() if mask is not None: if values.ndim == 1: values = values[mask] else: if values.dtype.kind == 'f': values[~mask] = np.nan elif values.dtype.kind == 'i': values[~mask] = 0 elif values.dtype.kind == 'S': values[~mask] = '' else: warnings.warn("Unknown data type in HDF5 export: {0}".format(values.dtype)) continue print(values) f.create_dataset(cid.label, data=values) f.close()
def test_write_only(self): """ User block only allowed for write """ name = self.mktemp() f = File(name, 'w') f.close() with self.assertRaises(ValueError): f = h5py.File(name, 'r', userblock_size=512) with self.assertRaises(ValueError): f = h5py.File(name, 'r+', userblock_size=512)
def test_readonly(self): """ Mode 'r' opens file in readonly mode """ fname = self.mktemp() fid = File(fname, 'w') fid.close() self.assertFalse(fid) fid = File(fname, 'r') self.assertTrue(fid) with self.assertRaises(ValueError): fid.create_group('foo') fid.close()
def test_readwrite(self): """ Mode 'r+' opens existing file in readwrite mode """ fname = self.mktemp() fid = File(fname, 'w') fid.create_group('foo') fid.close() fid = File(fname, 'r+') assert 'foo' in fid fid.create_group('bar') assert 'bar' in fid fid.close()
def test_file_mode_generalizes(self): fname = self.mktemp() fid = File(fname, 'w', libver='latest') g = fid.create_group('foo') # fid and group member file attribute should have the same mode assert fid.mode == g.file.mode == 'r+' fid.swmr_mode = True # fid and group member file attribute should still be 'r+' # even though file intent has changed assert fid.mode == g.file.mode == 'r+' fid.close()
def test_write_only(self): """ User block only allowed for write """ name = self.mktemp() f = File(name, 'w') f.close() with self.assertRaises(ValueError): f = h5py.File(name, 'r', userblock_size=512) with self.assertRaises(ValueError): f = h5py.File(name, 'r+', userblock_size=512)
def test_readonly(self): """ Mode 'r' opens file in readonly mode """ fname = self.mktemp() fid = File(fname, 'w') fid.close() self.assertFalse(fid) fid = File(fname, 'r') self.assertTrue(fid) with self.assertRaises(ValueError): fid.create_group('foo') fid.close()
def test_readwrite(self): """ Mode 'r+' opens existing file in readwrite mode """ fname = self.mktemp() fid = File(fname, 'w') fid.create_group('foo') fid.close() fid = File(fname, 'r+') assert 'foo' in fid fid.create_group('bar') assert 'bar' in fid fid.close()
def test_sec2(self): """ Sec2 driver is supported on posix """ fid = File(self.mktemp(), 'w', driver='sec2') self.assertTrue(fid) self.assertEqual(fid.driver, 'sec2') fid.close() # Testing creation with append flag fid = File(self.mktemp(), 'a', driver='sec2') self.assertTrue(fid) self.assertEqual(fid.driver, 'sec2') fid.close()
def test_write(self, get_test_data): file_path = DATASET_PATH + "test_file.hdf5" writer = HDF5Writer(file_path) writer.write("ones", (3, 2), pt.ones(3, 2), "0.01") writer.write("twos", (3, 2), pt.ones(3, 2) * 2, "0.01") writer.write("threes", (3, 2), pt.ones(3, 2) * 3, "0.03") del writer hdf5_file = File(file_path, mode="a") assert os.path.isfile(file_path) assert list(hdf5_file["variable"].keys()) == ["0.01", "0.03"] hdf5_file.close() os.remove(file_path)
def test_readonly(self): """ Core driver can be used to open existing files """ fname = self.mktemp() fid = File(fname, 'w') fid.create_group('foo') fid.close() fid = File(fname, 'r', driver='core') self.assertTrue(fid) assert 'foo' in fid with self.assertRaises(ValueError): fid.create_group('bar') fid.close()
def test_backing(self): """ Core driver saves to file when backing store used """ fname = self.mktemp() fid = File(fname, 'w', driver='core', backing_store=True) fid.create_group('foo') fid.close() fid = File(fname, 'r') assert 'foo' in fid fid.close() # keywords for other drivers are invalid when using the default driver with self.assertRaises(TypeError): File(fname, 'w', backing_store=True)
def write_esh5_orbitals(cell, name, kpts=numpy.zeros((1, 3), dtype=numpy.float64)): """Writes periodic AO basis to hdf5 file. Parameters ---------- cell: PySCF get.Cell object PySCF cell object which contains information of the system, including AO basis set, FFT mesh, unit cell information, etc. name: string Name of hdf5 file. kpts: array. Default: numpy.zeros((1,3) K-point array of dimension (nkpts, 3) dtype: datatype. Default: numpy.float64 Datatype of orbitals in file. """ def to_qmcpack_complex(array): shape = array.shape return array.view(numpy.float64).reshape(shape + (2, )) nao = cell.nao_nr() fh5 = File(name, 'w') coords = cell.gen_uniform_grids(cell.mesh) kpts = numpy.asarray(kpts) nkpts = len(kpts) norbs = numpy.zeros((nkpts, ), dtype=int) norbs[:] = nao grp = fh5.create_group("OrbsG") dset = grp.create_dataset("reciprocal_vectors", data=cell.reciprocal_vectors()) dset = grp.create_dataset("number_of_kpoints", data=len(kpts)) dset = grp.create_dataset("kpoints", data=kpts) dset = grp.create_dataset("number_of_orbitals", data=norbs) dset = grp.create_dataset("fft_grid", data=cell.mesh) dset = grp.create_dataset("grid_type", data=int(0)) nnr = cell.mesh[0] * cell.mesh[1] * cell.mesh[2] # loop over kpoints later for (ik, k) in enumerate(kpts): ao = numint.KNumInt().eval_ao(cell, coords, k)[0] fac = numpy.exp(-1j * numpy.dot(coords, k)) for i in range(norbs[ik]): aoi = fac * numpy.asarray(ao[:, i].T, order='C') aoi_G = tools.fft(aoi, cell.mesh) aoi_G = aoi_G.reshape(cell.mesh).transpose(2, 1, 0).reshape(nnr) dset = grp.create_dataset('kp' + str(ik) + '_b' + str(i), data=to_qmcpack_complex(aoi_G)) fh5.close()
def analyzeExperiment(self, experimentResults): # write to data file super(Origin, self).toHDF5(experimentResults[self.settings]) # and to settings file try: f = File('settings.hdf5', 'a') super(Origin, self).toHDF5(f['settings/experiment']) f.flush() # write changes except Exception as e: logger.exception('Uncaught Exception in origin.postExperiment.') finally: f.close() # close the file return 0
def __init__(self, split): print('==> initializing 2D {} data.'.format(split)) annot = {} tags = ['imgname', 'part', 'center', 'scale'] f = File(h5_path, 'r') for tag in tags: annot[tag] = np.asarray(f[tag]).copy() f.close() print('Loaded 2D {} {} samples'.format(split, len(annot['scale']))) self.split = split self.annot = annot
def check_progress_in_file(self): try: file = File(self.filePath, 'r') last_time = file['Source timer'] last_time = float(np.array(last_time)) state = None file.close() print(f'\tПрогресс восстановлен') except Exception: last_time = None state = None finally: return last_time, state
def __init__(self, opt, split): print('util2==> initializing 3D {} data.'.format(split)) annot = {} f_dir = os.path.join( '/data/ai/xxy/mask_rcnn/maskrcnn_3d/data/annotSampleTest.h5') f = File(f_dir, 'r') tags = [ 'action', 'bbox', 'camera', 'id', 'joint_2d', 'joint_3d_mono', 'subaction', 'subject', 'istrain' ] # tags=["category_id", "num_keypoints", "is_crowd", "keypoints", "keypoints3D", "bbox","image_id" ] if split == 'train': label = f['istrain'][:].tolist() ids = [i for i, x in enumerate(label) if x == 1][0:100000] else: label = f['istrain'][:].tolist() ids = [i for i, x in enumerate(label) if x == 0] for tag in tags: annot[tag] = np.asarray(f[tag]).copy() f.close() # ids = np.arange(annot['image_id'].shape[0]) for tag in tags: annot[tag] = annot[tag][ids] self.num_joints = 16 self.num_eval_joints = 16 self.acc_idxs = [0, 1, 2, 3, 4, 5, 10, 11, 14, 15] self.mean_bone_length = 4296.99233013 self.edges = [[0, 1], [1, 2], [2, 6], [6, 3], [3, 4], [4, 5], [10, 11], [11, 12], [12, 8], [8, 13], [13, 14], [14, 15], [6, 8], [8, 9]] self.edges_3d = [[0, 1], [1, 2], [2, 6], [6, 3], [3, 4], [4, 5], [10, 11], [11, 12], [12, 8], [8, 13], [13, 14], [14, 15], [6, 8], [8, 9]] self.shuffle_ref = [[0, 5], [1, 4], [2, 3], [10, 15], [11, 14], [12, 13]] self.mean = np.array([0.485, 0.456, 0.406], np.float32).reshape(1, 1, 3) self.std = np.array([0.229, 0.224, 0.225], np.float32).reshape(1, 1, 3) self.aspect_ratio = 1.0 * opt.input_w / opt.input_h #### 1 self.root = 7 self.split = split self.opt = opt self.annot = annot self.nSamples = len(self.annot['id']) # self.idxs = np.arange(self.nSamples) if split == 'train' else np.arange(0, self.nSamples, 1 if opt.full_test else 10) self.idxs = np.arange(self.nSamples) print('Loaded 3D {} {} samples'.format(split, len(self.annot['id'])))
def store_h5py(self, x_train, y_train, x_val, y_val, x_test, y_test): """ HDF5 storage. Has the pros of much faster I/O and compressed size than SQL storage and the dis of memory vs the solid storage.""" try: from h5py import File hdf = File(self.config.data.HDFS_INTERNAL_DATA_FILENAME, "w") except IOError as e: TextProcessing.logToFile.error( "The internal file failed to open for write in <TextProcessing/store_h5py" ) TextProcessing.logToFile.error(e) else: try: group_data = hdf.create_group("dataset") group_train = group_data.create_group("train") group_val = group_data.create_group("val") group_test = group_data.create_group("test") group_train.create_dataset("x_trainset", data=x_train, compression="gzip") group_train.create_dataset("y_trainset", data=y_train, compression="gzip") group_val.create_dataset("x_valset", data=x_val, compression="gzip") group_val.create_dataset("y_valset", data=y_val, compression="gzip") group_test.create_dataset("x_testset", data=x_test, compression="gzip") group_test.create_dataset("y_testset", data=y_test, compression="gzip") except IOError as e: TextProcessing.logToFile.error( "Failed to store in hdfs in <TextProcessing/store_h5py") TextProcessing.logToFile.error(e) else: hdf.close() TextProcessing.logToFile.logger.info( "Successful creation of data file with in-house text processing." ) TextProcessing.logToStream.logger.info( "Successful creation of data file with in-house text processing." )
def test_write(self, get_test_data): file_path = get_test_data.test_path + "test_file.hdf5" writer = HDF5Writer(file_path) writer.write("ones", (3, 2), pt.ones(3, 2), "0.01") writer.write("twos", (3, 2), pt.ones(3, 2) * 2, "0.01") writer.write("threes", (3, 2), pt.ones(3, 2) * 3, "0.03") del writer hdf5_file = File(file_path, mode="a", driver="mpio", comm=MPI.COMM_WORLD) assert os.path.isfile(file_path) assert list(hdf5_file["variable"].keys()) == ["0.01", "0.03"] hdf5_file.close()
def __init__(self, opt, split, returnMeta=False): print(f"==> initializing 2D {split} data.") annot = {} tags = ['imgname', 'part', 'center', 'scale'] f = File('{}/mpii/annot/{}.h5'.format(ref.dataDir, split), 'r') for tag in tags: annot[tag] = np.asarray(f[tag]).copy() f.close() print(f"Loaded 2D {split} {len(annot['scale'])} samples.") self.split = split self.opt = opt self.annot = annot self.returnMeta = returnMeta
def __init__(self, opt, split): print('==> initializing 2D {} data.'.format(split)) annot = {} tags = ['imgname','part','center','scale'] f = File('{}/mpii/annot/{}.h5'.format(ref.dataDir, split), 'r') for tag in tags: annot[tag] = np.asarray(f[tag]).copy() f.close() print('Loaded 2D {} {} samples'.format(split, len(annot['scale']))) self.split = split self.opt = opt self.annot = annot
class SortedQueryTest(AbstractQueryMixin, TestCase): path = 'SortedQueryTest.h5' def getDataset(self): self.f = File(self.path, 'w') self.input = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] self.d = self.f.create_dataset('dset', data=self.input) return SortedArray(self.d) def tearDown(self): self.f.close() if os.path.exists(self.path): os.remove(self.path)
def test_core(self): """ Core driver is supported (no backing store) """ fname = self.mktemp() fid = File(fname, 'w', driver='core', backing_store=False) self.assertTrue(fid) self.assertEqual(fid.driver, 'core') fid.close() self.assertFalse(os.path.exists(fname)) # Testing creation with append flag fid = File(self.mktemp(), 'a', driver='core') self.assertTrue(fid) self.assertEqual(fid.driver, 'core') fid.close()
def test_match_existing(self): """ User block size must match that of file when opening for append """ name = self.mktemp() f = File(name, 'w', userblock_size=512) f.close() with self.assertRaises(ValueError): f = File(name, 'a', userblock_size=1024) f = File(name, 'a', userblock_size=512) try: self.assertEqual(f.userblock_size, 512) finally: f.close()
class BaseMapping(BaseGroup): """ Base class for mapping tests """ def setUp(self): self.f = File(self.mktemp(), 'w') self.groups = ('a', 'b', 'c', 'd') for x in self.groups: self.f.create_group(x) self.f['x'] = h5py.SoftLink('/mongoose') self.groups = self.groups + ('x', ) def tearDown(self): if self.f: self.f.close()
def __init__(self, split): print('==> Initializing MPI_INF %s data' % (split)) annot = {} tags = ['idx', 'pose2d', 'pose3d', 'bbox', 'cam_f', 'cam_c', 'subject', 'sequence', 'video'] f = File('%s/inf/inf_%s.h5' % (conf.data_dir, split), 'r') for tag in tags: annot[tag] = np.asarray(f[tag]).copy() f.close() self.split = split self.annot = annot self.num_samples = self.annot['pose2d'].shape[0] print('Load %d MPI_INF %s samples' % (self.num_samples, self.split))
def main(): index_ids, index_vectors = jl.load('data/index.bin') test_ids, test_vectors = jl.load('data/test.bin') file = File('data/distances.h5', 'r') dataset = file['result'] result = jl.Parallel(n_jobs=-1, backend='threading')( jl.delayed(process)(i, v, id_, index_ids, dataset) for i, (id_, v) in tqdm(enumerate(zip(test_ids, test_vectors)), desc='test', total=len(test_ids))) file.close() result = pd.DataFrame(result) result.to_csv('result/retrieval.csv', index=False)
def pp_keys(self): data_file = File(self.data_path + self.log, 'r', libver='latest', swmr=True) try: keys = [key for key in data_file.keys()] pp_key_vals = [] for vals in keys: if vals != 'tpts' and vals != 'p_tot': pp_key_vals.append(vals) else: pass data_file.close() return pp_key_vals except: self.exit_handler() raise Exception('ERROR')
def load_namespaces(cls, namespace_catalog, path, namespaces=None): ''' Load cached namespaces from a file. ''' f = File(path, 'r') spec_group = f[f.attrs[SPEC_LOC_ATTR]] if namespaces is None: namespaces = list(spec_group.keys()) for ns in namespaces: ns_group = spec_group[ns] latest_version = list(ns_group.keys())[-1] ns_group = ns_group[latest_version] reader = H5SpecReader(ns_group) namespace_catalog.load_namespaces('namespace', reader=reader) f.close()
def __init__(self): print('==> Initializing MPII data') annot = {} tags = ['imgname', 'part', 'center', 'scale'] f1 = File('%s/mpii/annot/%s.h5' % (conf.data_dir, 'train'), 'r') f2 = File('%s/mpii/annot/%s.h5' % (conf.data_dir, 'val'), 'r') for tag in tags: annot[tag] = np.concatenate((np.asarray(f1[tag]).copy(), np.asarray(f2[tag]).copy()), axis=0) f1.close() f2.close() self.annot = annot self.num_samples = len(self.annot['scale']) print('Load %d MPII samples' % (len(annot['scale'])))
def new_model(self, filename): def add_group(group, si): attrs = group.attrs.items() attr_si = QStandardItem('attrs') attr_si.setEditable(False) for i in range(len(attrs)): attr_child = QStandardItem(attrs[i][0]) attr_child.setEditable(False) val = QStandardItem(str(attrs[i][1])) val.setEditable(False) attr_child.setChild(0, val) attr_si.setChild(i, attr_child) si.setChild(0, attr_si) groups = group.items() # Add 1 b/c attributes is 0 for i in range(len(groups)): gname = groups[i][0] ## Add leading zeros for nicer sorting... #if gname.startswith('trajectory'): # gname = "trajectory %06d"%(int(gname.split(' ')[-1])) newthing = QStandardItem(gname) newthing.setEditable(False) try: add_group(groups[i][1], newthing) si.setChild(i + 1, newthing) except: try: val = QStandardItem("Dataset - shape: " + str(groups[i][1].value.shape)) val.setData(groups[i][1].ref) val.setEditable(False) newthing.setChild(0, val) si.setChild(i + 1, newthing) except: print groups[i][0] self.model = QStandardItemModel(self.viewer) try: f = File(filename, 'r') dataset = QStandardItem(f.filename) dataset.setEditable(False) add_group(f, dataset) f.close() self.model.appendRow(dataset) except: pass
class TestSimpleSlicing(TestCase): """ Feature: Simple NumPy-style slices (start:stop:step) are supported. """ def setUp(self): self.f = File(self.mktemp(), 'w') self.arr = np.arange(10) self.dset = self.f.create_dataset('x', data=self.arr) def tearDown(self): if self.f: self.f.close() def test_negative_stop(self): """ Negative stop indexes work as they do in NumPy """ self.assertArrayEqual(self.dset[2:-2], self.arr[2:-2])
def __init__(self, protocol, split, dense=False, scale=False, noise=0, std_train=0, std_test=0, noise_path=None): print('==> Initializing H36M %s data' % (split)) annot = {} tags = ['idx', 'pose2d', 'pose3d', 'bbox', 'cam_f', 'cam_c', 'cam_R', 'cam_T', 'subject', 'action', 'subaction', 'camera'] if split == 'train': #f = File('%s/h36m/protocol2/h36m17_protocol%d.h5' % (conf.data_dir,protocol), 'r') f = File('%s/data/h36m17_new.h5' % (conf.data_dir), 'r') elif split == 'test' or split == 'val': f = File('%s/data/h36m17_protocol2_4.h5' % (conf.data_dir), 'r') for tag in tags: annot[tag] = np.asarray(f[tag]).copy() f.close() if dense == False: idxs = np.mod(annot['idx'], 50) == 1 idxs = np.arange(annot['idx'].shape[0])[idxs] for tag in tags: annot[tag] = annot[tag][idxs] idxs = np.full(annot['idx'].shape[0], False) subject = subject_list[protocol-1][1-int(split=='train' or split=='test_train')] for i in range(len(subject)): idxs = idxs + (annot['subject']==subject[i]) idxs = np.arange(annot['idx'].shape[0])[idxs] for tag in tags: annot[tag] = annot[tag][idxs] self.protocol = protocol self.split = split self.dense = dense self.scale = scale self.noise = noise self.std_train = std_train self.std_test = std_test self.noise_path = noise_path self.annot = annot self.num_samples = len(self.annot['idx']) # image size self.width = 256 self.height = 256 # load error statistics self.load_error_stat() print('Load %d H36M %s samples' % (self.num_samples, self.split))
def test_readonly_delete_exception(self): """ Deleting object in readonly file raises KeyError """ # Note: it is impossible to restore the old behavior (ValueError) # without breaking the above test (non-existing objects) fname = self.mktemp() hfile = File(fname, 'w') try: hfile.create_group('foo') finally: hfile.close() hfile = File(fname, 'r') try: with self.assertRaises(KeyError): del hfile['foo'] finally: hfile.close()