Example #1
0
 def test05_ReadFromNonexistingNodeRaises(self):
     # write using the filename as node name
     filenode.save_to_filenode(self.testh5fname, self.datafname, "/")
     # and read again
     self.assertRaises(tables.NoSuchNodeError, filenode.read_from_filenode,
                       self.testh5fname, self.testdir, "/",
                       name="THISNODEDOESNOTEXIST")
Example #2
0
def create_excel_dataset():
    # Excelファイルを入れ込む (画像の時と同じ方法)
    filepath = OUTPUT_DIR.joinpath('excel_data.hdf5')
    with tables.open_file(filepath, mode='w') as f:
        image_path = INPUT_DIR.joinpath('mybook.xlsx')
        filenode.save_to_filenode(f,
                                  filename=image_path,
                                  where='/',
                                  name='excel_node')
Example #3
0
def create_image_dataset():
    # 画像ファイルを入れ込む
    filepath = OUTPUT_DIR.joinpath('image_data.hdf5')
    with tables.open_file(filepath, mode='w') as f:
        image_path = INPUT_DIR.joinpath('shinanogold.png')

        filenode.save_to_filenode(f,
                                  filename=image_path,
                                  where='/',
                                  name='image_node')
        # nameを指定しないと、拡張子の `.` を `_` に変換したものがNode名になる
        filenode.save_to_filenode(f, filename=image_path, where='/')
Example #4
0
 def test03_AutomaticNameGuessing(self):
     # write using the filename as node name
     filenode.save_to_filenode(self.testh5fname, self.datafname, "/")
     # and read again
     datafname = os.path.split(self.datafname)[1]
     filenode.read_from_filenode(self.testh5fname, self.testdir, "/",
                                 name=datafname)
     # test if the output file really has the expected name
     self.assertEqual(os.access(os.path.join(self.testdir, datafname),
                                os.R_OK), True)
     # and compare result to what it should be
     with open(os.path.join(self.testdir, datafname), "rb") as fd:
         self.assertEqual(fd.read(), self.data)
Example #5
0
 def test04_AutomaticNameGuessingWithFilenameAttribute(self):
     # write using the filename as node name
     filenode.save_to_filenode(self.testh5fname, self.datafname, "/")
     # and read again
     datafname = os.path.split(self.datafname)[1]
     filenode.read_from_filenode(self.testh5fname, self.testdir, "/",
                                 name=datafname)
     # test if the output file really has the expected name
     self.assertEqual(os.access(os.path.join(self.testdir, datafname),
                                os.R_OK), True)
     # and compare result to what it should be
     with open(os.path.join(self.testdir, datafname), "rb") as fd:
         self.assertEqual(fd.read(), self.data)
Example #6
0
 def test02_WriteToHDF5File(self):
     # write contents of datafname to h5 testfile
     filenode.save_to_filenode(self.h5file, self.datafname, "/test1")
     # make sure writing to an existing node doesn't work ...
     self.assertRaises(IOError, filenode.save_to_filenode, self.h5file,
                       self.datafname, "/test1")
     # ... except if overwrite is True
     filenode.save_to_filenode(self.h5file, self.datafname, "/test1",
                               overwrite=True)
     # read from test h5file
     filenode.read_from_filenode(self.h5file, self.testfname, "/test1")
     # and compare result to what it should be
     with open(self.testfname, "rb") as fd:
         self.assertEqual(fd.read(), self.data)
     # make sure extracting to an existing file doesn't work ...
     self.assertRaises(IOError, filenode.read_from_filenode, self.h5file,
                       self.testfname, "/test1")
     # make sure the original h5file is still alive and kicking
     self.assertEqual(isinstance(self.h5file, tables.file.File), True)
     self.assertEqual(self.h5file.mode, "w")
Example #7
0
 def test02_WriteToHDF5File(self):
     # write contents of datafname to h5 testfile
     filenode.save_to_filenode(self.h5file, self.datafname, "/test1")
     # make sure writing to an existing node doesn't work ...
     self.assertRaises(IOError, filenode.save_to_filenode, self.h5file,
                       self.datafname, "/test1")
     # ... except if overwrite is True
     filenode.save_to_filenode(self.h5file, self.datafname, "/test1",
                               overwrite=True)
     # read from test h5file
     filenode.read_from_filenode(self.h5file, self.testfname, "/test1")
     # and compare result to what it should be
     with open(self.testfname, "rb") as fd:
         self.assertEqual(fd.read(), self.data)
     # make sure extracting to an existing file doesn't work ...
     self.assertRaises(IOError, filenode.read_from_filenode, self.h5file,
                       self.testfname, "/test1")
     # make sure the original h5file is still alive and kicking
     self.assertEqual(isinstance(self.h5file, tables.file.File), True)
     self.assertEqual(self.h5file.mode, "w")
Example #8
0
 def test01_WriteToFilename(self):
     # write contents of datafname to h5 testfile
     filenode.save_to_filenode(self.testh5fname, self.datafname, "/test1")
     # make sure writing to an existing node doesn't work ...
     self.assertRaises(IOError, filenode.save_to_filenode, self.testh5fname,
                       self.datafname, "/test1")
     # ... except if overwrite is True
     filenode.save_to_filenode(self.testh5fname, self.datafname, "/test1",
                               overwrite=True)
     # write again, this time specifying a name
     filenode.save_to_filenode(self.testh5fname, self.datafname, "/",
                               name="test2")
     # read from test h5file
     filenode.read_from_filenode(self.testh5fname, self.testfname, "/test1")
     # and compare result to what it should be
     with open(self.testfname, "rb") as fd:
         self.assertEqual(fd.read(), self.data)
     # make sure extracting to an existing file doesn't work ...
     self.assertRaises(IOError, filenode.read_from_filenode,
                       self.testh5fname, self.testfname, "/test1")
     # except overwrite is True.  And try reading with a name
     filenode.read_from_filenode(self.testh5fname, self.testfname, "/",
                                 name="test2", overwrite=True)
     # and compare to what it should be
     with open(self.testfname, "rb") as fd:
         self.assertEqual(fd.read(), self.data)
     # cleanup
     os.remove(self.testfname)
     os.remove(self.testh5fname)
Example #9
0
 def test01_WriteToFilename(self):
     # write contents of datafname to h5 testfile
     filenode.save_to_filenode(self.testh5fname, self.datafname, "/test1")
     # make sure writing to an existing node doesn't work ...
     self.assertRaises(IOError, filenode.save_to_filenode, self.testh5fname,
                       self.datafname, "/test1")
     # ... except if overwrite is True
     filenode.save_to_filenode(self.testh5fname, self.datafname, "/test1",
                               overwrite=True)
     # write again, this time specifying a name
     filenode.save_to_filenode(self.testh5fname, self.datafname, "/",
                               name="test2")
     # read from test h5file
     filenode.read_from_filenode(self.testh5fname, self.testfname, "/test1")
     # and compare result to what it should be
     with open(self.testfname, "rb") as fd:
         self.assertEqual(fd.read(), self.data)
     # make sure extracting to an existing file doesn't work ...
     self.assertRaises(IOError, filenode.read_from_filenode,
                       self.testh5fname, self.testfname, "/test1")
     # except overwrite is True.  And try reading with a name
     filenode.read_from_filenode(self.testh5fname, self.testfname, "/",
                                 name="test2", overwrite=True)
     # and compare to what it should be
     with open(self.testfname, "rb") as fd:
         self.assertEqual(fd.read(), self.data)
     # cleanup
     os.remove(self.testfname)
     os.remove(self.testh5fname)
Example #10
0
def compress_big_file():
    # 10MBくらいのバイナリファイルを作成し、圧縮効果を見る
    big_size_path = INPUT_DIR.joinpath('big.bin')
    if not big_size_path.exists():
        data = bytearray(
            random.getrandbits(8) for _ in range(10 * 1000 * 1000))

        with big_size_path.open(mode='wb') as f:
            f.write(data)

    print(f'original file: {os.path.getsize(big_size_path)}')  # => 10000000

    # HDF5ファイルに入れる & filter設定なし
    uncompression_file_path = OUTPUT_DIR.joinpath(
        'big_size_data_without_compression.hdf5')
    with tables.open_file(uncompression_file_path, mode='w') as f:
        filenode.save_to_filenode(f,
                                  filename=big_size_path,
                                  where='/',
                                  name='big_size_data')

    print(f'uncompression file: {os.path.getsize(uncompression_file_path)}'
          )  # => 10039272

    # # HDF5ファイルに入れる & filter 'lzo' を使用
    compression_file_path = OUTPUT_DIR.joinpath(
        'big_size_data_with_compression.hdf5')
    with tables.open_file(compression_file_path, mode='w') as f:
        filters = tables.Filters(complib='lzo')
        filenode.save_to_filenode(f,
                                  filename=big_size_path,
                                  where='/',
                                  name='big_size_data',
                                  filters=filters)

    print(f'compression file: {os.path.getsize(compression_file_path)}'
          )  # => 10039272