def test_chunk_size_edges(self):
with self.assertRaises(ValueError):
cd = ChunkedData(self.data, chunk_size=0)
cd = ChunkedData(self.data, chunk_size=1)
chunks_expected = [self.data[i:i + 1] for i in range(len(self.data))]
self.verify_chunking(cd, chunks_expected)
cd = ChunkedData(self.data, chunk_size=len(self.data))
chunks_expected = [self.data]
self.verify_chunking(cd, chunks_expected)
def test_chunks_from_dqm(self):
try:
from dimod import DQM
except ImportError:
self.skipTest("dimod.DQM unavailable")
# serialize a DQM via .to_file
dqm = DQM()
dqm.add_variable(1)
dqm_file = dqm.to_file() # returns SpooledTemporaryFile subclass
chunk_size = 100
cd = ChunkedData(dqm_file, chunk_size=chunk_size)
# verify chunks
dqm_file.seek(0)
raw = dqm_file.read()
chunks_expected = [
raw[i:i + chunk_size] for i in range(0, len(raw), chunk_size)
]
self.verify_chunking(cd, chunks_expected)
def test_chunks_from_bqm(self):
try:
from dimod import AdjVectorBQM
AdjVectorBQM.to_file
except (ImportError, AttributeError):
self.skipTest("dimod.AdjVectorBQM with .to_file() unavailable")
# serialize a BQM via .to_file
bqm = AdjVectorBQM.from_ising({'a': 1}, {})
bqmfile = bqm.to_file() # returns dimod's FileView
chunk_size = 10
cd = ChunkedData(bqmfile, chunk_size=chunk_size)
# verify chunks
bqmfile.seek(0)
raw = bqmfile.read()
chunks_expected = [
raw[i:i + chunk_size] for i in range(0, len(raw), chunk_size)
]
self.verify_chunking(cd, chunks_expected)
def test_chunks_from_bqm(self, bqm_cls_name):
try:
import dimod
bqm_cls = getattr(dimod, bqm_cls_name)
bqm_cls.to_file
except (ImportError, AttributeError):
self.skipTest(f"dimod.{bqm_cls_name}.to_file() unavailable")
# serialize a BQM via .to_file
bqm = bqm_cls.from_ising({'a': 1}, {})
bqm_file = bqm.to_file() # returns dimod's FileView
chunk_size = 10
cd = ChunkedData(bqm_file, chunk_size=chunk_size)
# verify chunks
bqm_file.seek(0)
raw = bqm_file.read()
chunks_expected = [
raw[i:i + chunk_size] for i in range(0, len(raw), chunk_size)
]
self.verify_chunking(cd, chunks_expected)
def test_chunk_generators(self):
cd = ChunkedData(self.data, chunk_size=3)
chunks_expected = [b'012', b'345', b'678', b'9']
chunks_generated = [g().read() for g in cd.generators()]
self.assertListEqual(chunks_expected, chunks_generated)
def test_chunks_from_memory_file(self):
data = io.BytesIO(self.data)
cd = ChunkedData(data, chunk_size=3)
chunks_expected = [b'012', b'345', b'678', b'9']
self.verify_chunking(cd, chunks_expected)
def test_chunks_from_str(self):
cd = ChunkedData(self.data.decode('ascii'), chunk_size=3)
chunks_expected = [b'012', b'345', b'678', b'9']
self.verify_chunking(cd, chunks_expected)
def test_chunks_from_bytearray(self):
cd = ChunkedData(bytearray(self.data), chunk_size=3)
chunks_expected = [b'012', b'345', b'678', b'9']
self.verify_chunking(cd, chunks_expected)