def _run_roundtrip_tst(self, nimages, aryShape, dtypeSpec, sizeSpec):
testArrays = TestSeriesBinaryWriteFromStack.generateTestImages(nimages, aryShape, dtypeSpec)
loader = SeriesLoader(self.sc)
series = loader.fromArraysAsImages(testArrays)
blocks = series.toBlocks(sizeSpec)
roundtrippedSeries = blocks.toSeries(newDType=series.dtype)
packedSeries = series.pack()
packedRoundtrippedSeries = roundtrippedSeries.pack()
assert_true(array_equal(packedSeries, packedRoundtrippedSeries))
def _run_roundtrip_tst(self, testIdx, nimages, aryShape, dtypeSpec, npartitions):
testArrays = TestSeriesBinaryWriteFromStack.generateTestImages(nimages, aryShape, dtypeSpec)
loader = SeriesLoader(self.sc)
series = loader.fromArraysAsImages(testArrays)
saveDirPath = os.path.join(self.outputdir, 'save%d' % testIdx)
series.repartition(npartitions) # note: this does an elementwise shuffle! won't be in sorted order
series.saveAsBinarySeries(saveDirPath)
nnonemptyPartitions = 0
for partitionList in series.rdd.glom().collect():
if partitionList:
nnonemptyPartitions += 1
del partitionList
nsaveFiles = len(glob.glob(saveDirPath + os.sep + "*.bin"))
roundtrippedSeries = loader.fromBinary(saveDirPath)
with open(os.path.join(saveDirPath, "conf.json"), 'r') as fp:
conf = json.load(fp)
# sorting is required here b/c of the randomization induced by the repartition.
# orig and roundtripped will in general be different from each other, since roundtripped
# will have (0, 0, 0) index as first element (since it will be the lexicographically first
# file) while orig has only a 1 in npartitions chance of starting with (0, 0, 0) after repartition.
expectedPackedAry = series.pack(sorting=True)
actualPackedAry = roundtrippedSeries.pack(sorting=True)
assert_true(array_equal(expectedPackedAry, actualPackedAry))
assert_equals(nnonemptyPartitions, nsaveFiles)
assert_equals(len(aryShape), conf["nkeys"])
assert_equals(nimages, conf["nvalues"])
assert_equals("int16", conf["keytype"])
assert_equals(str(series.dtype), conf["valuetype"])
# check that we have converted ourselves to an appropriate float after reloading
assert_equals(str(smallestFloatType(series.dtype)), str(roundtrippedSeries.dtype))
def _run_roundtrip_exception_tst(self, nimages, aryShape, dtypeSpec, sizeSpec):
testArrays = TestSeriesBinaryWriteFromStack.generateTestImages(nimages, aryShape, dtypeSpec)
loader = SeriesLoader(self.sc)
series = loader.fromArraysAsImages(testArrays)
assert_raises(ValueError, series.toBlocks, sizeSpec)
