def _run_roundtrip_tst(self, nimages, aryShape, dtypeSpec, sizeSpec): testArrays = TestSeriesBinaryWriteFromStack.generateTestImages(nimages, aryShape, dtypeSpec) loader = SeriesLoader(self.sc) series = loader.fromArraysAsImages(testArrays) blocks = series.toBlocks(sizeSpec) roundtrippedSeries = blocks.toSeries(newDType=series.dtype) packedSeries = series.pack() packedRoundtrippedSeries = roundtrippedSeries.pack() assert_true(array_equal(packedSeries, packedRoundtrippedSeries))
def _run_roundtrip_tst(self, testIdx, nimages, aryShape, dtypeSpec, npartitions): testArrays = TestSeriesBinaryWriteFromStack.generateTestImages(nimages, aryShape, dtypeSpec) loader = SeriesLoader(self.sc) series = loader.fromArraysAsImages(testArrays) saveDirPath = os.path.join(self.outputdir, 'save%d' % testIdx) series.repartition(npartitions) # note: this does an elementwise shuffle! won't be in sorted order series.saveAsBinarySeries(saveDirPath) nnonemptyPartitions = 0 for partitionList in series.rdd.glom().collect(): if partitionList: nnonemptyPartitions += 1 del partitionList nsaveFiles = len(glob.glob(saveDirPath + os.sep + "*.bin")) roundtrippedSeries = loader.fromBinary(saveDirPath) with open(os.path.join(saveDirPath, "conf.json"), 'r') as fp: conf = json.load(fp) # sorting is required here b/c of the randomization induced by the repartition. # orig and roundtripped will in general be different from each other, since roundtripped # will have (0, 0, 0) index as first element (since it will be the lexicographically first # file) while orig has only a 1 in npartitions chance of starting with (0, 0, 0) after repartition. expectedPackedAry = series.pack(sorting=True) actualPackedAry = roundtrippedSeries.pack(sorting=True) assert_true(array_equal(expectedPackedAry, actualPackedAry)) assert_equals(nnonemptyPartitions, nsaveFiles) assert_equals(len(aryShape), conf["nkeys"]) assert_equals(nimages, conf["nvalues"]) assert_equals("int16", conf["keytype"]) assert_equals(str(series.dtype), conf["valuetype"]) # check that we have converted ourselves to an appropriate float after reloading assert_equals(str(smallestFloatType(series.dtype)), str(roundtrippedSeries.dtype))
def _run_roundtrip_exception_tst(self, nimages, aryShape, dtypeSpec, sizeSpec): testArrays = TestSeriesBinaryWriteFromStack.generateTestImages(nimages, aryShape, dtypeSpec) loader = SeriesLoader(self.sc) series = loader.fromArraysAsImages(testArrays) assert_raises(ValueError, series.toBlocks, sizeSpec)