def writeToS3Test(self, bytecount, pageSizeOverride=1024*1024, workers=1, memoryLimitMb=45 * 1024,threadCount=30): text = """Vector.range(__bytecount__, {UInt8(_%100)}).paged""" s3 = ActualS3Interface.ActualS3InterfaceFactory() keyGuid = "bigbox-test-key-" + str(uuid.uuid4()) try: setupText = text.replace('__bytecount__', str(bytecount)) setupResults, simulation = self.computeUsingSeveralWorkers( setupText, s3, workers, memoryLimitMb=memoryLimitMb, threadCount=threadCount, returnSimulation=True, ioTaskThreadOverride=8, useInMemoryCache=False, timeout=30, objectStore=self.createObjectStore(s3), pageSizeOverride=pageSizeOverride ) result = simulation.executeExternalIoTask( CumulusNative.ExternalIoTask.WriteCharBigvecToS3( setupResults.asResult.result.getVectorBigvecGuid(), CumulusNative.S3KeyAndCredentials( self.getTestDataBucket(), keyGuid, "", "", "" ) ), timeout=60 ) self.assertTrue(result.isSuccess(), result) assert s3().getKeySize(self.getTestDataBucket(), keyGuid) == bytecount finally: try: s3().deleteKey(self.getTestDataBucket(), keyGuid) except: logging.warn("Failed to cleanup the test key: %s", traceback.format_exc())
def trigger(self): if self.successOrError is not None: return if self.computedValue.valueIVC is None: self.successOrError = { 'success': False, 'message': "Tried to trigger write before calculation was finished." } return if not self.computedValue.valueIVC.isVectorOfChar(): self.successOrError = { 'success': False, 'message': "Result should have been a string." } return if self.computedValue.isException: self.successOrError = { 'success': False, 'message': "Result should have been a string. Got an exception instead." } return def callback(result): if result.isSuccess(): self.successOrError = {'success': True} else: self.successOrError = { 'success': False, 'message': str(result) } ComputedValueGateway.getGateway().createExternalIoTask( CumulusNative.ExternalIoTask.WriteCharBigvecToS3( self.computedValue.valueIVC.getVectorBigvecGuid(), CumulusNative.S3KeyAndCredentials(self.bucketname, self.keyname, "", "", "")), callback)