def writeToS3Test(self, bytecount, pageSizeOverride=1024*1024, workers=1, memoryLimitMb=45 * 1024,threadCount=30):
text = """Vector.range(__bytecount__, {UInt8(_%100)}).paged"""
s3 = ActualS3Interface.ActualS3InterfaceFactory()
keyGuid = "bigbox-test-key-" + str(uuid.uuid4())
try:
setupText = text.replace('__bytecount__', str(bytecount))
setupResults, simulation = self.computeUsingSeveralWorkers(
setupText,
s3,
workers,
memoryLimitMb=memoryLimitMb,
threadCount=threadCount,
returnSimulation=True,
ioTaskThreadOverride=8,
useInMemoryCache=False,
timeout=30,
objectStore=self.createObjectStore(s3),
pageSizeOverride=pageSizeOverride
)
result = simulation.executeExternalIoTask(
CumulusNative.ExternalIoTask.WriteCharBigvecToS3(
setupResults.asResult.result.getVectorBigvecGuid(),
CumulusNative.S3KeyAndCredentials(
self.getTestDataBucket(),
keyGuid,
"",
"",
""
)
),
timeout=60
)
self.assertTrue(result.isSuccess(), result)
assert s3().getKeySize(self.getTestDataBucket(), keyGuid) == bytecount
finally:
try:
s3().deleteKey(self.getTestDataBucket(), keyGuid)
except:
logging.warn("Failed to cleanup the test key: %s", traceback.format_exc())
def trigger(self):
if self.successOrError is not None:
return
if self.computedValue.valueIVC is None:
self.successOrError = {
'success':
False,
'message':
"Tried to trigger write before calculation was finished."
}
return
if not self.computedValue.valueIVC.isVectorOfChar():
self.successOrError = {
'success': False,
'message': "Result should have been a string."
}
return
if self.computedValue.isException:
self.successOrError = {
'success':
False,
'message':
"Result should have been a string. Got an exception instead."
}
return
def callback(result):
if result.isSuccess():
self.successOrError = {'success': True}
else:
self.successOrError = {
'success': False,
'message': str(result)
}
ComputedValueGateway.getGateway().createExternalIoTask(
CumulusNative.ExternalIoTask.WriteCharBigvecToS3(
self.computedValue.valueIVC.getVectorBigvecGuid(),
CumulusNative.S3KeyAndCredentials(self.bucketname,
self.keyname, "", "", "")),
callback)
