def _s3_stash_objects_parallel(keyvalueiter, bucket, vcores=1, threads=5,
acl='private', encrypt=True, **xargs):
s3_stash_thrdd_prtl = partial(s3_stash_objects_threaded, bucket=bucket, threads=threads,
acl=acl, encrypt=encrypt, **xargs)
LOG.info('\t'.join(['keyvalueiter size:', str(len(keyvalueiter)),
'keyvalueiter type:', str(type(keyvalueiter))]))
for output in flatten_array_like_strct_gen(map_easy(s3_stash_thrdd_prtl,
list(keyvalueiter),
n_jobs=vcores)):
yield output
def _place_on_outerstack(self, stack):
LOG.info('Stack type:\t' + str(type(stack)))
self.outerstack.append(stack)
if self.outerstack >= self.vcores:
LOG.info('\t'.join(['outerstack size:', str(len(self.outerstack)),
'outerstack type:', str(type(self.outerstack))]))
for output in flatten_array_like_strct_gen(self.post_it(tuple(self.outerstack))):
LOG.debug(str(output))
# del(self.outerstack)
self.outerstack = []
def s3_stash_objects_threaded(keyvalueiter, bucket, threads=5,
acl='private', encrypt=True, **xargs):
client = boto3.client('s3', config=Config(signature_version='s3v4'))
s3_stash_thrdd_prtl = partial(_s3_stash_object_kv_spltr, bucket=bucket, client=client,
acl=acl, encrypt=encrypt, **xargs)
LOG.info('\t'.join(['keyvalueiter size:', str(len(keyvalueiter)),
'keyvalueiter type:', str(type(keyvalueiter))]))
kvgen = (i for i in keyvalueiter)
return [output for output in flatten_array_like_strct_gen(threading_easy(s3_stash_thrdd_prtl,
kvgen,
n_threads=threads))]