def put_all(self, _table, kvs: Iterable, use_serialize=True, chunk_size=100000, skip_chunk=0):
skipped_chunk = 0
for chunked_iter in split_every(kvs, chunk_size=chunk_size):
if skipped_chunk < skip_chunk:
skipped_chunk += 1
else:
self.kv_stub.putAll(self.__generate_operand(chunked_iter, use_serialize=use_serialize), metadata=_get_meta(_table))
def put_all(self,
_table,
kvs: Iterable,
use_serialize=True,
chunk_size=100000,
skip_chunk=0):
global gc_tag
gc_tag = False
skipped_chunk = 0
chunk_size = self.chunk_size
if chunk_size < CHUNK_SIZE_MIN:
chunk_size = CHUNK_SIZE_DEFAULT
host = self.host
port = self.port
process_pool_size = cpu_count()
with ProcessPoolExecutor(process_pool_size) as executor:
if isinstance(kvs, Sequence): # Sequence
for chunked_iter in split_every_yield(kvs, chunk_size):
if skipped_chunk < skip_chunk:
skipped_chunk += 1
else:
future = executor.submit(_EggRoll.action, _table, host,
port, chunked_iter,
use_serialize)
else: # other Iterable types
try:
index = 0
while True:
chunked_iter = split_every(kvs, index, chunk_size,
skip_chunk)
chunked_iter_ = copy.deepcopy(chunked_iter)
next(chunked_iter_)
future = executor.submit(_EggRoll.action, _table, host,
port, chunked_iter,
use_serialize)
index += 1
except StopIteration as e:
LOGGER.debug("StopIteration")
executor.shutdown(wait=True)
gc_tag = True
def put_all(self,
_table,
data: Iterable,
use_serialize=True,
chunk_size=100000,
skip_chunk=0,
include_key=True,
single_process=False):
global gc_tag
gc_tag = False
skipped_chunk = 0
if include_key == True:
kvs = data
else:
kvs = enumerate(data)
chunk_size = self.chunk_size
if chunk_size < CHUNK_SIZE_MIN:
chunk_size = CHUNK_SIZE_DEFAULT
host = self.host
port = self.port
process_pool_size = cpu_count()
if single_process is True:
for chunked_iter in split_every_yield(kvs, chunk_size=chunk_size):
if skipped_chunk < skip_chunk:
skipped_chunk += 1
else:
self.kv_stub.putAll(self.__generate_operand(
chunked_iter, use_serialize=use_serialize),
metadata=_get_meta(_table))
else:
with ProcessPoolExecutor(process_pool_size) as executor:
if isinstance(kvs, Sequence): # Sequence
for chunked_iter in split_every_yield(kvs, chunk_size):
if skipped_chunk < skip_chunk:
skipped_chunk += 1
else:
future = executor.submit(_EggRoll.action, _table,
host, port, chunked_iter,
use_serialize)
elif isinstance(kvs, Generator) or isinstance(data, Generator):
index = 0
while True:
chunked_iter = split_every_generator(
kvs, chunk_size, skip_chunk)
if chunked_iter == []:
break
future = executor.submit(_EggRoll.action, _table, host,
port, chunked_iter,
use_serialize)
index += 1
else: # other Iterable types
try:
index = 0
while True:
chunked_iter = split_every(kvs, index, chunk_size,
skip_chunk)
chunked_iter_ = copy.deepcopy(chunked_iter)
next(chunked_iter_)
future = executor.submit(_EggRoll.action, _table,
host, port, chunked_iter,
use_serialize)
index += 1
except StopIteration as e:
LOGGER.debug("StopIteration")
executor.shutdown(wait=True)
gc_tag = True
def put_all(self,
_table,
data: Iterable,
use_serialize=True,
chunk_size=100000,
skip_chunk=0,
include_key=True,
single_process=False):
global gc_tag
gc_tag = False
skipped_chunk = 0
if include_key == True:
kvs = data
else:
kvs = enumerate(data)
chunk_size = self.chunk_size
if chunk_size < CHUNK_SIZE_MIN:
chunk_size = CHUNK_SIZE_DEFAULT
host = self.host
port = self.port
process_pool_size = cpu_count()
if single_process is True:
for chunked_iter in split_every_yield(kvs, chunk_size=chunk_size):
if skipped_chunk < skip_chunk:
skipped_chunk += 1
else:
self.kv_stub.putAll(self.__generate_operand(
chunked_iter, use_serialize=use_serialize),
metadata=_get_meta(_table))
else:
_EggRoll.get_instance().get_channel().close()
with ProcessPoolExecutor(process_pool_size) as executor:
if isinstance(kvs, Sequence): # Sequence
for chunked_iter in split_every_yield(kvs, chunk_size):
if skipped_chunk < skip_chunk:
skipped_chunk += 1
else:
future = executor.submit(_EggRoll.action, _table,
host, port, chunked_iter,
use_serialize)
elif isinstance(kvs, Generator) or isinstance(data, Generator):
index = 0
while True:
chunked_iter = split_every_generator(
kvs, chunk_size, skip_chunk)
if chunked_iter == []:
break
future = executor.submit(_EggRoll.action, _table, host,
port, chunked_iter,
use_serialize)
index += 1
else: # other Iterable types
try:
index = 0
while True:
chunked_iter = split_every(kvs, index, chunk_size,
skip_chunk)
chunked_iter_ = copy.deepcopy(chunked_iter)
next(chunked_iter_)
future = executor.submit(_EggRoll.action, _table,
host, port, chunked_iter,
use_serialize)
index += 1
except StopIteration as e:
LOGGER.debug("StopIteration")
executor.shutdown(wait=True)
_EggRoll.get_instance().channel = grpc.insecure_channel(
target="{}:{}".format(host, port),
options=[('grpc.max_send_message_length', -1),
('grpc.max_receive_message_length', -1)])
_EggRoll.get_instance().kv_stub = kv_pb2_grpc.KVServiceStub(
_EggRoll.get_instance().get_channel())
_EggRoll.get_instance(
).proc_stub = processor_pb2_grpc.ProcessServiceStub(
_EggRoll.get_instance().get_channel())
_EggRoll.get_instance(
).session_stub = node_manager_pb2_grpc.SessionServiceStub(
_EggRoll.get_instance().get_channel())
gc_tag = True