def _split_batch_list(args, batch_list):
new_list = []
for batch in batch_list.batches:
new_list.append(batch)
if len(new_list) == args.batch_size_limit:
yield batch_pb2.BatchList(batches=new_list)
new_list = []
if new_list:
yield batch_pb2.BatchList(batches=new_list)
def do_submit(args):
with open(args.filename, mode='rb') as fd:
batches = batch_pb2.BatchList()
batches.ParseFromString(fd.read())
rest_client = RestClient(args.url)
start = time.time()
for batch_list in _split_batch_list(args, batches):
rest_client.send_batches(batch_list)
stop = time.time()
print("batches: {} batch/sec: {}".format(
str(len(batches.batches)),
len(batches.batches) / (stop - start)))
def do_batch_submit(args):
try:
with open(args.filename, mode='rb') as fd:
batches = batch_pb2.BatchList()
batches.ParseFromString(fd.read())
except IOError as e:
raise CliException(e)
rest_client = RestClient(args.url, args.user)
start = time.time()
for batch_list in _split_batch_list(args, batches):
rest_client.send_batches(batch_list)
stop = time.time()
print('batches: {}, batch/sec: {}'.format(
str(len(batches.batches)),
len(batches.batches) / (stop - start)))
if args.wait and args.wait > 0:
batch_ids = [b.header_signature for b in batches.batches]
wait_time = 0
start_time = time.time()
while wait_time < args.wait:
statuses = rest_client.get_statuses(
batch_ids,
args.wait - int(wait_time))
wait_time = time.time() - start_time
if all(s['status'] == 'COMMITTED' for s in statuses):
print('All batches committed in {:.6} sec'.format(wait_time))
return
# Wait a moment so as not to send another request immediately
time.sleep(0.2)
print('Wait timed out! Some batches have not yet been committed...')
for batch_id, status in statuses.items():
print('{:128.128} {:10.10}'.format(batch_id, status))
exit(1)
