def _split_batch_list(batch_list):
new_list = []
for batch in batch_list.batches:
new_list.append(batch)
if len(new_list) == 100:
yield batch_pb2.BatchList(batches=new_list)
new_list = []
if len(new_list) > 0:
yield batch_pb2.BatchList(batches=new_list)
def do_load(args):
print("Do load")
with open(args.filename, 'rb') as fd:
batches = batch_pb2.BatchList()
batches.ParseFromString(fd.read())
stream = Stream(args.url)
futures = []
start = time.time()
for batch_list in _split_batch_list(batches):
future = stream.send(message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST,
content=batch_list.SerializeToString())
futures.append(future)
for future in futures:
result = future.result()
assert (result.message_type == Message.CLIENT_BATCH_SUBMIT_RESPONSE)
stop = time.time()
print("batches: {} batch/sec: {}".format(
str(len(batches.batches)),
len(batches.batches) / (stop - start)))
stream.close()
def do_populate(args):
private_key = signing.generate_private_key()
public_key = signing.generate_public_key(private_key)
words = generate_word_list(args.pool_size)
batches = []
total_txn_count = 0
txns = []
for i in range(0, len(words)):
txn = create_intkey_transaction(verb='set',
name=words[i],
value=random.randint(9000, 100000),
private_key=private_key,
public_key=public_key)
total_txn_count += 1
txns.append(txn)
batch = create_batch(transactions=txns,
private_key=private_key,
public_key=public_key)
batches.append(batch)
batch_list = batch_pb2.BatchList(batches=batches)
print("Writing to {}...".format(args.output))
with open(args.output, "wb") as fd:
fd.write(batch_list.SerializeToString())
def submit_next_batch(self):
batch_list_bytes = self.make_batch(len(self.batches))
batch_list = batch_pb2.BatchList()
batch_list.ParseFromString(batch_list_bytes)
self.batches.append(batch_list.batches[0])
self._post_batch(batch_list_bytes)
return len(self.batches) - 1
def create_doctor(self, name, surname, wait=None):
# batch_key = txn_key = self._signer.get_public_key().as_hex()
#
# address = helper.make_doctor_address(doctor_pkey=txn_key)
#
# doctor = payload_pb2.CreateDoctor(
# public_key=txn_key,
# name=name,
# surname=surname)
#
# payload = payload_pb2.TransactionPayload(
# payload_type=payload_pb2.TransactionPayload.CREATE_DOCTOR,
# create_doctor=doctor)
#
# return self._send_healthcare_txn(txn_key, batch_key, [address], [address], payload,
# wait=wait,
# auth_user=auth_user,
# auth_password=auth_password)
batch, batch_id = transaction.create_doctor(txn_signer=self._signer,
batch_signer=self._signer,
name=name,
surname=surname)
batch_list = batch_pb2.BatchList(batches=[batch])
return self._send_batches(batch_list=batch_list,
batch_id=batch_id,
wait=wait)
def create_claim(self, txn_signer, batch_signer, claim_id, patient_pkey):
# clinic_pkey = txn_key.get_public_key().as_hex()
# clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey)
# claim_hex = helper.make_claim_address(claim_id=claim_id, clinic_pkey=clinic_pkey)
#
# claim = payload_pb2.CreateClaim(
# claim_id=claim_id,
# clinic_pkey=clinic_pkey,
# patient_pkey=patient_pkey)
#
# payload = payload_pb2.TransactionPayload(
# payload_type=payload_pb2.TransactionPayload.CREATE_CLAIM,
# create_claim=claim)
#
# batch, signature = self._create_txn_and_batch(txn_key, BATCH_KEY, [claim_hex, clinic_hex], [claim_hex], payload)
#
# batch_list = batch_pb2.BatchList(batches=[batch])
batch, batch_id = transaction.register_claim(txn_signer=txn_signer,
batch_signer=batch_signer,
claim_id=claim_id,
patient_pkey=patient_pkey)
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([batch_id], wait=40)
def on_batch_committed(self, batch_id):
with self._lock:
key = self._pending_batches.pop(batch_id, None)
if key is not None:
if key.value < 1000000:
txn = create_intkey_transaction(verb="inc",
name=key.name,
value=1,
deps=[self._deps[key.name]],
private_key=self._private_key,
public_key=self._public_key)
batch = create_batch(transactions=[txn],
private_key=self._private_key,
public_key=self._public_key)
batch_id = batch.header_signature
batch_list = batch_pb2.BatchList(batches=[batch])
key.stream.send(
message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST,
content=batch_list.SerializeToString())
with self._lock:
self._pending_batches[batch.header_signature] = \
IntKeyState(
name=key.name,
stream=key.stream,
value=key.value + 1)
self.delegate.on_new_batch(batch_id, key.stream)
else:
LOGGER.debug('Key %s completed', key.name)
self._create_new_key()
def next_visit(self, txn_signer, batch_signer, claim_id, doctor_pkey):
# clinic_pkey = txn_key.get_public_key().as_hex()
# clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey)
# claim_hex = helper.make_claim_address(claim_id=claim_id, clinic_pkey=clinic_pkey)
# current_times_str = str(time.time())
# event_hex = helper.make_event_address(claim_id=claim_id, clinic_pkey=clinic_pkey, event_time=current_times_str)
#
# next_visit = payload_pb2.ActionOnClaim(
# claim_id=claim_id,
# clinic_pkey=clinic_pkey,
# description="Doctor: {}, completed next visit for claim: {}".format(doctor_pkey, claim_hex),
# event_time=current_times_str
# )
#
# payload = payload_pb2.TransactionPayload(
# payload_type=payload_pb2.TransactionPayload.NEXT_VISIT,
# next_visit=next_visit)
#
# batch, signature = self._create_txn_and_batch(txn_key, BATCH_KEY, [claim_hex, event_hex, clinic_hex],
# [event_hex], payload)
current_times_str = str(time.time())
batch, batch_id = transaction.next_visit(
txn_signer=txn_signer,
batch_signer=batch_signer,
claim_id=claim_id,
description="Doctor: {}, completed next visit for claim: {}".
format(doctor_pkey, claim_id),
event_time=current_times_str)
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([batch_id], wait=80)
def do_populate(args):
context = create_context('secp256k1')
signer = CryptoFactory(context).new_signer(
context.new_random_private_key())
words = generate_word_list(args.pool_size)
batches = []
total_txn_count = 0
txns = []
for i in range(0, len(words)):
txn = create_intkey_transaction(verb='set',
name=words[i],
value=random.randint(9000, 100000),
signer=signer)
total_txn_count += 1
txns.append(txn)
batch = create_batch(transactions=txns, signer=signer)
batches.append(batch)
batch_list = batch_pb2.BatchList(batches=batches)
print("Writing to {}...".format(args.output))
with open(args.output, "wb") as fd:
fd.write(batch_list.SerializeToString())
def do_load(args):
with open(args.filename, mode='rb') as fd:
batches = batch_pb2.BatchList()
batches.ParseFromString(fd.read())
stream = Stream(args.url)
futures = []
start = time.time()
for batch_list in _split_batch_list(batches):
future = stream.send(
message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST,
content=batch_list.SerializeToString())
futures.append(future)
for future in futures:
result = future.result()
try:
assert result.message_type == Message.CLIENT_BATCH_SUBMIT_RESPONSE
except ValidatorConnectionError as vce:
LOGGER.warning("the future resolved to %s", vce)
stop = time.time()
print("batches: {} batch/sec: {}".format(
str(len(batches.batches)),
len(batches.batches) / (stop - start)))
stream.close()
def eat_pills(self, txn_key, claim_id):
clinic_pkey = txn_key.get_public_key().as_hex()
clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey)
claim_hex = helper.make_claim_address(claim_id=claim_id,
clinic_pkey=clinic_pkey)
current_times_str = str(time.time())
event_hex = helper.make_event_address(claim_id=claim_id,
clinic_pkey=clinic_pkey,
event_time=current_times_str)
pills = payload_pb2.ActionOnClaim(
claim_id=claim_id,
clinic_pkey=clinic_pkey,
description="Eat pills in scope of claim: {}".format(claim_hex),
event_time=current_times_str)
payload = payload_pb2.TransactionPayload(
payload_type=payload_pb2.TransactionPayload.EAT_PILLS,
eat_pills=pills)
batch, signature = self._create_txn_and_batch(
txn_key, BATCH_KEY, [claim_hex, event_hex, clinic_hex],
[event_hex], payload)
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([signature], wait=80)
def first_visit(self, txn_key, claim_id, doctor_pkey):
clinic_pkey = txn_key.get_public_key().as_hex()
clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey)
claim_hex = helper.make_claim_address(claim_id=claim_id,
clinic_pkey=clinic_pkey)
current_times_str = str(time.time())
event_hex = helper.make_event_address(claim_id=claim_id,
clinic_pkey=clinic_pkey,
event_time=current_times_str)
first_visit = payload_pb2.ActionOnClaim(
claim_id=claim_id,
clinic_pkey=clinic_pkey,
description="Doctor: {}, completed first visit for claim: {}, \
need to pass procedures and eat pills".format(
doctor_pkey, claim_hex),
event_time=current_times_str)
payload = payload_pb2.TransactionPayload(
payload_type=payload_pb2.TransactionPayload.FIRST_VISIT,
first_visit=first_visit)
batch, signature = self._create_txn_and_batch(
txn_key, BATCH_KEY, [claim_hex, event_hex, clinic_hex],
[event_hex], payload)
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([signature], wait=80)
def assign_doctor(self, txn_key, claim_id, doctor_pkey):
clinic_pkey = txn_key.get_public_key().as_hex()
clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey)
claim_hex = helper.make_claim_address(claim_id=claim_id,
clinic_pkey=clinic_pkey)
current_times_str = str(time.time())
event_hex = helper.make_event_address(claim_id=claim_id,
clinic_pkey=clinic_pkey,
event_time=current_times_str)
assign = payload_pb2.ActionOnClaim(
claim_id=claim_id,
clinic_pkey=clinic_pkey,
description="Doctor: {}, assigned to claim: {}".format(
doctor_pkey, claim_hex),
event_time=current_times_str)
payload = payload_pb2.TransactionPayload(
payload_type=payload_pb2.TransactionPayload.ASSIGN_DOCTOR,
assign_doctor=assign)
batch, signature = self._create_txn_and_batch(
txn_key, BATCH_KEY, [claim_hex, event_hex, clinic_hex],
[event_hex], payload)
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([signature], wait=80)
def do_load(args):
print "Do load"
with open(args.filename) as fd:
batches = batch_pb2.BatchList()
batches.ParseFromString(fd.read())
stream = Stream(args.url)
stream.connect()
futures = []
start = time.time()
for batch_list in _split_batch_list(batches):
future = stream.send(message_type='system/load',
content=batch_list.SerializeToString())
futures.append(future)
for future in futures:
result = future.result()
assert (result.message_type == 'system/load-response')
stop = time.time()
print "batches: {} batch/sec: {}".format(
str(len(batches.batches)),
len(batches.batches) / (stop - start))
stream.close()
def _create_new_key(self):
with self._lock:
url = random.choice(self._urls) if \
len(self._urls) > 0 else None
batch_id = None
if url is not None:
name = datetime.now().isoformat()
txn = create_intkey_transaction(verb="set",
name=name,
value=0,
deps=[],
private_key=self._private_key,
public_key=self._public_key)
batch = create_batch(transactions=[txn],
private_key=self._private_key,
public_key=self._public_key)
self._deps[name] = txn.header_signature
batch_id = batch.header_signature
batch_list = batch_pb2.BatchList(batches=[batch])
post_batches(url, batch_list)
with self._lock:
self._pending_batches[batch_id] = \
IntKeyState(name=name, url=url, value=0)
self.delegate.on_new_batch(batch_id, url)
def add_lab_test(self,
height,
weight,
gender,
a_g_ratio,
albumin,
alkaline_phosphatase,
appearance,
bilirubin,
casts,
color,
wait=None):
batch, batch_id = transaction.add_lab_test(
txn_signer=self._signer,
batch_signer=self._signer,
height=height,
weight=weight,
gender=gender,
a_g_ratio=a_g_ratio,
albumin=albumin,
alkaline_phosphatase=alkaline_phosphatase,
appearance=appearance,
bilirubin=bilirubin,
casts=casts,
color=color)
batch_list = batch_pb2.BatchList(batches=[batch])
return self._send_batches(batch_list=batch_list,
batch_id=batch_id,
wait=wait)
def on_batch_committed(self, batch_id):
with self._lock:
key = self._pending_batches.pop(batch_id, None)
if key is not None:
if key.value < 1000000:
txn = create_intkey_transaction(verb="inc",
name=key.name,
value=1,
deps=[self._deps[key.name]],
signer=self._signer)
batch = create_batch(transactions=[txn], signer=self._signer)
batch_id = batch.header_signature
batch_list = batch_pb2.BatchList(batches=[batch])
(code, _) = post_batches(key.url,
batch_list,
auth_info=self._auth_info)
if code == 202:
with self._lock:
self._pending_batches[batch.header_signature] = \
IntKeyState(
name=key.name,
url=key.url,
value=key.value + 1)
self.delegate.on_new_batch(batch_id, key.url)
else:
LOGGER.debug('Key %s completed', key.name)
self._create_new_key()
def _create_new_key(self):
with self._lock:
url = random.choice(self._urls) if self._urls else None
batch_id = None
if url is not None:
name = datetime.now().isoformat()[-20:]
txn = create_intkey_transaction(verb="set",
name=name,
value=0,
deps=[],
signer=self._signer)
batch = create_batch(transactions=[txn], signer=self._signer)
self._deps[name] = txn.header_signature
batch_id = batch.header_signature
batch_list = batch_pb2.BatchList(batches=[batch])
(code, _) = post_batches(url,
batch_list,
auth_info=self._auth_info)
if code == 202:
with self._lock:
self._pending_batches[batch_id] = \
IntKeyState(name=name, url=url, value=0)
self.delegate.on_new_batch(batch_id, url)
def _create_new_key(self):
with self._lock:
stream = random.choice(self._streams) if \
len(self._streams) > 0 else None
batch_id = None
if stream is not None:
name = datetime.now().isoformat()
txn = create_intkey_transaction(verb="set",
name=name,
value=0,
deps=[],
private_key=self._private_key,
public_key=self._public_key)
batch = create_batch(transactions=[txn],
private_key=self._private_key,
public_key=self._public_key)
self._deps[name] = txn.header_signature
batch_id = batch.header_signature
batch_list = batch_pb2.BatchList(batches=[batch])
stream.send(message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST,
content=batch_list.SerializeToString())
with self._lock:
self._pending_batches[batch_id] = \
IntKeyState(name=name, stream=stream, value=0)
self.delegate.on_new_batch(batch_id, stream)
def create_clinic(self, name, wait=None):
batch, batch_id = transaction.create_clinic(txn_signer=self._signer,
batch_signer=self._signer,
name=name)
batch_list = batch_pb2.BatchList(batches=[batch])
# inputs = outputs = helper.make_clinic_address(clinic_pkey=txn_key)
#
# clinic = payload_pb2.CreateClinic(
# public_key=txn_key,
# name=name)
#
# payload = payload_pb2.TransactionPayload(
# payload_type=payload_pb2.TransactionPayload.CREATE_CLINIC,
# create_clinic=clinic)
#
# return self._send_healthcare_txn(txn_key, batch_key, [inputs], [outputs], payload,
# wait=wait,
# auth_user=auth_user,
# auth_password=auth_password)
return self._send_batches(batch_list=batch_list,
batch_id=batch_id,
wait=wait)
def eat_pills(self, txn_signer, batch_signer, claim_id):
# clinic_pkey = txn_signer.get_public_key().as_hex()
# clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey)
# claim_hex = helper.make_claim_address(claim_id=claim_id, clinic_pkey=clinic_pkey)
# current_times_str = str(time.time())
# event_hex = helper.make_event_address(claim_id=claim_id, clinic_pkey=clinic_pkey, event_time=current_times_str)
#
# pills = payload_pb2.ActionOnClaim(
# claim_id=claim_id,
# clinic_pkey=clinic_pkey,
# description="Eat pills in scope of claim: {}".format(claim_hex),
# event_time=current_times_str
# )
#
# payload = payload_pb2.TransactionPayload(
# payload_type=payload_pb2.TransactionPayload.EAT_PILLS,
# eat_pills=pills)
#
# batch, batch_id = self._create_txn_and_batch(txn_signer, BATCH_KEY, [claim_hex, event_hex, clinic_hex],
# [event_hex], payload)
current_times_str = str(time.time())
batch, batch_id = transaction.eat_pills(
txn_signer=txn_signer,
batch_signer=batch_signer,
claim_id=claim_id,
description="Eat pills in scope of claim: {}".format(claim_id),
event_time=current_times_str)
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([batch_id], wait=80)
def _assert_validator_transaction(self, public_key, target_file):
filename = os.path.join(self._temp_dir, target_file)
batch_list = batch_pb.BatchList()
with open(filename, 'rb') as batch_file:
batch_list.ParseFromString(batch_file.read())
self.assertEqual(1, len(batch_list.batches))
batch = batch_list.batches[0]
self.assertEqual(1, len(batch.transactions))
batch_header = batch_pb.BatchHeader()
batch_header.ParseFromString(batch.header)
self.assertEqual(public_key, batch_header.signer_public_key)
txn = batch.transactions[0]
txn_header = txn_pb.TransactionHeader()
txn_header.ParseFromString(txn.header)
self.assertEqual(public_key, txn_header.signer_public_key)
self.assertEqual('sawtooth_validator_registry', txn_header.family_name)
payload = vr_pb.ValidatorRegistryPayload()
payload.ParseFromString(txn.payload)
self._assert_key_state(payload.signup_info.poet_public_key)
def _send_batch(self, batch):
"""Submits a batch to the validator.
After submission, the client will query the REST API again to
retrieve the commit status of the batch.
"""
batch_id = batch.header_signature
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([batch_id], wait=10)
def add_pulse(self, pulse, timestamp, wait=None):
batch, batch_id = transaction.add_pulse(txn_signer=self._signer,
batch_signer=self._signer,
pulse=pulse,
timestamp=timestamp)
batch_list = batch_pb2.BatchList(batches=[batch])
return self._send_batches(batch_list=batch_list,
batch_id=batch_id,
wait=wait)
def do_generate(args):
private_key = signing.generate_private_key()
public_key = signing.generate_public_key(private_key)
words = generate_word_list(args.pool_size)
batches = []
start = time.time()
total_txn_count = 0
for i in range(0, args.count):
txns = []
for _ in range(0, random.randint(1, args.batch_max_size)):
txn = create_intkey_transaction(
verb=random.choice(['inc', 'dec']),
name=random.choice(words),
value=1,
private_key=private_key,
public_key=public_key)
total_txn_count += 1
txns.append(txn)
batch = create_batch(
transactions=txns,
private_key=private_key,
public_key=public_key)
batches.append(batch)
if i % 100 == 0 and i != 0:
stop = time.time()
txn_count = 0
for batch in batches[-100:]:
txn_count += len(batch.transactions)
fmt = 'batches {}, batch/sec: {:.2f}, txns: {}, txns/sec: {:.2f}'
print(fmt.format(
str(i),
100 / (stop - start),
str(total_txn_count),
txn_count / (stop - start)))
start = stop
batch_list = batch_pb2.BatchList(batches=batches)
print("Writing to {}...".format(args.output))
with open(args.output, "wb") as fd:
fd.write(batch_list.SerializeToString())
def create_clinic(self, txn_key, name):
clinic = payload_pb2.CreateClinic(
public_key=txn_key.get_public_key().as_hex(), name=name)
payload = payload_pb2.TransactionPayload(
payload_type=payload_pb2.TransactionPayload.CREATE_CLINIC,
create_clinic=clinic)
clinic_hex = helper.make_clinic_address(
clinic_pkey=txn_key.get_public_key().as_hex())
batch, signature = self._create_txn_and_batch(txn_key, BATCH_KEY,
[clinic_hex],
[clinic_hex], payload)
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([signature], wait=40)
def _send_batch(self):
with self._lock:
url = random.choice(self._urls) if \
len(self._urls) > 0 else None
batch_id = None
if url is not None:
txns = []
for _ in range(0, 1):
txns.append(create_noop_transaction(self._signer))
batch = create_batch(transactions=txns, signer=self._signer)
batch_id = batch.header_signature
batch_list = batch_pb2.BatchList(batches=[batch])
post_batches(url, batch_list)
self.delegate.on_new_batch(batch_id, url)
def do_load(args):
with open(args.filename, mode='rb') as fd:
batches = batch_pb2.BatchList()
batches.ParseFromString(fd.read())
start = time.time()
futures = []
executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)
for batch_list in _split_batch_list(batches):
fut = executor.submit(post_batches, args.url, batch_list)
futures.append(fut)
# Wait until all futures are complete
wait(futures)
stop = time.time()
print("batches: {} batch/sec: {}".format(
str(len(batches.batches)),
len(batches.batches) / (stop - start)))
def create_patient(self, txn_key, name, surname):
patient = payload_pb2.CreatePatient(
public_key=txn_key.get_public_key().as_hex(),
name=name,
surname=surname)
payload = payload_pb2.TransactionPayload(
payload_type=payload_pb2.TransactionPayload.CREATE_PATIENT,
create_patient=patient)
patient_hex = helper.make_patient_address(
patient_pkey=txn_key.get_public_key().as_hex())
batch, signature = self._create_txn_and_batch(txn_key, BATCH_KEY,
[patient_hex],
[patient_hex], payload)
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([signature], wait=40)
def create_clinic(self, txn_signer, batch_signer, name):
# clinic = payload_pb2.CreateClinic(
# public_key=txn_signer.get_public_key().as_hex(),
# name=name)
#
# payload = payload_pb2.TransactionPayload(
# payload_type=payload_pb2.TransactionPayload.CREATE_CLINIC,
# create_clinic=clinic)
#
# clinic_hex = helper.make_clinic_address(clinic_pkey=txn_signer.get_public_key().as_hex())
#
# batch, batch_id = self._create_txn_and_batch(txn_signer, BATCH_KEY, [clinic_hex], [clinic_hex], payload)
batch, batch_id = transaction.create_clinic(txn_signer=txn_signer,
batch_signer=batch_signer,
name=name)
batch_list = batch_pb2.BatchList(batches=[batch])
self._client.send_batches(batch_list)
return self._client.get_statuses([batch_id], wait=40)
