def _split_batch_list(batch_list): new_list = [] for batch in batch_list.batches: new_list.append(batch) if len(new_list) == 100: yield batch_pb2.BatchList(batches=new_list) new_list = [] if len(new_list) > 0: yield batch_pb2.BatchList(batches=new_list)
def do_load(args): print("Do load") with open(args.filename, 'rb') as fd: batches = batch_pb2.BatchList() batches.ParseFromString(fd.read()) stream = Stream(args.url) futures = [] start = time.time() for batch_list in _split_batch_list(batches): future = stream.send(message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST, content=batch_list.SerializeToString()) futures.append(future) for future in futures: result = future.result() assert (result.message_type == Message.CLIENT_BATCH_SUBMIT_RESPONSE) stop = time.time() print("batches: {} batch/sec: {}".format( str(len(batches.batches)), len(batches.batches) / (stop - start))) stream.close()
def do_populate(args): private_key = signing.generate_private_key() public_key = signing.generate_public_key(private_key) words = generate_word_list(args.pool_size) batches = [] total_txn_count = 0 txns = [] for i in range(0, len(words)): txn = create_intkey_transaction(verb='set', name=words[i], value=random.randint(9000, 100000), private_key=private_key, public_key=public_key) total_txn_count += 1 txns.append(txn) batch = create_batch(transactions=txns, private_key=private_key, public_key=public_key) batches.append(batch) batch_list = batch_pb2.BatchList(batches=batches) print("Writing to {}...".format(args.output)) with open(args.output, "wb") as fd: fd.write(batch_list.SerializeToString())
def submit_next_batch(self): batch_list_bytes = self.make_batch(len(self.batches)) batch_list = batch_pb2.BatchList() batch_list.ParseFromString(batch_list_bytes) self.batches.append(batch_list.batches[0]) self._post_batch(batch_list_bytes) return len(self.batches) - 1
def create_doctor(self, name, surname, wait=None): # batch_key = txn_key = self._signer.get_public_key().as_hex() # # address = helper.make_doctor_address(doctor_pkey=txn_key) # # doctor = payload_pb2.CreateDoctor( # public_key=txn_key, # name=name, # surname=surname) # # payload = payload_pb2.TransactionPayload( # payload_type=payload_pb2.TransactionPayload.CREATE_DOCTOR, # create_doctor=doctor) # # return self._send_healthcare_txn(txn_key, batch_key, [address], [address], payload, # wait=wait, # auth_user=auth_user, # auth_password=auth_password) batch, batch_id = transaction.create_doctor(txn_signer=self._signer, batch_signer=self._signer, name=name, surname=surname) batch_list = batch_pb2.BatchList(batches=[batch]) return self._send_batches(batch_list=batch_list, batch_id=batch_id, wait=wait)
def create_claim(self, txn_signer, batch_signer, claim_id, patient_pkey): # clinic_pkey = txn_key.get_public_key().as_hex() # clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey) # claim_hex = helper.make_claim_address(claim_id=claim_id, clinic_pkey=clinic_pkey) # # claim = payload_pb2.CreateClaim( # claim_id=claim_id, # clinic_pkey=clinic_pkey, # patient_pkey=patient_pkey) # # payload = payload_pb2.TransactionPayload( # payload_type=payload_pb2.TransactionPayload.CREATE_CLAIM, # create_claim=claim) # # batch, signature = self._create_txn_and_batch(txn_key, BATCH_KEY, [claim_hex, clinic_hex], [claim_hex], payload) # # batch_list = batch_pb2.BatchList(batches=[batch]) batch, batch_id = transaction.register_claim(txn_signer=txn_signer, batch_signer=batch_signer, claim_id=claim_id, patient_pkey=patient_pkey) batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([batch_id], wait=40)
def on_batch_committed(self, batch_id): with self._lock: key = self._pending_batches.pop(batch_id, None) if key is not None: if key.value < 1000000: txn = create_intkey_transaction(verb="inc", name=key.name, value=1, deps=[self._deps[key.name]], private_key=self._private_key, public_key=self._public_key) batch = create_batch(transactions=[txn], private_key=self._private_key, public_key=self._public_key) batch_id = batch.header_signature batch_list = batch_pb2.BatchList(batches=[batch]) key.stream.send( message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST, content=batch_list.SerializeToString()) with self._lock: self._pending_batches[batch.header_signature] = \ IntKeyState( name=key.name, stream=key.stream, value=key.value + 1) self.delegate.on_new_batch(batch_id, key.stream) else: LOGGER.debug('Key %s completed', key.name) self._create_new_key()
def next_visit(self, txn_signer, batch_signer, claim_id, doctor_pkey): # clinic_pkey = txn_key.get_public_key().as_hex() # clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey) # claim_hex = helper.make_claim_address(claim_id=claim_id, clinic_pkey=clinic_pkey) # current_times_str = str(time.time()) # event_hex = helper.make_event_address(claim_id=claim_id, clinic_pkey=clinic_pkey, event_time=current_times_str) # # next_visit = payload_pb2.ActionOnClaim( # claim_id=claim_id, # clinic_pkey=clinic_pkey, # description="Doctor: {}, completed next visit for claim: {}".format(doctor_pkey, claim_hex), # event_time=current_times_str # ) # # payload = payload_pb2.TransactionPayload( # payload_type=payload_pb2.TransactionPayload.NEXT_VISIT, # next_visit=next_visit) # # batch, signature = self._create_txn_and_batch(txn_key, BATCH_KEY, [claim_hex, event_hex, clinic_hex], # [event_hex], payload) current_times_str = str(time.time()) batch, batch_id = transaction.next_visit( txn_signer=txn_signer, batch_signer=batch_signer, claim_id=claim_id, description="Doctor: {}, completed next visit for claim: {}". format(doctor_pkey, claim_id), event_time=current_times_str) batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([batch_id], wait=80)
def do_populate(args): context = create_context('secp256k1') signer = CryptoFactory(context).new_signer( context.new_random_private_key()) words = generate_word_list(args.pool_size) batches = [] total_txn_count = 0 txns = [] for i in range(0, len(words)): txn = create_intkey_transaction(verb='set', name=words[i], value=random.randint(9000, 100000), signer=signer) total_txn_count += 1 txns.append(txn) batch = create_batch(transactions=txns, signer=signer) batches.append(batch) batch_list = batch_pb2.BatchList(batches=batches) print("Writing to {}...".format(args.output)) with open(args.output, "wb") as fd: fd.write(batch_list.SerializeToString())
def do_load(args): with open(args.filename, mode='rb') as fd: batches = batch_pb2.BatchList() batches.ParseFromString(fd.read()) stream = Stream(args.url) futures = [] start = time.time() for batch_list in _split_batch_list(batches): future = stream.send( message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST, content=batch_list.SerializeToString()) futures.append(future) for future in futures: result = future.result() try: assert result.message_type == Message.CLIENT_BATCH_SUBMIT_RESPONSE except ValidatorConnectionError as vce: LOGGER.warning("the future resolved to %s", vce) stop = time.time() print("batches: {} batch/sec: {}".format( str(len(batches.batches)), len(batches.batches) / (stop - start))) stream.close()
def eat_pills(self, txn_key, claim_id): clinic_pkey = txn_key.get_public_key().as_hex() clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey) claim_hex = helper.make_claim_address(claim_id=claim_id, clinic_pkey=clinic_pkey) current_times_str = str(time.time()) event_hex = helper.make_event_address(claim_id=claim_id, clinic_pkey=clinic_pkey, event_time=current_times_str) pills = payload_pb2.ActionOnClaim( claim_id=claim_id, clinic_pkey=clinic_pkey, description="Eat pills in scope of claim: {}".format(claim_hex), event_time=current_times_str) payload = payload_pb2.TransactionPayload( payload_type=payload_pb2.TransactionPayload.EAT_PILLS, eat_pills=pills) batch, signature = self._create_txn_and_batch( txn_key, BATCH_KEY, [claim_hex, event_hex, clinic_hex], [event_hex], payload) batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([signature], wait=80)
def first_visit(self, txn_key, claim_id, doctor_pkey): clinic_pkey = txn_key.get_public_key().as_hex() clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey) claim_hex = helper.make_claim_address(claim_id=claim_id, clinic_pkey=clinic_pkey) current_times_str = str(time.time()) event_hex = helper.make_event_address(claim_id=claim_id, clinic_pkey=clinic_pkey, event_time=current_times_str) first_visit = payload_pb2.ActionOnClaim( claim_id=claim_id, clinic_pkey=clinic_pkey, description="Doctor: {}, completed first visit for claim: {}, \ need to pass procedures and eat pills".format( doctor_pkey, claim_hex), event_time=current_times_str) payload = payload_pb2.TransactionPayload( payload_type=payload_pb2.TransactionPayload.FIRST_VISIT, first_visit=first_visit) batch, signature = self._create_txn_and_batch( txn_key, BATCH_KEY, [claim_hex, event_hex, clinic_hex], [event_hex], payload) batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([signature], wait=80)
def assign_doctor(self, txn_key, claim_id, doctor_pkey): clinic_pkey = txn_key.get_public_key().as_hex() clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey) claim_hex = helper.make_claim_address(claim_id=claim_id, clinic_pkey=clinic_pkey) current_times_str = str(time.time()) event_hex = helper.make_event_address(claim_id=claim_id, clinic_pkey=clinic_pkey, event_time=current_times_str) assign = payload_pb2.ActionOnClaim( claim_id=claim_id, clinic_pkey=clinic_pkey, description="Doctor: {}, assigned to claim: {}".format( doctor_pkey, claim_hex), event_time=current_times_str) payload = payload_pb2.TransactionPayload( payload_type=payload_pb2.TransactionPayload.ASSIGN_DOCTOR, assign_doctor=assign) batch, signature = self._create_txn_and_batch( txn_key, BATCH_KEY, [claim_hex, event_hex, clinic_hex], [event_hex], payload) batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([signature], wait=80)
def do_load(args): print "Do load" with open(args.filename) as fd: batches = batch_pb2.BatchList() batches.ParseFromString(fd.read()) stream = Stream(args.url) stream.connect() futures = [] start = time.time() for batch_list in _split_batch_list(batches): future = stream.send(message_type='system/load', content=batch_list.SerializeToString()) futures.append(future) for future in futures: result = future.result() assert (result.message_type == 'system/load-response') stop = time.time() print "batches: {} batch/sec: {}".format( str(len(batches.batches)), len(batches.batches) / (stop - start)) stream.close()
def _create_new_key(self): with self._lock: url = random.choice(self._urls) if \ len(self._urls) > 0 else None batch_id = None if url is not None: name = datetime.now().isoformat() txn = create_intkey_transaction(verb="set", name=name, value=0, deps=[], private_key=self._private_key, public_key=self._public_key) batch = create_batch(transactions=[txn], private_key=self._private_key, public_key=self._public_key) self._deps[name] = txn.header_signature batch_id = batch.header_signature batch_list = batch_pb2.BatchList(batches=[batch]) post_batches(url, batch_list) with self._lock: self._pending_batches[batch_id] = \ IntKeyState(name=name, url=url, value=0) self.delegate.on_new_batch(batch_id, url)
def add_lab_test(self, height, weight, gender, a_g_ratio, albumin, alkaline_phosphatase, appearance, bilirubin, casts, color, wait=None): batch, batch_id = transaction.add_lab_test( txn_signer=self._signer, batch_signer=self._signer, height=height, weight=weight, gender=gender, a_g_ratio=a_g_ratio, albumin=albumin, alkaline_phosphatase=alkaline_phosphatase, appearance=appearance, bilirubin=bilirubin, casts=casts, color=color) batch_list = batch_pb2.BatchList(batches=[batch]) return self._send_batches(batch_list=batch_list, batch_id=batch_id, wait=wait)
def on_batch_committed(self, batch_id): with self._lock: key = self._pending_batches.pop(batch_id, None) if key is not None: if key.value < 1000000: txn = create_intkey_transaction(verb="inc", name=key.name, value=1, deps=[self._deps[key.name]], signer=self._signer) batch = create_batch(transactions=[txn], signer=self._signer) batch_id = batch.header_signature batch_list = batch_pb2.BatchList(batches=[batch]) (code, _) = post_batches(key.url, batch_list, auth_info=self._auth_info) if code == 202: with self._lock: self._pending_batches[batch.header_signature] = \ IntKeyState( name=key.name, url=key.url, value=key.value + 1) self.delegate.on_new_batch(batch_id, key.url) else: LOGGER.debug('Key %s completed', key.name) self._create_new_key()
def _create_new_key(self): with self._lock: url = random.choice(self._urls) if self._urls else None batch_id = None if url is not None: name = datetime.now().isoformat()[-20:] txn = create_intkey_transaction(verb="set", name=name, value=0, deps=[], signer=self._signer) batch = create_batch(transactions=[txn], signer=self._signer) self._deps[name] = txn.header_signature batch_id = batch.header_signature batch_list = batch_pb2.BatchList(batches=[batch]) (code, _) = post_batches(url, batch_list, auth_info=self._auth_info) if code == 202: with self._lock: self._pending_batches[batch_id] = \ IntKeyState(name=name, url=url, value=0) self.delegate.on_new_batch(batch_id, url)
def _create_new_key(self): with self._lock: stream = random.choice(self._streams) if \ len(self._streams) > 0 else None batch_id = None if stream is not None: name = datetime.now().isoformat() txn = create_intkey_transaction(verb="set", name=name, value=0, deps=[], private_key=self._private_key, public_key=self._public_key) batch = create_batch(transactions=[txn], private_key=self._private_key, public_key=self._public_key) self._deps[name] = txn.header_signature batch_id = batch.header_signature batch_list = batch_pb2.BatchList(batches=[batch]) stream.send(message_type=Message.CLIENT_BATCH_SUBMIT_REQUEST, content=batch_list.SerializeToString()) with self._lock: self._pending_batches[batch_id] = \ IntKeyState(name=name, stream=stream, value=0) self.delegate.on_new_batch(batch_id, stream)
def create_clinic(self, name, wait=None): batch, batch_id = transaction.create_clinic(txn_signer=self._signer, batch_signer=self._signer, name=name) batch_list = batch_pb2.BatchList(batches=[batch]) # inputs = outputs = helper.make_clinic_address(clinic_pkey=txn_key) # # clinic = payload_pb2.CreateClinic( # public_key=txn_key, # name=name) # # payload = payload_pb2.TransactionPayload( # payload_type=payload_pb2.TransactionPayload.CREATE_CLINIC, # create_clinic=clinic) # # return self._send_healthcare_txn(txn_key, batch_key, [inputs], [outputs], payload, # wait=wait, # auth_user=auth_user, # auth_password=auth_password) return self._send_batches(batch_list=batch_list, batch_id=batch_id, wait=wait)
def eat_pills(self, txn_signer, batch_signer, claim_id): # clinic_pkey = txn_signer.get_public_key().as_hex() # clinic_hex = helper.make_clinic_address(clinic_pkey=clinic_pkey) # claim_hex = helper.make_claim_address(claim_id=claim_id, clinic_pkey=clinic_pkey) # current_times_str = str(time.time()) # event_hex = helper.make_event_address(claim_id=claim_id, clinic_pkey=clinic_pkey, event_time=current_times_str) # # pills = payload_pb2.ActionOnClaim( # claim_id=claim_id, # clinic_pkey=clinic_pkey, # description="Eat pills in scope of claim: {}".format(claim_hex), # event_time=current_times_str # ) # # payload = payload_pb2.TransactionPayload( # payload_type=payload_pb2.TransactionPayload.EAT_PILLS, # eat_pills=pills) # # batch, batch_id = self._create_txn_and_batch(txn_signer, BATCH_KEY, [claim_hex, event_hex, clinic_hex], # [event_hex], payload) current_times_str = str(time.time()) batch, batch_id = transaction.eat_pills( txn_signer=txn_signer, batch_signer=batch_signer, claim_id=claim_id, description="Eat pills in scope of claim: {}".format(claim_id), event_time=current_times_str) batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([batch_id], wait=80)
def _assert_validator_transaction(self, public_key, target_file): filename = os.path.join(self._temp_dir, target_file) batch_list = batch_pb.BatchList() with open(filename, 'rb') as batch_file: batch_list.ParseFromString(batch_file.read()) self.assertEqual(1, len(batch_list.batches)) batch = batch_list.batches[0] self.assertEqual(1, len(batch.transactions)) batch_header = batch_pb.BatchHeader() batch_header.ParseFromString(batch.header) self.assertEqual(public_key, batch_header.signer_public_key) txn = batch.transactions[0] txn_header = txn_pb.TransactionHeader() txn_header.ParseFromString(txn.header) self.assertEqual(public_key, txn_header.signer_public_key) self.assertEqual('sawtooth_validator_registry', txn_header.family_name) payload = vr_pb.ValidatorRegistryPayload() payload.ParseFromString(txn.payload) self._assert_key_state(payload.signup_info.poet_public_key)
def _send_batch(self, batch): """Submits a batch to the validator. After submission, the client will query the REST API again to retrieve the commit status of the batch. """ batch_id = batch.header_signature batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([batch_id], wait=10)
def add_pulse(self, pulse, timestamp, wait=None): batch, batch_id = transaction.add_pulse(txn_signer=self._signer, batch_signer=self._signer, pulse=pulse, timestamp=timestamp) batch_list = batch_pb2.BatchList(batches=[batch]) return self._send_batches(batch_list=batch_list, batch_id=batch_id, wait=wait)
def do_generate(args): private_key = signing.generate_private_key() public_key = signing.generate_public_key(private_key) words = generate_word_list(args.pool_size) batches = [] start = time.time() total_txn_count = 0 for i in range(0, args.count): txns = [] for _ in range(0, random.randint(1, args.batch_max_size)): txn = create_intkey_transaction( verb=random.choice(['inc', 'dec']), name=random.choice(words), value=1, private_key=private_key, public_key=public_key) total_txn_count += 1 txns.append(txn) batch = create_batch( transactions=txns, private_key=private_key, public_key=public_key) batches.append(batch) if i % 100 == 0 and i != 0: stop = time.time() txn_count = 0 for batch in batches[-100:]: txn_count += len(batch.transactions) fmt = 'batches {}, batch/sec: {:.2f}, txns: {}, txns/sec: {:.2f}' print(fmt.format( str(i), 100 / (stop - start), str(total_txn_count), txn_count / (stop - start))) start = stop batch_list = batch_pb2.BatchList(batches=batches) print("Writing to {}...".format(args.output)) with open(args.output, "wb") as fd: fd.write(batch_list.SerializeToString())
def create_clinic(self, txn_key, name): clinic = payload_pb2.CreateClinic( public_key=txn_key.get_public_key().as_hex(), name=name) payload = payload_pb2.TransactionPayload( payload_type=payload_pb2.TransactionPayload.CREATE_CLINIC, create_clinic=clinic) clinic_hex = helper.make_clinic_address( clinic_pkey=txn_key.get_public_key().as_hex()) batch, signature = self._create_txn_and_batch(txn_key, BATCH_KEY, [clinic_hex], [clinic_hex], payload) batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([signature], wait=40)
def _send_batch(self): with self._lock: url = random.choice(self._urls) if \ len(self._urls) > 0 else None batch_id = None if url is not None: txns = [] for _ in range(0, 1): txns.append(create_noop_transaction(self._signer)) batch = create_batch(transactions=txns, signer=self._signer) batch_id = batch.header_signature batch_list = batch_pb2.BatchList(batches=[batch]) post_batches(url, batch_list) self.delegate.on_new_batch(batch_id, url)
def do_load(args): with open(args.filename, mode='rb') as fd: batches = batch_pb2.BatchList() batches.ParseFromString(fd.read()) start = time.time() futures = [] executor = concurrent.futures.ThreadPoolExecutor(max_workers=5) for batch_list in _split_batch_list(batches): fut = executor.submit(post_batches, args.url, batch_list) futures.append(fut) # Wait until all futures are complete wait(futures) stop = time.time() print("batches: {} batch/sec: {}".format( str(len(batches.batches)), len(batches.batches) / (stop - start)))
def create_patient(self, txn_key, name, surname): patient = payload_pb2.CreatePatient( public_key=txn_key.get_public_key().as_hex(), name=name, surname=surname) payload = payload_pb2.TransactionPayload( payload_type=payload_pb2.TransactionPayload.CREATE_PATIENT, create_patient=patient) patient_hex = helper.make_patient_address( patient_pkey=txn_key.get_public_key().as_hex()) batch, signature = self._create_txn_and_batch(txn_key, BATCH_KEY, [patient_hex], [patient_hex], payload) batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([signature], wait=40)
def create_clinic(self, txn_signer, batch_signer, name): # clinic = payload_pb2.CreateClinic( # public_key=txn_signer.get_public_key().as_hex(), # name=name) # # payload = payload_pb2.TransactionPayload( # payload_type=payload_pb2.TransactionPayload.CREATE_CLINIC, # create_clinic=clinic) # # clinic_hex = helper.make_clinic_address(clinic_pkey=txn_signer.get_public_key().as_hex()) # # batch, batch_id = self._create_txn_and_batch(txn_signer, BATCH_KEY, [clinic_hex], [clinic_hex], payload) batch, batch_id = transaction.create_clinic(txn_signer=txn_signer, batch_signer=batch_signer, name=name) batch_list = batch_pb2.BatchList(batches=[batch]) self._client.send_batches(batch_list) return self._client.get_statuses([batch_id], wait=40)