def resolve_data(self, node, guids, phase):
""" request unreceived verifications from node, notify node of already received verifications,
store received verifications from node, find replications and store them in transfers table """
received, unreceived = self.split_items(
lambda guid: verification_db.get(guid) is not None, guids)
verifications = node.client.transfer_data(node.pass_phrase, received,
unreceived)
node.last_transfer_time = int(time.time())
for verification in verifications:
try:
verification = thrift_converter.convert_thrift_verification(
verification)
verification_db.insert_verification(
verification, verification['verification_id'])
replicated_verifications = verification_db.get_all_replication(
verification['block_id'], phase, verification['origin_id'])
for replicated_ver in replicated_verifications:
vr_transfers_db.insert_transfer(
replicated_ver['origin_id'],
replicated_ver['signature']['signatory'],
verification['verification_id'])
except Exception as ex:
template = "An exception of type {0} occured. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
logger().warning(message)
def resolve_data(self, verifications, phase):
""" store received verifications from node, find replications and store them in transfers table """
for verification in verifications:
try:
verification = thrift_converter.convert_thrift_verification(
verification)
if verification['signature'][
'signatory'] is not self.this_node.node_id:
verification_db.insert_verification(
verification, verification['verification_id'])
# check if there are nodes further down the chain interested in this record
replicated_verifications = verification_db.get_all_replication(
verification['block_id'], phase,
verification['origin_id'])
for replicated_ver in replicated_verifications:
vr_transfers_db.insert_transfer(
replicated_ver['origin_id'],
replicated_ver['signature']['signatory'],
verification['verification_id'])
# check if there are active subscriptions interested in this record
self.update_subscription_response(verification)
except Exception as ex:
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
logger().warning(message)
def insert_verifications(self, verification_records):
""" insert given verification records in database """
for ver in verification_records:
try:
verification_db.insert_verification(ver, ver['verification_id'])
except Exception as ex:
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
logger().error(message)
continue
def _execute_phase_5(self, config, verification):
""" public, Bitcoin bridge phase """
phase = 5
verification_record = verification['record']
verification_info = verification['verification_info']
verification_record['verification_info'] = verification_info
# set block_id and origin_id to None for the reason that the records can come from any phase
if validate_verification_record(verification_record, verification_info):
timestamp_db.insert_verification(verification_record)
verification_db.insert_verification(verification_record)
print "phase 5 executed"
def _execute_phase_4(self, config, phase_3_info):
""" external partner notary phase """
phase = 4
phase_3_record = phase_3_info['record']
p3_verification_info = phase_3_info['verification_info']
phase_3_record['verification_info'] = p3_verification_info
prior_block_hash = self.get_prior_hash(phase_3_record[ORIGIN_ID], phase)
# validate phase_3's verification record
if validate_verification_record(phase_3_record, p3_verification_info):
# storing valid verification record
verification_db.insert_verification(phase_3_record)
# updating record phase
phase_3_record[PHASE] = phase
lower_hash = phase_3_record[SIGNATURE][HASH]
verification_info = lower_hash
# sign verification and rewrite record
block_info = sign_verification_record(self.network.this_node.node_id,
prior_block_hash,
lower_hash,
self.service_config['public_key'],
self.service_config['private_key'],
phase_3_record[BLOCK_ID],
phase_3_record[PHASE],
phase_3_record[ORIGIN_ID],
int(time.time()),
phase_3_record['public_transmission'],
verification_info
)
# inserting verification info after signing
verification_id = str(uuid.uuid4())
verification_db.insert_verification(block_info['verification_record'], verification_id)
# inserting receipt of signed verification for data transfer
vr_transfers_db.insert_transfer(phase_3_record['origin_id'], phase_3_record['signature']['signatory'], verification_id)
# send block info off for public transmission if configured to do so
if phase_3_record['public_transmission']['p4_pub_trans']:
self.network.public_broadcast(block_info, phase)
print "phase 4 executed"
def resolve_data(self, verifications, phase):
""" store received verifications from node, find replications and store them in transfers table """
for verification in verifications:
try:
verification = thrift_converter.convert_thrift_verification(verification)
if verification['signature']['signatory'] is not self.this_node.node_id:
# run broadcast smart contract (BSC)
self.processing_node.sch.execute_bsc(phase, verification)
verification_db.insert_verification(verification, verification['verification_id'])
# check if there are nodes further down the chain interested in this record
replicated_verifications = verification_db.get_all_replication(verification['block_id'], phase, verification['origin_id'])
for replicated_ver in replicated_verifications:
vr_transfers_db.insert_transfer(replicated_ver['origin_id'], replicated_ver['signature']['signatory'], verification['verification_id'])
# check if there are active subscriptions interested in this record
self.update_subscription_response(verification)
except Exception as ex:
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
logger().warning(message)
def _execute_timestamping(self, config):
"""
verification_info = {
'verification_records': {
origin_id: {
'timestamp_id': hash
}
},
'blockchain_type': "BTC"
}
"""
hashes = []
verification_info = {
'verification_records': {},
'blockchain_type': "BTC"
}
pending_records = timestamp_db.get_pending_timestamp()
# returns out of the function if there are no records waiting to be broadcast
if not pending_records:
return
# creates a list of hashes as well as builds the verification_info structure
for r in pending_records:
hashes.append(r['signature']['hash'])
# organizes the verification records by origin_id with a dictionary of timestamp_ids and hashes
if r['origin_id'] not in verification_info['verification_records']:
verification_info['verification_records'][r['origin_id']] = {}
verification_info['verification_records'][r['origin_id']][r['timestamp_id']] = r['signature']['hash']
# takes the list of hashes to be transmitted and hashes with 256 bit to get in form to send
transaction_hash = final_hash(hashes, type=256)
# normal SHA512 hash for all lower VR contents
lower_hash = final_hash(hashes)
# sets the hash in the verification_info structure to the hash we just generated
verification_info['hash'] = transaction_hash
stamper = BitcoinTimestamper(self.service_config['bitcoin_network'], BitcoinFeeProvider())
bitcoin_tx_id = stamper.persist("%s%s" % (LEVEL5_PREFIX, transaction_hash))
bitcoin_tx_id = b2lx(bitcoin_tx_id).encode('utf-8')
verification_info['public_transaction_id'] = bitcoin_tx_id
prior_block_hash = None
# This is the hash of all of the lower elements
block_id = None
phase = 5
origin_id = None
public_transmission = False
block_info = sign_verification_record(self.network.this_node.node_id,
prior_block_hash,
lower_hash,
self.service_config['public_key'],
self.service_config['private_key'],
block_id,
phase,
origin_id,
int(time.time()),
public_transmission,
verification_info
)
# inserts a new verification record for the new record created to be sent
verification_db.insert_verification(block_info['verification_record'])
# sets the timestamp_receipt to true to indicate the records have been sent
for origin_id in verification_info['verification_records'].keys():
for verification_id in verification_info['verification_records'][origin_id]:
timestamp_db.set_transaction_timestamp_proof(verification_id)
# dictionary where the key is origin_id and the value is list of signatories sent from that origin_id
unique_vr_transfers = {}
# creates a verification_record for each unique origin_id-signatory pair
for verification_record in pending_records:
if not verification_record['origin_id'] in unique_vr_transfers:
unique_vr_transfers[verification_record['origin_id']] = []
# inserts a single record per origin_id
if verification_record['signature']['signatory'] not in unique_vr_transfers[verification_record['origin_id']]:
vr_transfers_db.insert_transfer(verification_record['origin_id'],
verification_record['signature']['signatory'],
verification_record['timestamp_id'])
unique_vr_transfers[verification_record['origin_id']].append(verification_record['signature']['signatory'])
def _execute_phase_3(self, config, phase_2_info):
"""
* At this point, any participating processing nodes will have appended signed Phase 2 verification proof to the block.
* Processing nodes participating in Phase 3 Verification may be a different set of nodes than the set of nodes participating in Phase 1 and Phase 2 Verification processes.
* Processing nodes may be defined for the sole purpose of Phase 3 verification (e.g. for independent blockchain verification auditing purposes).
* A participating node will verify that no invalid transaction has been included in the set of approved transaction.
* A participating node will verify that all "approved" transactions are signed by their respective owner.
* A node may perform extra validation steps on all transactions and verification units.
* All signed "Phase 3 Signature Structures" will be grouped, concatenated, and cryptographically signed by the node.
"""
phase = 3
phase_2_record = phase_2_info['record']
p2_verification_info = phase_2_info['verification_info']
phase_2_record['verification_info'] = p2_verification_info
prior_block_hash = self.get_prior_hash(phase_2_record[ORIGIN_ID], phase)
# validate phase_2's verification record
if validate_verification_record(phase_2_record, p2_verification_info):
# storing valid verification record
verification_db.insert_verification(phase_2_record)
# retrieve all phase 2 records for current block
phase_2_records = self.get_sig_records(phase_2_record)
signatories, businesses, locations = self.get_verification_diversity(phase_2_records)
# checking if passed requirements to move on to next phase
if len(signatories) >= P2_COUNT_REQ and len(businesses) >= P2_BUS_COUNT_REQ and len(locations) >= P2_LOC_COUNT_REQ:
# updating record phase
phase_2_record[PHASE] = phase
lower_hashes = [record[SIGNATURE]['signatory'] + ":" + record[SIGNATURE][HASH] for record in phase_2_records]
verification_info = {
'lower_hashes': lower_hashes,
'p2_count': len(signatories),
'businesses': list(businesses),
'deploy_locations': list(locations)
}
lower_hash = str(final_hash(lower_hashes))
# sign verification and rewrite record
block_info = sign_verification_record(self.network.this_node.node_id,
prior_block_hash,
lower_hash,
self.service_config['public_key'],
self.service_config['private_key'],
phase_2_record[BLOCK_ID],
phase_2_record[PHASE],
phase_2_record[ORIGIN_ID],
int(time.time()),
phase_2_record['public_transmission'],
verification_info
)
# inserting verification info after signing
verification_id = str(uuid.uuid4())
verification_db.insert_verification(block_info['verification_record'], verification_id)
# inserting receipt for each phase 2 record received
for record in phase_2_records:
vr_transfers_db.insert_transfer(record['origin_id'], record['signature']['signatory'], verification_id)
# send block info off for public transmission if configured to do so
if phase_2_record['public_transmission']['p3_pub_trans']:
self.network.public_broadcast(block_info, phase)
# send block info for phase 4 validation
self.network.send_block(self.network.phase_3_broadcast, block_info, phase)
print "phase 3 executed"
def _execute_phase_2(self, config, phase_1_info):
"""
* At this point, any participating processing nodes will have appended signed "approvals" of their respectively owned transactions and signed "validations" of un-owned transactions (see Phase 1 Verification Process).
* Processing nodes participating in Phase 2 Verification may be a different set of nodes than the set of nodes participating in Phase 1 Verification.
* Processing nodes may be defined for the sole purpose of Phase 2 verification (without any "owned" transactions in the system).
* A node participating in Phase 2 verification will verify that all transactions in the prospective block are "approved" by their respective owners and are not declared invalid by a system configured portion (e.g. percentage, plurality, or majority).
* Any transactions which are not approved will be taken out of the prospective block and "bumped" to the next block now - (d * T) for Phase 1 processing.
* If a non-approved transaction which is older than a system configured time for the transaction type and owner, the transaction will not be "bumped" to the next block, but instead be placed in a system "pool" or "queue" for later handling or alternate processing.
* All signed "approval" verification units will be grouped, concatenated, and cryptographically signed.
* A "Phase 2 signature structure" is created and appended to the block.
"""
"""
- check block sig
- req: tx_type, tx_id, tx_ts, owner (origin - original phase_1 node_id), trans_sig
- tx minus the payload
"""
phase = 2
phase_1_record = phase_1_info["record"]
p1_verification_info = phase_1_info["verification_info"]
phase_1_record['verification_info'] = p1_verification_info
prior_block_hash = self.get_prior_hash(phase_1_record[ORIGIN_ID], phase)
# validate phase_1's verification record
if validate_verification_record(phase_1_record, p1_verification_info):
# storing valid verification record
verification_db.insert_verification(phase_1_record)
# updating record phase
phase_1_record[PHASE] = phase
valid_transactions, invalid_transactions = self.check_tx_requirements(p1_verification_info)
verification_info = {
'valid_txs': valid_transactions,
'invalid_txs': invalid_transactions,
'business': self.network.business,
'deploy_location': self.network.deploy_location
}
lower_hash = phase_1_record[SIGNATURE][HASH]
# sign verification and rewrite record
block_info = sign_verification_record(self.network.this_node.node_id,
prior_block_hash,
lower_hash,
self.service_config['public_key'],
self.service_config['private_key'],
phase_1_record[BLOCK_ID],
phase_1_record[PHASE],
phase_1_record[ORIGIN_ID],
int(time.time()),
phase_1_record['public_transmission'],
verification_info
)
# inserting verification info after signing
verification_id = str(uuid.uuid4())
verification_db.insert_verification(block_info['verification_record'], verification_id)
# inserting receipt of signed verification for data transfer
vr_transfers_db.insert_transfer(phase_1_record['origin_id'], phase_1_record['signature']['signatory'], verification_id)
# send block info off for public transmission if configured to do so
if phase_1_record['public_transmission']['p2_pub_trans']:
self.network.public_broadcast(block_info, phase)
# send block info for phase 3 validation
self.network.send_block(self.network.phase_2_broadcast, block_info, phase)
print "phase_2 executed"
def _execute_phase_1(self, config, current_block_id):
"""
TODO update all EXEC comments/docs
* Each node gathers all transactions that may be included in the prospective block and groups them by transaction owner.
* All transactions owned (or sourced from) a respective node's business unit or system (owned) are grouped for approval.
* All transactions not owned by the node's business unit or system (others) are grouped for validation.
* All owned transactions are verified per business rules, configurable, (e.g, existence or non-existence of particular fields, with field value validation logic).
* All owned and verified transactions are "approved" by executing the Transaction Verification Signing Process defined below.
* Any transactions deemed "unapproved" will be taken out of the prospective block from a node's perspective by non-inclusion in the signing process, and sent to a system "pool" or "queue" for analysis and alternate processing
* All other (non-owned) transactions are validated to system wide rules agreed upon for all nodes through business and system processes.
* All other (non-owned) transactions are declared "valid" by the node by executing the Transaction Verification Signing Process defined below.
* Any transactions deemed "invalid" will be taken out of the prospective block from a node's perspective by non-inclusion in the signing process, and sent to a system "pool" or "queue" for analysis and alternate processing.
"""
print("Phase 1 Verify Start.")
# Group transactions for last 5 seconds into current block id
block_bound_lower_ts = get_block_time(current_block_id - BLOCK_FIXATE_OFFSET)
print ("""Time bounds: %i - %i""" % (block_bound_lower_ts, block_bound_lower_ts + BLOCK_INTERVAL))
transaction_db.fixate_block(block_bound_lower_ts, block_bound_lower_ts + BLOCK_INTERVAL, current_block_id)
transactions = transaction_db.get_all(block_id=current_block_id)
# Validate the schema and structure of the transactions
valid_transactions, invalid_transactions = self.split_items(valid_transaction_sig, transactions)
rejected_transactions = []
approved_transactions = []
for txn in valid_transactions:
if self.handle_transaction(txn):
approved_transactions.append(txn)
else:
rejected_transactions.append(txn)
if len(approved_transactions) > 0:
# update status of approved transactions
for tx in approved_transactions:
tx["header"]["status"] = "approved"
transaction_db.update_transaction(tx)
# stripping payload from all transactions before signing
self.strip_payload(approved_transactions)
phase = 1
# signatory equals origin_id in phase 1
signatory = origin_id = self.network.this_node.node_id
prior_block_hash = self.get_prior_hash(origin_id, phase)
verification_info = approved_transactions
lower_hash = str(final_hash([0]))
# sign approved transactions
block_info = sign_verification_record(signatory,
prior_block_hash,
lower_hash,
self.service_config['public_key'],
self.service_config['private_key'],
current_block_id,
phase,
origin_id,
int(time.time()),
self.public_transmission,
verification_info)
# store signed phase specific data
verification_id = str(uuid.uuid4())
verification_db.insert_verification(block_info['verification_record'], verification_id)
# send block info off for public transmission if configured to do so
if block_info['verification_record']['public_transmission']['p1_pub_trans']:
self.network.public_broadcast(block_info, phase)
# send block info for phase 2 validation
self.network.send_block(self.network.phase_1_broadcast, block_info, phase)
print("Phase 1 signed " + str(len(approved_transactions)) + " transactions")
# update status transactions that were rejected
if len(rejected_transactions) > 0:
for tx in rejected_transactions:
tx["header"]["status"] = "rejected"
transaction_db.update_transaction(tx)