def __init__(self):
self.NODE_ID = None
self.NODE_NAME = None
self.NODE_TYPE = None
self.setup_node()
# Create the blockchain instance for this node if one doesn't exist
if not os.path.exists("blockchain.pickle"):
self.blockchain = Blockchain.instance(node_id=self.NODE_ID, node_name=self.NODE_NAME,
node_type=self.NODE_TYPE)
print("First time booting! Creating genesis block...")
self.blockchain.create_genesis()
else:
print("Successfully identified blockchain on this device. Loading into memory...")
loaded_blockchain = self.load_blockchain()
loaded_pending_transactions = self.load_pending_transactions()
self.blockchain = Blockchain.instance(loaded_blockchain, loaded_pending_transactions, self.NODE_ID,
self.NODE_NAME, self.NODE_TYPE)
API.instance()
Consensus.instance(self.NODE_NAME)
self.node_comm = Communication.instance(self.NODE_ID, self.NODE_NAME, self.blockchain, self.NODE_TYPE)
self.blockchain.set_communication(self.node_comm)
print(self.NODE_ID, self.NODE_NAME)
# Send a connection request to other distributor nodes in network
if self.NODE_TYPE == 'DISTRIBUTOR':
self.node_comm.notify_connection('connect')
def update_consensus_for_system(self, systemId, consensus_id, subconsensus_id, in_agents, agents_id):
# First we get the agents and latest measures
measures = json.loads(requests.get(select_measures_consensus_url.format(consensus=consensus_id, subConsensus=subconsensus_id)).content)
measures = sorted(measures,key=lambda m: m['agent'].split('#')[1])
all_measures = [m['measure'] for m in measures]
new_consensus = Consensus(self.agents_rep)
# agreed_measures = new_consensus.get_average_consensus(all_measures, in_agents, len(all_measures)+1)
agreed_measures = new_consensus.get_consensus(all_measures, len(all_measures)+1, in_agents, agents_id)
# agreed_measures = new_consensus.get_reputation_consensus(all_measures, len(all_measures)+1, in_agents, agents_id)
print('Agreed measures for Subconsensus {} should be {}'.format(subconsensus_id, agreed_measures))
return agreed_measures
def run(self):
# Phase 1 - Detection of BarCode
self.bc = BarCode(self.bam)
sys.stderr.write("[%s] Starting BarCode Analysis \n" % (self.get_time(),))
self.bc.simple_approach()
sys.stderr.write("[%s] Analyzed BarCodes \n" % (self.get_time(),))
self.bc.write_barcodes(self.barcodes)
sys.stderr.write("[%s] Wrote BarCodes\n" % (self.get_time(),))
# Phase 2 - Rewrite BAM
sys.stderr.write("[%s] Starting Sort and Rewrite BAM\n" % (self.get_time(),))
self.bc.load_barcodes(self.barcodes)
sys.stderr.write("[%s] Loaded BarCodes\n" % (self.get_time(),))
self.bc.bam.reset()
self.bc.sort_and_rewrite_bam(self.rewritten_bam)
pysam.sort("-n", self.rewritten_bam, self.rewritten_sorted_bam.replace(".bam", ""))
sys.stderr.write("[%s] Sort and Rewrite BAM\n" % (self.get_time(),))
# Phase 3 - Build Consensus
self.consensus = Consensus(self.rewritten_sorted_bam, self.ref)
sys.stderr.write("[%s] Starting Consensus Building\n" % (self.get_time(),))
self.consensus.build()
sys.stderr.write("[%s] Built and Calculated Consensus\n" % (self.get_time(),))
self.consensus.infer_consensus(self.consensus_reference)
sys.stderr.write("[%s] Inferred Consensus\n" % (self.get_time(),))
# Phase 4 - Call Variants and Haplotypes
self.consensus.output_consensus_genomes(self.consensus_genomes)
sys.stderr.write("[%s] Output Consensus Genomes\n" % (self.get_time(),))
self.consensus.output_haplotype_distribution(self.haplotype_distribution)
sys.stderr.write("[%s] Output Haplotype Distribution\n" % (self.get_time(),))
self.ovcf = VCF(self.vcf, crossmap=self.crossmap)
self.ovcf.get_variants(self.ref.sequence,
self.consensus.consensus_genomes)
self.ovcf.output_vcf(self.ref.sequence)
sys.stderr.write("[%s] Output VCF\n" % (self.get_time(),))
# Phase 5 - Summary Statistics and Chain Files
f_out = open(self.out, "w")
self.consensus.output_consensus_coverage(f_out)
self.ovcf.output_variants_distribution(f_out)
self.bc.output_reads_in_barcode_distribution(f_out)
f_out.close()
sys.stderr.write("[%s] Output Summary Statistics\n" % (self.get_time(),))
self.ochain = Chain(self.chain)
self.ochain.output_chain(self.ref,
self.consensus.inferred_consensus,
self.consensus.inferred_structure)
sys.stderr.write("[%s] Output Chain File\n" % (self.get_time(),))
def resolve():
'''
Runs the Consensus algorithm to resolve any conflicts
and ensure a node has the correct chain.
'''
replaced = Consensus.resolve_conflicts(nodes.nodes, blockchain)
if replaced:
response = {
'message': 'our chain is replaced',
'new_chain': blockchain.chain
}
else:
response = {'message': 'our chain wins', 'chain': blockchain.chain}
return jsonify(response), 200
def __consensus(self, params):
if len(params) > 0 and params[0] in ["?", "HELP"]:
self.__show_command_help(
usage="consensus [node]",
description="Displays the consensus power of all nodes, or a certain node if specified in the first parameter."
)
else:
consensus_power = Consensus.instance().get_consensus_power()
if len(params) > 0:
node = params[0]
if node in consensus_power:
print(node + " : " + str(consensus_power[node]))
else:
print("Node " + node + " does not exist on the network. Please try again.")
else:
print(consensus_power)
def main():
server_name = f"server{sys.argv[1]}"
print(f"Starting as : {server_name}")
rpcServer = RPCServer(address=cluster_info_by_name[server_name],
authkey=b"peekaboo")
consensusModule = Consensus()
raft_server = RaftServer(name=server_name,
rpcServer=rpcServer,
consensusModule=consensusModule)
raft_server.start_rpc()
raft_server.connect_all_peers()
print(f"{server_name} : Attempting to connect to peers")
consensusModule.set_server(raft_server)
consensusModule.run_election_timer()
time.sleep(100)
def assemble_consensus_genomes(self):
# build consensus
self.consensus = Consensus(self.rewritten_sorted_bam, self.ref)
sys.stderr.write("[%s] Starting Consensus Building\n" % (self.get_time(),))
self.consensus.build(debug=self.debug)
sys.stderr.write("[%s] Built and Calculated Consensus\n" % (self.get_time(),))
self.consensus.infer_consensus(self.consensus_reference)
sys.stderr.write("[%s] Inferred Consensus\n" % (self.get_time(),))
self.consensus.output_consensus_genomes(self.consensus_genomes)
sys.stderr.write("[%s] Output Consensus Genomes\n" % (self.get_time(),))
self.consensus.output_haplotype_distribution(self.haplotype_distribution)
sys.stderr.write("[%s] Output Haplotype Distribution\n" % (self.get_time(),))
self.quark = Quark(self.ref.sequence)
self.quark.distance_matrix(sorted(self.consensus.freq_distribution.items(),
key=lambda q: q[1],
reverse=True))
self.quark.graph_it()
self.quark.rank_it(self.rank)
sys.stderr.write("[%s] Output Quark Analysis\n" % (self.get_time(),))
self.ovcf = VCF(self.vcf, crossmap=self.crossmap)
self.ovcf.get_variants(self.ref.sequence,
self.consensus.consensus_genomes)
self.ovcf.output_vcf(self.ref.sequence)
sys.stderr.write("[%s] Output VCF\n" % (self.get_time(),))
self.summary_statistics()
sys.stderr.write("[%s] Output Summary Statistics\n" % (self.get_time(),))
self.ochain = Chain(self.chain)
self.ochain.output_chain(self.ref,
self.consensus.inferred_consensus,
self.consensus.inferred_structure)
sys.stderr.write("[%s] Output Chain File\n" % (self.get_time(),))
from flask import Flask, jsonify, request
from blockchain import Blockchain
from consensus import Consensus
from hash import Hash
from nodes import Nodes
from proof import Proof
# Initialise a Node and generate a globally unique address
app = Flask(__name__)
node_id = str(uuid4()).replace('-', '')
blockchain = Blockchain()
proof = Proof()
nodes = Nodes()
consensus = Consensus()
@app.route('/mine', methods=['GET'])
def mine():
'''
Mine a new block.
'''
# Get the previous block and proof and find the new proof.
previous_block = blockchain.last_block
previous_proof = previous_block["proof"]
new_proof = proof.proof_of_work(previous_proof)
# Because the user is mining a block, we want to reward them with a block,
# so we create a transaction which will be added to the blockchain
for filename, params in self._inputConfig.items():
try:
print(self.flag.taskStarted.format(filename))
path = self.getAbsPath('/in', filename)
if not os.path.isfile(path):
if os.path.isfile(self.getAbsPath('/', filename)):
raise customExceptions.FileTypeNotSupportedError(
filename)
raise customExceptions.FileNotFoundError(filename)
TTS().textToSpeech(path=path,
voice=params['voice'],
latency=params['latency'],
out=self.getAbsPath('/out', filename,
self._EXTENSION))
print(self.flag.taskEnded)
except customExceptions.CustomError:
pass
except Exception as e:
print(e)
self.renameInputFiles()
print(self.flag.executionEnded)
if __name__ == '__main__':
app = App()
app.main()
Consensus(datadir=app.datadir, outputdir=app.out)
from uuid import uuid4
from flask import Flask, jsonify, request
from consensus import Consensus
from transaction import Transaction
import pdb
# Instantiate the Node
web_app = Flask(__name__, static_url_path = "")
my_app = Consensus()
# Generate a globally unique address for this node
node_identifier = str(uuid4()).replace('-', '')
# transaction/new
@web_app.route('/transaction/new', methods=['POST'])
def new_transaction():
values = request.get_json()
# Check that the required fields are in the POST'ed data
required = ['sender', 'recipient', 'code']
if not all(k in values for k in required):
return 'Missing values', 400
# Create a new Transaction
transaction = Transaction(values['sender'], values['recipient'], values['code'])
approved = my_app.new_transaction(transaction)
if not approved:
return 'unauthorized transaction', 400
imagePath)
raise customExceptions.FileNotFoundError(imagePath)
text = OCR().imageToString(path=imagePath,
lang=params['language'])
textFilePath = self.getAbsPath('/text', imageName,
self._TEXT_EXTENSION)
self.save(path=textFilePath, text=text)
soundFilePath = self.getAbsPath('/sound', imageName,
self._SOUND_EXTENSION)
TTS().textToSpeech(path=textFilePath,
voice=params['voice'],
latency=params['latency'],
out=soundFilePath)
print(self.flag.taskEnded)
except customExceptions.CustomError:
pass
except Exception as e:
print(e)
self.renameInputFiles()
print(self.flag.executionEnded)
if __name__ == '__main__':
app = App()
app.main()
Consensus(datadir=app.datadir, textDir=app.textDir, soundDir=app.soundDir)
nodes = {}
for node in [ConsensusNode(i) for i in range(len(peers))]:
nodes[node.node_index] = node
byz_quorum = args.byz_quorum
rc_threshold = args.rc_threshold
round_duration = datetime.timedelta(seconds=args.round_duration)
start_time = datetime.datetime.fromtimestamp(args.start_time)
node_identity = args.node_identity
if node_identity < 0:
node_identity = None
store = ConsensusStore(node_identity, peers)
consensus_instance = Consensus(nodes,
byz_quorum,
rc_threshold,
round_duration,
start_time,
store,
node_identity=node_identity)
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='ibft_node_' + str(node_identity))
scheduler = BackgroundScheduler()
atexit.register(lambda: scheduler.shutdown())
scheduler.start()
def start_timer(start_time):
if scheduler.get_job('round_timeout') is not None:
def init_consensus(blockhash):
return Consensus(blockhash)
class BaseSeq(Helper):
def __init__(self, bam, barcodes=None, out=None, ref=None,
rewritten_bam=None,
consensus_reference=None,
consensus_genomes=None,
haplotype_distribution=None,
vcf=None,
chain=None,
crossmap=None,
export=None,
rank=None,
debug=None):
self.bam = bam
self.barcodes = barcodes
self.out = out
self.ref = Reference(ref)
self.rewritten_bam = rewritten_bam
self.rewritten_sorted_bam = rewritten_bam.replace(".bam", ".sorted.bam") if rewritten_bam else None
self.consensus_reference = consensus_reference
self.consensus_genomes = consensus_genomes
self.haplotype_distribution = haplotype_distribution
self.vcf = vcf
self.chain = chain
self.crossmap = crossmap
self.export = export
self.rank = rank
self.debug = int(debug)
def get_barcodes(self):
# simple approach - align, take soft-clipped, and use the arbitrary 20 bases
# intermediate approach - use the seed and extend approach
out = open(self.out, "w")
self.bc = BarCode(self.bam)
self.bc.simple_approach()
for k, v in sorted(self.bc.barcode_to_read.items()):
q = sorted(v)
out.write("%s\t%s\n" % (k, ",".join(q)))
out.close()
def error_correction_barcodes(self):
# start analysis
self.bc = BarCode(self.bam)
sys.stdout.write("[%s] Starting Error Correction Analysis\n" % (self.get_time(),))
# load barcodes
self.bc.load_barcodes(self.barcodes)
sys.stdout.write("[%s] Loaded BarCodes\n" % (self.get_time(),))
# cluster barcodes
self.bc.cluster_barcodes()
sys.stdout.write("[%s] Clustered BarCodes\n" % (self.get_time(),))
def filter_barcodes(self, barcode, export="fastq"):
list_of_ids = []
with open(self.barcodes, "r") as f:
for line in f:
data = line.strip("\r\n").split("\t")
if barcode == data[0]:
list_of_ids = data[1].split(",")
break
self.bc = BarCode(self.bam)
self.bc.filter_and_export(list_of_ids, self.out, export=export)
def sort_and_rewrite_bam(self):
self.bc = BarCode(self.bam)
sys.stderr.write("[%s] Starting Sort and Rewrite BAM\n" % (self.get_time(),))
self.bc.load_barcodes(self.barcodes)
sys.stderr.write("[%s] Loaded BarCodes\n" % (self.get_time(),))
self.bc.sort_and_rewrite_bam(self.rewritten_bam)
pysam.sort("-n", self.rewritten_bam, self.rewritten_sorted_bam.replace(".bam", ""))
sys.stderr.write("[%s] Sort and Rewrite BAM\n" % (self.get_time(),))
def split_bam_by_barcode(self):
self.bc = BarCode(self.bam)
sys.stderr.write("[%s] Starting procedure to split BAM by barcode\n" % (self.get_time(),))
self.bc.split_bam_into_barcodes(self.ref, self.out, self.export)
sys.stderr.write("[%s] Finished splitting BAM by barcode id\n" % (self.get_time(),))
def assemble_consensus_genomes(self):
# build consensus
self.consensus = Consensus(self.rewritten_sorted_bam, self.ref)
sys.stderr.write("[%s] Starting Consensus Building\n" % (self.get_time(),))
self.consensus.build(debug=self.debug)
sys.stderr.write("[%s] Built and Calculated Consensus\n" % (self.get_time(),))
self.consensus.infer_consensus(self.consensus_reference)
sys.stderr.write("[%s] Inferred Consensus\n" % (self.get_time(),))
self.consensus.output_consensus_genomes(self.consensus_genomes)
sys.stderr.write("[%s] Output Consensus Genomes\n" % (self.get_time(),))
self.consensus.output_haplotype_distribution(self.haplotype_distribution)
sys.stderr.write("[%s] Output Haplotype Distribution\n" % (self.get_time(),))
self.quark = Quark(self.ref.sequence)
self.quark.distance_matrix(sorted(self.consensus.freq_distribution.items(),
key=lambda q: q[1],
reverse=True))
self.quark.graph_it()
self.quark.rank_it(self.rank)
sys.stderr.write("[%s] Output Quark Analysis\n" % (self.get_time(),))
self.ovcf = VCF(self.vcf, crossmap=self.crossmap)
self.ovcf.get_variants(self.ref.sequence,
self.consensus.consensus_genomes)
self.ovcf.output_vcf(self.ref.sequence)
sys.stderr.write("[%s] Output VCF\n" % (self.get_time(),))
self.summary_statistics()
sys.stderr.write("[%s] Output Summary Statistics\n" % (self.get_time(),))
self.ochain = Chain(self.chain)
self.ochain.output_chain(self.ref,
self.consensus.inferred_consensus,
self.consensus.inferred_structure)
sys.stderr.write("[%s] Output Chain File\n" % (self.get_time(),))
def summary_statistics(self):
# coverage per genome
# variants per genome
# estimate PCR and sequencing errors
# barcode distribution
f_out = open(self.out, "w")
self.bc = BarCode(self.bam) #TEMP
self.bc.load_barcodes(self.barcodes) #TEMP
self.consensus.output_consensus_coverage(f_out)
self.ovcf.output_variants_distribution(f_out)
self.bc.output_reads_in_barcode_distribution(f_out)
f_out.close()
def run(self):
# Phase 1 - Detection of BarCode
self.bc = BarCode(self.bam)
sys.stderr.write("[%s] Starting BarCode Analysis \n" % (self.get_time(),))
self.bc.simple_approach()
sys.stderr.write("[%s] Analyzed BarCodes \n" % (self.get_time(),))
self.bc.write_barcodes(self.barcodes)
sys.stderr.write("[%s] Wrote BarCodes\n" % (self.get_time(),))
# Phase 2 - Rewrite BAM
sys.stderr.write("[%s] Starting Sort and Rewrite BAM\n" % (self.get_time(),))
self.bc.load_barcodes(self.barcodes)
sys.stderr.write("[%s] Loaded BarCodes\n" % (self.get_time(),))
self.bc.bam.reset()
self.bc.sort_and_rewrite_bam(self.rewritten_bam)
pysam.sort("-n", self.rewritten_bam, self.rewritten_sorted_bam.replace(".bam", ""))
sys.stderr.write("[%s] Sort and Rewrite BAM\n" % (self.get_time(),))
# Phase 3 - Build Consensus
self.consensus = Consensus(self.rewritten_sorted_bam, self.ref)
sys.stderr.write("[%s] Starting Consensus Building\n" % (self.get_time(),))
self.consensus.build()
sys.stderr.write("[%s] Built and Calculated Consensus\n" % (self.get_time(),))
self.consensus.infer_consensus(self.consensus_reference)
sys.stderr.write("[%s] Inferred Consensus\n" % (self.get_time(),))
# Phase 4 - Call Variants and Haplotypes
self.consensus.output_consensus_genomes(self.consensus_genomes)
sys.stderr.write("[%s] Output Consensus Genomes\n" % (self.get_time(),))
self.consensus.output_haplotype_distribution(self.haplotype_distribution)
sys.stderr.write("[%s] Output Haplotype Distribution\n" % (self.get_time(),))
self.ovcf = VCF(self.vcf, crossmap=self.crossmap)
self.ovcf.get_variants(self.ref.sequence,
self.consensus.consensus_genomes)
self.ovcf.output_vcf(self.ref.sequence)
sys.stderr.write("[%s] Output VCF\n" % (self.get_time(),))
# Phase 5 - Summary Statistics and Chain Files
f_out = open(self.out, "w")
self.consensus.output_consensus_coverage(f_out)
self.ovcf.output_variants_distribution(f_out)
self.bc.output_reads_in_barcode_distribution(f_out)
f_out.close()
sys.stderr.write("[%s] Output Summary Statistics\n" % (self.get_time(),))
self.ochain = Chain(self.chain)
self.ochain.output_chain(self.ref,
self.consensus.inferred_consensus,
self.consensus.inferred_structure)
sys.stderr.write("[%s] Output Chain File\n" % (self.get_time(),))
def assemble_genomes(self):
pass
def assemble_genomes_from_fastq(self):
pass