def datagramReceived(self, datagram, address):
time_keeper = TimeKeeper()
if self.noisy:
log.msg("received datagram from %s" % repr(address))
if len(datagram) < 22:
log.msg("received datagram too small from %s, ignoring" %
repr(address))
return
msgID = datagram[1:21]
time_keeper.start_clock()
data = umsgpack.unpackb(datagram[21:])
time_keeper.stop_clock("time_unpack_msg")
# self.log.debug("[BENCH] LOW RPC RECEIVE -> %s " % (time_keeper.get_summary(),))
if datagram[:1] == b'\x00':
self._acceptRequest(msgID, data, address)
elif datagram[:1] == b'\x01':
self._acceptResponse(msgID, data, address)
else:
# otherwise, don't know the format, don't do anything
log.msg("Received unknown message from %s, ignoring" %
repr(address))
def func(address, node_id, *args):
time_keeper = TimeKeeper()
msgID = sha1(os.urandom(32)).digest()
assert len(node_id) == 20
time_keeper.start_clock()
data = umsgpack.packb([str(name), node_id, args])
time_keeper.stop_clock("time_data_pack")
if len(data) > self.max_packet_size:
msg = "Total length of function name and arguments cannot exceed 8K"
raise MalformedMessage(msg)
txdata = b'\x00' + msgID + data
if self.noisy:
log.msg("calling remote function %s on %s (msgid %s)" %
(name, address, b64encode(msgID)))
time_keeper.start_clock()
self.transport.write(txdata, address)
time_keeper.stop_clock("time_write_socket")
# self.log.debug("[BENCH] LOW RPC SEND TIMES %s -> %s " % (str(name), time_keeper.get_summary()))
d = defer.Deferred()
timeout = reactor.callLater(self._waitTimeout, self._timeout,
msgID)
self._outstanding[msgID] = (d, timeout)
return d
def render_POST(self, request):
if len(request.prepath) < 4:
request.setResponseCode(400)
return json.dumps({'error': "Illegal URL"})
try:
time_keeper = TimeKeeper()
total_time_id = time_keeper.start_clock_unique()
nodeid = unhexlify(request.prepath[1])
source_ip = request.client.host
source_port = int(request.prepath[2])
kad_key = unhexlify(request.prepath[3])
source = Node(nodeid, source_ip, source_port)
time_keeper.start_clock()
self.rpc_protocol.welcomeIfNewNode(source)
time_keeper.stop_clock(ENTRY_TIME_WELCOME_NODE)
encoded_chunk = request.content.read()
chunk = CloudChunk.decode(encoded_chunk)
if not digest(chunk.key) == kad_key:
request.setResponseCode(400)
return json.dumps({'error': "key missmatch"})
def handle_policy(policy):
time_keeper.stop_clock(ENTRY_FETCH_POLICY)
id = time_keeper.start_clock_unique()
self.storage.store_check_chunk(chunk,
None,
policy,
time_keeper=time_keeper)
time_keeper.stop_clock_unique(ENTRY_STORE_CHECK, id)
time_keeper.stop_clock_unique(ENTRY_TOTAL_STORE_LOCAL,
total_time_id)
self.log.debug("%s %s %s" % (BENCH_TAG, TYPE_STORE_CHUNK_LOCAL,
time_keeper.get_summary()))
request.write(json.dumps({'value': "ok"}))
request.finish()
time_keeper.start_clock()
self.talos_vc.get_policy_with_txid(
chunk.get_tag_hex()).addCallback(handle_policy)
return NOT_DONE_YET
except InvalidChunkError as e:
request.setResponseCode(400)
return json.dumps({'error': e.value})
except TalosVCRestClientError:
request.setResponseCode(400)
return "ERROR: No policy found"
except:
request.setResponseCode(400)
return json.dumps({'error': "Error occured"})
def run_benchmark_fetch_par(num_rounds, data_path, num_entries, granularity, fetch_granularity, num_threads, out_logger,
private_key=BitcoinVersionedPrivateKey(PRIVATE_KEY),
policy_nonce=base64.b64decode(NONCE), stream_id=STREAMID,
txid=TXID, ip=IP, port=PORT):
key = os.urandom(32)
identifier = DataStreamIdentifier(private_key.public_key().address(), stream_id, policy_nonce, txid)
dht_api_client = DHTRestClient(dhtip=ip, dhtport=port)
for block_id, chunk_data in enumerate(extract_eth_smartmeter_data(data_path, granularity, max_entries=num_entries)):
try:
chunk = generate_cloud_chunk(chunk_data, private_key, block_id, identifier, key=key)
dht_api_client.store_chunk(chunk)
print "Store chunk %d Num entries: %d" % (block_id, len(chunk_data.entries))
except DHTRestClientException as e:
print "Store round %d error: %s" % (block_id, e)
num_fetches = num_entries / granularity
if not num_entries % granularity == 0:
num_fetches += 1
for x in fetch_granularity:
num_fetches = x / granularity
if not x % granularity == 0:
num_fetches += 1
for round in range(num_rounds):
time_keeper = TimeKeeper()
results = [[]] * num_threads
if num_fetches < num_threads:
temp_threads = num_fetches
print "Num Fetches: %d Temp_threads %d" % (num_fetches, temp_threads)
threads = [Fetchjob(idx, results, DHTRestClient(dhtip=ip, dhtport=port), block_id, private_key, identifier)
for idx, block_id in enumerate(splitting(range(num_fetches), temp_threads))]
else:
threads = [Fetchjob(idx, results, DHTRestClient(dhtip=ip, dhtport=port), block_id, private_key, identifier)
for idx, block_id in enumerate(splitting(range(num_fetches), num_threads))]
time_keeper.start_clock()
map(lambda x: x.start(), threads)
map(lambda x: x.join(), threads)
time_keeper.stop_clock("time_fetch_all")
time_keeper.store_value("num_entries", x)
time_keeper.store_value("num_blocks", num_fetches)
time_keeper.store_value("round", round)
chunks = [item for sublist in results for item in sublist]
if len(chunks) == num_fetches:
print "Round %d ok Num results: %d" % (round, num_fetches)
else:
print "Round %d ok Num results: %d" % (round, num_fetches)
for idx, chunk in enumerate(chunks):
if chunk is None:
print "No result for chunk %d " % idx
out_logger.log_times_keeper(time_keeper)
out_logger.flush_to_db()
print "DONE"
def run_benchmark_fetch_par(
num_rounds,
num_fetches,
num_threads,
out_logger,
private_key=BitcoinVersionedPrivateKey(PRIVATE_KEY),
policy_nonce=base64.b64decode(NONCE),
stream_id=STREAMID,
txid=TXID,
ip=IP,
port=PORT,
chunk_size=100000):
key = os.urandom(32)
identifier = DataStreamIdentifier(private_key.public_key().address(),
stream_id, policy_nonce, txid)
dht_api_client = DHTRestClient(dhtip=ip, dhtport=port)
for round_bench in range(num_fetches):
try:
chunk = generate_random_chunk(private_key,
round_bench,
identifier,
key=key,
size=chunk_size)
dht_api_client.store_chunk(chunk)
print "Store chunk %d" % (round_bench, )
except DHTRestClientException as e:
print "Store round %d error: %s" % (round_bench, e)
for round in range(num_rounds):
time_keeper = TimeKeeper()
results = [[]] * num_threads
threads = [
Fetchjob(idx, results, DHTRestClient(dhtip=ip, dhtport=port),
block_id, private_key, identifier)
for idx, block_id in enumerate(
splitting(range(num_fetches), num_threads))
]
time_keeper.start_clock()
map(lambda x: x.start(), threads)
map(lambda x: x.join(), threads)
time_keeper.stop_clock("time_fetch_all")
chunks = [item for sublist in results for item in sublist]
if len(chunks) == num_fetches:
print "Round %d ok Num results: %d" % (round, num_fetches)
else:
print "Round %d ok Num results: %d" % (round, num_fetches)
for idx, chunk in enumerate(chunks):
if chunk is None:
print "No result for chunk %d " % idx
out_logger.log_times_keeper(time_keeper)
out_logger.flush_to_db()
print "DONE"
def render_POST(self, request):
msg = json.loads(request.content.read())
timekeeper = TimeKeeper()
total_time_id = timekeeper.start_clock_unique()
try:
timekeeper.start_clock()
token = get_and_check_query_token(msg)
check_query_token_valid(token)
timekeeper.stop_clock(ENTRY_CHECK_TOKEN_VALID)
# Check nonce ok
if not self._check_cache(token.nonce):
raise InvalidQueryToken("Nonce not valid")
def handle_policy(policy):
timekeeper.stop_clock(ENTRY_FETCH_POLICY)
if policy is None:
request.setResponseCode(400)
request.write("No Policy Found")
request.finish()
# check policy for correctness
id = timekeeper.start_clock_unique()
chunk = self.storage.get_check_chunk(token.chunk_key,
token.pubkey,
policy,
time_keeper=timekeeper)
timekeeper.stop_clock_unique(ENTRY_GET_AND_CHECK, id)
timekeeper.stop_clock_unique(ENTRY_TOTAL_LOCAL_QUERY,
total_time_id)
self.log.debug("%s %s %s" % (BENCH_TAG, TYPE_QUERY_CHUNK_LOCAL,
timekeeper.get_summary()))
request.write(chunk.encode())
request.finish()
timekeeper.start_clock()
self.talos_vc.get_policy(token.owner,
token.streamid).addCallback(handle_policy)
return NOT_DONE_YET
except InvalidQueryToken:
request.setResponseCode(400)
return "ERROR: token verification failure"
except TalosVCRestClientError:
request.setResponseCode(400)
return "ERROR: No policy found"
except:
request.setResponseCode(400)
return "ERROR: error occured"
def rpc_store(self, sender, nodeid, key, value):
source = Node(nodeid, sender[0], sender[1])
time_keeper = TimeKeeper()
total_time_id = time_keeper.start_clock_unique()
time_keeper.start_clock()
self.welcomeIfNewNode(source)
time_keeper.stop_clock(ENTRY_TIME_WELCOME_NODE)
self.log.debug("got a store request from %s, storing value" %
str(sender))
try:
chunk = CloudChunk.decode(value)
if not digest(chunk.key) == key:
return {'error': 'key missmatch'}
def handle_policy(policy):
time_keeper.stop_clock(ENTRY_FETCH_POLICY)
# Hack no chunk id given -> no key checks, key is in the encoded chunk
id = time_keeper.start_clock_unique()
self.storage.store_check_chunk(chunk,
None,
policy,
time_keeper=time_keeper)
time_keeper.stop_clock_unique(ENTRY_STORE_CHECK, id)
time_keeper.stop_clock_unique(ENTRY_TOTAL_STORE_LOCAL,
total_time_id)
self.log.debug("%s %s %s" % (BENCH_TAG, TYPE_STORE_CHUNK_LOCAL,
time_keeper.get_summary()))
return {'value': 'ok'}
time_keeper.start_clock()
return self.talos_vc.get_policy_with_txid(
chunk.get_tag_hex()).addCallback(handle_policy)
except InvalidChunkError as e:
return {'error': e.value}
except TalosVCRestClientError:
return {'error': "No policy found"}
def run_benchmark_s3_talos_fetch(
num_rounds,
num_gets,
out_logger,
bucket_name,
private_key=BitcoinVersionedPrivateKey(PRIVATE_KEY),
policy_nonce=base64.b64decode(NONCE),
stream_id=STREAMID,
txid=TXID,
chunk_size=100000,
num_threads=None,
do_store=True,
do_delete=True,
avoid_token_create=True):
key = os.urandom(32)
owner = private_key.public_key().address()
identifier = DataStreamIdentifier(owner, stream_id, policy_nonce, txid)
vc_client = TalosVCRestClient()
storage = TalosS3Storage(bucket_name)
num_threads = num_threads or num_gets
if do_store:
print "Store in S3"
for iter in range(num_gets):
chunk = generate_random_chunk(private_key,
iter,
identifier,
key=key,
size=chunk_size)
store_chunk(storage, vc_client, chunk)
if avoid_token_create:
token_storage = []
for block_id in range(num_gets):
token = generate_query_token(
identifier.owner, identifier.streamid, str(bytearray(16)),
identifier.get_key_for_blockid(block_id), private_key)
token_storage.append(token)
else:
token_storage = None
for round in range(num_rounds):
try:
time_keeper = TimeKeeper()
results = [[]] * num_threads
threads = [
FetchTalosThread(idx,
results,
TalosS3Storage(bucket_name),
block_id,
private_key,
identifier,
vc_client,
token_store=token_storage)
for idx, block_id in enumerate(
splitting(range(num_gets), num_threads))
]
time_keeper.start_clock()
map(lambda x: x.start(), threads)
map(lambda x: x.join(), threads)
time_keeper.stop_clock("time_fetch_all")
chunks = [item for sublist in results for item in sublist]
if len(chunks) == num_gets:
print "Round %d ok Num results: %d" % (round, num_gets)
else:
print "Round %d ok Num results: %d" % (round, num_gets)
out_logger.log_times_keeper(time_keeper)
except Exception as e:
print "Round %d error: %s" % (round, e)
print "DONE"
if do_delete:
clean_bucket(storage.s3, bucket_name)