def render_GET(self, request):
time_keeper = TimeKeeper()
time_id = time_keeper.start_clock_unique()
def respond(result):
time_keeper.stop_clock_unique(ENTRY_TOTAL_QUERY_CHUNK, time_id)
self.log.debug(
"%s %s %s" %
(BENCH_TAG, TYPE_QUERY_CHUNK_ADDR, time_keeper.get_summary()))
if result is None:
request.setResponseCode(400)
request.write("No Result found")
else:
request.setResponseCode(200)
request.write(result)
request.finish()
if len(request.prepath) < 2:
request.setResponseCode(400)
return json.dumps({'error': "Illegal URL"})
try:
chunk_key = binascii.unhexlify(request.prepath[1])
self.dhtstorage.get_addr_chunk(
chunk_key, time_keeper=time_keeper).addCallback(respond)
return NOT_DONE_YET
except TalosVCRestClientError:
request.setResponseCode(400)
return "ERROR: No Policy found"
except:
request.setResponseCode(400)
return "ERROR"
def render_POST(self, request):
time_keeper = TimeKeeper()
time_id = time_keeper.start_clock_unique()
def respond(result):
time_keeper.stop_clock_unique(ENTRY_TOTAL_ADD_CHUNK, time_id)
self.log.debug(
"%s %s %s" %
(BENCH_TAG, TYPE_ADD_CHUNK, time_keeper.get_summary()))
if not result is None:
request.setResponseCode(200)
request.write("OK")
else:
request.setResponseCode(400)
request.write("ERROR")
request.finish()
encoded_chunk = request.content.read()
try:
chunk = CloudChunk.decode(encoded_chunk)
self.dhtstorage.store_chunk(
chunk, time_keeper=time_keeper).addCallback(respond)
return NOT_DONE_YET
except InvalidChunkError:
request.setResponseCode(400)
return "ERROR: Invalid Chunk"
except TalosVCRestClientError:
request.setResponseCode(400)
return "ERROR: No Policy found"
except:
request.setResponseCode(400)
return "ERROR"
def run_loop(self,
image_capture,
time_file,
sym_key="a" * 16,
interval=3600):
while True:
try:
timer_chunk = TimeKeeper()
total_id = timer_chunk.start_clock_unique()
timestamp_data = int(time.time())
block_id = (timestamp_data - self.start_time) / interval
# Take a picture
picture_name = "%s%d.jpg" % (self.name, block_id)
image_capture(picture_name)
print picture_name
# load image
with open(picture_name, 'r') as f:
picture = f.read()
chunk_tmp = ChunkData()
chunk_tmp.add_entry(
PictureEntry(timestamp_data,
picture_name,
picture,
time_keeper=timer_chunk))
cur_time = timer()
cloud_chunk = self._generate_cloud_chunk(
block_id, sym_key, chunk_tmp, timer_chunk)
chunk_creation = timer() - cur_time
len_normal = len(chunk_tmp.encode(use_compression=False))
len_compressed = len(
compress_data(chunk_tmp.encode(use_compression=True)))
cloud_chunk_encoded = cloud_chunk.encode()
length_final = len(cloud_chunk_encoded)
cur_time = timer()
self._store_to_cloud(cloud_chunk_encoded)
chunk_store = timer() - cur_time
times = timer_chunk.logged_times
time_file.write(
"%s, %s, %s, %s, %s, %s, %d, %d, %d,\n" %
(times['chunk_compression'], times['gcm_encryption'],
times['ecdsa_signature'],
times['time_lepton_compression'], chunk_creation * 1000,
chunk_store * 1000, len_normal, len_compressed,
length_final))
time_file.flush()
timer_chunk.stop_clock_unique('time_total', total_id)
time.sleep(interval -
int(timer_chunk.logged_times['time_total'] / 1000))
except RuntimeError as e:
print e.message
logging.error("Exception occured %s" % e.message)
def render_POST(self, request):
if len(request.prepath) < 4:
request.setResponseCode(400)
return json.dumps({'error': "Illegal URL"})
try:
time_keeper = TimeKeeper()
total_time_id = time_keeper.start_clock_unique()
nodeid = unhexlify(request.prepath[1])
source_ip = request.client.host
source_port = int(request.prepath[2])
kad_key = unhexlify(request.prepath[3])
source = Node(nodeid, source_ip, source_port)
time_keeper.start_clock()
self.rpc_protocol.welcomeIfNewNode(source)
time_keeper.stop_clock(ENTRY_TIME_WELCOME_NODE)
encoded_chunk = request.content.read()
chunk = CloudChunk.decode(encoded_chunk)
if not digest(chunk.key) == kad_key:
request.setResponseCode(400)
return json.dumps({'error': "key missmatch"})
def handle_policy(policy):
time_keeper.stop_clock(ENTRY_FETCH_POLICY)
id = time_keeper.start_clock_unique()
self.storage.store_check_chunk(chunk,
None,
policy,
time_keeper=time_keeper)
time_keeper.stop_clock_unique(ENTRY_STORE_CHECK, id)
time_keeper.stop_clock_unique(ENTRY_TOTAL_STORE_LOCAL,
total_time_id)
self.log.debug("%s %s %s" % (BENCH_TAG, TYPE_STORE_CHUNK_LOCAL,
time_keeper.get_summary()))
request.write(json.dumps({'value': "ok"}))
request.finish()
time_keeper.start_clock()
self.talos_vc.get_policy_with_txid(
chunk.get_tag_hex()).addCallback(handle_policy)
return NOT_DONE_YET
except InvalidChunkError as e:
request.setResponseCode(400)
return json.dumps({'error': e.value})
except TalosVCRestClientError:
request.setResponseCode(400)
return "ERROR: No policy found"
except:
request.setResponseCode(400)
return json.dumps({'error': "Error occured"})
def callStore(self, nodeToAsk, key, value):
address = (nodeToAsk.ip, nodeToAsk.port)
time_keeper = TimeKeeper()
id = time_keeper.start_clock_unique()
if len(value) < MAX_UDP_SIZE:
d = self.store(address, self.sourceNode.id, key, value)
else:
d = self.http_client.call_store_large_chunk(nodeToAsk, key, value)
return d.addCallback(self.handleTimedCallResponse, nodeToAsk,
time_keeper, id, ENTRY_STORE_ONE_NODE)
def render_POST(self, request):
msg = json.loads(request.content.read())
timekeeper = TimeKeeper()
total_time_id = timekeeper.start_clock_unique()
try:
timekeeper.start_clock()
token = get_and_check_query_token(msg)
check_query_token_valid(token)
timekeeper.stop_clock(ENTRY_CHECK_TOKEN_VALID)
# Check nonce ok
if not self._check_cache(token.nonce):
raise InvalidQueryToken("Nonce not valid")
def handle_policy(policy):
timekeeper.stop_clock(ENTRY_FETCH_POLICY)
if policy is None:
request.setResponseCode(400)
request.write("No Policy Found")
request.finish()
# check policy for correctness
id = timekeeper.start_clock_unique()
chunk = self.storage.get_check_chunk(token.chunk_key,
token.pubkey,
policy,
time_keeper=timekeeper)
timekeeper.stop_clock_unique(ENTRY_GET_AND_CHECK, id)
timekeeper.stop_clock_unique(ENTRY_TOTAL_LOCAL_QUERY,
total_time_id)
self.log.debug("%s %s %s" % (BENCH_TAG, TYPE_QUERY_CHUNK_LOCAL,
timekeeper.get_summary()))
request.write(chunk.encode())
request.finish()
timekeeper.start_clock()
self.talos_vc.get_policy(token.owner,
token.streamid).addCallback(handle_policy)
return NOT_DONE_YET
except InvalidQueryToken:
request.setResponseCode(400)
return "ERROR: token verification failure"
except TalosVCRestClientError:
request.setResponseCode(400)
return "ERROR: No policy found"
except:
request.setResponseCode(400)
return "ERROR: error occured"
def rpc_store(self, sender, nodeid, key, value):
source = Node(nodeid, sender[0], sender[1])
time_keeper = TimeKeeper()
total_time_id = time_keeper.start_clock_unique()
time_keeper.start_clock()
self.welcomeIfNewNode(source)
time_keeper.stop_clock(ENTRY_TIME_WELCOME_NODE)
self.log.debug("got a store request from %s, storing value" %
str(sender))
try:
chunk = CloudChunk.decode(value)
if not digest(chunk.key) == key:
return {'error': 'key missmatch'}
def handle_policy(policy):
time_keeper.stop_clock(ENTRY_FETCH_POLICY)
# Hack no chunk id given -> no key checks, key is in the encoded chunk
id = time_keeper.start_clock_unique()
self.storage.store_check_chunk(chunk,
None,
policy,
time_keeper=time_keeper)
time_keeper.stop_clock_unique(ENTRY_STORE_CHECK, id)
time_keeper.stop_clock_unique(ENTRY_TOTAL_STORE_LOCAL,
total_time_id)
self.log.debug("%s %s %s" % (BENCH_TAG, TYPE_STORE_CHUNK_LOCAL,
time_keeper.get_summary()))
return {'value': 'ok'}
time_keeper.start_clock()
return self.talos_vc.get_policy_with_txid(
chunk.get_tag_hex()).addCallback(handle_policy)
except InvalidChunkError as e:
return {'error': e.value}
except TalosVCRestClientError:
return {'error': "No policy found"}
def run_benchmark_s3_talos(num_rounds,
out_logger,
bucket_name,
private_key=BitcoinVersionedPrivateKey(PRIVATE_KEY),
policy_nonce=base64.b64decode(NONCE),
stream_id=STREAMID,
txid=TXID,
chunk_size=100000,
do_delete=True,
do_sig=False):
key = os.urandom(32)
owner = private_key.public_key().address()
identifier = DataStreamIdentifier(owner, stream_id, policy_nonce, txid)
vc_client = TalosVCRestClient()
storage = TalosS3Storage(bucket_name)
for round_bench in range(num_rounds):
try:
time_keeper = TimeKeeper()
#chunk_data = generate_data(size=chunk_size)
chunk_data = DummyData(8500)
global_id = time_keeper.start_clock_unique()
chunk = generate_random_chunk_from_data(chunk_data,
private_key,
round_bench,
identifier,
time_keeper=time_keeper)
store_chunk(storage,
vc_client,
chunk,
time_keeper=time_keeper,
do_sig=do_sig)
time_keeper.stop_clock_unique("time_s3_store_chunk", global_id)
token_time_id = time_keeper.start_clock_unique()
token = generate_query_token(owner, stream_id, str(bytearray(16)),
chunk.key, private_key)
time_keeper.stop_clock_unique("time_token_create", token_time_id)
global_id = time_keeper.start_clock_unique()
chunk = fetch_chunk(storage,
vc_client,
token,
global_id=global_id,
time_keeper=time_keeper)
data = chunk.get_and_check_chunk_data(aes_key,
compression_used=False,
time_keeper=time_keeper,
do_decode=False)
time_keeper.stop_clock_unique("time_s3_get_chunk", global_id)
if chunk is None:
print "Round %d error" % round_bench
else:
print "Round %d ok Chunk size: %d" % (round_bench,
len(chunk.encode()))
out_logger.log_times_keeper(time_keeper)
except Exception as e:
print "Round %d error: %s" % (round_bench, e)
print "DONE"
if do_delete:
clean_bucket(storage.s3, bucket_name)
