def render_GET(self, request): time_keeper = TimeKeeper() time_id = time_keeper.start_clock_unique() def respond(result): time_keeper.stop_clock_unique(ENTRY_TOTAL_QUERY_CHUNK, time_id) self.log.debug( "%s %s %s" % (BENCH_TAG, TYPE_QUERY_CHUNK_ADDR, time_keeper.get_summary())) if result is None: request.setResponseCode(400) request.write("No Result found") else: request.setResponseCode(200) request.write(result) request.finish() if len(request.prepath) < 2: request.setResponseCode(400) return json.dumps({'error': "Illegal URL"}) try: chunk_key = binascii.unhexlify(request.prepath[1]) self.dhtstorage.get_addr_chunk( chunk_key, time_keeper=time_keeper).addCallback(respond) return NOT_DONE_YET except TalosVCRestClientError: request.setResponseCode(400) return "ERROR: No Policy found" except: request.setResponseCode(400) return "ERROR"
def render_POST(self, request): time_keeper = TimeKeeper() time_id = time_keeper.start_clock_unique() def respond(result): time_keeper.stop_clock_unique(ENTRY_TOTAL_ADD_CHUNK, time_id) self.log.debug( "%s %s %s" % (BENCH_TAG, TYPE_ADD_CHUNK, time_keeper.get_summary())) if not result is None: request.setResponseCode(200) request.write("OK") else: request.setResponseCode(400) request.write("ERROR") request.finish() encoded_chunk = request.content.read() try: chunk = CloudChunk.decode(encoded_chunk) self.dhtstorage.store_chunk( chunk, time_keeper=time_keeper).addCallback(respond) return NOT_DONE_YET except InvalidChunkError: request.setResponseCode(400) return "ERROR: Invalid Chunk" except TalosVCRestClientError: request.setResponseCode(400) return "ERROR: No Policy found" except: request.setResponseCode(400) return "ERROR"
def run_loop(self, image_capture, time_file, sym_key="a" * 16, interval=3600): while True: try: timer_chunk = TimeKeeper() total_id = timer_chunk.start_clock_unique() timestamp_data = int(time.time()) block_id = (timestamp_data - self.start_time) / interval # Take a picture picture_name = "%s%d.jpg" % (self.name, block_id) image_capture(picture_name) print picture_name # load image with open(picture_name, 'r') as f: picture = f.read() chunk_tmp = ChunkData() chunk_tmp.add_entry( PictureEntry(timestamp_data, picture_name, picture, time_keeper=timer_chunk)) cur_time = timer() cloud_chunk = self._generate_cloud_chunk( block_id, sym_key, chunk_tmp, timer_chunk) chunk_creation = timer() - cur_time len_normal = len(chunk_tmp.encode(use_compression=False)) len_compressed = len( compress_data(chunk_tmp.encode(use_compression=True))) cloud_chunk_encoded = cloud_chunk.encode() length_final = len(cloud_chunk_encoded) cur_time = timer() self._store_to_cloud(cloud_chunk_encoded) chunk_store = timer() - cur_time times = timer_chunk.logged_times time_file.write( "%s, %s, %s, %s, %s, %s, %d, %d, %d,\n" % (times['chunk_compression'], times['gcm_encryption'], times['ecdsa_signature'], times['time_lepton_compression'], chunk_creation * 1000, chunk_store * 1000, len_normal, len_compressed, length_final)) time_file.flush() timer_chunk.stop_clock_unique('time_total', total_id) time.sleep(interval - int(timer_chunk.logged_times['time_total'] / 1000)) except RuntimeError as e: print e.message logging.error("Exception occured %s" % e.message)
def render_POST(self, request): if len(request.prepath) < 4: request.setResponseCode(400) return json.dumps({'error': "Illegal URL"}) try: time_keeper = TimeKeeper() total_time_id = time_keeper.start_clock_unique() nodeid = unhexlify(request.prepath[1]) source_ip = request.client.host source_port = int(request.prepath[2]) kad_key = unhexlify(request.prepath[3]) source = Node(nodeid, source_ip, source_port) time_keeper.start_clock() self.rpc_protocol.welcomeIfNewNode(source) time_keeper.stop_clock(ENTRY_TIME_WELCOME_NODE) encoded_chunk = request.content.read() chunk = CloudChunk.decode(encoded_chunk) if not digest(chunk.key) == kad_key: request.setResponseCode(400) return json.dumps({'error': "key missmatch"}) def handle_policy(policy): time_keeper.stop_clock(ENTRY_FETCH_POLICY) id = time_keeper.start_clock_unique() self.storage.store_check_chunk(chunk, None, policy, time_keeper=time_keeper) time_keeper.stop_clock_unique(ENTRY_STORE_CHECK, id) time_keeper.stop_clock_unique(ENTRY_TOTAL_STORE_LOCAL, total_time_id) self.log.debug("%s %s %s" % (BENCH_TAG, TYPE_STORE_CHUNK_LOCAL, time_keeper.get_summary())) request.write(json.dumps({'value': "ok"})) request.finish() time_keeper.start_clock() self.talos_vc.get_policy_with_txid( chunk.get_tag_hex()).addCallback(handle_policy) return NOT_DONE_YET except InvalidChunkError as e: request.setResponseCode(400) return json.dumps({'error': e.value}) except TalosVCRestClientError: request.setResponseCode(400) return "ERROR: No policy found" except: request.setResponseCode(400) return json.dumps({'error': "Error occured"})
def callStore(self, nodeToAsk, key, value): address = (nodeToAsk.ip, nodeToAsk.port) time_keeper = TimeKeeper() id = time_keeper.start_clock_unique() if len(value) < MAX_UDP_SIZE: d = self.store(address, self.sourceNode.id, key, value) else: d = self.http_client.call_store_large_chunk(nodeToAsk, key, value) return d.addCallback(self.handleTimedCallResponse, nodeToAsk, time_keeper, id, ENTRY_STORE_ONE_NODE)
def render_POST(self, request): msg = json.loads(request.content.read()) timekeeper = TimeKeeper() total_time_id = timekeeper.start_clock_unique() try: timekeeper.start_clock() token = get_and_check_query_token(msg) check_query_token_valid(token) timekeeper.stop_clock(ENTRY_CHECK_TOKEN_VALID) # Check nonce ok if not self._check_cache(token.nonce): raise InvalidQueryToken("Nonce not valid") def handle_policy(policy): timekeeper.stop_clock(ENTRY_FETCH_POLICY) if policy is None: request.setResponseCode(400) request.write("No Policy Found") request.finish() # check policy for correctness id = timekeeper.start_clock_unique() chunk = self.storage.get_check_chunk(token.chunk_key, token.pubkey, policy, time_keeper=timekeeper) timekeeper.stop_clock_unique(ENTRY_GET_AND_CHECK, id) timekeeper.stop_clock_unique(ENTRY_TOTAL_LOCAL_QUERY, total_time_id) self.log.debug("%s %s %s" % (BENCH_TAG, TYPE_QUERY_CHUNK_LOCAL, timekeeper.get_summary())) request.write(chunk.encode()) request.finish() timekeeper.start_clock() self.talos_vc.get_policy(token.owner, token.streamid).addCallback(handle_policy) return NOT_DONE_YET except InvalidQueryToken: request.setResponseCode(400) return "ERROR: token verification failure" except TalosVCRestClientError: request.setResponseCode(400) return "ERROR: No policy found" except: request.setResponseCode(400) return "ERROR: error occured"
def rpc_store(self, sender, nodeid, key, value): source = Node(nodeid, sender[0], sender[1]) time_keeper = TimeKeeper() total_time_id = time_keeper.start_clock_unique() time_keeper.start_clock() self.welcomeIfNewNode(source) time_keeper.stop_clock(ENTRY_TIME_WELCOME_NODE) self.log.debug("got a store request from %s, storing value" % str(sender)) try: chunk = CloudChunk.decode(value) if not digest(chunk.key) == key: return {'error': 'key missmatch'} def handle_policy(policy): time_keeper.stop_clock(ENTRY_FETCH_POLICY) # Hack no chunk id given -> no key checks, key is in the encoded chunk id = time_keeper.start_clock_unique() self.storage.store_check_chunk(chunk, None, policy, time_keeper=time_keeper) time_keeper.stop_clock_unique(ENTRY_STORE_CHECK, id) time_keeper.stop_clock_unique(ENTRY_TOTAL_STORE_LOCAL, total_time_id) self.log.debug("%s %s %s" % (BENCH_TAG, TYPE_STORE_CHUNK_LOCAL, time_keeper.get_summary())) return {'value': 'ok'} time_keeper.start_clock() return self.talos_vc.get_policy_with_txid( chunk.get_tag_hex()).addCallback(handle_policy) except InvalidChunkError as e: return {'error': e.value} except TalosVCRestClientError: return {'error': "No policy found"}
def run_benchmark_s3_talos(num_rounds, out_logger, bucket_name, private_key=BitcoinVersionedPrivateKey(PRIVATE_KEY), policy_nonce=base64.b64decode(NONCE), stream_id=STREAMID, txid=TXID, chunk_size=100000, do_delete=True, do_sig=False): key = os.urandom(32) owner = private_key.public_key().address() identifier = DataStreamIdentifier(owner, stream_id, policy_nonce, txid) vc_client = TalosVCRestClient() storage = TalosS3Storage(bucket_name) for round_bench in range(num_rounds): try: time_keeper = TimeKeeper() #chunk_data = generate_data(size=chunk_size) chunk_data = DummyData(8500) global_id = time_keeper.start_clock_unique() chunk = generate_random_chunk_from_data(chunk_data, private_key, round_bench, identifier, time_keeper=time_keeper) store_chunk(storage, vc_client, chunk, time_keeper=time_keeper, do_sig=do_sig) time_keeper.stop_clock_unique("time_s3_store_chunk", global_id) token_time_id = time_keeper.start_clock_unique() token = generate_query_token(owner, stream_id, str(bytearray(16)), chunk.key, private_key) time_keeper.stop_clock_unique("time_token_create", token_time_id) global_id = time_keeper.start_clock_unique() chunk = fetch_chunk(storage, vc_client, token, global_id=global_id, time_keeper=time_keeper) data = chunk.get_and_check_chunk_data(aes_key, compression_used=False, time_keeper=time_keeper, do_decode=False) time_keeper.stop_clock_unique("time_s3_get_chunk", global_id) if chunk is None: print "Round %d error" % round_bench else: print "Round %d ok Chunk size: %d" % (round_bench, len(chunk.encode())) out_logger.log_times_keeper(time_keeper) except Exception as e: print "Round %d error: %s" % (round_bench, e) print "DONE" if do_delete: clean_bucket(storage.s3, bucket_name)