def run_benchmark_fetch_par(num_rounds, data_path, num_entries, granularity, fetch_granularity, num_threads, out_logger,
private_key=BitcoinVersionedPrivateKey(PRIVATE_KEY),
policy_nonce=base64.b64decode(NONCE), stream_id=STREAMID,
txid=TXID, ip=IP, port=PORT):
key = os.urandom(32)
identifier = DataStreamIdentifier(private_key.public_key().address(), stream_id, policy_nonce, txid)
dht_api_client = DHTRestClient(dhtip=ip, dhtport=port)
for block_id, chunk_data in enumerate(extract_eth_smartmeter_data(data_path, granularity, max_entries=num_entries)):
try:
chunk = generate_cloud_chunk(chunk_data, private_key, block_id, identifier, key=key)
dht_api_client.store_chunk(chunk)
print "Store chunk %d Num entries: %d" % (block_id, len(chunk_data.entries))
except DHTRestClientException as e:
print "Store round %d error: %s" % (block_id, e)
num_fetches = num_entries / granularity
if not num_entries % granularity == 0:
num_fetches += 1
for x in fetch_granularity:
num_fetches = x / granularity
if not x % granularity == 0:
num_fetches += 1
for round in range(num_rounds):
time_keeper = TimeKeeper()
results = [[]] * num_threads
if num_fetches < num_threads:
temp_threads = num_fetches
print "Num Fetches: %d Temp_threads %d" % (num_fetches, temp_threads)
threads = [Fetchjob(idx, results, DHTRestClient(dhtip=ip, dhtport=port), block_id, private_key, identifier)
for idx, block_id in enumerate(splitting(range(num_fetches), temp_threads))]
else:
threads = [Fetchjob(idx, results, DHTRestClient(dhtip=ip, dhtport=port), block_id, private_key, identifier)
for idx, block_id in enumerate(splitting(range(num_fetches), num_threads))]
time_keeper.start_clock()
map(lambda x: x.start(), threads)
map(lambda x: x.join(), threads)
time_keeper.stop_clock("time_fetch_all")
time_keeper.store_value("num_entries", x)
time_keeper.store_value("num_blocks", num_fetches)
time_keeper.store_value("round", round)
chunks = [item for sublist in results for item in sublist]
if len(chunks) == num_fetches:
print "Round %d ok Num results: %d" % (round, num_fetches)
else:
print "Round %d ok Num results: %d" % (round, num_fetches)
for idx, chunk in enumerate(chunks):
if chunk is None:
print "No result for chunk %d " % idx
out_logger.log_times_keeper(time_keeper)
out_logger.flush_to_db()
print "DONE"
else:
logger = SQLLiteBenchmarkLogger(args.log_db, LOGGING_FIELDS, "%s" % (args.name,))
try:
for chunk_size in args.chunk_sizes:
time_keeper = TimeKeeper()
size_plain = 0
size_compressed = 0
print "Chunk size: %d" % chunk_size
if args.do_smartmeter:
for chunk in extract_eth_smartmeter_data(args.data_path, chunk_size):
encoded = chunk.encode()
data_compressed = compress_data(encoded)
#print "Before: %d After: %d" % (len(encoded), len(data_compressed))
size_plain += len(encoded)
size_compressed += len(data_compressed)
else:
for chunk in extract_eth_plug_data(args.data_path, chunk_size, 1, 1):
encoded = chunk.encode()
data_compressed = compress_data(encoded)
print "Before: %d After: %d" % (len(encoded), len(data_compressed))
size_plain += len(encoded)
size_compressed += len(data_compressed)
print "Chunk size: %d DONE" % chunk_size
time_keeper.store_value("num_chunk_entries", chunk_size)
time_keeper.store_value("size_before", size_plain)
time_keeper.store_value("size_compressed", size_compressed)
logger.log_times_keeper(time_keeper)
finally:
logger.close()
