def check_check_func(self):
key = BitcoinVersionedPrivateKey(
"cN5YgNRq8rbcJwngdp3fRzv833E7Z74TsF8nB6GhzRg8Gd9aGWH1")
client = TalosVCRestClient()
chunk = generate_random_chunk(1)
policy = client.get_policy_with_txid(chunk.get_tag_hex())
def get_bitcoin_address_for_pubkey_tmp(hex_pubkey):
before = timer()
priv = extract_bin_bitcoin_pubkey(hex_pubkey)
hash_priv = get_bin_hash160(priv)
addr = bin_hash160_to_address(hash_priv, version_byte=111)
print "Time creation %s" % ((timer() - before) * 1000, )
return addr
def check_access_allowed_tmp(hex_pubkey, policy):
before = timer()
addr = get_bitcoin_address_for_pubkey_tmp(str(hex_pubkey))
print "Bitcoin_lib %s" % ((timer() - before) * 1000, )
if addr == policy.owner:
return True
if addr in policy.shares:
return True
return False
self.assertTrue(
check_access_allowed_tmp(key.public_key().to_hex(), policy))
def test_storage1(self):
key = BitcoinVersionedPrivateKey(
"cN5YgNRq8rbcJwngdp3fRzv833E7Z74TsF8nB6GhzRg8Gd9aGWH1")
talosStorage = TalosLevelDBDHTStorage("db_tmp")
client = TalosVCRestClient()
num_iter = 100
for i in range(num_iter):
chunk = generate_random_chunk(i)
policy = client.get_policy_with_txid(chunk.get_tag_hex())
before = timer()
talosStorage.store_check_chunk(chunk, i, policy)
print "Time store %s" % ((timer() - before) * 1000, )
keeper = TimeKeeper()
before = timer()
talosStorage.get_check_chunk(chunk.key,
key.public_key().to_hex(),
policy,
time_keeper=keeper)
print "Time get %s" % ((timer() - before) * 1000, )
count = 0
for (key, value) in talosStorage.iteritemsOlderThan(100):
count += 1
self.assertEquals(0, count)
count = 0
for (key, value) in talosStorage.iteritems():
count += 1
self.assertEquals(num_iter, count)
time.sleep(6)
count = 0
for (key, value) in talosStorage.iteritemsOlderThan(5):
count += 1
self.assertEquals(num_iter, count)
def test_chunk1(self):
client = TalosVCRestClient()
for i in range(100):
chunk = generate_random_chunk(i)
policy = client.get_policy_with_txid(chunk.get_tag_hex())
before = timer()
check_chunk_valid(chunk, policy, chunk_id=i)
print "Time check %s" % ((timer() - before) * 1000, )
def test_cross(self):
client = TalosVCRestClient()
for i in range(100):
chunk = generate_random_chunk(i)
policy = client.get_policy_with_txid(chunk.get_tag_hex())
before = timer()
pub_key = get_crypto_ecdsa_pubkey_from_bitcoin_hex(
str(policy.owner_pk))
print "Time check %s" % ((timer() - before) * 1000, )
def __init__(self,
sourceNode,
storage,
ksize,
talos_vc=TalosVCRestClient()):
TalosRPCProtocol.__init__(self)
self.router = TalosKademliaRoutingTable(self, ksize, sourceNode)
self.storage = storage
self.sourceNode = sourceNode
self.log = Logger(system=self)
self.talos_vc = talos_vc
self.http_client = None
def __init__(self,
storage,
talos_vc=TalosVCRestClient(),
max_nonce_cache=1000,
nonce_ttl=10):
Resource.__init__(self)
self.storage = storage
self.log = Logger(system=self)
self.talos_vc = talos_vc
self.nonce_cache = TTLCache(max_nonce_cache, nonce_ttl)
self.refreshLoop = LoopingCall(self.nonce_cache.expire).start(3600)
self.sem = Semaphore(1)
def __init__(self,
ecdsa_privkey,
sourceNode,
storage,
ksize,
talos_vc=TalosVCRestClient(),
cbits=10,
bench_mode=True):
TalosWeakSignedRPCProtocol.__init__(self,
ecdsa_privkey,
sourceNode.id,
cbits=cbits)
self.router = TalosKademliaRoutingTable(self, ksize, sourceNode)
self.storage = storage
self.sourceNode = sourceNode
self.log = Logger(system=self)
self.talos_vc = talos_vc
self.http_client = None
self.bench_mode = bench_mode
def test_storage(self):
key = BitcoinVersionedPrivateKey(
"cN5YgNRq8rbcJwngdp3fRzv833E7Z74TsF8nB6GhzRg8Gd9aGWH1")
talosStorage = LevelDBStorage("db_tmp")
client = TalosVCRestClient()
for i in range(100):
chunk = generate_random_chunk(i)
policy = client.get_policy_with_txid(chunk.get_tag_hex())
before = timer()
talosStorage.store_check_chunk(chunk, i, policy)
print "Time store %s" % ((timer() - before) * 1000, )
keeper = TimeKeeper()
before = timer()
talosStorage.get_check_chunk(chunk.key,
key.public_key().to_hex(),
policy,
time_keeper=keeper)
print "Time get %s" % ((timer() - before) * 1000, )
for (key, value) in talosStorage.db.RangeIter():
print base64.b64encode(key)
print base64.b64encode(value)
def run_benchmark_s3_talos_fetch(
num_rounds,
num_gets,
out_logger,
bucket_name,
private_key=BitcoinVersionedPrivateKey(PRIVATE_KEY),
policy_nonce=base64.b64decode(NONCE),
stream_id=STREAMID,
txid=TXID,
chunk_size=100000,
num_threads=None,
do_store=True,
do_delete=True,
avoid_token_create=True):
key = os.urandom(32)
owner = private_key.public_key().address()
identifier = DataStreamIdentifier(owner, stream_id, policy_nonce, txid)
vc_client = TalosVCRestClient()
storage = TalosS3Storage(bucket_name)
num_threads = num_threads or num_gets
if do_store:
print "Store in S3"
for iter in range(num_gets):
chunk = generate_random_chunk(private_key,
iter,
identifier,
key=key,
size=chunk_size)
store_chunk(storage, vc_client, chunk)
if avoid_token_create:
token_storage = []
for block_id in range(num_gets):
token = generate_query_token(
identifier.owner, identifier.streamid, str(bytearray(16)),
identifier.get_key_for_blockid(block_id), private_key)
token_storage.append(token)
else:
token_storage = None
for round in range(num_rounds):
try:
time_keeper = TimeKeeper()
results = [[]] * num_threads
threads = [
FetchTalosThread(idx,
results,
TalosS3Storage(bucket_name),
block_id,
private_key,
identifier,
vc_client,
token_store=token_storage)
for idx, block_id in enumerate(
splitting(range(num_gets), num_threads))
]
time_keeper.start_clock()
map(lambda x: x.start(), threads)
map(lambda x: x.join(), threads)
time_keeper.stop_clock("time_fetch_all")
chunks = [item for sublist in results for item in sublist]
if len(chunks) == num_gets:
print "Round %d ok Num results: %d" % (round, num_gets)
else:
print "Round %d ok Num results: %d" % (round, num_gets)
out_logger.log_times_keeper(time_keeper)
except Exception as e:
print "Round %d error: %s" % (round, e)
print "DONE"
if do_delete:
clean_bucket(storage.s3, bucket_name)
def run_benchmark_s3_talos(num_rounds,
out_logger,
bucket_name,
private_key=BitcoinVersionedPrivateKey(PRIVATE_KEY),
policy_nonce=base64.b64decode(NONCE),
stream_id=STREAMID,
txid=TXID,
chunk_size=100000,
do_delete=True,
do_sig=False):
key = os.urandom(32)
owner = private_key.public_key().address()
identifier = DataStreamIdentifier(owner, stream_id, policy_nonce, txid)
vc_client = TalosVCRestClient()
storage = TalosS3Storage(bucket_name)
for round_bench in range(num_rounds):
try:
time_keeper = TimeKeeper()
#chunk_data = generate_data(size=chunk_size)
chunk_data = DummyData(8500)
global_id = time_keeper.start_clock_unique()
chunk = generate_random_chunk_from_data(chunk_data,
private_key,
round_bench,
identifier,
time_keeper=time_keeper)
store_chunk(storage,
vc_client,
chunk,
time_keeper=time_keeper,
do_sig=do_sig)
time_keeper.stop_clock_unique("time_s3_store_chunk", global_id)
token_time_id = time_keeper.start_clock_unique()
token = generate_query_token(owner, stream_id, str(bytearray(16)),
chunk.key, private_key)
time_keeper.stop_clock_unique("time_token_create", token_time_id)
global_id = time_keeper.start_clock_unique()
chunk = fetch_chunk(storage,
vc_client,
token,
global_id=global_id,
time_keeper=time_keeper)
data = chunk.get_and_check_chunk_data(aes_key,
compression_used=False,
time_keeper=time_keeper,
do_decode=False)
time_keeper.stop_clock_unique("time_s3_get_chunk", global_id)
if chunk is None:
print "Round %d error" % round_bench
else:
print "Round %d ok Chunk size: %d" % (round_bench,
len(chunk.encode()))
out_logger.log_times_keeper(time_keeper)
except Exception as e:
print "Round %d error: %s" % (round_bench, e)
print "DONE"
if do_delete:
clean_bucket(storage.s3, bucket_name)
#© 2017-2020, ETH Zurich, D-INFK, [email protected] from flask import Flask from flask import request, g from talosstorage.checks import check_query_token_valid, get_and_check_query_token from talosstorage.chunkdata import CloudChunk from talosstorage.storage import InvalidChunkError, InvalidAccess, InvalidQueryToken from talosvc.talosclient.restapiclient import TalosVCRestClient app = Flask("Talos-Storage-LevelDB") MAX_TIME = 10 client = TalosVCRestClient() storage_impl = None def set_storage_impl(impl): global storage_impl storage_impl = impl def set_vc_client(vc_client): global client client = vc_client def get_storage(): state = getattr(g, '_storage', None)
def __init__(self, storage, rpc_protocol, talos_vc=TalosVCRestClient()):
Resource.__init__(self)
self.storage = storage
self.log = Logger(system=self)
self.talos_vc = talos_vc
self.rpc_protocol = rpc_protocol
import argparse
from talosvc.talosclient.restapiclient import TalosVCRestClient
from talosstorage.restapi import app, set_storage_impl, set_vc_client
from talosstorage.storage import LevelDBStorage
if __name__ == "__main__":
parser = argparse.ArgumentParser("Run storage server client")
parser.add_argument('--vcport', type=int, help='dir', default=5000, required=False)
parser.add_argument('--vcserver', type=str, help='server', default="127.0.0.1", required=False)
parser.add_argument('--port', type=int, help='dir', default=12000, required=False)
parser.add_argument('--server', type=str, help='server', default="127.0.0.1", required=False)
args = parser.parse_args()
VC_IP = args.vcserver
VC_PORT = args.vcport
client = TalosVCRestClient(ip=args.vcserver, port=args.vcport)
set_storage_impl(LevelDBStorage("./leveldb"))
set_vc_client(client)
app.run(debug=False, host=args.server, port=args.port)