class TestServerThread(threading.Thread):
def __init__(self, port, home_dir, neighbour=None, is_monitor=False, \
config={}, ks_path='', clear_home=True):
threading.Thread.__init__(self)
if clear_home and os.path.exists(home_dir):
shutil.rmtree(home_dir)
if not os.path.exists(home_dir):
os.mkdir(home_dir)
self.port = port
self.home_dir = home_dir
self.stopped = True
self.operator = None
self.neighbour = neighbour
self.config = config
self.is_monitor = is_monitor
self.ks_path = ks_path
self.ks = None if not ks_path else init_keystore(ks_path, PASSWD)
self.__lock = threading.Lock()
def run(self):
os.system('cp ./tests/ks/certs.ca %s'%self.home_dir)
address = '127.0.0.1:%s'%self.port
if self.is_monitor:
node_type = 'TestMonitor'
else:
node_type = 'DHT'
config = {'INIT_DHT_WAIT_NEIGHBOUR_TIMEOUT': 0.1,
'MONITOR_DHT_RANGES_TIMEOUT': 1,
'CHECK_HASH_TABLE_TIMEOUT': 1,
'WAIT_FILE_MD_TIMEDELTA': 0.1,
'WAIT_DHT_TABLE_UPDATE': 0.2}
config.update(self.config)
node = Node('127.0.0.1', self.port, self.home_dir, 'node_%s'%self.port,
ks_path=self.ks_path, ks_passwd=PASSWD, node_type=node_type, config=config)
node.set_auth_key(None)
node.start(self.neighbour)
self.__lock.acquire()
try:
self.operator = OperatorClient('node_%s'%self.port)
self.stopped = False
finally:
self.__lock.release()
while not self.stopped:
time.sleep(0.1)
node.stop()
def stop(self):
self.stopped = True
self.join()
def wait_oper_status(self, status, errmsg=''):
for i in xrange(20):
if self.get_status() == status:
return
time.sleep(.25)
raise Exception('[%s].wait_oper_status(%s) timeouted! %s'%(self.port, status, errmsg))
def get_range_dir(self):
return os.path.join(self.home_dir, 'dht_range/%s'%FSMappedDHTRange.DBCT_MASTER)
def get_replicas_dir(self):
return os.path.join(self.home_dir, 'dht_range/%s'%FSMappedDHTRange.DBCT_REPLICA)
def get_tmp_dir(self):
return os.path.join(self.home_dir, 'dht_range/%s'%FSMappedDHTRange.DBCT_TEMP)
def get_stat(self):
packet_obj = FabnetPacketRequest(method='NodeStatistic')
client = FriClient(self.ks)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
if ret_packet.ret_code != 0:
raise Exception('cant get node statistic. details: %s'%ret_packet.ret_message)
return ret_packet.ret_parameters
def get_status(self):
self.__lock.acquire()
try:
if self.operator is None:
return None
return self.operator.get_status()
finally:
self.__lock.release()
def put_data_block(self, data, key, dbct=FSMappedDHTRange.DBCT_MASTER, user_id_hash='', \
careful_save=False, init_block=False, stored_unixtime=None):
params = {'key': key, 'dbct': dbct, 'user_id_hash': user_id_hash, 'init_block': init_block,\
'carefully_save': careful_save, 'stored_unixtime': stored_unixtime}
req = FabnetPacketRequest(method='PutDataBlock',\
binary_data=data, parameters=params)
client = FriClient(self.ks)
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
return resp
def put(self, data, wait_writes=3, init_block=True, client_ks=None, key=None, replica_count=2):
if not client_ks:
client_ks = self.ks
client = FriClient(client_ks)
params = {'wait_writes_count': wait_writes, 'replica_count': replica_count, \
'init_block': init_block, 'key': key}
data = RamBasedBinaryData(data, 20)
packet_obj = FabnetPacketRequest(method='ClientPutData', parameters=params, binary_data=data, sync=True)
#print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count())
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
#print '========SENDED DATA BLOCK'
return ret_packet
def get_keys_info(self, key, replica_count=2):
client = FriClient(self.ks)
packet = FabnetPacketRequest(method='GetKeysInfo', \
parameters={'key': key, 'replica_count': replica_count}, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet)
if ret_packet.ret_code != 0:
raise Exception('GetKeysInfo error: %s'%ret_packet.ret_message)
keys_info = ret_packet.ret_parameters['keys_info']
return keys_info
def get_data_block(self, key, dbct=FSMappedDHTRange.DBCT_MASTER, client_ks=None, user_id_hash=''):
params = {'key': key, 'dbct': dbct, 'user_id_hash': user_id_hash}
req = FabnetPacketRequest(method='GetDataBlock', parameters=params)
client = FriClient(client_ks)
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
return resp
def delete(self, key, client_ks=None, replica_count=2):
client = FriClient(client_ks)
params = {'key': key, 'replica_count': replica_count}
packet_obj = FabnetPacketRequest(method='ClientDeleteData', parameters=params, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
return ret_packet
def check_data_block(self, key, dbct=FSMappedDHTRange.DBCT_MASTER, checksum=None):
client = FriClient(self.ks)
params = {'key': key, 'dbct': dbct, 'checksum': checksum}
packet_obj = FabnetPacketRequest(method='CheckDataBlock', parameters=params)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
return ret_packet
def update_user_md(self, user_id_hash, storage_size):
client = FriClient(self.ks)
params = {'user_id_hash': user_id_hash, 'storage_size': storage_size}
packet_obj = FabnetPacketRequest(method='UpdateUserProfile', parameters=params)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
return ret_packet
def update_md(self, user_id_hash, add_list, rm_list=[]):
client = FriClient(self.ks)
params = {'user_id_hash': user_id_hash, 'add_list': add_list, 'rm_list': rm_list}
packet_obj = FabnetPacketRequest(method='UpdateMetadata', parameters=params)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
return ret_packet
def put_object_part(self, obj_path, data, seek=0, wait_writes=3, init_block=True, client_ks=None, key=None, replica_count=2):
if not client_ks:
client_ks = self.ks
client = FriClient(client_ks)
params = {'wait_writes_count': wait_writes, 'replica_count': replica_count, \
'init_block': init_block, 'key': key, 'obj_path': obj_path, 'seek': seek}
data = RamBasedBinaryData(data, 20)
packet_obj = FabnetPacketRequest(method='PutObjectPart', parameters=params, binary_data=data, sync=True)
#print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count())
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
#print '========SENDED DATA BLOCK'
return ret_packet
def get_object_info(self, obj_path, client_ks, req_user_info=True):
params = {'obj_path': obj_path, 'req_user_info': req_user_info}
req = FabnetPacketRequest(method='GetObjectInfo', parameters=params)
client = FriClient(client_ks)
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
return resp
def get(self, key, client_ks=None):
client = FriClient(client_ks)
params = {'key': key, 'replica_count': 2}
packet_obj = FabnetPacketRequest(method='ClientGetData', parameters=params, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
return ret_packet
class TestServerThread(threading.Thread):
SESSION_ID = 1234322321
def __init__(self, port, home_dir, neighbour=None, is_monitor=False, config={}):
threading.Thread.__init__(self)
self.port = port
self.home_dir = home_dir
self.stopped = True
self.operator = None
self.neighbour = neighbour
self.config = config
self.is_monitor = is_monitor
self.__lock = threading.Lock()
def run(self):
address = '127.0.0.1:%s'%self.port
if self.is_monitor:
node_type = 'Monitor'
else:
node_type = 'DHT'
config = {'WAIT_RANGE_TIMEOUT': 0.1,
'INIT_DHT_WAIT_NEIGHBOUR_TIMEOUT': 0.1,
'MONITOR_DHT_RANGES_TIMEOUT': 1,
'CHECK_HASH_TABLE_TIMEOUT': 1,
'WAIT_FILE_MD_TIMEDELTA': 0.1,
'WAIT_DHT_TABLE_UPDATE': 0.2}
config.update(self.config)
node = Node('127.0.0.1', self.port, self.home_dir, 'node_%s'%self.port,
ks_path=None, ks_passwd=None, node_type=node_type, config=config)
node.start(self.neighbour)
self.__lock.acquire()
try:
self.operator = OperatorClient('node_%s'%self.port)
self.stopped = False
finally:
self.__lock.release()
while not self.stopped:
time.sleep(0.1)
node.stop()
def stop(self):
self.stopped = True
self.join()
def get_stat(self):
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
client = FriClient()
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
if ret_packet.ret_code != 0:
raise Exception('cant get node statistic. details: %s'%ret_packet.ret_message)
return ret_packet.ret_parameters
def put_data_block(self, data, key, is_replica=False, npkey=False, user_id=0):
checksum = hashlib.sha1(data).hexdigest()
params = {'key': key, 'checksum': checksum, 'is_replica':is_replica, \
'carefully_save': True, 'replica_count': 2, 'user_id': user_id}
if not npkey:
params['primary_key'] = key
req = FabnetPacketRequest(method='PutDataBlock',\
binary_data=data, sync=True,
parameters=params)
client = FriClient()
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
return resp
def get_data_block(self, key, is_replica=False, user_id=0):
params = {'key': key, 'is_replica':is_replica}
req = FabnetPacketRequest(method='GetDataBlock',\
sync=True, parameters=params)
client = FriClient(session_id=user_id or self.SESSION_ID)
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
if resp.ret_code == constants.RC_NO_DATA:
return None
if resp.ret_code != 0:
raise Exception('GetDataBlock operation failed on %s. Details: %s'%('127.0.0.1:%s'%self.port, resp.ret_message))
return resp.binary_data
def put(self, data, wait_writes=3):
client = FriClient(session_id=self.SESSION_ID)
checksum = hashlib.sha1(data).hexdigest()
params = {'checksum': checksum, 'wait_writes_count': wait_writes, 'replica_count':2}
data = RamBasedBinaryData(data, 20)
packet_obj = FabnetPacketRequest(method='ClientPutData', parameters=params, binary_data=data, sync=True)
#print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count())
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
#print '========SENDED DATA BLOCK'
if ret_packet.ret_code != 0:
raise Exception('put data failed: %s'%ret_packet.ret_message)
return ret_packet.ret_parameters['key']
def get(self, key):
client = FriClient(session_id=self.SESSION_ID)
params = {'key': key, 'replica_count': 2}
packet_obj = FabnetPacketRequest(method='ClientGetData', parameters=params, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
return ret_packet
def delete(self, key, session_id=None):
client = FriClient(session_id=session_id or self.SESSION_ID)
params = {'key': key, 'replica_count': 2}
packet_obj = FabnetPacketRequest(method='ClientDeleteData', parameters=params, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
return ret_packet
def get_range_dir(self):
stat = self.get_stat()
h_start = stat['DHTInfo']['range_start']
h_end = stat['DHTInfo']['range_end']
return os.path.join(self.home_dir, 'dht_range/%s_%s'%(h_start, h_end))
def get_keys_info(self, key):
client = FriClient(session_id=self.SESSION_ID)
packet = FabnetPacketRequest(method='GetKeysInfo', \
parameters={'key': key, 'replica_count': 2}, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet)
keys_info = ret_packet.ret_parameters['keys_info']
return keys_info
def get_replicas_dir(self):
return os.path.join(self.home_dir, 'dht_range/replica_data')
def get_tmp_dir(self):
return os.path.join(self.home_dir, 'dht_range/tmp')
def get_status(self):
self.__lock.acquire()
try:
if self.operator is None:
return None
return self.operator.get_status()
finally:
self.__lock.release()
