class TestServerThread(threading.Thread): def __init__(self, port, home_dir, neighbour=None, is_monitor=False, \ config={}, ks_path='', clear_home=True): threading.Thread.__init__(self) if clear_home and os.path.exists(home_dir): shutil.rmtree(home_dir) if not os.path.exists(home_dir): os.mkdir(home_dir) self.port = port self.home_dir = home_dir self.stopped = True self.operator = None self.neighbour = neighbour self.config = config self.is_monitor = is_monitor self.ks_path = ks_path self.ks = None if not ks_path else init_keystore(ks_path, PASSWD) self.__lock = threading.Lock() def run(self): os.system('cp ./tests/ks/certs.ca %s'%self.home_dir) address = '127.0.0.1:%s'%self.port if self.is_monitor: node_type = 'TestMonitor' else: node_type = 'DHT' config = {'INIT_DHT_WAIT_NEIGHBOUR_TIMEOUT': 0.1, 'MONITOR_DHT_RANGES_TIMEOUT': 1, 'CHECK_HASH_TABLE_TIMEOUT': 1, 'WAIT_FILE_MD_TIMEDELTA': 0.1, 'WAIT_DHT_TABLE_UPDATE': 0.2} config.update(self.config) node = Node('127.0.0.1', self.port, self.home_dir, 'node_%s'%self.port, ks_path=self.ks_path, ks_passwd=PASSWD, node_type=node_type, config=config) node.set_auth_key(None) node.start(self.neighbour) self.__lock.acquire() try: self.operator = OperatorClient('node_%s'%self.port) self.stopped = False finally: self.__lock.release() while not self.stopped: time.sleep(0.1) node.stop() def stop(self): self.stopped = True self.join() def wait_oper_status(self, status, errmsg=''): for i in xrange(20): if self.get_status() == status: return time.sleep(.25) raise Exception('[%s].wait_oper_status(%s) timeouted! %s'%(self.port, status, errmsg)) def get_range_dir(self): return os.path.join(self.home_dir, 'dht_range/%s'%FSMappedDHTRange.DBCT_MASTER) def get_replicas_dir(self): return os.path.join(self.home_dir, 'dht_range/%s'%FSMappedDHTRange.DBCT_REPLICA) def get_tmp_dir(self): return os.path.join(self.home_dir, 'dht_range/%s'%FSMappedDHTRange.DBCT_TEMP) def get_stat(self): packet_obj = FabnetPacketRequest(method='NodeStatistic') client = FriClient(self.ks) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) if ret_packet.ret_code != 0: raise Exception('cant get node statistic. details: %s'%ret_packet.ret_message) return ret_packet.ret_parameters def get_status(self): self.__lock.acquire() try: if self.operator is None: return None return self.operator.get_status() finally: self.__lock.release() def put_data_block(self, data, key, dbct=FSMappedDHTRange.DBCT_MASTER, user_id_hash='', \ careful_save=False, init_block=False, stored_unixtime=None): params = {'key': key, 'dbct': dbct, 'user_id_hash': user_id_hash, 'init_block': init_block,\ 'carefully_save': careful_save, 'stored_unixtime': stored_unixtime} req = FabnetPacketRequest(method='PutDataBlock',\ binary_data=data, parameters=params) client = FriClient(self.ks) resp = client.call_sync('127.0.0.1:%s'%self.port, req) return resp def put(self, data, wait_writes=3, init_block=True, client_ks=None, key=None, replica_count=2): if not client_ks: client_ks = self.ks client = FriClient(client_ks) params = {'wait_writes_count': wait_writes, 'replica_count': replica_count, \ 'init_block': init_block, 'key': key} data = RamBasedBinaryData(data, 20) packet_obj = FabnetPacketRequest(method='ClientPutData', parameters=params, binary_data=data, sync=True) #print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count()) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) #print '========SENDED DATA BLOCK' return ret_packet def get_keys_info(self, key, replica_count=2): client = FriClient(self.ks) packet = FabnetPacketRequest(method='GetKeysInfo', \ parameters={'key': key, 'replica_count': replica_count}, sync=True) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet) if ret_packet.ret_code != 0: raise Exception('GetKeysInfo error: %s'%ret_packet.ret_message) keys_info = ret_packet.ret_parameters['keys_info'] return keys_info def get_data_block(self, key, dbct=FSMappedDHTRange.DBCT_MASTER, client_ks=None, user_id_hash=''): params = {'key': key, 'dbct': dbct, 'user_id_hash': user_id_hash} req = FabnetPacketRequest(method='GetDataBlock', parameters=params) client = FriClient(client_ks) resp = client.call_sync('127.0.0.1:%s'%self.port, req) return resp def delete(self, key, client_ks=None, replica_count=2): client = FriClient(client_ks) params = {'key': key, 'replica_count': replica_count} packet_obj = FabnetPacketRequest(method='ClientDeleteData', parameters=params, sync=True) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) return ret_packet def check_data_block(self, key, dbct=FSMappedDHTRange.DBCT_MASTER, checksum=None): client = FriClient(self.ks) params = {'key': key, 'dbct': dbct, 'checksum': checksum} packet_obj = FabnetPacketRequest(method='CheckDataBlock', parameters=params) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) return ret_packet def update_user_md(self, user_id_hash, storage_size): client = FriClient(self.ks) params = {'user_id_hash': user_id_hash, 'storage_size': storage_size} packet_obj = FabnetPacketRequest(method='UpdateUserProfile', parameters=params) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) return ret_packet def update_md(self, user_id_hash, add_list, rm_list=[]): client = FriClient(self.ks) params = {'user_id_hash': user_id_hash, 'add_list': add_list, 'rm_list': rm_list} packet_obj = FabnetPacketRequest(method='UpdateMetadata', parameters=params) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) return ret_packet def put_object_part(self, obj_path, data, seek=0, wait_writes=3, init_block=True, client_ks=None, key=None, replica_count=2): if not client_ks: client_ks = self.ks client = FriClient(client_ks) params = {'wait_writes_count': wait_writes, 'replica_count': replica_count, \ 'init_block': init_block, 'key': key, 'obj_path': obj_path, 'seek': seek} data = RamBasedBinaryData(data, 20) packet_obj = FabnetPacketRequest(method='PutObjectPart', parameters=params, binary_data=data, sync=True) #print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count()) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) #print '========SENDED DATA BLOCK' return ret_packet def get_object_info(self, obj_path, client_ks, req_user_info=True): params = {'obj_path': obj_path, 'req_user_info': req_user_info} req = FabnetPacketRequest(method='GetObjectInfo', parameters=params) client = FriClient(client_ks) resp = client.call_sync('127.0.0.1:%s'%self.port, req) return resp def get(self, key, client_ks=None): client = FriClient(client_ks) params = {'key': key, 'replica_count': 2} packet_obj = FabnetPacketRequest(method='ClientGetData', parameters=params, sync=True) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) return ret_packet
class TestServerThread(threading.Thread): SESSION_ID = 1234322321 def __init__(self, port, home_dir, neighbour=None, is_monitor=False, config={}): threading.Thread.__init__(self) self.port = port self.home_dir = home_dir self.stopped = True self.operator = None self.neighbour = neighbour self.config = config self.is_monitor = is_monitor self.__lock = threading.Lock() def run(self): address = '127.0.0.1:%s'%self.port if self.is_monitor: node_type = 'Monitor' else: node_type = 'DHT' config = {'WAIT_RANGE_TIMEOUT': 0.1, 'INIT_DHT_WAIT_NEIGHBOUR_TIMEOUT': 0.1, 'MONITOR_DHT_RANGES_TIMEOUT': 1, 'CHECK_HASH_TABLE_TIMEOUT': 1, 'WAIT_FILE_MD_TIMEDELTA': 0.1, 'WAIT_DHT_TABLE_UPDATE': 0.2} config.update(self.config) node = Node('127.0.0.1', self.port, self.home_dir, 'node_%s'%self.port, ks_path=None, ks_passwd=None, node_type=node_type, config=config) node.start(self.neighbour) self.__lock.acquire() try: self.operator = OperatorClient('node_%s'%self.port) self.stopped = False finally: self.__lock.release() while not self.stopped: time.sleep(0.1) node.stop() def stop(self): self.stopped = True self.join() def get_stat(self): packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True) client = FriClient() ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) if ret_packet.ret_code != 0: raise Exception('cant get node statistic. details: %s'%ret_packet.ret_message) return ret_packet.ret_parameters def put_data_block(self, data, key, is_replica=False, npkey=False, user_id=0): checksum = hashlib.sha1(data).hexdigest() params = {'key': key, 'checksum': checksum, 'is_replica':is_replica, \ 'carefully_save': True, 'replica_count': 2, 'user_id': user_id} if not npkey: params['primary_key'] = key req = FabnetPacketRequest(method='PutDataBlock',\ binary_data=data, sync=True, parameters=params) client = FriClient() resp = client.call_sync('127.0.0.1:%s'%self.port, req) return resp def get_data_block(self, key, is_replica=False, user_id=0): params = {'key': key, 'is_replica':is_replica} req = FabnetPacketRequest(method='GetDataBlock',\ sync=True, parameters=params) client = FriClient(session_id=user_id or self.SESSION_ID) resp = client.call_sync('127.0.0.1:%s'%self.port, req) if resp.ret_code == constants.RC_NO_DATA: return None if resp.ret_code != 0: raise Exception('GetDataBlock operation failed on %s. Details: %s'%('127.0.0.1:%s'%self.port, resp.ret_message)) return resp.binary_data def put(self, data, wait_writes=3): client = FriClient(session_id=self.SESSION_ID) checksum = hashlib.sha1(data).hexdigest() params = {'checksum': checksum, 'wait_writes_count': wait_writes, 'replica_count':2} data = RamBasedBinaryData(data, 20) packet_obj = FabnetPacketRequest(method='ClientPutData', parameters=params, binary_data=data, sync=True) #print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count()) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) #print '========SENDED DATA BLOCK' if ret_packet.ret_code != 0: raise Exception('put data failed: %s'%ret_packet.ret_message) return ret_packet.ret_parameters['key'] def get(self, key): client = FriClient(session_id=self.SESSION_ID) params = {'key': key, 'replica_count': 2} packet_obj = FabnetPacketRequest(method='ClientGetData', parameters=params, sync=True) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) return ret_packet def delete(self, key, session_id=None): client = FriClient(session_id=session_id or self.SESSION_ID) params = {'key': key, 'replica_count': 2} packet_obj = FabnetPacketRequest(method='ClientDeleteData', parameters=params, sync=True) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj) return ret_packet def get_range_dir(self): stat = self.get_stat() h_start = stat['DHTInfo']['range_start'] h_end = stat['DHTInfo']['range_end'] return os.path.join(self.home_dir, 'dht_range/%s_%s'%(h_start, h_end)) def get_keys_info(self, key): client = FriClient(session_id=self.SESSION_ID) packet = FabnetPacketRequest(method='GetKeysInfo', \ parameters={'key': key, 'replica_count': 2}, sync=True) ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet) keys_info = ret_packet.ret_parameters['keys_info'] return keys_info def get_replicas_dir(self): return os.path.join(self.home_dir, 'dht_range/replica_data') def get_tmp_dir(self): return os.path.join(self.home_dir, 'dht_range/tmp') def get_status(self): self.__lock.acquire() try: if self.operator is None: return None return self.operator.get_status() finally: self.__lock.release()