def __deserialize(self, filename):
raft_data = None
with self.__lock:
try:
self.__logger.info('deserializer has started')
with zipfile.ZipFile(filename, 'r') as zf:
# extract the federation data
zf.extract('federation.bin', path=self.__data_dir)
self.__data = pickle.loads(zf.read('federation.bin'))
self.__logger.info('federation.bin has restored')
# restore the raft data
raft_data = pickle.loads(zf.read(RAFT_DATA_FILE))
self.__logger.info(
'raft.{0} has restored'.format(RAFT_DATA_FILE))
self.__logger.info('snapshot has restored')
except Exception as ex:
self.__logger.error(
'failed to restore indices: {0}'.format(ex))
finally:
self.__logger.info('deserializer has stopped')
return raft_data
def test_unpickle():
data = {'foo': 'bar', 'command': b'\xfa', 'entries': [b'\xfb', b'\xfc']}
python2_cpickle = b'\x80\x02}q\x01(U\x03fooq\x02U\x03barq\x03U\x07commandq\x04U\x01\xfaU\x07entriesq\x05]q\x06(U\x01\xfbU\x01\xfceu.'
python2_pickle = b'\x80\x02}q\x00(U\x03fooq\x01U\x03barq\x02U\x07commandq\x03U\x01\xfaq\x04U\x07entriesq\x05]q\x06(U\x01\xfbq\x07U\x01\xfcq\x08eu.'
python3_pickle = b'\x80\x02}q\x00(X\x03\x00\x00\x00fooq\x01X\x03\x00\x00\x00barq\x02X\x07\x00\x00\x00commandq\x03c_codecs\nencode\nq\x04X\x02\x00\x00\x00\xc3\xbaq\x05X\x06\x00\x00\x00latin1q\x06\x86q\x07Rq\x08X\x07\x00\x00\x00entriesq\t]q\n(h\x04X\x02\x00\x00\x00\xc3\xbbq\x0bh\x06\x86q\x0cRq\rh\x04X\x02\x00\x00\x00\xc3\xbcq\x0eh\x06\x86q\x0fRq\x10eu.'
python2_cpickle_data = pickle.loads(python2_cpickle)
assert data == python2_cpickle_data, 'Failed to unpickle data pickled by python2 cPickle'
python2_pickle_data = pickle.loads(python2_pickle)
assert data == python2_pickle_data, 'Failed to unpickle data pickled by python2 pickle'
python3_pickle_data = pickle.loads(python3_pickle)
assert data == python3_pickle_data, 'Failed to unpickle data pickled by python3 pickle'
def __open_index(self, index_name, index_config=None):
start_time = time.time()
index = None
try:
# open the index
index = self.__indices.get(index_name)
if index is None:
self.__logger.debug('opening {0}'.format(index_name))
if index_config is None:
# set saved index config
with open(
os.path.join(
self.__file_storage.folder,
self.get_index_config_file(index_name)),
'rb') as f:
self.__index_configs[index_name] = pickle.loads(
f.read())
else:
# set given index config
self.__index_configs[index_name] = index_config
if self.__index_configs[index_name].get_storage_type(
) == 'ram':
index = self.__ram_storage.open_index(
indexname=index_name,
schema=self.__index_configs[index_name].get_schema())
else:
index = self.__file_storage.open_index(
indexname=index_name,
schema=self.__index_configs[index_name].get_schema())
self.__indices[index_name] = index
self.__logger.info('{0} has opened'.format(index_name))
# open the index writer
self.__open_writer(index_name)
except Exception as ex:
self.__logger.error('failed to open {0}: {1}'.format(
index_name, ex))
finally:
self.__record_metrics(start_time, 'open_index')
return index
def __processParseMessage(self):
if len(self.__readBuffer) < 4:
return None
l = struct.unpack('i', self.__readBuffer[:4])[0]
if len(self.__readBuffer) - 4 < l:
return None
data = self.__readBuffer[4:4 + l]
try:
if self.encryptor:
data = self.encryptor.decrypt(data)
message = pickle.loads(zlib.decompress(data))
if self.recvRandKey:
randKey, message = message
assert randKey == self.recvRandKey
except:
self.disconnect()
return None
self.__readBuffer = self.__readBuffer[4 + l:]
return message
def __deserialize(self, filename):
with self.__lock:
try:
self.__logger.debug('deserializer has started')
with zipfile.ZipFile(filename, 'r') as zf:
# get file names in snapshot file
filenames = list(zf.namelist())
# get index names in snapshot file
index_names = []
pattern_toc = re.compile(r'^_(.+)_\d+\.toc$')
for f in filenames:
match = pattern_toc.search(f)
if match and match.group(1) not in index_names:
index_names.append(match.group(1))
for index_name in index_names:
# extract the index config first
zf.extract(self.get_index_config_file(index_name),
path=self.__file_storage.folder)
index_config = pickle.loads(
zf.read(self.get_index_config_file(index_name)))
# get index files
pattern_toc = re.compile(r'^_{0}_(\d+)\..+$'.format(
index_name)) # ex) _myindex_0.toc
pattern_seg = re.compile(
r'^{0}_([a-z0-9]+)\..+$'.format(index_name)
) # ex) myindex_zseabukc2nbpvh0u.seg
pattern_lock = re.compile(r'^{0}_WRITELOCK$'.format(
index_name)) # ex) myindex_WRITELOCK
index_files = []
for file_name in filenames:
if re.match(pattern_toc, file_name):
index_files.append(file_name)
elif re.match(pattern_seg, file_name):
index_files.append(file_name)
elif re.match(pattern_lock, file_name):
index_files.append(file_name)
# extract the index files
for index_file in index_files:
if index_config.get_storage_type() == 'ram':
with self.__ram_storage.create_file(
index_file) as r:
r.write(zf.read(index_file))
else:
zf.extract(index_file,
path=self.__file_storage.folder)
self.__logger.debug(
'{0} has restored from {1}'.format(
index_file, filename))
self.__logger.debug(
'{0} has restored'.format(index_name))
# extract the raft data
raft_data = pickle.loads(zf.read(RAFT_DATA_FILE))
self.__logger.debug(
'{0} has restored'.format(RAFT_DATA_FILE))
return raft_data
except Exception as ex:
self.__logger.error(
'failed to restore indices: {0}'.format(ex))
finally:
self.__logger.debug('deserializer has stopped')
