def load_train_snapshot(executor, program, file_path):
assert os.path.exists(file_path), "[%s] cann't be found." % file_path
io.load_persistables(executor=executor, dirname=file_path, main_program=program)
if os.path.exists(file_path + ".json"):
info = file_utils.read_file(file_name=file_path + ".json", file_format="json")
return info
return False
def read_from_pickle(self, cachename):
"""
读取examples_list的缓存
:return:
"""
name = cachename
assert name != "", "if use read_from_pickle, must pass in example_file_name when initialize dataset!"
self.examples = read_file(file_type='example', file_name=name, file_format='pickle')
def load_features(self):
if self.cache != "":
if os.path.exists(
get_fullurl(file_type='datap',
file_name=self.cache,
file_format='json')):
self.logger.info('load features from file')
features = read_file('datap', self.cache, 'json')
self.logger.info('{} features loaded'.format(len(features)))
return features
return None
def parse(self, file_name):
content = read_file("keyword", file_name, "")
for keyword in content:
self.add(keyword.strip().encode("utf-8").lower())