def stop( self ):
''' 停止，批量装载内存 '''
obj_list = [ Data.table_list, Data.ut2remote, Data.ut2seatid, Data.tableid2match ]
Pc.dump( self.saveFile, obj_list )
Re.delete( SERVER_ADDR_PREFIX, self.servername )
def output(self, outfile, format):
out = open(outfile, 'wb')
if format == 'text':
out.write('%s\n' % self.files)
out.write('%s\n' % self.matrix)
elif format == 'pickle':
Pickle.dump(self.files, out)
Pickle.dump(self.matrix, out)
else:
sys.stderr.write('invalid output format\n')
sys.exit(1)
out.close()
def test_simple_object2():
serializer=Pickle.Pickle()
strdata=serializer.dumps(SIMPLE_OBJECTS)
fromstrdata=serializer.loads(strdata)
print(dir(fromstrdata),'5')
print(dir(SIMPLE_OBJECTS),'6')
assert dir(fromstrdata)==dir(SIMPLE_OBJECTS)
def test_simple_object():
serializer=Pickle.Pickle()
print(test_simple_hash, 'hdfgdgf')
strdata=serializer.dumps(test_simple_hash)
print(strdata, 'hj')
fromstrdata=serializer.loads(strdata)
print(fromstrdata)
strdata1=serializer.dumps(fromstrdata)
fromstrdata1=serializer.loads(strdata1)
assert fromstrdata1==test_simple_hash
def test_simple_object3():
serializer=Pickle.Pickle()
strdata=serializer.dumps(sum)
print(strdata, 'hi')
fromstrdata=serializer.loads(strdata)
print(fromstrdata(5))
print(fromstrdata, '3 usual')
print(dir(fromstrdata),'5 dir')
print(dir(fromstrdata),'5 dir')
print(dir(sum),'6 dir')
assert roune._function_equals(sum, fromstrdata)
def gt_roidb(self):
"""
Return the database of ground-truth regions of interest.
This function loads/saves from/to a cache file to speed up future calls.
"""
cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
if os.path.exists(cache_file):
with open(cache_file, 'rb') as fid:
roidb = Pickle.load(fid)
print('{} gt roidb loaded from {}'.format(self.name, cache_file))
return roidb
gt_roidb = [
self._load_clutteredMNIST_annotation(index)
for index in self.image_index
]
with open(cache_file, 'wb') as fid:
Pickle.dump(gt_roidb, fid)
print('wrote gt roidb to {}'.format(cache_file))
return gt_roidb
def start( self ):
''' 启动,批量加载序列 '''
obj_list = Pc.load( self.saveFile )
if obj_list != list():
Data.table_list, Data.ut2remote, Data.ut2seatid , Data.tableid2match = obj_list
Re.set( SERVER_ADDR_PREFIX, self.servername, 50 )
print Data.table_list
return Data.table_list
def deserialize(self, format, filepath):
self.deserialize = None
if format == 'JSON':
self.deserialize = Json.Json()
elif format == 'TOML':
self.deserialize = Toml.TomlSerializer()
elif format == 'YAML':
self.deserialize = Yaml.Yaml()
elif format == 'PICLE':
self.deserialize = Pickle.Pickle()
with open(filepath, 'rb') as f:
return self.deserialize.load(f)
else:
logging.error("Unsuported type from dump ")
exit()
return self.deserialize.load(filepath)
def loads(self, string, complex_convert=True):
result = TOML_converter.convert('[Start]', string)
if complex_convert:
if "function_type" in result and len(result.keys()) == 1:
des = Pickle.Pickle()
return des.loads(result["function_type"])
if 'class' in result.get('__type__', ''):
print(result, 2)
obj = globals()[result['__name__']]()
for k, b in result.items():
if not k.startswith('__'):
obj.__dict__[k] = b
return obj
else:
return result
else:
return result
def serialize(self, obj, format, filepath):
self.serialize = None
print(format)
if format == 'JSON':
self.serialize = Json.Json()
elif format == 'TOML':
self.serialize = Toml.TomlSerializer()
elif format == 'YAML':
self.serialize = Yaml.Yaml()
elif format == 'PICLE':
self.serialize = Pickle.Pickle()
with open(filepath, 'wb') as f:
return self.serialize.dump(obj, f)
else:
logging.error("Unsuported type from load ")
exit()
return self.serialize.dump(obj, filepath)
def main(args):
max_q_length = 25
max_a_length = 90
with open(os.path.join(processed_data_path, 'pointwise_corpus.pkl'),
'r') as fr:
train_corpus, val_corpus, test_corpus = pkl.load(fr)
embeddings = build_embedding(embedding_path, word2id)
train_qids, train_q, train_aids, train_ap, train_labels = zip(
*train_corpus)
train_q = padding(train_q, max_q_length)
train_ql = np.sum((train_q > 0).astype(int), axis=1)
train_ap = padding(train_ap, max_a_length)
train_al = np.sum((train_ap > 0).astype(int), axis=1)
train_corpus = zip(train_qids, train_q, train_ql, train_aids, train_ap,
train_al, train_labels)
val_qids, val_q, val_aids, val_ap, labels = zip(*val_corpus)
val_q = padding(val_q, max_q_length)
val_ql = np.sum((val_q > 0).astype(int), axis=1)
val_ap = padding(val_ap, max_a_length)
val_al = np.sum((val_ap > 0).astype(int), axis=1)
val_corpus = zip(val_qids, val_q, val_ql, val_aids, val_ap, val_al, labels)
test_qids, test_q, test_aids, test_ap, labels = zip(*test_corpus)
test_q = padding(test_q, max_q_length)
test_ql = np.sum((test_q > 0).astype(int), axis=1)
test_ap = padding(test_ap, max_a_length)
test_al = np.sum((test_ap > 0).astype(int), axis=1)
test_corpus = zip(test_qids, test_q, test_ql, test_aids, test_ap, test_al,
labels)
config = BiMPMConfig(max(word2id.values()) + 1, embeddings=embeddings)
config.max_q_length = max_q_length
config.max_a_length = max_a_length
if args.train:
train(deepcopy(train_corpus), config, val_corpus,
deepcopy(train_corpus))
elif args.test:
test(test_corpus, config)
def main(args, model_type='cnn'):
# pretrained embeddings load
embeddings = build_embedding(embedding_path, word2id)
# config
if model_type == 'cnn':
config = CNNConfig(max(word2id.values()) + 1, embeddings=embeddings)
elif model_type == 'rnn':
config = RNNConfig(max(word2id.values()) + 1, embeddings=embeddings)
max_q_length = config.max_q_length
max_a_length = config.max_a_length
# corpus data: pointwise形式的数据，即(Q, A, label)
with open(os.path.join(processed_data_path, 'pointwise_corpus.pkl'),
'r') as fr:
train_corpus, val_corpus, test_corpus = pkl.load(fr)
# padding
train_qids, train_q, train_aids, train_ap, train_labels = zip(
*train_corpus)
train_q = padding(train_q, max_q_length)
train_ap = padding(train_ap, max_a_length)
train_corpus = zip(train_qids, train_q, train_aids, train_ap, train_labels)
val_qids, val_q, val_aids, val_ap, labels = zip(*val_corpus)
val_q = padding(val_q, max_q_length)
val_ap = padding(val_ap, max_a_length)
val_corpus = zip(val_qids, val_q, val_aids, val_ap, labels)
test_qids, test_q, test_aids, test_ap, labels = zip(*test_corpus)
test_q = padding(test_q, max_q_length)
test_ap = padding(test_ap, max_a_length)
test_corpus = zip(test_qids, test_q, test_aids, test_ap, labels)
# run
if args.train:
train(deepcopy(train_corpus), config, val_corpus,
deepcopy(train_corpus))
elif args.test:
test(test_corpus, config)
def _prepare(self, obj):
if isinstance(obj, (int, float, int, bool, str)) or \
obj is None:
return obj
elif isinstance(obj, list):
for index in range(len(obj)):
obj[index] = self._prepare(obj[index])
return obj
elif isinstance(obj, types.FunctionType):
ser = Pickle.Pickle()
return {"function_type": ser.dumps(obj)}
elif isinstance(obj, dict):
for key, value in obj.items():
obj[key] = self._prepare(value)
return obj
else:
value = dict()
try:
for k, b in obj.__dict__.items():
if not k.startswith('__'):
value[k] = b
if obj.__class__.__name__ == 'type':
name = obj.__name__
else:
name = obj.__class__.__name__
value["__type__"] = str(type(obj))
value["__name__"] = name
return value
except Exception as e:
traceback.print_exc()
logging.error(repr(obj) + ' is not serializable' + repr(e))
exit()
elif args.test:
test(test_corpus, config)
if __name__ == "__main__":
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
# Get ENV
ENVIRON = os.environ.copy()
# data path process
raw_data_path = './data/WikiQA/raw'
processed_data_path = './data/WikiQA/processed'
processed_data_pkl = os.path.join(processed_data_path, 'vocab.pkl')
embedding_path = './data/embedding/glove.6B.300d.txt'
model_path = 'models'
if 'GLOVE_EMBEDDING_6B' in ENVIRON:
embedding_path = ENVIRON['GLOVE_EMBEDDING_6B']
print("embedding file: %s" % embedding_path)
if not os.path.exists(processed_data_pkl):
raise BaseException(
"data [{processed_data_pkl}] not exist, run preprocess_wiki.py first."
)
with open(processed_data_pkl, 'r') as fr:
word2id, id2word = pkl.load(fr)
# run
main(args)
pointwise_train_corpus = pointwise_data(transformed_train_corpus,
keep_ids=True)
pairwise_train_corpus = pairwise_data(transformed_train_corpus)
listwise_train_corpus = listwise_data(transformed_train_corpus)
transformed_val_corpus = transform(val_processed_qa, word2id)
pointwise_val_corpus = pointwise_data(transformed_val_corpus, keep_ids=True)
pairwise_val_corpus = pointwise_data(transformed_val_corpus, keep_ids=True)
listwise_val_corpus = listwise_data(transformed_val_corpus)
transformed_test_corpus = transform(test_processed_qa, word2id)
pointwise_test_corpus = pointwise_data(transformed_test_corpus, keep_ids=True)
pairwise_test_corpus = pointwise_data(transformed_test_corpus, keep_ids=True)
listwise_test_corpus = listwise_data(transformed_test_corpus)
with open(os.path.join(processed_data_path, 'vocab.pkl'), 'w') as fw:
pkl.dump([word2id, id2word], fw)
with open(os.path.join(processed_data_path, 'pointwise_corpus.pkl'),
'w') as fw:
pkl.dump(
[pointwise_train_corpus, pointwise_val_corpus, pointwise_test_corpus],
fw)
with open(os.path.join(processed_data_path, 'pairwise_corpus.pkl'), 'w') as fw:
pkl.dump(
[pairwise_train_corpus, pairwise_val_corpus, pairwise_test_corpus], fw)
with open(os.path.join(processed_data_path, 'listwise_corpus.pkl'), 'w') as fw:
pkl.dump(
[listwise_train_corpus, listwise_val_corpus, listwise_test_corpus], fw)
print('done!')