def build_words_model(query_tokens_count, rules_count, nodes_count,
words_count):
words_query_encoder, words_query_encoder_pc = build_query_encoder_for_words(
query_tokens_count, batch_size=BATCH_SIZE)
words_encoder, words_encoder_pc = build_words_encoder(
rules_count, nodes_count)
words_decoder, words_decoder_pc = build_words_decoder(words_encoder)
copy_mechanism, copy_mechanism_pc = build_copy_mechanism(
words_query_encoder.all_states, words_decoder.all_states, words_count)
with variable_scope('loss'):
words_loss, words_stats = build_words_loss(copy_mechanism,
words_decoder_pc,
copy_mechanism_pc)
stacked_words_loss = tf.stack(list(words_loss.values()))
loss = tf.reduce_sum(stacked_words_loss)
with variable_scope('l2_loss'):
variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
l2_variables = [
v for v in variables if v.name.split('/')[-1].startswith('kernel')
]
l2_losses = [tf.nn.l2_loss(v) for v in l2_variables]
l2_loss = L2_COEFFICIENT * tf.reduce_sum(l2_losses)
loss_with_l2 = loss + l2_loss
placeholders = WordsModelPlaceholders(words_query_encoder_pc,
words_encoder_pc, words_decoder_pc,
copy_mechanism_pc)
stats = dict_to_object(words_stats)
model = WordsModel(placeholders, loss, loss_with_l2, stats)
return model
def syns_all(save_data=False):
best_chain = sync_local()
for peer in cnst.PEERS:
peer_block_chain_url = peer + 'blockchain'
try:
req = requests.get(peer_block_chain_url)
peer_blockchain_dict = req.json()
peer_blocks = [
dict_to_object(bcDict) for bcDict in peer_blockchain_dict
]
peer_chain = BlockChain()
peer_chain.chainList = peer_blocks
if block_chain.isValid(chainList=peer_blocks) and len(
peer_chain.chainList) > len(best_chain.chainList):
best_chain = peer_chain
except requests.exceptions.ConnectionError as e:
print(e)
if save_data:
best_chain.save_block_chain()
return best_chain
def train(term_args):
myalgo_args = args.copy()
method = 'direct'
for init_std, y2_max in product([0.3, 1.0], [0.025, 0.050, 0.100]):
myalgo_args.update({
'label': term_args.label,
'env_name': term_args.env,
'method': method,
'init_std': init_std,
'y2_max': y2_max,
})
myalgo_args = dict_to_object(myalgo_args)
test(myalgo_args, term_args.label)
myalgo_args = args.copy()
method = 'ES'
for init_std, step_size in product([0.3, 1.0], [0.05, 0.10, 0.50, 1.00]):
myalgo_args.update({
'label': term_args.label,
'env_name': term_args.env,
'method': method,
'init_std': init_std,
'step_size': y2_max,
})
myalgo_args = dict_to_object(myalgo_args)
test(myalgo_args, term_args.label)
myalgo_args = args.copy()
method = 'ES-MA1'
for init_std, step_size, mean_ratio, n_points in product(
[0.3, 1.0], [0.05, 0.10, 0.50, 1.00], [0.1, 1.0], [2, 5]):
myalgo_args.update({
'label': term_args.label,
'env_name': term_args.env,
'method': method,
'init_std': init_std,
'step_size': y2_max,
'mean_ratio': mean_ratio,
'n_points': n_points,
})
myalgo_args = dict_to_object(myalgo_args)
test(myalgo_args, term_args.label)
def main():
train_dir = "origin_data/-Doi_song_train.muc"
dev_dir = "origin_data/Doi_song_dev.muc"
test_dir = "origin_data/Doi_song_test.muc"
counter = Counter()
print("read train file")
sentences_train = read_file(train_dir, counter)
print("read dev file")
sentences_dev = read_file(dev_dir, counter)
print("read test file")
sentences_test = read_file(test_dir, counter)
print(counter)
print("longest sentence: %s" % str(counter.longest_sen))
print("longest word: %s" % counter.longest_word())
word2idx = read_word_embedding()
char2idx = tags2idx(counter.char_vocab)
pos2idx = tags2idx(counter.pos_tags)
chunk2idx = tags2idx(counter.chunk_tags)
ner2idx = tags2idx(counter.ner_tags)
utils.json_dump(char2idx, "embedding/char2idx.json")
utils.json_dump(pos2idx, "embedding/pos2idx.json")
utils.json_dump(chunk2idx, "embedding/chunk2idx.json")
utils.json_dump(ner2idx, "embedding/ner2idx.json")
print("encoding data")
encoder = {
"max_sen_len": counter.max_sen_len,
"max_word_len": counter.max_word_len,
**utils.make_dict(word2idx, char2idx, pos2idx, chunk2idx, ner2idx)
}
encoder = utils.dict_to_object(encoder)
chars_train, words_train, pos_train, chunk_train, ner_train = encode_sens(
sentences_train, encoder)
chars_dev, words_dev, pos_dev, chunk_dev, ner_dev = encode_sens(
sentences_dev, encoder)
chars_test, words_test, pos_test, chunk_test, ner_test = encode_sens(
sentences_test, encoder)
print("saving data")
data = utils.make_dict(chars_train, words_train, pos_train, chunk_train,
ner_train, chars_dev, words_dev, pos_dev, chunk_dev,
ner_dev, chars_test, words_test, pos_test,
chunk_test, ner_test)
os.makedirs("data", exist_ok=True)
for k, d in data.items():
np.save("data/%s.npy" % k, d)
def sync_local():
local_chain = BlockChain()
if os.path.exists(cnst.BLOCK_CHAIN_DIR):
for filepath in glob.glob(os.path.join(cnst.BLOCK_CHAIN_DIR,
'*.json')):
with open(filepath, 'r') as block_file:
try:
block_info = json.load(block_file)
except:
print(filepath)
block_object = dict_to_object(block_info=block_info)
local_chain.chainList.append(block_object)
return local_chain
def load_data():
files = glob.glob("data/*.npy")
data = {}
for f in files:
name = utils.get_file_name(f)
data[name] = np.load(f)
data = utils.dict_to_object(data)
data.max_sen_len, data.max_word_len = data.words_train.shape[1:]
pos2idx = utils.json_load("embedding/pos2idx")
data.pos_alphabet_size = len(pos2idx)
char2idx = utils.json_load("embedding/char2idx")
data.char_alphabet_size = len(char2idx)
ner2idx = utils.json_load("embedding/ner2idx")
data.num_labels = len(ner2idx)
import os
import yaml
from utils import dict_to_object
# 项目路径
BASE_PATH = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'application')
# 获取静态配置参数
SETTING_DICT = yaml.safe_load(
open(os.path.join(os.path.dirname(BASE_PATH), 'config', 'conf.yaml')))
# 添加动态配置参数
SETTING_DYNAMIC = {
# JWT加密key
'JWT_KEY': os.urandom(24),
# 文章md文件位置
'ARTICLE_FILE': os.path.join(os.path.dirname(BASE_PATH),
'static/article_file')
}
# 更新配置参数
SETTING_DICT.update(SETTING_DYNAMIC)
SETTING = dict_to_object(SETTING_DICT)