def start_container_on_node(spec):
(out, err) = CLIUtils.run(
START_POD_CONTAINER_SELF_CMD.format(spec["node"],
spec["container"]["port"],
spec["container"]["image"],
spec["container"]["name"]))
logger.debug('started container id: {}'.format(out))
def test_logger_debug(self, mock_log):
obj = object()
logger.debug('test', request=obj)
self.assertEqual(mock_log.call_args, (
('[{0}] test'.format(id(obj)), ),
{ 'extra': {} },
))
def load_config():
config = json.load(open('config.json', 'r'))
path_keys_to_expand = ['data_dir', 'save_dir', 'model_save_path', 'albert_cache_dir']
for path_key in path_keys_to_expand:
new_path = os.path.abspath(os.path.expanduser(config[path_key]))
logger.debug("expand {} to {}".format(config[path_key], new_path))
config[path_key] = new_path
max_seq_len = config['max_seq_len']
return config
def make_get_request(self, router_name: str):
url = f'{self.base_url}{router_name}'
try:
logger.info(f'request {url}')
output = urllib.request.urlopen(url).read().decode()
logger.debug(f'output: {output}')
objs = json.loads(output)
return objs
except Exception as ex:
logger.exception(f'error on {url}', exc_info=ex)
def forward(self, batch: SquadBatch):
memory, dec_init_hidden, tgt_embed, src_mask, src_ext_index = self.encode(
batch)
logger.debug("memory mask shape {}".format(src_mask))
logger.debug("memory shape {}".format(memory.shape))
gen_output, atten_output = self.decoder(memory, src_mask,
src_ext_index, tgt_embed,
dec_init_hidden)
# (tgt_len - 1, B, vocab + oov), not probability
return gen_output
def parse(self, response):
tr_list = response.xpath("//table[@class='tablelist']/tr")[1:-1]
for tr in tr_list:
item = TencentItem()
item["title"] = tr.xpath("./td[1]/a/text()").extract_first()
item["position"] = tr.xpath("./td[2]/text()").extract_first()
item["publish_date"] = tr.xpath("./td[5]/text()").extract_first()
logger.debug("item")
yield item
def make_post_request(self, router_name: str, values):
url = f'{self.base_url}{router_name}'
headers = {
"Content-Type": "application/json",
"Accept": "application/json",
}
data = json.dumps(values).encode("utf-8")
try:
req = urllib.request.Request(url, data, headers)
with urllib.request.urlopen(req) as f:
output = f.read()
logger.debug(f'output: {output}')
return json.loads(output.decode())
except Exception as ex:
logger.exception(f'error on {url}', exc_info=ex)
def concat_attn(self, query, key, value, mask):
"""
:param query: (B, query_size)
:param key: (B, src_len, attn_size)
:param value: (B, src_len, value_size)
:param mask: (B, src_len)
:return:
"""
query = self.gru_hidden_attn_proj(query).unsqueeze(
1) # (B, 1, attn_size)
tmp = torch.add(key, query) # (B, src_len, attn_size)
e = self.engy_attn_proj(torch.tanh(tmp)).squeeze(2) # (B, src_len)
if mask is not None:
e = e * mask + (1 - mask) * (-1000000)
score = torch.softmax(e, dim=1) # (B, src_len)
logger.debug("\nattention score\n{}".format(score))
ctxt = torch.bmm(score.unsqueeze(1),
value).squeeze(1) # (B, 1, value_size)
return ctxt, score, e
def train(config, model, optim:Optim, train_instances, dev_instances, word_vocab, bio_vocab, feat_vocab):
model.train()
start_time = time.time()
batch_num = 0
num_trial = 0
report_start_time = start_time
report_loss, report_words_num = 0, 0
for epoch in range(config['epoch']):
for batch in batch_iter(train_instances, config['batch_size'],
word_vocab=word_vocab,
bio_vocab=bio_vocab,
feat_vocab=feat_vocab):
logger.debug("src_tokens\n{}".format(batch.src))
logger.debug("ans_tokens\n{}".format(batch.ans))
batch_num += 1
model.zero_grad()
gen_output = model(batch) # (tgt_len-1, B, vocab)
gen_output = gen_output.transpose(0, 1).contiguous() # (B, tgt_len-1, vocab)
lprobs = torch.log_softmax(gen_output, dim=-1)
batch_size = gen_output.size(0)
if config['max_out_cpy']:
gold = torch.tensor([x[1:] for x in batch.tgt_extended_index], dtype=torch.long, device=model.device)
else:
gold = torch.tensor([x[1:] for x in batch.tgt_index], dtype=torch.long, device=model.device)
# (B, tgt_len-1)
batch_loss = lloss(lprobs, gold, ignore_index=word_vocab.pad_idx)
if config['ulloss']:
batch_loss += config['ulloss_weight'] * ulloss(lprobs, gold, ignore_index=word_vocab.pad_idx)
if config['seq_ulloss'] and torch.rand(1).item() < config['seq_ulloss_rate']:
batch_loss += ulloss_seq(lprobs, config['seq_ulloss_ngram'], config['seq_ulloss_seq_type'],
mask_p=config['seq_ulloss_mask_p'])
report_loss += batch_loss.item()
report_words_num += sum(batch.tgt_len) - batch_size
batch_loss.backward()
optim.step()
if batch_num % config['log_per_batches'] == 0:
logger.info('epoch {}|batch {}|avg.loss {:.4f}|ppl {:.3f}|lr {}|t {}|total t {}'.format(
epoch,
batch_num,
report_loss/report_words_num,
math.exp(report_loss / report_words_num),
optim.lr,
user_friendly_time_since(report_start_time),
user_friendly_time_since(start_time)
))
report_loss = report_words_num = 0
report_start_time = time.time()
if batch_num > config['start_validate_after_batches'] and batch_num % config['validate_per_batches'] == 0:
ppl = evaluate_ppl(model, dev_instances,
word_vocab=word_vocab,
bio_vocab=bio_vocab,
feat_vocab=feat_vocab)
if optim.is_better(ppl):
model.save(config['model_save_path'])
logger.info("model saved!")
hit_trial = optim.update_lr(ppl)
optim.metric_history.append(ppl)
logger.info('eval ppl {}|patience {}|current lr {}|best metric {}'.format(
ppl, optim.patience, optim.lr, optim.best_metric))
if hit_trial:
num_trial += 1
logger.info("hit trial: [{}]".format(num_trial))
if num_trial >= config['max_num_trial']:
logger.info("early stop")
exit(0)
logger.info('restoring parameters')
state = torch.load(config['model_save_path'])
model.load_state_dict(state['model_state'])
model.to(device)
import random
test_instances = random.sample(train_instances, 100)
bleus = evaluate_bleu(model, test_instances, config, word_vocab)
logger.info("BLEU_1 {} BLEU_2 {} BLEU_3 {} BLEU_4 {} BLEU {}".format(*bleus))
def process_item(self, item, spider):
logger.debug(item)
return item
}
}
num_nodes = 5
num_replicas = 3
port = START_PORT
# for i in range(int(num_replicas)):
# cont_port = START_PORT
# image_name = "nginx"
# # for i in range(int(num_replicas)):
# # cont_name = "{}_{}".format(image_name, cont_port)
# # (out, err) = CLIUtils.run(START_POD_CONTAINER_CMD.format(port, cont_port, image_name, cont_name))
# # logger.debug('started container id: {}'.format(out))
# # cont_port = cont_port + 1
# cont_name = "{}_{}".format(image_name, cont_port)
# (out, err) = CLIUtils.run(START_POD_CONTAINER_CMD.format(port, cont_port, image_name, cont_name))
# logger.debug('started container id: {}'.format(out))
# port = port + 1
(out, err) = CLIUtils.run(
START_POD_CONTAINER_CMD.format(cont_spec["node"],
cont_spec["container"]["port"],
cont_spec["container"]["image"],
cont_spec["container"]["name"]))
logger.debug('started container id: {}'.format(out))
port = START_PORT
for i in range(int(num_nodes)):
logger.info("Containers on Node co_node_{}: ".format(port))
(out, err) = CLIUtils.run(LIST_POD_CONTAINERS_CMD.format(port))
logger.info('list of containers: %s', out)
port = port + 1
channel = connection.channel()
channel.exchange_declare('co_topic')
port = START_PORT
for i in range(int(num_nodes)):
node_name = "co_node_{}".format(port)
na = NodeAgent(node_name)
na.nodes.insert_one({
"name": node_name,
"description": "worker node",
"heart_beat_time": datetime.datetime.utcnow(),
"free_mem": 10,
"free_cpu": 4,
"free_disk": 128
})
logger.debug('added worker node container')
# create queues
channel.queue_declare(node_name)
# bind the queue to exchange
channel.queue_bind(node_name, 'co_topic', routing_key=node_name)
# start node agent
cmd = "python3 /NodeAgent.py -n {} -c &".format(node_name)
d_cmd = "docker exec {} {}".format(node_name, cmd)
print(d_cmd)
CLIUtils.run_nb(d_cmd)
# start node agent poller
cmd = "python3 /NodeAgent.py -n {} -p &".format(node_name)
d_cmd = "docker exec {} {}".format(node_name, cmd)
print(d_cmd)
CLIUtils.run_nb(d_cmd)