class Dialog(object):
"""Dialogue runner."""
def __init__(self, agents, args):
# for now we only suppport dialog of 2 agents
assert len(agents) == 2
self.agents = agents
self.args = args
self.domain = domain.get_domain(args.domain)
self.metrics = MetricsContainer()
self._register_metrics()
def _register_metrics(self):
"""Registers valuable metrics."""
self.metrics.register_average('dialog_len')
self.metrics.register_average('sent_len')
self.metrics.register_percentage('agree')
self.metrics.register_average('advantage')
self.metrics.register_time('time')
self.metrics.register_average('comb_rew')
for agent in self.agents:
self.metrics.register_average('%s_rew' % agent.name)
self.metrics.register_percentage('%s_sel' % agent.name)
self.metrics.register_uniqueness('%s_unique' % agent.name)
# text metrics
ref_text = ' '.join(data.read_lines(self.args.ref_text))
self.metrics.register_ngram('full_match', text=ref_text)
def _is_selection(self, out):
return len(out) == 1 and out[0] == '<selection>'
def show_metrics(self):
return ' '.join(
['%s=%s' % (k, v) for k, v in self.metrics.dict().items()])
def get_loss(self,
inpt,
words,
lang_hs,
lang_h,
c=1,
bob_ends=True,
bob_out=None):
bob = self.agents[1]
bob.read(inpt, f_encode=False)
if bob_ends:
loss1, bob_out, _ = bob.write_selection(wb_attack=True)
bob_choice, classify_loss, _ = bob.choose()
t_loss = c * loss1 + classify_loss
else:
#if bob_out is None:
bob_out = bob.write(bob_ends)
_, out, _ = self.agents[0].write_selection(wb_attack=True,
alice=True)
bob.read(out)
bob_choice, classify_loss, _ = bob.choose()
t_loss = classify_loss
#t_loss.backward(retain_graph=True)
bob.words = copy.copy(words)
bob.lang_hs = copy.copy(lang_hs)
bob.lang_h = lang_h.clone()
if bob_ends:
return t_loss.item(), loss1, classify_loss, bob_out, bob_choice
else:
return t_loss.item(), bob_out, bob_choice
#return t_loss, None, bob_choice
def run(self, ctxs, logger):
"""Runs one instance of the dialogue."""
assert len(self.agents) == len(ctxs)
# initialize agents by feeding in the contexes
#for agent, ctx in zip(self.agents, ctxs):
# agent.feed_context(ctx)
# logger.dump_ctx(agent.name, ctx)
self.agents[0].feed_context(ctxs[0])
logger.dump_ctx(self.agents[0].name, ctxs[0])
self.agents[1].feed_context(ctxs[1], ctxs[0])
logger.dump_ctx(self.agents[1].name, ctxs[1])
logger.dump('-' * 80)
# choose who goes first by random
if np.random.rand() < 0.5:
writer, reader = self.agents
else:
reader, writer = self.agents
writer, reader = self.agents
conv = []
# reset metrics
self.metrics.reset()
#### Minhao ####
count_turns = 0
with torch.no_grad():
while True:
# produce an utterance
if count_turns > self.args.max_turns - 1:
if writer == self.agents[0]:
inpt, lang_hs, lang_h, words = writer.write_white(
reader)
#print(writer.words[-1][0].grad)
### need padding in the input_emb
break
#else:
else:
out = writer.write()
self.metrics.record('sent_len', len(out))
self.metrics.record('full_match', out)
self.metrics.record('%s_unique' % writer.name, out)
# append the utterance to the conversation
conv.append(out)
# make the other agent to read it
reader.read(out)
if not writer.human:
logger.dump_sent(writer.name, out)
# check if the end of the conversation was generated
if self._is_selection(out):
self.metrics.record('%s_sel' % writer.name, 1)
self.metrics.record('%s_sel' % reader.name, 0)
break
writer, reader = reader, writer
count_turns += 1
##### add selection mark if exceeding the max_turns
### Minhao: need to design loss focusing on the choices
### No evalution in the conversation????
bob = self.agents[1]
choices = []
#class_losses = []
# generate choices for each of the agents
#flag_loss=False
c = 1
#print(words)
all_index_n = len(self.agents[0].model.word_dict)
#print(inpt)
bob_ends = True
#fixed_lang_h = bob.lang_h.copy()
fixed_ctx_h = bob.ctx_h.clone()
if True:
iterations = 100
#mask= [0] * (inpt_emb.size()[0]-1)
for iter_idx in range(iterations):
# projection
min_inpt = None
#temp_inpt = inpt.clone()
min_loss_a = []
min_inpt_a = []
if bob_ends:
t_loss, _, _, bob_out, bob_choice = self.get_loss(
inpt, words, lang_hs, lang_h)
else:
#bob_out = bob.write(bob_ends)
t_loss, bob_out, bob_choice = self.get_loss(
inpt,
words,
lang_hs,
lang_h,
bob_ends=bob_ends,
bob_out=None)
print(iter_idx, t_loss)
if t_loss <= -3.0:
print("get legimate adversarial example")
print(self.agents[0]._decode(
inpt, bob.model.word_dict)) ### bug still?????
break
for emb_idx in range(1, inpt.size()[0] - 1):
min_loss = t_loss
for candi in range(1, all_index_n):
temp_inpt = inpt.clone()
temp_inpt[emb_idx] = candi
if bob_ends:
loss, _, _, _, _ = self.get_loss(
temp_inpt, words, lang_hs, lang_h)
else:
#bob_out = bob.write(bob_ends)
loss, _, _ = self.get_loss(temp_inpt,
words,
lang_hs,
lang_h,
bob_ends=bob_ends,
bob_out=None)
if loss < 0:
sum_loss = 0
for _ in range(10):
loss, _, _ = self.get_loss(
temp_inpt,
words,
lang_hs,
lang_h,
bob_ends=bob_ends,
bob_out=None)
sum_loss += loss
#print(loss)
loss = sum_loss / 10
#if loss<0:
# print("first loss",loss, "bob_choice", bob_choice, "bob_out", bob_out)
#print(temp_inpt,bob.words,bob.lang_hs,bob.lang_h.size())
# print("sec loss",self.get_loss(temp_inpt, words, lang_hs, lang_h, bob_ends=bob_ends,bob_out=bob_out))
#print(temp_inpt,bob.words,bob.lang_hs,bob.lang_h.size())
# print("third loss",self.get_loss(temp_inpt, words, lang_hs, lang_h, bob_ends=bob_ends,bob_out=bob_out))
#print(temp_inpt,bob.words,bob.lang_hs,bob.lang_h.size())
if loss < min_loss:
min_loss = loss
min_inpt = temp_inpt.clone()
#print(min_loss)
min_loss_a.append(min_loss)
min_inpt_a.append(min_inpt)
if len(min_loss_a) != 0:
min_idx_in_a = np.argmin(min_loss_a)
inpt = min_inpt_a[min_idx_in_a].clone()
print(min_loss_a)
print(inpt)
#print(loss)
else:
break
print("attack finished")
else:
if bob_ends:
bob.read_emb(inpt_emb, inpt)
_, bob_out, _ = bob.write_selection(wb_attack=True)
bob.words = words.copy()
bob_choice, _, _ = bob.choose(inpt_emb=inpt_emb,
wb_attack=True)
else:
bob.read_emb(inpt_emb, inpt)
bob_out = bob.write(bob_ends)
out = self.agents[0].write_selection()
bob.read(out)
bob.words = words.copy()
bob_choice, _, _ = bob.choose(inpt_emb=inpt_emb,
bob_ends=bob_ends,
bob_out=bob_out,
wb_attack=True)
if bob_ends:
logger.dump_sent(self.agents[0].name,
self.agents[0]._decode(inpt, bob.model.word_dict))
logger.dump_sent(bob.name, ['<selection>'])
else:
logger.dump_sent(
self.agents[0].name,
self.agents[0]._decode(inpt, self.agents[0].model.word_dict))
logger.dump_sent(bob.name, bob._decode(bob_out,
bob.model.word_dict))
logger.dump_sent(self.agents[0].name, ['<selection>'])
#####
choices.append(bob_choice)
alice_choice = bob_choice[:]
for indx in range(3):
alice_choice[
indx +
3], alice_choice[indx] = alice_choice[indx], alice_choice[indx
+ 3]
choices.append(alice_choice) ######## always agree
choices[1], choices[0] = choices[0], choices[1]
#print(choices)
#for agent in self.agents:
# choice, class_loss = agent.choose(flag=flag_loss)
# class_losses.append(class_loss)
# choices.append(choice)
# logger.dump_choice(agent.name, choice[: self.domain.selection_length() // 2])
# flag_loss=True
# evaluate the choices, produce agreement and a reward
#print(choices,ctxs)
agree, rewards = self.domain.score_choices(choices, ctxs)
logger.dump('-' * 80)
logger.dump_agreement(agree)
#print(rewards)
# perform update, in case if any of the agents is learnable
# let the difference become new reward
## how to combine the loss to the reward
'''
diff = rewards[0]-rewards[1]
flag = True
agree = 1
#print(5 - classify_loss.item())
for agent, reward in zip(self.agents, rewards):
if flag:
logger.dump_reward(agent.name, agree, reward)
#agent.update(agree, 50-class_losses[1].item())
agent.update(agree, 5-classify_loss.item())
#agent.update(agree, diff - 0.05 * class_losses[1].data[0])
#agent.update(agree, diff)
else:
logger.dump_reward(agent.name, agree, reward)
if not self.args.fixed_bob:
agent.update(agree, reward)
flag=False
'''
agree = 1
for agent, reward in zip(self.agents, rewards):
logger.dump_reward(agent.name, agree, reward)
logging.debug("%s : %s : %s" %
(str(agent.name), str(agree), str(rewards)))
#agent.update(agree, 5-classify_loss.item())
if agree:
self.metrics.record('advantage', rewards[0] - rewards[1])
self.metrics.record('time')
self.metrics.record('dialog_len', len(conv))
self.metrics.record('agree', int(agree))
self.metrics.record('comb_rew', np.sum(rewards) if agree else 0)
for agent, reward in zip(self.agents, rewards):
self.metrics.record('%s_rew' % agent.name, reward if agree else 0)
logger.dump('-' * 80)
logger.dump(self.show_metrics())
logger.dump('-' * 80)
for ctx, choice in zip(ctxs, choices):
logger.dump('debug: %s %s' % (' '.join(ctx), ' '.join(choice)))
return conv, agree, rewards
class Dialog(object):
"""Dialogue runner."""
def __init__(self, agents, args):
# for now we only suppport dialog of 2 agents
assert len(agents) == 2
self.agents = agents
self.args = args
self.domain = domain.get_domain(args.domain)
self.metrics = MetricsContainer()
self._register_metrics()
def _register_metrics(self):
"""Registers valuable metrics."""
self.metrics.register_average('dialog_len')
self.metrics.register_average('sent_len')
self.metrics.register_percentage('agree')
self.metrics.register_average('advantage')
self.metrics.register_time('time')
self.metrics.register_average('comb_rew')
for agent in self.agents:
self.metrics.register_average('%s_rew' % agent.name)
self.metrics.register_percentage('%s_sel' % agent.name)
self.metrics.register_uniqueness('%s_unique' % agent.name)
# text metrics
ref_text = ' '.join(data.read_lines(self.args.ref_text))
self.metrics.register_ngram('full_match', text=ref_text)
def _is_selection(self, out):
return len(out) == 1 and out[0] == '<selection>'
def show_metrics(self):
return ' '.join(['%s=%s' % (k, v) for k, v in self.metrics.dict().items()])
def run(self, ctxs, logger):
"""Runs one instance of the dialogue."""
assert len(self.agents) == len(ctxs)
# initialize agents by feeding in the contexes
for agent, ctx in zip(self.agents, ctxs):
agent.feed_context(ctx)
logger.dump_ctx(agent.name, ctx)
logger.dump('-' * 80)
# choose who goes first by random
if np.random.rand() < 0.5:
writer, reader = self.agents
else:
reader, writer = self.agents
conv = []
# reset metrics
self.metrics.reset()
while True:
# produce an utterance
out = writer.write()
self.metrics.record('sent_len', len(out))
self.metrics.record('full_match', out)
self.metrics.record('%s_unique' % writer.name, out)
# append the utterance to the conversation
conv.append(out)
# make the other agent to read it
reader.read(out)
if not writer.human:
logger.dump_sent(writer.name, out)
# check if the end of the conversation was generated
if self._is_selection(out):
self.metrics.record('%s_sel' % writer.name, 1)
self.metrics.record('%s_sel' % reader.name, 0)
break
writer, reader = reader, writer
choices = []
# generate choices for each of the agents
for agent in self.agents:
choice = agent.choose()
choices.append(choice)
logger.dump_choice(agent.name, choice[: self.domain.selection_length() // 2])
# evaluate the choices, produce agreement and a reward
agree, rewards = self.domain.score_choices(choices, ctxs)
logger.dump('-' * 80)
logger.dump_agreement(agree)
# perform update, in case if any of the agents is learnable
for agent, reward in zip(self.agents, rewards):
logger.dump_reward(agent.name, agree, reward)
agent.update(agree, reward)
if agree:
self.metrics.record('advantage', rewards[0] - rewards[1])
self.metrics.record('time')
self.metrics.record('dialog_len', len(conv))
self.metrics.record('agree', int(agree))
self.metrics.record('comb_rew', np.sum(rewards) if agree else 0)
for agent, reward in zip(self.agents, rewards):
self.metrics.record('%s_rew' % agent.name, reward if agree else 0)
logger.dump('-' * 80)
logger.dump(self.show_metrics())
logger.dump('-' * 80)
for ctx, choice in zip(ctxs, choices):
logger.dump('debug: %s %s' % (' '.join(ctx), ' '.join(choice)))
return conv, agree, rewards
