def __init__(self,
transcripts_paths,
survey_paths,
stats_path,
price_tracker_model,
liwc_path,
max_examples=None):
transcripts = self._read_transcripts(transcripts_paths, max_examples)
self.dataset = utils.filter_rejected_chats(transcripts)
dialogue_scores = self._read_surveys(survey_paths)
self.price_tracker = PriceTracker(price_tracker_model)
self.liwc = LIWC.from_pkl(liwc_path)
# group chats depending on whether the seller or the buyer wins
#self.buyer_wins, self.seller_wins = self.group_outcomes_and_roles()
self.stats_path = stats_path
if not os.path.exists(self.stats_path):
os.makedirs(self.stats_path)
self.examples = [
Dialogue.from_dict(raw, dialogue_scores.get(raw['uuid'], {}),
self.price_tracker) for raw in self.dataset
]
def get_system(name, args, schema=None, timed=False, model_path=None):
from core.price_tracker import PriceTracker
lexicon = PriceTracker(args.price_tracker_model)
if name == 'rulebased':
from rulebased_system import RulebasedSystem
from model.generator import Templates, Generator
from model.manager import Manager
templates = Templates.from_pickle(args.templates)
generator = Generator(templates)
manager = Manager.from_pickle(args.policy)
return RulebasedSystem(lexicon, generator, manager, timed)
elif name == 'hybrid':
from hybrid_system import HybridSystem
templates = Templates.from_pickle(args.templates)
manager = PytorchNeuralSystem(args, schema, lexicon, model_path, timed)
generator = Generator(templates)
return HybridSystem(lexicon, generator, manager, timed)
elif name == 'cmd':
from cmd_system import CmdSystem
return CmdSystem()
elif name == 'pt-neural':
from neural_system import PytorchNeuralSystem
assert model_path
return PytorchNeuralSystem(args, schema, lexicon, model_path, timed)
else:
raise ValueError('Unknown system %s' % name)
def get_system(name, args, schema=None, timed=False, model_path=None):
lexicon = PriceTracker(args.price_tracker_model)
if name == 'rulebased':
templates = Templates.from_pickle(args.templates)
generator = Generator(templates)
manager = Manager.from_pickle(args.policy)
return RulebasedSystem(lexicon, generator, manager, timed)
#elif name == 'config-rulebased':
# configs = read_json(args.rulebased_configs)
# return ConfigurableRulebasedSystem(configs, lexicon, timed_session=timed, policy=args.config_search_policy, max_chats_per_config=args.chats_per_config, db=args.trials_db, templates=templates)
elif name == 'cmd':
return CmdSystem()
elif name.startswith('ranker'):
# TODO: hack
#retriever1 = Retriever(args.index+'-1', context_size=args.retriever_context_len, num_candidates=args.num_candidates)
#retriever2 = Retriever(args.index+'-2', context_size=args.retriever_context_len, num_candidates=args.num_candidates)
retriever = Retriever(args.index, context_size=args.retriever_context_len, num_candidates=args.num_candidates)
if name == 'ranker-ir':
return IRRankerSystem(schema, lexicon, retriever)
elif name == 'ranker-ir1':
return IRRankerSystem(schema, lexicon, retriever1)
elif name == 'ranker-ir2':
return IRRankerSystem(schema, lexicon, retriever2)
elif name == 'ranker-neural':
return NeuralRankerSystem(schema, lexicon, retriever, model_path, args.mappings)
else:
raise ValueError
elif name in ('neural-gen', 'neural-sel'):
assert model_path
return NeuralSystem(schema, lexicon, model_path, args.mappings, args.decoding, index=args.index, num_candidates=args.num_candidates, retriever_context_len=args.retriever_context_len, timed_session=timed)
else:
raise ValueError('Unknown system %s' % name)
def get_data_generator(args, model_args, schema, test=False):
from cocoa.core.scenario_db import ScenarioDB
from cocoa.core.dataset import read_dataset
from cocoa.core.util import read_json
from core.scenario import Scenario
from core.price_tracker import PriceTracker
from .preprocess import DataGenerator, Preprocessor
import os.path
# TODO: move this to dataset
dataset = read_dataset(args, Scenario)
mappings_path = model_args.mappings
lexicon = PriceTracker(model_args.price_tracker_model)
preprocessor = Preprocessor(schema,
lexicon,
model_args.entity_encoding_form,
model_args.entity_decoding_form,
model_args.entity_target_form,
model=model_args.model)
if test:
model_args.dropout = 0
train, dev, test = None, None, dataset.test_examples
else:
train, dev, test = dataset.train_examples, dataset.test_examples, None
data_generator = DataGenerator(train,
dev,
test,
preprocessor,
schema,
mappings_path,
cache=args.cache,
ignore_cache=args.ignore_cache,
num_context=model_args.num_context,
batch_size=args.batch_size,
model=model_args.model,
dia_num=args.dia_num,
state_length=args.state_length)
return data_generator
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--transcripts',
nargs='*',
help='JSON transcripts to extract templates')
parser.add_argument('--max-examples', default=-1, type=int)
parser.add_argument('--templates', help='Path to load templates')
parser.add_argument('--policy', help='Path to load model')
parser.add_argument('--schema-path', help='Path to schema')
parser.add_argument(
'--agent', help='Only consider examples with the given type of agent')
add_price_tracker_arguments(parser)
args = parser.parse_args()
lexicon = PriceTracker(args.price_tracker_model)
#templates = Templates.from_pickle(args.templates)
templates = Templates()
manager = Manager.from_pickle(args.policy)
analyzer = Analyzer(lexicon)
# TODO: skip examples
examples = read_examples(args.transcripts, args.max_examples, Scenario)
agent = args.agent
if agent is not None:
examples = [e for e in examples if agent in e.agents.values()]
analyzer.example_stats(examples, agent=agent)
#import sys; sys.exit()
parsed_dialogues = []
for example in examples:
def to_float_price(entity):
return float('{:.2f}'.format(PriceTracker.get_price(entity)))
parser.add_argument('--transcripts',
nargs='*',
help='JSON transcripts to extract templates')
parser.add_argument('--price-tracker-model')
parser.add_argument('--max-examples', default=-1, type=int)
parser.add_argument('--output', help='Path to save templates')
parser.add_argument('--output-transcripts',
help='Path to JSON examples with templates')
parser.add_argument('--templates', help='Path to load templates')
parser.add_argument('--debug', default=False, action='store_true')
args = parser.parse_args()
if args.templates:
templates = Templates.from_pickle(args.templates)
else:
price_tracker = PriceTracker(args.price_tracker_model)
template_extractor = TemplateExtractor(price_tracker)
template_extractor.extract_templates(args.transcripts,
args.max_examples)
write_pickle(template_extractor.templates, args.output)
templates = Templates(template_extractor.templates)
t = templates.templates
response_tags = set(t.response_tag.values)
tag_counts = []
for tag in response_tags:
tag_counts.append(
(tag, t[t.response_tag == tag].shape[0] / float(t.shape[0])))
tag_counts = sorted(tag_counts, key=lambda x: x[1], reverse=True)
for x in tag_counts:
print x
def get_data_generator(args, model_args, mappings, schema):
from cocoa.core.scenario_db import ScenarioDB
from cocoa.core.dataset import read_dataset, EvalExample
from cocoa.core.util import read_json
from core.scenario import Scenario
from core.price_tracker import PriceTracker
from core.slot_detector import SlotDetector
from retriever import Retriever
from preprocess import DataGenerator, LMDataGenerator, EvalDataGenerator, Preprocessor
import os.path
# TODO: move this to dataset
if args.eval:
dataset = []
for path in args.eval_examples_paths:
dataset.extend(
[EvalExample.from_dict(schema, e) for e in read_json(path)])
else:
dataset = read_dataset(args, Scenario)
lexicon = PriceTracker(model_args.price_tracker_model)
slot_detector = SlotDetector(slot_scores_path=model_args.slot_scores)
# Model config tells data generator which batcher to use
model_config = {}
if args.retrieve or model_args.model in ('ir', 'selector'):
model_config['retrieve'] = True
if args.predict_price:
model_config['price'] = True
# For retrieval-based models only: whether to add ground truth response in the candidates
if model_args.model in ('selector', 'ir'):
if 'loss' in args.eval_modes and 'generation' in args.eval_modes:
print '"loss" requires ground truth reponse to be added to the candidate set. Please evaluate "loss" and "generation" separately.'
raise ValueError
if (not args.test) or args.eval_modes == ['loss']:
add_ground_truth = True
else:
add_ground_truth = False
print 'Ground truth response {} be added to the candidate set.'.format(
'will' if add_ground_truth else 'will not')
else:
add_ground_truth = False
# TODO: hacky
if args.model == 'lm':
DataGenerator = LMDataGenerator
if args.retrieve or args.model in ('selector', 'ir'):
retriever = Retriever(args.index,
context_size=args.retriever_context_len,
num_candidates=args.num_candidates)
else:
retriever = None
preprocessor = Preprocessor(schema,
lexicon,
model_args.entity_encoding_form,
model_args.entity_decoding_form,
model_args.entity_target_form,
slot_filling=model_args.slot_filling,
slot_detector=slot_detector)
trie_path = os.path.join(model_args.mappings, 'trie.pkl')
if args.eval:
data_generator = EvalDataGenerator(dataset, preprocessor, mappings,
model_args.num_context)
else:
if args.test:
model_args.dropout = 0
train, dev, test = None, None, dataset.test_examples
else:
train, dev, test = dataset.train_examples, dataset.test_examples, None
data_generator = DataGenerator(train,
dev,
test,
preprocessor,
schema,
mappings,
retriever=retriever,
cache=args.cache,
ignore_cache=args.ignore_cache,
candidates_path=args.candidates_path,
num_context=model_args.num_context,
trie_path=trie_path,
batch_size=args.batch_size,
model_config=model_config,
add_ground_truth=add_ground_truth)
return data_generator
class StrategyAnalyzer(object):
sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
def __init__(self,
transcripts_paths,
survey_paths,
stats_path,
price_tracker_model,
liwc_path,
max_examples=None):
transcripts = self._read_transcripts(transcripts_paths, max_examples)
self.dataset = utils.filter_rejected_chats(transcripts)
dialogue_scores = self._read_surveys(survey_paths)
self.price_tracker = PriceTracker(price_tracker_model)
self.liwc = LIWC.from_pkl(liwc_path)
# group chats depending on whether the seller or the buyer wins
#self.buyer_wins, self.seller_wins = self.group_outcomes_and_roles()
self.stats_path = stats_path
if not os.path.exists(self.stats_path):
os.makedirs(self.stats_path)
self.examples = [
Dialogue.from_dict(raw, dialogue_scores.get(raw['uuid'], {}),
self.price_tracker) for raw in self.dataset
]
def _read_transcripts(self, transcripts_paths, max_examples):
transcripts = []
for transcripts_path in transcripts_paths:
transcripts.extend(read_json(transcripts_path))
if max_examples is not None:
transcripts = transcripts[:max_examples]
return transcripts
def _read_surveys(self, survey_paths):
dialogue_scores = {}
for path in survey_paths:
dialogue_scores.update(read_json(path)[1])
return dialogue_scores
def label_dialogues(self, labels=('speech_act', 'stage', 'liwc')):
for dialogue in self.examples:
if 'speech_act' in labels:
dialogue.extract_keywords()
dialogue.label_speech_acts()
if 'stage' in labels:
dialogue.label_stage()
if 'liwc' in labels:
dialogue.label_liwc(self.liwc)
def summarize_tags(self):
tags = defaultdict(lambda: defaultdict(int))
for dialogue in self.examples:
for turn in dialogue.turns:
agent_name = dialogue.agents[turn.agent]
for tag in turn.tags:
tags[agent_name][tag] += 1
for system, labels in tags.iteritems():
print system.upper()
for k, v in labels.iteritems():
print k, v
def create_dataframe(self):
data = []
for dialogue in ifilter(lambda x: x.has_deal(), self.examples):
for turn in dialogue.turns:
for u in turn.iter_utterances():
row = {
'post_id': dialogue.post_id,
'chat_id': dialogue.chat_id,
'scenario_id': dialogue.scenario_id,
'buyer_target': dialogue.buyer_target,
'listing_price': dialogue.listing_price,
'margin_seller': dialogue.margins['seller'],
'margin_buyer': dialogue.margins['buyer'],
'stage': u.stage,
'role': turn.role,
'num_tokens': u.num_tokens(),
}
for a in u.speech_acts:
row['act_{}'.format(a[0].name)] = 1
for cat, word_count in u.categories.iteritems():
row['cat_{}'.format(cat)] = sum(word_count.values())
for q in dialogue.eval_questions:
for r in ('buyer', 'seller'):
key = 'eval_{question}_{role}'.format(question=q,
role=r)
try:
row[key] = dialogue.eval_scores[r][q]
except KeyError:
row[key] = -1
data.append(row)
df = pd.DataFrame(data).fillna(0)
return df
def summarize_liwc(self, k=10):
categories = defaultdict(lambda: defaultdict(int))
for dialogue in ifilter(lambda x: x.has_deal(), self.examples):
for u in dialogue.iter_utterances():
for cat, word_count in u.categories.iteritems():
for w, count in word_count.iteritems():
categories[cat][w] += count
cat_freq = {
c: sum(word_counts.values())
for c, word_counts in categories.iteritems()
}
cat_freq = sorted(cat_freq.items(), key=lambda x: x[1], reverse=True)
def topk(word_counts, k=10):
wc = sorted(word_counts.items(), key=lambda x: x[1],
reverse=True)[:k]
return [x[0] for x in wc]
for cat, count in cat_freq:
print cat, count, topk(categories[cat], k)
def html_visualize(self,
output,
img_path,
css_file=None,
mpld3_plugin=None):
examples = [ex for ex in self.examples if self.has_deal(ex)]
examples.sort(
key=lambda d: (d.scenario_id, d.outcome['offer']['price']))
include_scripts = []
include_scripts.append(
'<script type="text/javascript" src="http://d3js.org/d3.v3.min.js"></script>'
)
include_scripts.append(
'<script type="text/javascript" src="https://mpld3.github.io/js/mpld3.v0.2.js"></script>'
)
if mpld3_plugin:
with open(mpld3_plugin, 'r') as fin:
mpld3_script = fin.read()
include_scripts.append(
'<script type="text/javascript">{}</script>'.format(
mpld3_script))
css_style = """
table {
table-layout: fixed;
width: 600px;
border-collapse: collapse;
}
tr:nth-child(n) {
border: solid thin;
}
.fig {
height: 500px;
}
"""
if css_file:
with open(css_file, 'r') as fin:
css_style = '{}\n{}'.format(css_style, fin.read())
style = [
'<style type="text/css">', css_style, Dialogue.css, '</style>'
]
header = ['<head>'] + style + include_scripts + ['</head>']
plot_divs = []
plot_scripts = []
plot_scripts.append('<script type="text/javascript">')
for d in examples:
var_name = 'json_{}'.format(d.chat_id)
json_str = json.dumps(d.fig_dict())
div_name = 'fig_{}'.format(d.chat_id)
plot_divs.append('<div class="fig" id="{div_name}"></div>'.format(
div_name=div_name))
plot_scripts.append('var {var_name} = {json_str};'.format(
var_name=var_name, json_str=json_str))
plot_scripts.append(
'!function(mpld3) {{ mpld3.draw_figure("{div_name}", {var_name}); }}(mpld3);'
.format(div_name=div_name, var_name=var_name))
plot_scripts.append('</script>')
body = ['<body>']
for d, plot_div in izip(examples, plot_divs):
body.extend(
NegotiationHTMLVisualizer.render_scenario(None,
img_path=img_path,
kbs=d.kbs,
uuid=d.scenario_id))
body.append('<p>Final deal: {}</p>'.format(
d.outcome['offer']['price']))
body.append(plot_div)
body.extend(plot_scripts)
body.append('</body>')
html_lines = ['<html>'] + header + body + ['</html>']
outfile = open(output, 'w')
for line in html_lines:
outfile.write(line.encode('utf8') + "\n")
print 'Write to', output
outfile.close()
@classmethod
def get_price_trend(cls, price_tracker, chat, agent=None):
def _normalize_price(seen_price):
return (float(seller_target) - float(seen_price)) / (
float(seller_target) - float(buyer_target))
scenario = NegotiationScenario.from_dict(None, chat['scenario'])
# print chat['scenario']
kbs = scenario.kbs
roles = {
kbs[0].facts['personal']['Role']: 0,
kbs[1].facts['personal']['Role']: 1
}
buyer_target = kbs[roles[utils.BUYER]].facts['personal']['Target']
seller_target = kbs[roles[utils.SELLER]].facts['personal']['Target']
prices = []
for e in chat['events']:
if e['action'] == 'message':
if agent is not None and e['agent'] != agent:
continue
raw_tokens = tokenize(e['data'])
# link entity
linked_tokens = price_tracker.link_entity(raw_tokens,
kb=kbs[e['agent']])
for token in linked_tokens:
if isinstance(token, Entity):
try:
replaced = PriceScaler.unscale_price(
kbs[e['agent']], token)
except OverflowError:
print "Raw tokens: ", raw_tokens
print "Overflow error: {:s}".format(token)
print kbs[e['agent']].facts
print "-------"
continue
norm_price = _normalize_price(replaced.canonical.value)
if 0. <= norm_price <= 2.:
# if the number is greater than the list price or significantly lower than the buyer's
# target it's probably not a price
prices.append(norm_price)
# do some stuff here
elif e['action'] == 'offer':
norm_price = _normalize_price(e['data']['price'])
if 0. <= norm_price <= 2.:
prices.append(norm_price)
# prices.append(e['data']['price'])
# print "Chat: {:s}".format(chat['uuid'])
# print "Trend:", prices
return prices
@classmethod
def split_turn(cls, turn):
# a single turn can be comprised of multiple sentences
return cls.sent_tokenizer.tokenize(turn)
def get_speech_acts(self, ex):
stats = {0: [], 1: []}
kbs = ex.kbs
for e in ex.events:
if e.action != 'message':
continue
sentences = self.split_turn(e.data.lower())
for s in sentences:
tokens = tokenize(s)
linked_tokens = self.price_tracker.link_entity(tokens,
kb=kbs[e.agent])
act = SpeechActAnalyzer.get_speech_act(s, linked_tokens)
stats[e.agent].append(act)
return stats
@classmethod
def valid_price(cls, price):
return price <= MAX_PRICE and price >= MIN_PRICE
@classmethod
def valid_margin(cls, margin):
return margin <= MAX_MARGIN and margin >= MIN_MARGIN
def get_first_price(self, ex):
agents = {1: None, 0: None}
for e in ex.events:
if e.action == 'message':
for sent_tokens in e.tokens:
for token in sent_tokens:
if agents[1] and agents[0]:
return agents
# Return at the first mention
if is_entity(token):
price = token.canonical.value
agents[e.agent] = (e.role, price)
return agents
return agents
@classmethod
def get_margin(cls, ex, price, agent, role, remove_outlier=True):
agent_target = ex.scenario.kbs[agent].facts["personal"]["Target"]
partner_target = ex.scenario.kbs[1 - agent].facts["personal"]["Target"]
midpoint = (agent_target + partner_target) / 2.
norm_factor = np.abs(midpoint - agent_target)
if role == utils.SELLER:
margin = (price - midpoint) / norm_factor
else:
margin = (midpoint - price) / norm_factor
if remove_outlier and not cls.valid_margin(margin):
return None
return margin
@classmethod
def print_ex(cls, ex):
print '===================='
for e in ex.events:
print e.role.upper(), e.data
print '===================='
def get_basic_stats(self, ex):
stats = {0: None, 1: None}
for agent in (0, 1):
num_turns = ex.num_turns()
num_tokens = ex.num_tokens()
stats[agent] = {
'role': ex.kbs[agent].facts['personal']['Role'],
'num_turns': num_turns,
'num_tokens_per_turn': num_tokens / num_turns * 1.,
}
return stats
def is_good_negotiator(self, final_margin):
if final_margin > 0.8 and final_margin <= 1:
return 1
elif final_margin < -0.8 and final_margin <= -1:
return -1
else:
return 0
@classmethod
def has_deal(cls, ex):
if ex.outcome is None or ex.outcome['reward'] == 0 or ex.outcome.get(
'offer', None) is None or ex.outcome['offer']['price'] is None:
return False
return True
def plot_speech_acts(self, output='figures/speech_acts'):
data = defaultdict(list)
for ex in ifilter(self.has_deal, self.examples):
stats = self.get_speech_acts(ex)
final_price = ex.outcome['offer']['price']
for agent, acts in stats.iteritems():
role = ex.agent_to_role[agent]
final_margin = self.get_margin(ex, final_price, agent, role)
label = self.is_good_negotiator(final_margin)
for act in acts:
data['role'].append(role)
data['label'].append(label)
data['final_margin'].append(final_margin)
data['act'].append(act)
for role in ('seller', 'buyer'):
print role.upper()
print '=' * 40
good_seller_act = [
a for r, l, m, a in izip(data['role'], data['label'],
data['final_margin'], data['act'])
if r == role and l == 1
]
bad_seller_act = [
a for r, l, m, a in izip(data['role'], data['label'],
data['final_margin'], data['act'])
if r == role and l == -1
]
sum_act = lambda a, l: np.mean([1 if a in x else 0 for x in l])
print len(good_seller_act), len(bad_seller_act)
print '{:<20} {:<10} {:<10}'.format('ACT', 'GOOD', 'BAD')
print '-' * 40
for act in SpeechActs.ACTS:
print '{:<20} {:<10.4f} {:<10.4f}'.format(
act, sum_act(act, good_seller_act),
sum_act(act, bad_seller_act))
return
def plot_basic_stats(self, output='figures/basic_stats'):
data = {
'role': [],
'final_margin': [],
'num_turns': [],
'num_tokens_per_turn': [],
'label': []
}
for ex in ifilter(self.has_deal, self.examples):
stats = self.get_basic_stats(ex)
final_price = ex.outcome['offer']['price']
for agent, stats in stats.iteritems():
role = stats['role']
final_margin = self.get_margin(ex, final_price, agent, role)
label = self.is_good_negotiator(final_margin)
for k, v in stats.iteritems():
data[k].append(v)
data['label'].append(label)
data['final_margin'].append(final_margin)
fig = plt.figure()
df = pd.DataFrame(data)
#g = sns.lmplot(x='num_tokens_per_turn', y='final_margin', col='role', row='label', data=dataframe, scatter_kws={'alpha':0.5})
#g.savefig(output)
for role in ('buyer', 'seller'):
d1 = df.num_tokens_per_turn[(df['label'] == 1)
& (df['role'] == role)]
d2 = df.num_tokens_per_turn[(df.label == -1) & (df.role == role)]
sns.distplot(d1, label='good')
sns.distplot(d2, label='bad')
plt.legend()
plt.savefig('%s_%s.png' % (output, role))
plt.clf()
def plot_opening_vs_result(self, output='figures/opening_vs_result.png'):
data = {'role': [], 'init_margin': [], 'final_margin': []}
for ex in ifilter(self.has_deal, self.examples):
final_price = ex.outcome['offer']['price']
init_prices = self.get_first_price(ex)
for agent, p in init_prices.iteritems():
if p is None:
continue
role, price = p
init_margin = self.get_margin(ex, price, agent, role)
final_margin = self.get_margin(ex, final_price, agent, role)
if init_margin is None or final_margin is None:
continue
# NOTE: sometimes one is saying a price is not okay, i.e. negative mention
# TODO: detect negative vs positive mention
if init_margin == -1 and init_margin < final_margin:
continue
#if init_margin < final_margin:
# print role, (price, init_margin), (final_price, final_margin)
# self.print_ex(ex)
# import sys; sys.exit()
for k, v in izip(('role', 'init_margin', 'final_margin'),
(role, init_margin, final_margin)):
data[k].append(v)
dataframe = pd.DataFrame(data)
fig = plt.figure()
g = sns.lmplot(x='init_margin',
y='final_margin',
col='role',
data=dataframe,
scatter_kws={'alpha': 0.5})
g.savefig(output)
def group_outcomes_and_roles(self):
buyer_wins = []
seller_wins = []
ties = 0
total_chats = 0
for ex in self.dataset:
roles = {
0: ex["scenario"]["kbs"][0]["personal"]["Role"],
1: ex["scenario"]["kbs"][1]["personal"]["Role"]
}
winner = utils.get_winner(ex)
if winner is None:
continue
total_chats += 1
if winner == -1:
buyer_wins.append(ex)
seller_wins.append(ex)
ties += 1
elif roles[winner] == utils.BUYER:
buyer_wins.append(ex)
elif roles[winner] == utils.SELLER:
seller_wins.append(ex)
print "# of ties: {:d}".format(ties)
print "Total chats with outcomes: {:d}".format(total_chats)
return buyer_wins, seller_wins
def plot_length_vs_margin(self, out_name='turns_vs_margin.png'):
labels = ['buyer wins', 'seller wins']
plt.figure(figsize=(10, 6))
for (chats, lbl) in zip([self.buyer_wins, self.seller_wins], labels):
margins = defaultdict(list)
for ex in chats:
turns = utils.get_turns_per_agent(ex)
total_turns = turns[0] + turns[1]
margin = utils.get_margin(ex)
if margin > MAX_MARGIN or margin < 0.:
continue
margins[total_turns].append(margin)
sorted_keys = list(sorted(margins.keys()))
turns = []
means = []
errors = []
for k in sorted_keys:
if len(margins[k]) >= THRESHOLD:
turns.append(k)
means.append(np.mean(margins[k]))
errors.append(stats.sem(margins[k]))
plt.errorbar(turns, means, yerr=errors, label=lbl, fmt='--o')
plt.legend()
plt.xlabel('# of turns in dialogue')
plt.ylabel('Margin of victory')
save_path = os.path.join(self.stats_path, out_name)
plt.savefig(save_path)
def plot_margin_histograms(self):
for (lbl, group) in zip(['buyer_wins', 'seller_wins'],
[self.buyer_wins, self.seller_wins]):
margins = []
for ex in group:
winner = utils.get_winner(ex)
if winner is None:
continue
margin = utils.get_margin(ex)
if 0 <= margin <= MAX_MARGIN:
margins.append(margin)
b = np.linspace(0, MAX_MARGIN, num=int(MAX_MARGIN / 0.2) + 2)
print b
hist, bins = np.histogram(margins, bins=b)
width = np.diff(bins)
center = (bins[:-1] + bins[1:]) / 2
fig, ax = plt.subplots(figsize=(8, 3))
ax.bar(center, hist, align='center', width=width)
ax.set_xticks(bins)
save_path = os.path.join(
self.stats_path, '{:s}_wins_margins_histogram.png'.format(lbl))
plt.savefig(save_path)
def plot_length_histograms(self):
lengths = []
for ex in self.dataset:
winner = utils.get_winner(ex)
if winner is None:
continue
turns = utils.get_turns_per_agent(ex)
total_turns = turns[0] + turns[1]
lengths.append(total_turns)
hist, bins = np.histogram(lengths)
width = np.diff(bins)
center = (bins[:-1] + bins[1:]) / 2
fig, ax = plt.subplots(figsize=(8, 3))
ax.bar(center, hist, align='center', width=width)
ax.set_xticks(bins)
save_path = os.path.join(self.stats_path, 'turns_histogram.png')
plt.savefig(save_path)
def plot_price_trends(self, top_n=10):
labels = ['buyer_wins', 'seller_wins']
for (group, lbl) in zip([self.buyer_wins, self.seller_wins], labels):
plt.figure(figsize=(10, 6))
trends = []
for chat in group:
winner = utils.get_winner(chat)
margin = utils.get_margin(chat)
if margin > 1.0 or margin < 0.:
continue
if winner is None:
continue
# print "Winner: Agent {:d}\tWin margin: {:.2f}".format(winner, margin)
if winner == -1 or winner == 0:
trend = self.get_price_trend(self.price_tracker,
chat,
agent=0)
if len(trend) > 1:
trends.append((margin, chat, trend))
if winner == -1 or winner == 1:
trend = self.get_price_trend(self.price_tracker,
chat,
agent=1)
if len(trend) > 1:
trends.append((margin, chat, trend))
# print ""
sorted_trends = sorted(trends, key=lambda x: x[0], reverse=True)
for (idx, (margin, chat,
trend)) in enumerate(sorted_trends[:top_n]):
print '{:s}: Chat {:s}\tMargin: {:.2f}'.format(
lbl, chat['uuid'], margin)
print 'Trend: ', trend
print chat['scenario']['kbs']
print ""
plt.plot(trend, label='Margin={:.2f}'.format(margin))
plt.legend()
plt.xlabel('N-th price mentioned in chat')
plt.ylabel('Value of mentioned price')
out_path = os.path.join(self.stats_path,
'{:s}_trend.png'.format(lbl))
plt.savefig(out_path)
def _get_price_mentions(self, chat, agent=None):
scenario = NegotiationScenario.from_dict(None, chat['scenario'])
# print chat['scenario']
kbs = scenario.kbs
prices = 0
for e in chat['events']:
if agent is not None and e['agent'] != agent:
continue
if e['action'] == 'message':
raw_tokens = tokenize(e['data'])
# link entity
linked_tokens = self.price_tracker.link_entity(
raw_tokens, kb=kbs[e['agent']])
for token in linked_tokens:
if isinstance(token,
Entity) and token.canonical.type == 'price':
prices += 1
return prices
def plot_speech_acts_old(self):
labels = ['buyer_wins', 'seller_wins']
for (group, lbl) in zip([self.buyer_wins, self.seller_wins], labels):
plt.figure(figsize=(10, 6))
speech_act_counts = dict(
(act, defaultdict(list)) for act in SpeechActs.ACTS)
for chat in group:
winner = utils.get_winner(chat)
margin = utils.get_margin(chat)
if margin > MAX_MARGIN or margin < 0.:
continue
if winner is None:
continue
margin = round_partial(
margin
) # round the margin to the nearest 0.1 to reduce noise
if winner == -1 or winner == 0:
speech_acts = self.get_speech_acts(chat, agent=0)
# print "Chat {:s}\tWinner: {:d}".format(chat['uuid'], winner)
# print speech_acts
for act in SpeechActs.ACTS:
frac = float(speech_acts.count(act)) / float(
len(speech_acts))
speech_act_counts[act][margin].append(frac)
if winner == -1 or winner == 1:
speech_acts = self.get_speech_acts(chat, agent=1)
# print "Chat {:s}\tWinner: {:d}".format(chat['uuid'], winner)
# print speech_acts
for act in SpeechActs.ACTS:
frac = float(speech_acts.count(act)) / float(
len(speech_acts))
speech_act_counts[act][margin].append(frac)
for act in SpeechActs.ACTS:
counts = speech_act_counts[act]
margins = []
fracs = []
errors = []
bin_totals = 0.
for m in sorted(counts.keys()):
if len(counts[m]) > THRESHOLD:
bin_totals += len(counts[m])
margins.append(m)
fracs.append(np.mean(counts[m]))
errors.append(stats.sem(counts[m]))
print bin_totals / float(len(margins))
plt.errorbar(margins, fracs, yerr=errors, label=act, fmt='--o')
plt.xlabel('Margin of victory')
plt.ylabel('Fraction of speech act occurences')
plt.title('Speech act frequency vs. margin of victory')
plt.legend()
save_path = os.path.join(self.stats_path,
'{:s}_speech_acts.png'.format(lbl))
plt.savefig(save_path)
def plot_speech_acts_by_role(self):
labels = utils.ROLES
for lbl in labels:
plt.figure(figsize=(10, 6))
speech_act_counts = dict(
(act, defaultdict(list)) for act in SpeechActs.ACTS)
for chat in self.dataset:
if utils.get_winner(chat) is None:
# skip chats with no outcomes
continue
speech_acts = self.get_speech_acts(chat, role=lbl)
agent = 1 if chat['scenario']['kbs'][1]['personal'][
'Role'] == lbl else 0
margin = utils.get_margin(chat, agent=agent)
if margin > MAX_MARGIN:
continue
margin = round_partial(margin)
for act in SpeechActs.ACTS:
frac = float(speech_acts.count(act)) / float(
len(speech_acts))
speech_act_counts[act][margin].append(frac)
for act in SpeechActs.ACTS:
counts = speech_act_counts[act]
margins = []
fracs = []
errors = []
for m in sorted(counts.keys()):
if len(counts[m]) > THRESHOLD:
margins.append(m)
fracs.append(np.mean(counts[m]))
errors.append(stats.sem(counts[m]))
plt.errorbar(margins, fracs, yerr=errors, label=act, fmt='--o')
plt.xlabel('Margin of victory')
plt.ylabel('Fraction of speech act occurences')
plt.title('Speech act frequency vs. margin of victory')
plt.legend()
save_path = os.path.join(self.stats_path,
'{:s}_speech_acts.png'.format(lbl))
plt.savefig(save_path)
