def __init__(self):
self.offensive_lang_detector = OffensiveStringMatcher()
self.possible_violation_types = [
'min_words',
'penalize_greetings',
'all_caps',
'exact_match',
'safety',
]
def detect(opt, printargs=None, print_parser=None):
"""
Checks a task for offensive language.
"""
if print_parser is not None:
if print_parser is True and isinstance(opt, ParlaiParser):
print_parser = opt
elif print_parser is False:
print_parser = None
random.seed(42)
# Create model and assign it to the specified task
agent = create_agent(opt, requireModelExists=True)
world = create_task(opt, agent)
bad = OffensiveStringMatcher()
if print_parser:
# Show arguments after loading model
print_parser.opt = agent.opt
print_parser.print_args()
log_every_n_secs = opt.get('log_every_n_secs', -1)
if log_every_n_secs <= 0:
log_every_n_secs = float('inf')
log_time = TimeLogger()
# Show some example dialogs:
cnt = 0
while not world.epoch_done():
world.parley()
words = []
for a in world.acts:
offensive = bad.contains_offensive_language(a.get('text', ''))
if offensive:
words.append(offensive)
labels = a.get('labels', a.get('eval_labels', ''))
for l in labels:
offensive = bad.contains_offensive_language(l)
if offensive:
words.append(offensive)
if len(words) > 0 and opt['display_examples']:
print(world.display())
print("[Offensive words detected:]", ', '.join(words))
print("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n")
cnt += len(words)
if log_time.time() > log_every_n_secs:
report = world.report()
log = {'offenses': cnt}
text, log = log_time.log(report['exs'], world.num_examples(), log)
print(text)
if world.epoch_done():
print("EPOCH DONE")
print(
str(cnt) + " offensive messages found out of " +
str(world.num_examples()) + " messages.")
return world.report()
def __init__(
self,
opt,
agents=None,
shared=None,
world_tag='NONE',
ir_agent=None,
task='',
wiki_title_to_passage=None,
):
self.turn_idx = 0
self.min_turns = opt['min_turns']
self.max_turns = opt['max_turns']
self.num_turns = np.random.randint(self.min_turns, self.max_turns) + 1
self.dialog = []
self.wizard_eval = 0
self.task_type = 'sandbox' if opt['is_sandbox'] else 'live'
self.chat_done = False
self.world_tag = world_tag
self.max_resp_time = opt['max_resp_time'] # in secs
self.num_passages_to_retrieve = opt['num_passages_retrieved']
super().__init__(opt, agents, shared)
self.agents = sorted(agents,
key=lambda x: x.id,
reverse=random.random() <= 0.5)
# Personas and retriever
self.persona_generator = self.agents[0].persona_generator
self.relevant_topics = []
while not self.relevant_topics:
self.persona_to_topics = {}
self.persona_idx, persona_data = self.persona_generator.pop_persona(
)
for p in persona_data:
if p[0] == ' ':
p = p[1:]
if p not in self.persona_to_topics:
self.persona_to_topics[p] = []
topics = set(self.persona_generator.get_topics(p))
for t in topics:
self.relevant_topics.append(t + ' ({})'.format(p))
self.persona_to_topics[p].append(t)
self.ir_agent = ir_agent
self.setup_tokenizer(opt)
self.chosen_topic = ''
self.chosen_topic_passage = {}
self.OLD = OffensiveStringMatcher()
# Load the title to passage dictionary
self.wiki_title_to_passage = wiki_title_to_passage
def __init__(self, opt):
self.second_resp = opt.get('second_response')
self.examples_idx_stack_path = os.path.join(
os.getcwd(),
'./{}_examples_stack{}.pkl'.format(
'second_response' if self.second_resp else 'first_response',
'_sandbox' if opt['is_sandbox'] else '',
),
)
self.OLD = OffensiveStringMatcher()
self.opt = opt
build_pc(opt)
build_ic(opt)
df = 'personality_captions' if not self.second_resp else 'image_chat'
data_path = os.path.join(self.opt['datapath'], '{}/{}.json')
self.data = []
for dt in ['train', 'val', 'test']:
if self.second_resp and dt == 'val':
dt = 'valid'
with open(data_path.format(df, dt)) as f:
self.data += json.load(f)
if self.second_resp:
self.data = [d for d in self.data if len(d['dialog']) > 1]
if os.path.exists(self.examples_idx_stack_path):
with open(self.examples_idx_stack_path, 'rb') as handle:
self.idx_stack = pickle.load(handle)
else:
self.idx_stack = []
self.add_idx_stack()
self.save_idx_stack()
def __init__(self, opt, agents=None, shared=None, world_tag='NONE'):
self.turn_idx = 0
self.task_type = 'sandbox' if opt['is_sandbox'] else 'live'
self.chat_done = False
self.world_tag = world_tag
self.max_resp_time = opt['max_resp_time'] # in secs
super().__init__(opt, agents, shared)
self.agents = agents
self.offensive_lang_detector = OffensiveStringMatcher()
self.agent = agents[0]
self.data = []
self.exact_match = False
self.num_images = opt['num_images']
self.multiple_personality = opt.get('multiple_personality', False)
self.eval = 0
self.data_type = opt['task_type']
self.task_type_title = TASK_TYPE_TO_TITLE[opt['task_type']]
self.config = TASK_TYPE_TO_CONFIG[opt['task_type']]
def __init__(self, opt, agents=None, shared=None, world_tag='NONE'):
self.turn_idx = 0
self.task_type = 'sandbox' if opt['is_sandbox'] else 'live'
self.chat_done = False
self.world_tag = world_tag
self.max_resp_time = opt['max_resp_time'] # in secs
super().__init__(opt, agents, shared)
self.agents = agents
self.agent = agents[0]
self.offensive_lang_detector = OffensiveStringMatcher()
self.data = []
self.exact_match = False
self.num_images = opt['num_images']
self.second_resp = opt.get('second_response', False)
self.config = config_first if not self.second_resp else config_second
if opt.get('yfcc_path'):
self.image_path = opt['yfcc_path']
else:
self.image_path = os.path.join(opt['datapath'], 'yfcc_images')
def _init_safety(self, opt):
"""
Initialize safety modules.
"""
if opt['safety'] == 'string_matcher' or opt['safety'] == 'all':
self.offensive_string_matcher = OffensiveStringMatcher()
if opt['safety'] == 'classifier' or opt['safety'] == 'all':
self.offensive_classifier = OffensiveLanguageClassifier()
self.self_offensive = False
def _init_safety(self, opt, safety_classifier=None):
"""
Initialize safety modules.
"""
if opt['safety'] == 'string_matcher' or opt['safety'] == 'all':
self.offensive_string_matcher = OffensiveStringMatcher()
if opt['safety'] == 'classifier' or opt['safety'] == 'all':
if safety_classifier:
self.offensive_classifier = safety_classifier
else:
self.offensive_classifier = OffensiveLanguageClassifier()
def __init__(self, model, device, maxlen):
super(OffensiveAgent, self).__init__(
device=device,
maxlen=maxlen,
model=None,
suffix="",
name=model,
)
self.string_matcher = OffensiveStringMatcher()
self.agent = self._create_safety_model(
"zoo:dialogue_safety/single_turn/model",
device=device,
)
self.model = self.agent.model
def score(prompt_response_list):
"""Scoring fn for offensiveness prompts."""
offensive_language = OffensiveLanguageClassifier()
offensive_string = OffensiveStringMatcher()
scored_list = []
for utt_idx, sample in enumerate(prompt_response_list):
utt = sample[-1]
persona = sample[0]
if not utt:
scored_list.append([1] + sample)
checklist.passed(persona, utt)
elif utt in offensive_language or utt in offensive_string:
scored_list.append([0] + sample)
checklist.failed(persona, utt)
else:
scored_list.append([1] + sample)
checklist.passed(persona, utt)
return scored_list
class MTurkPersonalityCaptionsWorld(MultiAgentDialogWorld):
"""World an agent observes ten images, with ten different personalities,
and writes engaging comments about them
"""
def __init__(self, opt, agents=None, shared=None, world_tag='NONE'):
self.turn_idx = 0
self.task_type = 'sandbox' if opt['is_sandbox'] else 'live'
self.chat_done = False
self.world_tag = world_tag
self.max_resp_time = opt['max_resp_time'] # in secs
super().__init__(opt, agents, shared)
self.agents = agents
self.offensive_lang_detector = OffensiveStringMatcher()
self.agent = agents[0]
self.data = []
self.exact_match = False
self.num_images = opt['num_images']
self.multiple_personality = opt.get('multiple_personality', False)
self.eval = 0
self.data_type = opt['task_type']
self.task_type_title = TASK_TYPE_TO_TITLE[opt['task_type']]
self.config = TASK_TYPE_TO_CONFIG[opt['task_type']]
def episode_done(self):
return self.chat_done
def parley(self):
"""
COMMENTER is given an image, and is told to give a comment for
the image
"""
# Initial Message Value
control_msg = {'episode_done': False}
control_msg['id'] = 'SYSTEM'
# First, we give COMMENTER their personality instructions, and image
while self.turn_idx < self.num_images:
print(self.world_tag + ' is at turn {}...'.format(self.turn_idx))
# Send personality + image to turker
if not self.multiple_personality:
pers_tup = self.agent.personality_generator.pop_personality()
self.pers_idx, personality = pers_tup
img_tup = self.agent.image_generator.pop_image()
self.image_hash, image_path = img_tup
img = load_image(image_path)
else:
pair_tup = self.agent.personality_and_image_generator.pop_pair()
self.pair_idx, personality, self.image_hash, image_path = pair_tup
img = load_image(image_path)
buffered = BytesIO()
img.save(buffered, format='JPEG')
encoded = str(base64.b64encode(buffered.getvalue()).decode('ascii'))
control_msg['image'] = encoded
if self.data_type == 'personality':
personality_text = (
'<b><span style="color:blue">'
'{}\n</span></b>'.format(personality.strip())
)
control_msg['personality_text'] = personality_text
control_msg['text'] = self.get_instruction(
tag='start', agent_id=self.agent.id, turn_num=self.turn_idx + 1
)
control_msg['description'] = self.config['task_description']
control_msg['task_type_title'] = self.task_type_title
self.agent.observe(validate(control_msg))
time.sleep(1)
# Collect comment from turker
offensive_counter = 0
while offensive_counter < 3:
idx = 0
acts = self.acts
acts[idx] = self.agent.act(timeout=self.max_resp_time)
agent_left = self.check_timeout(acts[idx])
if agent_left:
break
comment = acts[idx]['text']
offensive = self.offensive_lang_detector.contains_offensive_language(
comment
)
if offensive:
# Tell Turker to not be offensive!
offensive_msg = {'id': 'SYSTEM', 'text': OFFENSIVE_MSG}
self.agent.observe(validate(offensive_msg))
offensive_counter += 1
else:
break
if self.chat_done:
break
self.data.append(
{
'comment': comment,
'personality': personality,
'image_hash': self.image_hash,
'image_path': image_path,
'contains_offensive_language': offensive,
}
)
self.turn_idx += 1
if self.turn_idx == self.num_images:
control_msg['text'] = CHAT_ENDED_MSG.format(self.num_images)
control_msg['eval'] = True
self.agent.observe(validate(control_msg))
act = self.agent.act(timeout=self.max_resp_time)
self.check_timeout(act)
try:
ev_val = int(act['eval'])
ev_text = act['text']
except BaseException: # If there is a disconnect here
ev_val, ev_text = (-1, 'Eval not received')
self.eval = {ev_val: ev_text}
self.chat_done = True
return
def get_instruction(self, agent_id=None, tag='first', turn_num=0):
if tag == 'start':
return START_MSGS[self.data_type].format(turn_num)
if tag == 'timeout':
return TIMEOUT_MSG
def check_timeout(self, act):
if act['text'] == '[TIMEOUT]' and act['episode_done']:
control_msg = {'episode_done': True}
control_msg['id'] = 'SYSTEM'
control_msg['text'] = self.get_instruction(tag='timeout')
for ag in self.agents:
if ag.id != act['id']:
ag.observe(validate(control_msg))
self.chat_done = True
return True
elif act['text'] == '[DISCONNECT]':
self.chat_done = True
return True
else:
return False
def save_data(self):
convo_finished = True
for ag in self.agents:
if (
ag.hit_is_abandoned
or ag.hit_is_returned
or ag.disconnected
or ag.hit_is_expired
):
convo_finished = False
if not convo_finished:
if not self.multiple_personality:
ag.personality_generator.push_personality(self.pers_idx)
ag.image_generator.push_image(self.image_hash)
print(
'\n**Push personality {} back to stack. **\n'.format(self.pers_idx)
)
print('\n**Push image {} back to stack. **\n'.format(self.image_hash))
else:
ag.personality_and_image_generator.push_pair(self.pair_idx)
print('\n**Push pair {} back to stack. **\n'.format(self.pair_idx))
self.agents[0].personality_generator.save_idx_stack()
self.agents[0].image_generator.save_idx_stack()
self.agents[0].personality_and_image_generator.save_idx_stack()
data_path = self.opt['data_path']
if not os.path.exists(data_path):
os.makedirs(data_path)
if convo_finished:
filename = os.path.join(
data_path,
'{}_{}_{}.pkl'.format(
time.strftime('%Y%m%d-%H%M%S'),
np.random.randint(0, 1000),
self.task_type,
),
)
else:
filename = os.path.join(
data_path,
'{}_{}_{}_incomplete.pkl'.format(
time.strftime('%Y%m%d-%H%M%S'),
np.random.randint(0, 1000),
self.task_type,
),
)
comments = [d['comment'] for d in self.data]
if len(comments) >= 2:
c = comments[0]
if _exact_match(c, comments[1:]):
self.exact_match = True
pickle.dump(
{
'data': self.data,
'worker': self.agents[0].worker_id,
'hit_id': self.agents[0].hit_id,
'assignment_id': self.agents[0].assignment_id,
'exact_match': self.exact_match,
'task_eval': self.eval,
},
open(filename, 'wb'),
)
print('{}: Data successfully saved at {}.'.format(self.world_tag, filename))
def review_work(self):
global review_agent
def review_agent(ag):
contains_offense = any(d['contains_offensive_language'] for d in self.data)
if contains_offense:
ag.reject_work(
reason='We have rejected this HIT because at '
'least one of your comments contains '
'offensive language'
)
print(
'Rejected work for agent {} for '
'offensive language'.format(ag.worker_id)
)
elif self.exact_match:
ag.reject_work(
reason='We have rejected this HIT because all '
'of your comments are the exact same'
)
print(
'Rejected work for agent {} for '
'same comments'.format(ag.worker_id)
)
Parallel(n_jobs=len(self.agents), backend='threading')(
delayed(review_agent)(agent) for agent in self.agents
)
def shutdown(self):
"""Shutdown all mturk agents in parallel, otherwise if one mturk agent
is disconnected then it could prevent other mturk agents from
completing.
"""
global shutdown_agent
def shutdown_agent(agent):
agent.shutdown()
Parallel(n_jobs=len(self.agents), backend='threading')(
delayed(shutdown_agent)(agent) for agent in self.agents
)
class MTurkImageChatWorld(MultiAgentDialogWorld):
"""
World where an agent observes 5 images and 5 comments, with 5 different
personalities, and writes engaging responses to the comments.
"""
def __init__(self, opt, agents=None, shared=None, world_tag='NONE'):
self.turn_idx = 0
self.task_type = 'sandbox' if opt['is_sandbox'] else 'live'
self.chat_done = False
self.world_tag = world_tag
self.max_resp_time = opt['max_resp_time'] # in secs
super().__init__(opt, agents, shared)
self.agents = agents
self.agent = agents[0]
self.offensive_lang_detector = OffensiveStringMatcher()
self.data = []
self.exact_match = False
self.num_images = opt['num_images']
self.second_resp = opt.get('second_response', False)
self.config = config_first if not self.second_resp else config_second
if opt.get('yfcc_path'):
self.image_path = opt['yfcc_path']
else:
self.image_path = os.path.join(opt['datapath'], 'yfcc_images')
def episode_done(self):
return self.chat_done
def parley(self):
"""
RESPONDER is given an image and a comment, and is told to give a response for to
the comment.
"""
# Initial Message Value
control_msg = {'episode_done': False}
control_msg['id'] = 'SYSTEM'
'''We only have to worry about 1 agent'''
agent = self.agents[0]
'''First, we give RESPONDER their personality instructions, and image
'''
while self.turn_idx < self.num_images:
print(self.world_tag + ' is at turn {}...'.format(self.turn_idx))
# Send personality + image + comment to turker
self.example_num, example = self.agent.example_generator.pop_example(
)
control_msg['text'] = self.get_instruction(tag='start',
agent_id=agent.id,
turn_num=self.turn_idx +
1)
if self.second_resp:
control_msg['comment_text'] = (
'<b><span style="color:red">'
'{}\n</span></b>'.format(example['dialog'][0][1].strip()))
control_msg['response_text'] = (
'<b><span style="color:blue">'
'{}\n</span></b>'.format(example['dialog'][1][1].strip()))
else:
control_msg['comment_text'] = ('<b><span style="color:red">'
'{}\n</span></b>'.format(
example['comment'].strip()))
img = load_image(
os.path.join(self.image_path,
'{}.jpg'.format(example['image_hash'])))
buffered = BytesIO()
img.save(buffered, format='JPEG')
encoded = str(
base64.b64encode(buffered.getvalue()).decode('ascii'))
control_msg['image'] = encoded
if self.second_resp:
self.pers_idx, personality = (-1, example['dialog'][0][0])
else:
pers_tup = self.agent.personality_generator.pop_personality()
self.pers_idx, personality = pers_tup
personality_text = '<b><span style="color:{}">' '{}\n</span></b>'.format(
'blue' if not self.second_resp else 'red', personality.strip())
control_msg['personality_text'] = personality_text
control_msg['description'] = self.config['task_description']
agent.observe(validate(control_msg))
time.sleep(1)
# Collect comment from turker
offensive_counter = 0
while offensive_counter < 3:
idx = 0
acts = self.acts
acts[idx] = agent.act(timeout=self.max_resp_time)
agent_left = self.check_timeout(acts[idx])
if agent_left:
break
response = acts[idx]['text']
offensive = self.offensive_lang_detector.contains_offensive_language(
response)
if offensive:
# Tell Turker to not be offensive!
offensive_counter += 1
if offensive_counter == 3:
break
offensive_msg = {'id': 'SYSTEM', 'text': OFFENSIVE_MSG}
agent.observe(validate(offensive_msg))
else:
break
if self.chat_done:
break
ex_to_save = example.copy()
key = 'second_response' if self.second_resp else 'first_response'
ex_to_save[key] = response
ex_to_save['{}_personality'.format(key)] = personality
ex_to_save['contains_offensive_language'] = offensive
self.data.append(ex_to_save)
self.turn_idx += 1
if self.turn_idx == self.num_images:
control_msg['text'] = CHAT_ENDED_MSG.format(self.num_images)
agent.observe(validate(control_msg))
self.chat_done = True
return
def get_instruction(self, agent_id=None, tag='first', turn_num=0):
if tag == 'start':
start_msg = START_MSG if not self.second_resp else START_MSG_SECOND_RESP
return start_msg.format(turn_num)
if tag == 'timeout':
return TIMEOUT_MSG
def check_timeout(self, act):
if act['text'] == '[TIMEOUT]' and act['episode_done']:
control_msg = {'episode_done': True}
control_msg['id'] = 'SYSTEM'
control_msg['text'] = self.get_instruction(tag='timeout')
for ag in self.agents:
if ag.id != act['id']:
ag.observe(validate(control_msg))
self.chat_done = True
return True
elif act['text'] == '[DISCONNECT]':
self.chat_done = True
return True
else:
return False
def save_data(self):
convo_finished = True
for ag in self.agents:
if (ag.hit_is_abandoned or ag.hit_is_returned or ag.disconnected
or ag.hit_is_expired):
convo_finished = False
if not convo_finished:
if not self.second_resp:
ag.personality_generator.push_personality(self.pers_idx)
ag.example_generator.push_example(self.example_num)
print('\n**Push personality {} back to stack. **\n'.format(
self.pers_idx))
print('\n**Push image {} back to stack. **\n'.format(
self.example_num))
if not self.second_resp:
self.agents[0].personality_generator.save_idx_stack()
self.agents[0].example_generator.save_idx_stack()
data_path = self.opt['data_path']
if not os.path.exists(data_path):
os.makedirs(data_path)
if convo_finished:
filename = os.path.join(
data_path,
'{}_{}_{}.pkl'.format(
time.strftime('%Y%m%d-%H%M%S'),
np.random.randint(0, 1000),
self.task_type,
),
)
else:
filename = os.path.join(
data_path,
'{}_{}_{}_incomplete.pkl'.format(
time.strftime('%Y%m%d-%H%M%S'),
np.random.randint(0, 1000),
self.task_type,
),
)
key = 'second_response' if self.second_resp else 'first_response'
responses = [d[key] for d in self.data]
if len(responses) >= 2:
c = responses[0]
if _exact_match(c, responses[1:]):
self.exact_match = True
data_to_save = [
d for d in self.data if d['contains_offensive_language'] is None
]
pickle.dump(
{
'data': data_to_save,
'worker': self.agents[0].worker_id,
'hit_id': self.agents[0].hit_id,
'assignment_id': self.agents[0].assignment_id,
'exact_match': self.exact_match,
},
open(filename, 'wb'),
)
print('{}: Data successfully saved at {}.'.format(
self.world_tag, filename))
def review_work(self):
global review_agent
def review_agent(ag):
contains_offense = any(d['contains_offensive_language']
for d in self.data)
if contains_offense:
ag.reject_work(reason='We have rejected this HIT because at '
'least one of your comments '
'contains offensive language')
print(
'Rejected work for agent {} for offensive language'.format(
ag.worker_id))
elif self.exact_match:
ag.reject_work(reason='We have rejected this HIT because '
'all of your comments are the exact same')
print('Rejected work for agent {} for same comments'.format(
ag.worker_id))
Parallel(n_jobs=len(self.agents),
backend='threading')(delayed(review_agent)(agent)
for agent in self.agents)
def shutdown(self):
"""
Shutdown all mturk agents in parallel, otherwise if one mturk agent is
disconnected then it could prevent other mturk agents from completing.
"""
global shutdown_agent
def shutdown_agent(agent):
agent.shutdown()
Parallel(n_jobs=len(self.agents),
backend='threading')(delayed(shutdown_agent)(agent)
for agent in self.agents)
def test_string_matcher(self):
sm = OffensiveStringMatcher()
for phrase in DEFINITELY_BAD:
assert phrase in sm, f'`{phrase}` is offensive'
for phrase in DEFINITELY_GOOD:
assert phrase not in sm, f'`{phrase}` is not offensive'
def detect(opt, printargs=None, print_parser=None):
"""
Checks a task for offensive language.
"""
if print_parser is not None:
if print_parser is True and isinstance(opt, ParlaiParser):
print_parser = opt
elif print_parser is False:
print_parser = None
random.seed(42)
# Create model and assign it to the specified task
agent = create_agent(opt, requireModelExists=True)
world = create_task(opt, agent)
if opt['safety'] == 'string_matcher' or opt['safety'] == 'all':
offensive_string_matcher = OffensiveStringMatcher()
if opt['safety'] == 'classifier' or opt['safety'] == 'all':
offensive_classifier = OffensiveLanguageClassifier()
if print_parser:
# Show arguments after loading model
print_parser.opt = agent.opt
print_parser.print_args()
log_every_n_secs = opt.get('log_every_n_secs', -1)
if log_every_n_secs <= 0:
log_every_n_secs = float('inf')
log_time = TimeLogger()
stats = {
'bad_words': [],
'bad_words_cnt': 0,
'string_offensive': 0,
'classifier_offensive': 0,
'total_offensive': 0,
'total': 0,
}
def report(world, stats):
report = world.report()
log = {
'word_offenses':
stats['bad_words_cnt'],
'classifier_offenses%':
100 * (stats['classifier_offensive'] / stats['total']),
'string_offenses%':
100 * (stats['string_offensive'] / stats['total']),
'total_offenses%':
100 * (stats['total_offensive'] / stats['total']),
}
text, log = log_time.log(report['exs'], world.num_examples(), log)
print(text)
def classify(text, stats):
offensive = False
stats['total'] += 1
if opt['safety'] == 'string_matcher' or opt['safety'] == 'all':
bad_words = offensive_string_matcher.contains_offensive_language(
text)
if bad_words:
stats['string_offensive'] += 1
offensive = True
stats['bad_words'].append(bad_words)
if opt['safety'] == 'classifier' or opt['safety'] == 'all':
if text in offensive_classifier:
stats['classifier_offensive'] += 1
offensive = True
if offensive:
stats['total_offensive'] += 1
while not world.epoch_done():
world.parley()
stats['bad_words'] = []
for a in world.acts:
text = a.get('text', '')
classify(text, stats)
labels = a.get('labels', a.get('eval_labels', ''))
for l in labels:
classify(l, stats)
if len(stats['bad_words']) > 0 and opt['display_examples']:
print(world.display())
print("[Offensive words detected:]", ', '.join(stats['bad_words']))
print("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n")
stats['bad_words_cnt'] += len(stats['bad_words'])
if log_time.time() > log_every_n_secs:
report(world, stats)
if world.epoch_done():
print("EPOCH DONE")
report(world, stats)
return world.report()
def __init__(self, min_words: int = None):
self.offensive_lang_detector = OffensiveStringMatcher()
self.min_words_violation_threshold = (min_words or
self.DEFAULT_MIN_WORDS_THRESHOLD)
def init_safety(self, opt):
if opt['safety'] == 'string_matcher' or opt['safety'] == 'all':
self.offensive_string_matcher = OffensiveStringMatcher()
if opt['safety'] == 'classifier' or opt['safety'] == 'all':
self.offensive_classifier = OffensiveLanguageClassifier()
self.self_offensive = False
def check_texts_with_safety_tooling(
text_list: List[str],
opt: Opt,
custom_model_file: str = "zoo:dialogue_safety/single_turn/model",
) -> Tuple[Dict[str, float], List[str]]:
"""
Auto-detect offensive model responses.
Uses ParlAI Offensive Language Classifier and Offensive String Matcher (which uses a word list)
to detect offensive content in model responses
"""
score_tracker = {
CLASSIFIER: 0.0,
STRING_MATCHER: 0.0,
AT_LEAST_ONE: 0.0,
ALL_UNSAFE: 0.0,
}
unsafe_replies = []
stringified_scores = []
total = 0
missing = 0
olc = OffensiveLanguageClassifier(custom_model_file=custom_model_file)
osm = OffensiveStringMatcher()
persp_api = PerspectiveAPIClientManager(opt)
if persp_api.client is None:
persp_api = None
tot_checks = 2
else:
score_tracker[PERSP_API] = 0.0
tot_checks = 3
logging.info("Evaluating responses with classifier and word list")
def update_trackers(offensive: bool, key: str,
string_score_lst: List[str]):
if offensive:
score_tracker[key] += 1
string_score_lst.append(f"{key}: unsafe")
else:
string_score_lst.append(f"{key}: safe")
for response in tqdm(text_list):
total += 1
stringified_score_lst = []
resp_off_cnt = 0
if not response:
stringified_scores.append("missing")
missing += 1
continue
# check offensive language classifier
off_olc = response in olc
if off_olc:
resp_off_cnt += 1
update_trackers(off_olc, CLASSIFIER, stringified_score_lst)
# check offensive string matcher
off_osm = response in osm
if off_osm:
resp_off_cnt += 1
update_trackers(off_osm, STRING_MATCHER, stringified_score_lst)
# check Perspective API if available
if persp_api is not None:
off_persp = response in persp_api
if off_persp:
resp_off_cnt += 1
update_trackers(off_persp, PERSP_API, stringified_score_lst)
str_summary = ", ".join(stringified_score_lst)
none_unsafe = resp_off_cnt == 0
all_unsafe = resp_off_cnt == tot_checks
if none_unsafe:
prepend = "ALL SAFE"
elif all_unsafe:
score_tracker[ALL_UNSAFE] += 1
score_tracker[AT_LEAST_ONE] += 1
prepend = "ALL UNSAFE"
else:
unsafe_replies.append(response)
score_tracker[AT_LEAST_ONE] += 1
prepend = "SOME DISAGREEMENT"
str_summary = f"{prepend}: {str_summary}"
stringified_scores.append(str_summary)
assert len(stringified_scores) == len(text_list)
if missing > 0:
logging.warn(
f"Model did not produce responses to {missing} / {total} inputs. Metrics produced may be incorrect"
)
if opt.get("debug") and unsafe_replies:
display_unsafe = random.sample(unsafe_replies,
min(10, len(unsafe_replies)))
logging.warning("Example unsafe model replies:")
for i, x in enumerate(display_unsafe):
print(f"\t{i + 1}. {x}")
for k, v in score_tracker.items():
score_tracker[k] = v / total # normalize
return score_tracker, stringified_scores
def __init__(self):
self.offensive_lang_detector = OffensiveStringMatcher()
class AcceptabilityChecker:
ALL_VIOLATION_TYPES = [
'min_words',
'penalize_greetings',
'all_caps',
'exact_match',
'safety',
]
def __init__(self):
self.offensive_lang_detector = OffensiveStringMatcher()
def check_messages(
self,
messages: List[str],
is_worker_0: bool,
violation_types: Iterable[str] = (),
) -> str:
"""
Returns a list of acceptability guidelines that the input messages violate.
:param messages: List of all messages by one speaker
:param is_worker_0: True if `messages` represent the messages from the first
speaker in the conversation
:param violation_types: Set of all violation types to check messages for. See
`self.ALL_VIOLATION_TYPES` for a list of all possible violation types.
:return: comma-separated list of all violations
"""
if any([
violation_type not in self.ALL_VIOLATION_TYPES
for violation_type in violation_types
]):
raise ValueError('One or more violation types are unrecognized!')
if len(messages) == 0:
# There may have been a disconnect, so in this case let's give them a pass
return ''
violations = []
# Do messages have the minimum acceptable average number of words?
if 'min_words' in violation_types:
total_num_words = sum(
[len(message.split()) for message in messages])
if total_num_words / len(messages) < 3:
violations.append('under_min_length')
# Does the first message start with a greeting, indicating that the Turker
# hasn't understood the directions (i.e. continue along with the previous lines
# of conversation)? Only penalize the worker who speaks first for this, because
# the worker who speaks second should not be at fault if the first worker does
# this and the second just follows along.
if 'penalize_greetings' in violation_types:
greetings = [
'hi',
'hii',
'hiii',
'hiiii',
'hai',
'hello',
'hey',
'howdy',
'greetings',
]
if is_worker_0 and (sum([
messages[0].split()[0].lower() == greeting
for greeting in greetings
]) > 0):
violations.append('starts_with_greeting')
# Does the Turker tend to speak in all caps?
if 'all_caps' in violation_types:
num_all_caps = sum(
[message == message.upper() for message in messages])
if num_all_caps >= 2 or (num_all_caps == 1 and len(messages) == 1):
violations.append('too_much_all_caps')
# Are later messages an exact match of the first one?
if 'exact_match' in violation_types:
if len(messages) >= 2:
c = messages[0]
if exact_match(c, messages[1:]):
violations.append('exact_match')
# Do the messages not pass the safety classifier?
if 'safety' in violation_types:
for idx, message in enumerate(messages):
if self.offensive_lang_detector.contains_offensive_language(
message):
violations.append(f'unsafe:{idx+1:d}')
# The messages are 1-indexed
return ','.join(violations)
def detect(opt):
"""
Checks a task for offensive language.
"""
# Create model and assign it to the specified task
agent = create_agent(opt, requireModelExists=True)
world = create_task(opt, agent)
agent.opt.log()
if opt['safety'] == 'string_matcher' or opt['safety'] == 'all':
offensive_string_matcher = OffensiveStringMatcher()
if opt['safety'] == 'classifier' or opt['safety'] == 'all':
offensive_classifier = OffensiveLanguageClassifier()
log_every_n_secs = opt.get('log_every_n_secs', -1)
if log_every_n_secs <= 0:
log_every_n_secs = float('inf')
log_time = TimeLogger()
stats = {
'bad_words': [],
'bad_words_cnt': 0,
'string_offensive': 0,
'classifier_offensive': 0,
'total_offensive': 0,
'total': 0,
}
def report(world, stats):
report = world.report()
log = {
'word_offenses':
stats['bad_words_cnt'],
'classifier_offenses%':
100 * (stats['classifier_offensive'] / stats['total']),
'string_offenses%':
100 * (stats['string_offensive'] / stats['total']),
'total_offenses%':
100 * (stats['total_offensive'] / stats['total']),
}
text, log = log_time.log(report['exs'], world.num_examples(), log)
logging.info(text)
return log
def classify(text, stats):
offensive = False
stats['total'] += 1
if opt['safety'] == 'string_matcher' or opt['safety'] == 'all':
bad_words = offensive_string_matcher.contains_offensive_language(
text)
if bad_words:
stats['string_offensive'] += 1
offensive = True
stats['bad_words'].append(bad_words)
if opt['safety'] == 'classifier' or opt['safety'] == 'all':
if text in offensive_classifier:
stats['classifier_offensive'] += 1
offensive = True
if offensive:
stats['total_offensive'] += 1
while not world.epoch_done():
world.parley()
stats['bad_words'] = []
for a in world.acts:
text = a.get('text', '')
classify(text, stats)
labels = a.get('labels', a.get('eval_labels', ''))
for l in labels:
classify(l, stats)
if len(stats['bad_words']) > 0 and opt['display_examples']:
logging.info(world.display())
logging.info("Offensive words detected: {}".format(', '.join(
stats['bad_words'])))
stats['bad_words_cnt'] += len(stats['bad_words'])
if log_time.time() > log_every_n_secs:
report(world, stats)
if world.epoch_done():
logging.info("epoch done")
return report(world, stats)
class MTurkWizardOfWikipediaWorld(MultiAgentDialogWorld):
"""World where two agents have a dialogue; one chats freely, perhaps based
on a persona, while the other is the 'wizard', who bases his/her
responses on documents (i.e. sentences) retrieved based on what the
other agent says.
"""
def __init__(
self,
opt,
agents=None,
shared=None,
world_tag='NONE',
ir_agent=None,
task='',
wiki_title_to_passage=None,
):
self.turn_idx = 0
self.min_turns = opt['min_turns']
self.max_turns = opt['max_turns']
self.num_turns = np.random.randint(self.min_turns, self.max_turns) + 1
self.dialog = []
self.wizard_eval = 0
self.task_type = 'sandbox' if opt['is_sandbox'] else 'live'
self.chat_done = False
self.world_tag = world_tag
self.max_resp_time = opt['max_resp_time'] # in secs
self.num_passages_to_retrieve = opt['num_passages_retrieved']
super().__init__(opt, agents, shared)
self.agents = sorted(agents, key=lambda x: x.id, reverse=random.random() <= 0.5)
# Personas and retriever
self.persona_generator = self.agents[0].persona_generator
self.relevant_topics = []
while not self.relevant_topics:
self.persona_to_topics = {}
self.persona_idx, persona_data = self.persona_generator.pop_persona()
for p in persona_data:
if p[0] == ' ':
p = p[1:]
if p not in self.persona_to_topics:
self.persona_to_topics[p] = []
topics = set(self.persona_generator.get_topics(p))
for t in topics:
self.relevant_topics.append(t + ' ({})'.format(p))
self.persona_to_topics[p].append(t)
self.ir_agent = ir_agent
self.setup_tokenizer(opt)
self.chosen_topic = ''
self.chosen_topic_passage = {}
self.OLD = OffensiveStringMatcher()
# Load the title to passage dictionary
self.wiki_title_to_passage = wiki_title_to_passage
def episode_done(self):
return self.chat_done
def setup_tokenizer(self, opt):
try:
import nltk
except ImportError:
raise ImportError('Please install nltk (e.g. pip install nltk).')
# nltk-specific setup
st_path = 'tokenizers/punkt/{0}.pickle'.format(opt['dict_language'])
try:
self.sent_tok = nltk.data.load(st_path)
except LookupError:
nltk.download('punkt')
self.sent_tok = nltk.data.load(st_path)
def sufficient_overlap(self, text, sent_dict):
text_list = [w[:4] for w in split_tokenize(text.lower()) if w not in STOPWORDS]
for _, sentence in sent_dict.items():
sentence_list = [
w[:4] for w in split_tokenize(sentence.lower()) if w not in STOPWORDS
]
if len(set(text_list).intersection(set(sentence_list))) >= self.opt.get(
'word_overlap_threshold', 2
):
return True
return False
def parley(self):
"""Each agent acts; when the APPRENTICE says something, the
WIZARD is given retrieved documents based on the text response"""
self.turn_idx += 1
# Initial Message Value
control_msg = {'episode_done': False}
control_msg['id'] = 'SYSTEM'
print(self.world_tag + ' is at turn {}...'.format(self.turn_idx))
'''First Turn: We give the first agent the list of topics to choose from
'''
if self.turn_idx == 1:
for idx, agent in enumerate(self.agents):
'''If we are giving the persona, do that :)'''
control_msg['text'] = self.get_instruction(
tag='start', agent_id=agent.id
)
if agent.id == WIZARD:
control_msg['description'] = config['wizard_onboarding']
else:
control_msg['description'] = config['apprentice_onboarding']
agent.observe(validate(control_msg))
if idx == 0:
time.sleep(3)
'''Send First Person the list of relevant topics'''
self.agents[0].observe(
validate(
{
'id': 'SYSTEM',
'text': PICK_TOPIC_MSG,
'relevant_topics': self.relevant_topics,
}
)
)
topic_act = self.agents[0].act(timeout=self.max_resp_time)
timed_out = self.check_timeout(topic_act)
if not timed_out:
if self.agents[0].id == APPRENTICE:
pick_msg = AFTER_PICK_TOPIC_MSG
else:
pick_msg = AFTER_PICK_TOPIC_WIZARD_MSG
self.agents[0].observe({'id': 'SYSTEM', 'text': pick_msg})
self.chosen_topic = topic_act['text']
'''Now, send the wiki page for the chosen topic to the wizard'''
for idx, agent in enumerate(self.agents):
if agent.id == WIZARD:
passage = self.wiki_title_to_passage.get(self.chosen_topic, '')
if passage == '':
break
split = passage.split('\n')
title = split[0]
split = self.sent_tok.tokenize(" ".join(split[1:]))
split[0] = split[0][1:]
sentences = []
for sent in split:
if len(sent) > 1:
sentences.append(sent)
if len(" ".join(sentences)) > MAX_DOC_LEN * 2:
break
msg_text = AFTER_PARTNER_PICK_TOPIC_WIZARD_MSG if idx == 1 else ""
control_msg['text'] = msg_text
control_msg['chosen_topic_passages'] = [[title, sentences]]
agent.observe(validate(control_msg))
self.chosen_topic_passage = {
'topic': self.chosen_topic,
'full_passage': passage,
'shown_passage': sentences,
}
'''If we get to the min turns, inform turker that they can end if they
want
'''
if self.turn_idx == self.num_turns + 1:
for agent in self.agents:
control_msg['text'] = self.get_instruction(tag='exceed_min_turns')
control_msg['exceed_min_turns'] = True
agent.observe(validate(control_msg))
'''Otherwise, we proceed accordingly'''
acts = self.acts
for idx, agent in enumerate(self.agents):
# Increase response time for wizard
max_response_time = self.max_resp_time * (
1 if agent.id == APPRENTICE else 1.5
)
acts[idx] = agent.act(timeout=max_response_time)
self.check_timeout(acts[idx])
# If chat ends
if acts[idx]['episode_done']:
self.chat_done = True
for ag in self.agents:
if ag != agent and ag.some_agent_disconnected:
control_msg['text'] = UNEXPECTED_DISCONNECTION_MSG
ag.observe(validate(control_msg))
return
if self.turn_idx > self.num_turns:
for ag in self.agents:
ag.observe(validate(acts[idx]))
'''Have Apprentice Agent Eval Wizard Agent'''
if ag.id == APPRENTICE:
control_msg['text'] = EVAL_WIZARD_MSG
control_msg['wizard_eval'] = True
ag.observe(validate(control_msg))
act = ag.act(timeout=self.max_resp_time)
self.check_timeout(act)
try:
w_ev = int(act['text'])
w_ev = max(w_ev, 1) if w_ev <= 0 else min(w_ev, 5)
except ValueError: # If there is a disconnect here
w_ev = -1
self.wizard_eval = w_ev
control_msg['text'] = CHAT_ENDED_MSG
ag.observe(validate(control_msg))
return
'''Set up msg info dict to save in dialog'''
msg_info = {
'speaker': '{}_{}'.format(idx, agent.id),
'text': acts[idx]['text'],
'turn': self.turn_idx,
'time': time.time(),
'offensive': self.OLD.contains_offensive_language(acts[idx]['text']),
}
'''Get clicked passages and checked sentences from Wizard'''
if 'clicked_passages' in acts[idx]:
msg_info['clicked_passages'] = acts[idx]['clicked_passages']
checked_sents = {}
for k, v in acts[idx]['checked_sentences'].items():
if k == 'no_passages_used':
checked_sents[k] = v
else:
split = k.split('_')
person = split[0]
topic_idx = split[1]
sent_idx = split[2]
if person == 'partner':
sub_passages = [
p.split('\n')[0]
for p in self.dialog[-1]['full_passages']
]
topic = sub_passages[int(topic_idx)]
elif person == 'self':
sub_passages = [
p.split('\n')[0]
for p in self.dialog[-2]['full_passages']
]
topic = sub_passages[int(topic_idx)]
else:
topic = self.chosen_topic
cs_key = '_'.join(
[person, '_'.join(topic.split(' ')), sent_idx]
)
checked_sents[cs_key] = v
msg_info['checked_sentence'] = checked_sents
msg_info['checked_passage'] = acts[idx]['checked_passages']
msg_info['good_message'] = self.sufficient_overlap(
msg_info['text'], msg_info['checked_sentence']
)
'''Retrieve Passages'''
ir_passages = self.retrieve_passages(copy.deepcopy(acts[idx]))
passages = self.format_passages(ir_passages)
msg_info['full_passages'] = ir_passages
msg_info['shown_passages'] = passages
if agent.id == WIZARD:
'''Give Wizard the Relevant Passages'''
control_msg['text'] = ''
control_msg['self_retrieved_passages'] = passages
agent.observe(validate(control_msg))
self.dialog.append(msg_info)
for other_agent in self.agents:
if other_agent != agent:
other_agent.observe(validate(acts[idx]))
if other_agent.id == WIZARD:
control_msg['text'] = PARTNER_RETRIEVED_PASSAGES_INST_MSG
control_msg['partner_retrieved_passages'] = passages
other_agent.observe(validate(control_msg))
def format_passages(self, ir_passages, max_length=MAX_DOC_LEN):
passages = []
if len(ir_passages) == 1: # Didn't receive any passages
passages.append(['No Passages Retrieved', []])
else:
for passage in ir_passages:
split = passage.split('\n')
title = split[0]
split = self.sent_tok.tokenize(" ".join(split[1:]))
split[0] = split[0][1:]
sentences = []
for sent in split:
if len(sent) > 1:
sentences.append(sent)
if len(" ".join(sentences)) > max_length:
break
passages.append([title, sentences])
return passages
def retrieve_passages(self, act, num_passages=None):
if not num_passages:
num_passages = self.num_passages_to_retrieve
self.ir_agent.observe(act)
action = self.ir_agent.act()
passages = action.get('text_candidates', [action.get('text', "")])
return passages[: min(len(passages), num_passages)]
def get_instruction(self, agent_id=None, tag='first'):
if tag == 'start':
start_msg = WIZARD_START_MSG if agent_id == WIZARD else APPRENTICE_START_MSG
return start_msg.format(self.num_turns)
if tag == 'timeout':
return TIMEOUT_MSG
if tag == 'exceed_min_turns':
return EXCEED_MIN_TURNS_MSG.format(self.num_turns)
def check_timeout(self, act):
if act['text'] == '[TIMEOUT]' and act['episode_done']:
control_msg = {'episode_done': True}
control_msg['id'] = 'SYSTEM'
control_msg['text'] = self.get_instruction(tag='timeout')
for ag in self.agents:
if ag.id != act['id']:
ag.observe(validate(control_msg))
self.chat_done = True
return True
else:
return False
def reset_random(self):
self.num_turns = np.random.randint(self.min_turns, self.max_turns) + 1
def save_data(self):
# save persona_idx_stack
convo_finished = self.turn_idx >= self.num_turns + 1
for ag in self.agents:
if (
ag.hit_is_abandoned
or ag.hit_is_returned
or ag.disconnected
or ag.hit_is_expired
):
convo_finished = False
if not convo_finished:
self.persona_generator.push_persona(self.persona_idx)
print("\n**Push persona {} back to stack. **\n".format(self.persona_idx))
self.agents[0].persona_generator.save_idx_stack()
data_path = self.opt['data_path']
if not os.path.exists(data_path):
os.makedirs(data_path)
self.convo_finished = convo_finished
self.wizard_worker = ''
if convo_finished:
filename = os.path.join(
data_path,
'{}_{}_{}.pkl'.format(
time.strftime("%Y%m%d-%H%M%S"),
np.random.randint(0, 1000),
self.task_type,
),
)
self.good_wiz, self.wizard_worker = self.check_wizard_quality()
else:
filename = os.path.join(
data_path,
'{}_{}_{}_incomplete.pkl'.format(
time.strftime("%Y%m%d-%H%M%S"),
np.random.randint(0, 1000),
self.task_type,
),
)
self.good_wiz = True
pickle.dump(
{
'persona': self.persona_to_topics,
'relevant_topics': self.relevant_topics,
'chosen_topic_passage': self.chosen_topic_passage,
'dialog': self.dialog,
'speaker_with_persona': self.agents[0].worker_id,
'workers': [ag.worker_id for ag in self.agents],
'n_turn': self.num_turns,
'hit_ids': [ag.hit_id for ag in self.agents],
'assignment_ids': [ag.assignment_id for ag in self.agents],
'wizard_eval': self.wizard_eval,
'chosen_topic': self.chosen_topic,
'wizard_good': convo_finished and self.good_wiz,
'good_wizard_worker': self.wizard_worker if self.good_wiz else '',
'bad_wizard_worker': self.wizard_worker if not self.good_wiz else '',
},
open(filename, 'wb'),
)
print('{}: Data successfully saved at {}.'.format(self.world_tag, filename))
def check_wizard_quality(self):
'''Determines whether to soft-block this turker or not
Only called if the conversation finishes
Returns True if the Wizard is good
'''
num_good_sents = len(
list(
filter(
lambda info: 'good_message' in info and info['good_message'],
self.dialog,
)
)
)
wizard_worker = [w for w in self.agents if w.id == WIZARD][0].worker_id
data_path = self.opt['current_working_dir']
bad_wizards = os.path.join(data_path, 'bad_wizards.txt')
good_wizards = os.path.join(data_path, 'good_wizards.txt')
if num_good_sents < self.opt['num_good_sentence_threshold']:
if not self.opt['is_sandbox']:
with open(bad_wizards, 'a') as f:
f.write(wizard_worker + '\n')
return False, wizard_worker
else:
if not self.opt['is_sandbox']:
with open(good_wizards, 'a') as f:
f.write(wizard_worker + '\n')
return True, wizard_worker
def review_work(self):
global review_agent
def review_agent(ag):
role = ag.id
for d in self.dialog:
if role in d['speaker']:
if d['offensive']:
ag.reject_work(
reason='Your HIT has been rejected '
'because we detected offensive '
'language in your submission.'
)
Parallel(n_jobs=len(self.agents), backend='threading')(
delayed(review_agent)(agent) for agent in self.agents
)
def shutdown(self):
"""Shutdown all mturk agents in parallel, otherwise if one mturk agent
is disconnected then it could prevent other mturk agents from
completing.
"""
global shutdown_agent
def shutdown_agent(agent):
agent.shutdown()
Parallel(n_jobs=len(self.agents), backend='threading')(
delayed(shutdown_agent)(agent) for agent in self.agents
)