def setup_args():
"""
Set up args.
"""
parser = ParlaiParser(False, False)
parser.add_parlai_data_path()
parser.add_messenger_args()
return parser.parse_args()
def setup_args():
"""
Set up args.
"""
parser = ParlaiParser(False, False)
parser.add_parlai_data_path()
parser.add_websockets_args()
return parser.parse_args()
def setup_args():
"""
Set up args.
"""
parser = ParlaiParser(False, False)
parser.add_parlai_data_path()
reddit = parser.add_argument_group('Download Reddit Docs')
reddit.add_argument('-sy',
'--start_year',
default=2011,
type=int,
metavar='N',
help='starting year')
reddit.add_argument('-ey',
'--end_year',
default=2018,
type=int,
metavar='N',
help='end year')
reddit.add_argument('-sm',
'--start_month',
default=7,
type=int,
metavar='N',
help='starting year')
reddit.add_argument('-em',
'--end_month',
default=7,
type=int,
metavar='N',
help='end year')
reddit.add_argument(
'-sr_l',
'--subreddit_list',
default='["explainlikeimfive"]',
type=str,
help='subreddit name',
)
reddit.add_argument(
'-id_l',
'--id_list',
type=str,
help='json file path of base36 post IDs (in a list format)',
)
reddit.add_argument('-Q',
'--questions_only',
action='store_true',
help='only download submissions')
reddit.add_argument('-A',
'--answers_only',
action='store_true',
help='only download comments')
reddit.add_argument('-o',
'--output_dir',
default='eli5/',
type=str,
help='where to save the output')
return parser.parse_args()
def create_agent(opt, requireModelExists=False):
"""
Create an agent from the options ``model``, ``model_params`` and ``model_file``.
The input is either of the form
``parlai.agents.ir_baseline.agents:IrBaselineAgent`` (i.e. the path
followed by the class name) or else just ``ir_baseline`` which
assumes the path above, and a class name suffixed with 'Agent'.
If ``model-file`` is available in the options this function can also
attempt to load the model from that location instead. This avoids having to
specify all the other options necessary to set up the model including its
name as they are all loaded from the options file if it exists (the file
opt['model_file'] + '.opt' must exist and contain a pickled or json dict
containing the model's options).
"""
if opt.get('datapath', None) is None:
# add datapath, it is missing
from parlai.core.params import ParlaiParser, get_model_name
parser = ParlaiParser(add_parlai_args=False)
parser.add_parlai_data_path()
# add model args if they are missing
model = get_model_name(opt)
if model is not None:
parser.add_model_subargs(model)
opt_parser = parser.parse_args("", print_args=False)
for k, v in opt_parser.items():
if k not in opt:
opt[k] = v
if opt.get('model_file'):
opt['model_file'] = modelzoo_path(opt.get('datapath'), opt['model_file'])
if requireModelExists and not os.path.isfile(opt['model_file']):
raise RuntimeError(
'WARNING: Model file does not exist, check to make '
'sure it is correct: {}'.format(opt['model_file'])
)
# Attempt to load the model from the model file first (this way we do
# not even have to specify the model name as a parameter)
model = load_agent_module(opt)
if model is not None:
return model
else:
print("[ no model with opt yet at: " + opt.get('model_file') + "(.opt) ]")
if opt.get('model'):
model_class = get_agent_module(opt['model'])
# if we want to load weights from --init-model, compare opts with
# loaded ones
compare_init_model_opts(opt, opt)
model = model_class(opt)
if requireModelExists and hasattr(model, 'load') and not opt.get('model_file'):
# double check that we didn't forget to set model_file on loadable model
print('WARNING: model_file unset but model has a `load` function.')
return model
else:
raise RuntimeError('Need to set `model` argument to use create_agent.')
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
human_agent_2_id = 'human_2'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_1_id, mturk_agent_2_id],
all_agent_ids=[
human_agent_1_id, human_agent_2_id, mturk_agent_1_id,
mturk_agent_2_id
] # In speaking order
)
mturk_manager.init_aws(opt=opt)
global run_hit
def run_hit(hit_index, assignment_index, opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
# Create mturk agents
mturk_agent_1 = MTurkAgent(id=mturk_agent_1_id,
manager=mturk_manager,
conversation_id=conversation_id,
opt=opt)
mturk_agent_2 = MTurkAgent(id=mturk_agent_2_id,
manager=mturk_manager,
conversation_id=conversation_id,
opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = human_agent_2_id
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_messenger_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
# Initialize a SQuAD teacher agent, which we will get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
messenger_manager = MessengerManager(opt=opt)
messenger_manager.setup_server()
messenger_manager.init_new_state()
def get_overworld(agent):
return MessengerOverworld(None, agent)
def assign_agent_role(agent):
agent[0].disp_id = 'Agent'
def run_conversation(manager, opt, agents, task_id):
task = task_class(task_opt)
agent = agents[0]
world = QADataCollectionWorld(
opt=opt,
task=task,
agent=agent
)
while not world.episode_done():
world.parley()
world.shutdown()
# World with no onboarding
messenger_manager.set_onboard_functions({'default': None})
task_functions = {'default': run_conversation}
assign_agent_roles = {'default': assign_agent_role}
messenger_manager.set_agents_required({'default': 1})
messenger_manager.set_overworld_func(get_overworld)
messenger_manager.setup_socket()
try:
messenger_manager.start_new_run()
messenger_manager.start_task(
assign_role_functions=assign_agent_roles,
task_functions=task_functions,
)
except BaseException:
raise
finally:
messenger_manager.shutdown()
def main_bak():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# Initialize a SQuAD teacher agent, which we will get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = opt.copy()
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=[mturk_agent_id])
mturk_manager.setup_server()
mturk_manager.set_onboard_function(onboard_function=None)
mturk_manager.start_new_run()
mturk_manager.create_hits()
mturk_manager.ready_to_accept_workers()
def check_workers_eligibility(workers):
return workers
eligibility_function = {
'func': check_workers_eligibility,
'multiple': True,
}
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
world = QADataCollectionWorld(opt=opt, mturk_agent=mturk_agent)
btime = time.time()
world.parley()
etime = time.time()
logger.debug("DialTime: ", (etime - btime))
world.shutdown()
world.review_work()
mturk_manager.start_task(eligibility_function=eligibility_function,
assign_role_function=assign_worker_roles,
task_function=run_conversation)
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
opt.update(task_config)
global run_hit
def run_hit(i, opt, mturk_manager):
# Create mturk agents
mturk_agent_1 = MTurkAgent(id='mturk_agent_1',
manager=mturk_manager,
conversation_id=i,
opt=opt)
mturk_agent_2 = MTurkAgent(id='mturk_agent_2',
manager=mturk_manager,
conversation_id=i,
opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = 'human_1'
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = 'human_2'
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
# Since we are using the regular MultiAgentDialogWorld, we do the following outside of the world instead.
mturk_agent_ids = [mturk_agent_1.id, mturk_agent_2.id]
all_agent_ids = [human_agent_1.id, human_agent_2.id] + mturk_agent_ids
mturk_agent_1.mturk_agent_ids = mturk_agent_ids
mturk_agent_1.all_agent_ids = all_agent_ids
mturk_agent_2.mturk_agent_ids = mturk_agent_ids
mturk_agent_2.all_agent_ids = all_agent_ids
mturk_agent_1.create_hit()
mturk_agent_2.create_hit()
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager = MTurkManager()
mturk_manager.init_aws(opt=opt)
results = Parallel(n_jobs=opt['num_hits'], backend='threading')(
delayed(run_hit)(i, opt, mturk_manager)
for i in range(1, opt['num_hits'] + 1))
mturk_manager.review_hits()
mturk_manager.shutdown()
def setUp(self):
self.work_state_1 = WorkerState(TEST_WORKER_ID_1, 10)
self.work_state_2 = WorkerState(TEST_WORKER_ID_2)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args([])
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt, mturk_agent_ids=mturk_agent_ids)
self.worker_manager = WorkerManager(self.mturk_manager, self.opt)
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_manager = MTurkManager()
mturk_manager.init_aws(opt=opt)
mturk_agent_id = 'Worker'
mturk_manager.mturk_agent_ids = [mturk_agent_id]
mturk_manager.all_agent_ids = [
ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id
] # In speaking order
global run_hit
def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id,
manager=mturk_manager,
conversation_id=conversation_id,
opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, opt, task_opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.review_hits()
mturk_manager.shutdown()
def setUp(self):
self.agent_state1 = AssignState()
self.agent_state2 = AssignState(status=AssignState.STATUS_IN_TASK)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt,
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
opt.update(task_config)
# Initialize a SQuAD teacher agent, which we will get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
mturk_manager = MTurkManager()
mturk_manager.init_aws(opt=opt)
mturk_agent_id = 'Worker'
mturk_manager.mturk_agent_ids = [mturk_agent_id]
mturk_manager.all_agent_ids = [
QADataCollectionWorld.collector_agent_id, mturk_agent_id
] # In speaking order
global run_hit
def run_hit(hit_index, assignment_index, task_class, task_opt, opt,
mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
task = task_class(task_opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id,
manager=mturk_manager,
conversation_id=conversation_id,
opt=opt)
world = QADataCollectionWorld(opt=opt,
task=task,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.review_hits()
mturk_manager.shutdown()
def get_context_generator(
override_opt: Optional[Dict[str, Any]] = None) -> ContextGenerator:
"""
Return an object to return BlendedSkillTalk-style context info (personas, etc.).
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
if override_opt is not None:
argparser.set_params(**override_opt)
opt = argparser.parse_args([])
context_generator = ContextGenerator(opt, datatype='test', seed=0)
# We pull from the test set so that the model can't regurgitate
# memorized conversations
return context_generator
def add_datapath_and_model_args(opt: Opt):
# add datapath, it is missing
from parlai.core.params import ParlaiParser, get_model_name
parser = ParlaiParser(add_parlai_args=False)
parser.add_parlai_data_path()
# add model args if they are missing
model = get_model_name(opt)
if model is not None:
parser.add_model_subargs(model, opt)
opt_parser = parser.parse_args("")
for k, v in opt_parser.items():
if k not in opt:
opt[k] = v
def setup_args():
"""
Set up args.
:return: A parser that takes in command line arguments for chat services (debug, config-path, password), and a port.
"""
parser = ParlaiParser(False, False)
parser.add_parlai_data_path()
parser.add_chatservice_args()
parser_grp = parser.add_argument_group('Discord Chat')
parser_grp.add_argument(
'--port', default=35496, type=int, help='Port to run the discord chat server'
)
return parser.parse_args()
def setUp(self):
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args(print_args=False)
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt.copy(),
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
self.turk_agent = MTurkAgent(self.opt.copy(), self.mturk_manager,
TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1)
def main():
global run_hit
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
opt['mturk_agent_1_id'] = 'mturk_agent_1'
opt['mturk_agent_2_id'] = 'mturk_agent_2'
opt['human_agent_1_id'] = 'human_1'
opt['human_agent_2_id'] = 'human_2'
opt.update(task_config)
opt['mturk_agent_ids'] = [opt['mturk_agent_1_id'], opt['mturk_agent_2_id']]
opt['all_agent_ids'] = [opt['human_agent_1_id'], opt['human_agent_2_id']
] + opt['mturk_agent_ids']
opt['run_id'] = str(int(time.time()))
def run_hit(i, opt):
opt['conversation_id'] = str(i)
opt['agent_id'] = opt['mturk_agent_1_id']
mturk_agent_1 = MTurkAgent(opt=opt)
opt['agent_id'] = opt['mturk_agent_2_id']
mturk_agent_2 = MTurkAgent(opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = opt['human_agent_1_id']
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = opt['human_agent_2_id']
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
while not world.episode_done():
world.parley()
world.shutdown()
MTurkAgent.init_aws(opt)
results = Parallel(n_jobs=opt['num_hits'], backend='threading')(
delayed(run_hit)(i, copy.deepcopy(opt))
for i in range(1, opt['num_hits'] + 1))
MTurkAgent.review_hits(opt=opt)
def setup_args():
parser = ParlaiParser(False, False)
parser.add_parlai_data_path()
merge = parser.add_argument_group('Merge Support Docs')
merge.add_argument(
'-c',
'--c',
default=True,
type=str,
help='Finalize the output (or slice number to merge)',
)
merge.add_argument('-n',
'--name',
default='explainlikeimfive',
type=str,
help='Slices to merge')
return parser.parse_args()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_messenger_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt['password'] = '******' # If password is none anyone can chat
messenger_manager = MessengerManager(opt=opt)
messenger_manager.setup_server()
messenger_manager.init_new_state()
def get_overworld(agent):
return MessengerOverworld(None, agent)
onboard_functions = {
name: worlds[0].run
for (name, worlds) in MessengerOverworld.DEMOS.items()
}
messenger_manager.set_onboard_functions(onboard_functions)
task_functions = {
name: worlds[1].run
for (name, worlds) in MessengerOverworld.DEMOS.items()
}
assign_agent_roles = {
name: worlds[1].assign_roles
for (name, worlds) in MessengerOverworld.DEMOS.items()
}
agents_required = {
name: worlds[1].MAX_AGENTS
for (name, worlds) in MessengerOverworld.DEMOS.items()
}
messenger_manager.set_agents_required(agents_required)
messenger_manager.set_overworld_func(get_overworld)
messenger_manager.setup_socket()
try:
messenger_manager.start_new_run()
messenger_manager.start_task(
assign_role_functions=assign_agent_roles,
task_functions=task_functions,
)
except BaseException:
raise
finally:
messenger_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids = [mturk_agent_id],
all_agent_ids = [ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id] # In speaking order
)
mturk_manager.init_aws(opt=opt)
global run_hit
def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, opt, task_opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
human_agent_2_id = 'human_2'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id],
all_agent_ids = [human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id] # In speaking order
)
mturk_manager.init_aws(opt=opt)
global run_hit
def run_hit(hit_index, assignment_index, opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
# Create mturk agents
mturk_agent_1 = MTurkAgent(id=mturk_agent_1_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
mturk_agent_2 = MTurkAgent(id=mturk_agent_2_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = human_agent_2_id
world = MultiAgentDialogWorld(opt=opt, agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.shutdown()
def main():
global run_hit
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
# Initialize a SQuAD teacher agent, which we will later get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
# Create the MTurk agent which provides a chat interface to the Turker
opt.update(task_config)
mturk_agent_id = 'Worker'
opt['agent_id'] = mturk_agent_id
opt['mturk_agent_ids'] = [mturk_agent_id]
opt['all_agent_ids'] = [
QADataCollectionWorld.collector_agent_id, mturk_agent_id
]
opt['run_id'] = str(int(time.time()))
def run_hit(i, task_class, task_opt, opt):
task = task_class(task_opt)
opt['conversation_id'] = str(i)
mturk_agent = MTurkAgent(opt=opt)
world = QADataCollectionWorld(opt=opt,
task=task,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
MTurkAgent.init_aws(opt)
results = Parallel(n_jobs=opt['num_hits'], backend='threading')(
delayed(run_hit)(i, task_class, task_opt, copy.deepcopy(opt))
for i in range(1, opt['num_hits'] + 1))
MTurkAgent.review_hits(opt=opt)
def setup_args(parser=None):
if parser is None:
parser = ParlaiParser(True, True, 'Evaluate a model')
parser.add_parlai_data_path()
# Get command line arguments
parser.add_argument('-tmf', '--tourist-model-file', type=str)
parser.add_argument('-gmf', '--guide-model-file', type=str)
parser.add_argument('-ne', '--num-examples', type=int, default=-1)
parser.add_argument('-d', '--display-examples', type='bool', default=False)
parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=2)
parser.add_argument('--metrics',
type=str,
default="all",
help="list of metrics to show/compute, e.g. "
"ppl,f1,accuracy,hits@1."
"If 'all' is specified [default] all are shown.")
TensorboardLogger.add_cmdline_args(parser)
parser.set_defaults(datatype='valid')
return parser
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# Initialize a SQuAD teacher agent, which we will get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids = [mturk_agent_id],
all_agent_ids = [QADataCollectionWorld.collector_agent_id, mturk_agent_id] # In speaking order
)
mturk_manager.init_aws(opt=opt)
global run_hit
def run_hit(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
task = task_class(task_opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.shutdown()
def get_context_generator(
override_opt: Optional[Dict[str, Any]] = None,
task: Optional[str] = 'blended_skill_talk',
**kwargs,
) -> ContextGenerator:
"""
Return an object to return BlendedSkillTalk-style context info (personas, etc.).
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
if override_opt is not None:
argparser.set_params(**override_opt)
opt = argparser.parse_args([])
task_module = load_task_module(task)
context_generator_class = getattr(task_module, 'ContextGenerator', None)
context_generator = context_generator_class(opt, datatype='test', seed=0, **kwargs)
# We pull from the test set so that the model can't regurgitate
# memorized conversations
return context_generator
def main():
"""
Main function for the DMG pilot data collection task
:return: Nothing.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
opt = argparser.parse_args()
opt['task'] = 'dmg_pilot_dev'
opt['datatype'] = 'dmg_pilot_data_1'
opt.update(task_config)
local_agent_1 = LocalHumanAgent(opt=None)
local_agent_1.id = 'A'
local_agent_2 = LocalHumanAgent(opt=None)
local_agent_2.id = 'B'
agents = [local_agent_1, local_agent_2]
world = LocalDMGDialogWorld(opt=opt, agents=agents)
# Loop over all five rounds of the game
for r in range(5):
print("--- Starting round {} ---".format(r + 1))
while not world.episode_done():
world.parley()
# Write the log data to file
with open('logs/dmg_pilot_data_{}.json'.format(world.game_nr),
'w') as f:
json.dump(world.conversation_log, f)
# Reset the world for the next round
world.round_log['data'] = []
world.round_log['score'] = None
world.selections = defaultdict(lambda: dict())
world.turn_nr = -1
world.round_nr += 1
world.episodeDone = False
world.shutdown()
def setUp(self):
disconnect_path = os.path.join(parent_dir, 'disconnect-test.pickle')
if os.path.exists(disconnect_path):
os.remove(disconnect_path)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args([])
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(
opt=self.opt.copy(), mturk_agent_ids=mturk_agent_ids
)
self.worker_manager = self.mturk_manager.worker_manager
self.worker_state_1 = self.worker_manager.worker_alive(TEST_WORKER_ID_1)
self.worker_state_2 = self.worker_manager.worker_alive(TEST_WORKER_ID_2)
self.worker_state_3 = self.worker_manager.worker_alive(TEST_WORKER_ID_3)
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
human_agent_2_id = 'human_2'
# Create the MTurk agents
opt.update(task_config)
opt['conversation_id'] = str(int(time.time()))
opt['mturk_agent_ids'] = [mturk_agent_1_id, mturk_agent_2_id]
opt['all_agent_ids'] = [
human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id
]
opt['agent_id'] = mturk_agent_1_id
mturk_agent_1 = MTurkAgent(opt=opt)
opt['agent_id'] = mturk_agent_2_id
mturk_agent_2 = MTurkAgent(opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = human_agent_2_id
world = MultiAgentDialogWorld(
opt=opt,
agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
while not world.episode_done():
world.parley()
world.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
model_agent = IrBaselineAgent(opt=opt)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
# Create the MTurk agent which provides a chat interface to the Turker
opt.update(task_config)
mturk_agent_id = 'Worker'
opt['agent_id'] = mturk_agent_id
opt['mturk_agent_ids'] = [mturk_agent_id]
opt['all_agent_ids'] = [
ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id
]
opt['conversation_id'] = str(int(time.time()))
mturk_agent = MTurkAgent(opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
argparser.add_arg('--room_ablation', type=bool, default=False)
# ============ above copied from projects/graph_world2/train.py ============
argparser.add_argument('--num_machines', type=int, default=1)
argparser.add_argument('--job_timeout', type=float, default=3600*4)
argparser.add_argument('--split', action='store_true', default=False)
argparser.add_argument('--train', action='store_true', default=False)
argparser.add_argument('--eval', action='store_true', default=False)
argparser.add_argument('--seq2seq', action='store_true', default=False)
argparser.add_argument('--constrain', action='store_true', default=False)
argparser.add_argument('--rounds_breakdown', action='store_true', default=False)
argparser.add_argument('--data_breakdown', action='store_true', default=False)
argparser.add_argument('--ablation', action='store_true', default=False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
# ============ below copied from projects/graph_world2/train.py ============
opt['bidir'] = True
opt['action_type_emb_dim'] = 5
opt['counter_max'] = 3
opt['counter_emb_dim'] = 5
# ============ above copied from projects/graph_world2/train.py ============
if opt['split']:
overall_split(opt)
quit()
if opt['train']:
overall_run(opt, seq2seq=opt['seq2seq'])
def main():
"""
This task consists of one local human agent and two MTurk agents,
each MTurk agent will go through the onboarding step to provide
information about themselves, before being put into a conversation.
You can end the conversation by sending a message ending with
`[DONE]` from human_1.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id]
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = MTurkMultiAgentDialogOnboardWorld(opt=opt,
mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
# You can set onboard_function to None to skip onboarding
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
# Create mturk agents
mturk_agent_1 = workers[0]
mturk_agent_2 = workers[1]
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
world = MTurkMultiAgentDialogWorld(
opt=opt, agents=[human_agent_1, mturk_agent_1, mturk_agent_2])
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.start_task(eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""This task consists of one agent, model or MTurk worker, talking to an
MTurk worker to negotiate a deal.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('-min_t',
'--min_turns',
default=5,
type=int,
help='minimum number of turns')
argparser.add_argument('-mt',
'--max_turns',
default=10,
type=int,
help='maximal number of chat turns')
argparser.add_argument(
'-mx_rsp_time',
'--max_resp_time',
default=150,
type=int,
help='time limit for entering a dialog message',
)
argparser.add_argument(
'--ag_shutdown_time',
default=120,
type=int,
help='time limit for entering a dialog message',
)
argparser.add_argument(
'--persona-type',
default='both',
type=str,
choices=['both', 'self', 'other'],
help='Which personas to load from personachat',
)
opt = argparser.parse_args()
directory_path = os.path.dirname(os.path.abspath(__file__))
opt['task'] = os.path.basename(directory_path)
opt['extract_personas_path'] = os.path.join(opt['datapath'], opt['task'])
opt.update(task_config)
mturk_agent_ids = ['PERSON_1']
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
mturk_manager.setup_server(task_directory_path=directory_path)
personas_generator = PersonasGenerator(opt)
opt['personas_generator'] = personas_generator
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
if not opt['is_sandbox']:
# ADD BLOCKED WORKERS HERE
blocked_worker_list = []
for w in blocked_worker_list:
mturk_manager.block_worker(
w,
'We found that you have unexpected behaviors in our '
'previous HITs. For more questions please email us.',
)
def run_onboard(worker):
pass
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
worker = workers[0]
world = RephrasePersonaWorld(opt, worker)
while not world.episode_done():
world.parley()
world.save_data()
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation,
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def setUp(self):
patcher = mock.patch('builtins.input', return_value='y')
self.addCleanup(patcher.stop)
patcher.start()
# Mock functions that hit external APIs and such
self.server_utils = MTurkManagerFile.server_utils
self.mturk_utils = MTurkManagerFile.mturk_utils
self.server_utils.setup_server = mock.MagicMock(
return_value='https://127.0.0.1')
self.server_utils.setup_legacy_server = mock.MagicMock(
return_value='https://127.0.0.1')
self.server_utils.delete_server = mock.MagicMock()
self.mturk_utils.setup_aws_credentials = mock.MagicMock()
self.mturk_utils.calculate_mturk_cost = mock.MagicMock(return_value=1)
self.mturk_utils.check_mturk_balance = mock.MagicMock(
return_value=True)
self.mturk_utils.create_hit_config = mock.MagicMock()
self.mturk_utils.setup_sns_topic = mock.MagicMock(
return_value=TOPIC_ARN)
self.mturk_utils.delete_sns_topic = mock.MagicMock()
self.mturk_utils.delete_qualification = mock.MagicMock()
self.mturk_utils.find_or_create_qualification = mock.MagicMock(
return_value=QUALIFICATION_ID)
self.mturk_utils.find_qualification = mock.MagicMock(
return_value=QUALIFICATION_ID)
self.mturk_utils.give_worker_qualification = mock.MagicMock()
self.mturk_utils.remove_worker_qualification = mock.MagicMock()
self.mturk_utils.create_hit_type = mock.MagicMock(
return_value=HIT_TYPE_ID)
self.mturk_utils.subscribe_to_hits = mock.MagicMock()
self.mturk_utils.create_hit_with_hit_type = mock.MagicMock(
return_value=(MTURK_PAGE_URL, FAKE_HIT_ID, 'MTURK_HIT_DATA'))
self.mturk_utils.get_mturk_client = mock.MagicMock(
return_value=mock.MagicMock())
self.onboarding_agents = {}
self.worlds_agents = {}
# Set up an MTurk Manager and get it ready for accepting workers
self.fake_socket = MockSocket()
time.sleep(0.1)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 1
self.opt['hit_title'] = 'test_hit_title'
self.opt['hit_description'] = 'test_hit_description'
self.opt['task_description'] = 'test_task_description'
self.opt['hit_keywords'] = 'test_hit_keywords'
self.opt['reward'] = 0.1
self.opt['is_debug'] = True
self.opt['log_level'] = 0
self.opt['num_conversations'] = 1
self.mturk_agent_ids = ['mturk_agent_1', 'mturk_agent_2']
self.mturk_manager = MTurkManager(opt=self.opt,
mturk_agent_ids=self.mturk_agent_ids,
is_test=True)
self.mturk_manager.port = self.fake_socket.port
self.mturk_manager.setup_server()
self.mturk_manager.start_new_run()
self.mturk_manager.ready_to_accept_workers()
self.mturk_manager.set_onboard_function(self.onboard_agent)
self.mturk_manager.create_hits()
def assign_worker_roles(workers):
workers[0].id = 'mturk_agent_1'
workers[1].id = 'mturk_agent_2'
def run_task_wait():
self.mturk_manager.start_task(lambda w: True, assign_worker_roles,
self.run_conversation)
self.task_thread = threading.Thread(target=run_task_wait)
self.task_thread.start()
self.agent_1 = MockAgent(TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1, TASK_GROUP_ID_1)
self.agent_1_2 = MockAgent(TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_3,
TEST_WORKER_ID_1, TASK_GROUP_ID_1)
self.agent_2 = MockAgent(TEST_HIT_ID_2, TEST_ASSIGNMENT_ID_2,
TEST_WORKER_ID_2, TASK_GROUP_ID_1)
def main():
"""
This task consists of one local human agent and two MTurk agents,
each MTurk agent will go through the onboarding step to provide
information about themselves, before being put into a conversation.
You can end the conversation by sending a message ending with
`[DONE]` from human_1.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id]
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=mturk_agent_ids
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = MTurkMultiAgentDialogOnboardWorld(
opt=opt,
mturk_agent=worker
)
while not world.episode_done():
world.parley()
world.shutdown()
# You can set onboard_function to None to skip onboarding
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
eligibility_function = {
'func': check_worker_eligibility,
'multiple': False,
}
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
# Create mturk agents
mturk_agent_1 = workers[0]
mturk_agent_2 = workers[1]
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
world = MTurkMultiAgentDialogWorld(
opt=opt,
agents=[human_agent_1, mturk_agent_1, mturk_agent_2]
)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.start_task(
eligibility_function=eligibility_function,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
completed_workers = []
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_id]
)
mturk_manager.setup_server()
qual_name = 'ParlAIExcludeQual{}t{}'.format(
random.randint(10000, 99999), random.randint(10000, 99999))
qual_desc = (
'Qualification for a worker not correctly completing the '
'first iteration of a task. Used to filter to different task pools.'
)
qualification_id = \
mturk_utils.find_or_create_qualification(qual_name, qual_desc)
print('Created qualification: ', qualification_id)
def run_onboard(worker):
world = QualificationFlowOnboardWorld(opt, worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
try:
mturk_manager.start_new_run()
agent_qualifications = [{
'QualificationTypeId': qualification_id,
'Comparator': 'DoesNotExist',
'RequiredToPreview': True
}]
mturk_manager.create_hits(qualifications=agent_qualifications)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
world = QualificationFlowSoloWorld(
opt=opt,
mturk_agent=mturk_agent,
qualification_id=qualification_id,
firstTime=(mturk_agent.worker_id not in completed_workers),
)
while not world.episode_done():
world.parley()
completed_workers.append(mturk_agent.worker_id)
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_utils.delete_qualification(qualification_id)
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""This task consists of one agent, model or MTurk worker, talking to an
MTurk worker to negotiate a deal.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('--two_mturk_agents', dest='two_mturk_agents',
action='store_true', help='data collection mode '
'with converations between two MTurk agents')
opt = argparser.parse_args()
opt['task'] = 'dealnodeal'
opt['datatype'] = 'valid'
opt.update(task_config)
local_agent_1_id = 'local_1'
mturk_agent_ids = ['mturk_agent_1']
if opt['two_mturk_agents']:
mturk_agent_ids.append('mturk_agent_2')
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=mturk_agent_ids
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
mturk_manager.set_onboard_function(onboard_function=None)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
agents = workers[:]
# Create a local agent
if not opt['two_mturk_agents']:
if 'model' in opt:
local_agent = create_agent(opt)
else:
local_agent = LocalHumanAgent(opt=None)
local_agent.id = local_agent_1_id
agents.append(local_agent)
opt["batchindex"] = mturk_manager.started_conversations
world = MTurkDealNoDealDialogWorld(
opt=opt,
agents=agents
)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_id]
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = ModelEvaluatorOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
model_agent = IrBaselineAgent(opt=opt)
world = ModelEvaluatorWorld(
opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent
)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# Initialize a SQuAD teacher agent, which we will get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_id]
)
mturk_manager.setup_server()
def run_onboard(worker):
world = QADataCollectionOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=None)
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
mturk_manager.ready_to_accept_workers()
def check_workers_eligibility(workers):
return workers
eligibility_function = {
'func': check_workers_eligibility,
'multiple': True,
}
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
task = task_class(task_opt)
mturk_agent = workers[0]
world = QADataCollectionWorld(
opt=opt,
task=task,
mturk_agent=mturk_agent
)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=eligibility_function,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""This task consists of an MTurk agent evaluating a chit-chat model. They
are asked to chat to the model adopting a specific persona. After their
conversation, they are asked to evaluate their partner on several metrics.
"""
argparser = ParlaiParser(False, add_model_args=True)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('-mt', '--max-turns', default=10, type=int,
help='maximal number of chat turns')
argparser.add_argument('--max-resp-time', default=240,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('--max-persona-time', type=int,
default=300, help='time limit for turker'
'entering the persona')
argparser.add_argument('--ag-shutdown-time', default=120,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('--persona-type', default='both', type=str,
choices=['both', 'self', 'other'],
help='Which personas to load from personachat')
argparser.add_argument('--revised', default=False, type='bool',
help='Whether to use revised personas')
argparser.add_argument('-rt', '--range-turn', default='5,6',
help='sample range of number of turns')
argparser.add_argument('--auto-approve-delay', type=int,
default=3600 * 24 * 1,
help='how long to wait for auto approval')
argparser.add_argument('--only-masters', type='bool', default=False,
help='Set to True to use only master turks for this' +
' test eval, default is %(default)s')
# ADD MODEL ARGS HERE, UNCOMMENT TO USE KVMEMNN MODEL AS AN EXAMPLE
# argparser.set_defaults(
# model='projects.personachat.kvmemnn.kvmemnn:Kvmemnn',
# model_file='models:convai2/kvmemnn/model',
# )
opt = argparser.parse_args()
# add additional model args
opt['override'] = {
'no_cuda': True,
'interactive_mode': True,
'tensorboard_log': False
}
bot = create_agent(opt)
shared_bot_params = bot.share()
print(
'=== Actual bot opt === :\n {}'.format(
'\n'.join(["[{}] : {}".format(k, v) for k, v in bot.opt.items()])
)
)
folder_name = (
'master_{}_YOURCOMMENT__'.format(opt['only_masters']) +
'__'.join(['{}_{}'.format(k, v) for k, v in opt['override'].items()])
)
# this is mturk task, not convai2 task from ParlAI
opt['task'] = 'convai2:self'
if 'data_path' not in opt:
opt['data_path'] = os.getcwd() + '/data/' + folder_name
opt.update(task_config)
mturk_agent_ids = ['PERSON_1']
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=mturk_agent_ids
)
persona_generator = PersonasGenerator(opt)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
agent_qualifications = []
if opt['only_masters']:
if opt['is_sandbox']:
agent_qualifications.append(MASTER_QUALIF_SDBOX)
else:
agent_qualifications.append(MASTER_QUALIF)
mturk_manager.create_hits(qualifications=agent_qualifications)
if not opt['is_sandbox']:
# ADD SOFT-BLOCKED WORKERS HERE
# NOTE: blocking qual *must be* specified
blocked_worker_list = []
for w in blocked_worker_list:
print('Soft Blocking {}\n'.format(w))
mturk_manager.soft_block_worker(w)
time.sleep(0.1) # do the sleep to prevent amazon query drop
def run_onboard(worker):
worker.persona_generator = persona_generator
world = PersonaProfileWorld(opt, worker)
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
agents = workers[0]
conv_idx = mturk_manager.conversation_index
world = Convai2EvalWorld(
opt=opt,
agents=[agents],
range_turn=[int(s) for s in opt['range_turn'].split(',')],
max_turn=opt['max_turns'],
max_resp_time=opt['max_resp_time'],
model_agent_opt=shared_bot_params,
world_tag='conversation t_{}'.format(conv_idx),
agent_timeout_shutdown=opt['ag_shutdown_time'],
)
world.reset_random()
while not world.episode_done():
world.parley()
world.save_data()
world.shutdown()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id]
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids = mturk_agent_ids
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = MTurkMultiAgentDialogOnboardWorld(
opt=opt,
mturk_agent=worker
)
while not world.episode_done():
world.parley()
world.shutdown()
# You can set onboard_function to None to skip onboarding
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
# Create mturk agents
mturk_agent_1 = workers[0]
mturk_agent_2 = workers[1]
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
world = MTurkMultiAgentDialogWorld(
opt=opt,
agents=[human_agent_1, mturk_agent_1, mturk_agent_2]
)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
