def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser()
parser.add_argument('-n', '--num-examples', default=10)
parser.set_defaults(datatype='train:ordered')
ImageLoader.add_cmdline_args(parser)
opt = parser.parse_args()
opt['no_cuda'] = False
opt['gpu'] = 0
# create repeat label agent and assign it to the specified task
agent = RepeatLabelAgent(opt)
world = create_task(opt, agent)
# Show some example dialogs.
with world:
for k in range(int(opt['num_examples'])):
world.parley()
print(world.display() + '\n~~')
if world.epoch_done():
print('EPOCH DONE')
break
def main():
# Get command line arguments
parser = ParlaiParser()
parser.add_argument('-n', '--num-examples', default=10)
opt = parser.parse_args()
agent = Agent(opt)
opt['datatype'] = 'train'
world_train = create_task(opt, agent)
opt['datatype'] = 'valid'
world_valid = create_task(opt, agent)
start = time.time()
# train / valid loop
for _ in range(1):
print('[ training ]')
for _ in range(10): # train for a bit
world_train.parley()
print('[ training summary. ]')
print(world_train.report())
print('[ validating ]')
for _ in range(1): # check valid accuracy
world_valid.parley()
print('[ validation summary. ]')
print(world_valid.report())
print('finished in {} s'.format(round(time.time() - start, 2)))
def test_basic_parse(self):
"""Check that the dictionary is correctly adding and parsing short
sentence.
"""
from parlai.core.dict import DictionaryAgent
from parlai.core.params import ParlaiParser
argparser = ParlaiParser()
DictionaryAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
dictionary = DictionaryAgent(opt)
num_builtin = len(dictionary)
dictionary.observe({'text': 'hello world'})
dictionary.act()
assert len(dictionary) - num_builtin == 2
vec = dictionary.parse('hello world')
assert len(vec) == 2
assert vec[0] == num_builtin
assert vec[1] == num_builtin + 1
vec = dictionary.parse('hello world', vec_type=list)
assert len(vec) == 2
assert vec[0] == num_builtin
assert vec[1] == num_builtin + 1
vec = dictionary.parse('hello world', vec_type=tuple)
assert len(vec) == 2
assert vec[0] == num_builtin
assert vec[1] == num_builtin + 1
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser(True, True)
RemoteAgentAgent.add_cmdline_args(parser)
opt = parser.parse_args()
remote = RemoteAgentAgent(opt)
if opt.get('task'):
world = create_task(opt, [remote])
else:
if opt.get('model'):
local = create_agent(opt)
else:
local = LocalHumanAgent(opt)
# the remote-host goes **second**
agents = [local, remote] if not opt['remote_host'] else [remote, local]
world = DialogPartnerWorld(opt, agents)
# Talk to the remote agent
with world:
while True:
world.parley()
print(world.display())
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser()
parser.add_argument('-n', '--num-examples', default=10, type=int)
opt = parser.parse_args()
display_data(opt)
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_messenger_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
# Initialize a SQuAD teacher agent, which we will get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
messenger_manager = MessengerManager(opt=opt)
messenger_manager.setup_server()
messenger_manager.init_new_state()
def get_overworld(agent):
return MessengerOverworld(None, agent)
def assign_agent_role(agent):
agent[0].disp_id = 'Agent'
def run_conversation(manager, opt, agents, task_id):
task = task_class(task_opt)
agent = agents[0]
world = QADataCollectionWorld(
opt=opt,
task=task,
agent=agent
)
while not world.episode_done():
world.parley()
world.shutdown()
# World with no onboarding
messenger_manager.set_onboard_functions({'default': None})
task_functions = {'default': run_conversation}
assign_agent_roles = {'default': assign_agent_role}
messenger_manager.set_agents_required({'default': 1})
messenger_manager.set_overworld_func(get_overworld)
messenger_manager.setup_socket()
try:
messenger_manager.start_new_run()
messenger_manager.start_task(
assign_role_functions=assign_agent_roles,
task_functions=task_functions,
)
except BaseException:
raise
finally:
messenger_manager.shutdown()
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser(True, True)
parser.add_argument('-n', '--num-examples', default=100000000)
parser.add_argument('-d', '--display-examples', type='bool', default=False)
parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=2)
parser.set_defaults(datatype='valid')
opt = parser.parse_args(print_args=False)
eval_model(opt, parser)
def main():
# Get command line arguments
argparser = ParlaiParser(True, True)
build = argparser.add_argument_group('Data Building Args')
build.add_argument('--datafile',
help=('The file to be loaded, preprocessed, and saved'))
build.add_argument('--pytorch-buildteacher', type=str, default='',
help='Which teacher to use when building the pytorch data')
build.add_argument('--pytorch-preprocess', type='bool', default=True,
help='Whether the agent should preprocess the data while building'
'the pytorch data')
opt = argparser.parse_args()
build_data(opt)
def main():
parser = ParlaiParser(True, True)
ConvAIWorld.add_cmdline_args(parser)
opt = parser.parse_args()
agent = ConvAISampleAgent(opt)
world = ConvAIWorld(opt, [agent])
while True:
try:
world.parley()
except Exception as e:
print('Exception: {}'.format(e))
def test_fvqa(self):
from parlai.core.params import ParlaiParser
parser = ParlaiParser()
parser.add_task_args(['-t', 'fvqa'])
opt = parser.parse_args(args=self.args)
from parlai.tasks.fvqa.agents import DefaultTeacher
for dt in ['train:ordered', 'test']:
opt['datatype'] = dt
teacher = DefaultTeacher(opt)
reply = teacher.act()
check(opt, reply)
shutil.rmtree(self.TMP_PATH)
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids = [mturk_agent_id],
all_agent_ids = [ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id] # In speaking order
)
mturk_manager.init_aws(opt=opt)
global run_hit
def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, opt, task_opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
human_agent_2_id = 'human_2'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id],
all_agent_ids = [human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id] # In speaking order
)
mturk_manager.init_aws(opt=opt)
global run_hit
def run_hit(hit_index, assignment_index, opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
# Create mturk agents
mturk_agent_1 = MTurkAgent(id=mturk_agent_1_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
mturk_agent_2 = MTurkAgent(id=mturk_agent_2_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = human_agent_2_id
world = MultiAgentDialogWorld(opt=opt, agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# Initialize a SQuAD teacher agent, which we will get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids = [mturk_agent_id],
all_agent_ids = [QADataCollectionWorld.collector_agent_id, mturk_agent_id] # In speaking order
)
mturk_manager.init_aws(opt=opt)
global run_hit
def run_hit(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
task = task_class(task_opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.shutdown()
def test_upgrade_opt(self):
"""
Test whether upgrade_opt works.
"""
with testing_utils.tempdir() as tmp:
modfn = os.path.join(tmp, 'model')
with open(modfn, 'w') as f:
f.write('Test.')
optfn = modfn + '.opt'
base_opt = {
'model': 'tests.test_params:_ExampleUpgradeOptAgent',
'dict_file': modfn + '.dict',
'model_file': modfn,
}
with open(optfn, 'w') as f:
json.dump(base_opt, f)
pp = ParlaiParser(True, True)
opt = pp.parse_args(['--model-file', modfn])
agents.create_agent(opt)
def setUp(self):
disconnect_path = os.path.join(parent_dir, 'disconnect-test.pickle')
if os.path.exists(disconnect_path):
os.remove(disconnect_path)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args([])
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(
opt=self.opt.copy(), mturk_agent_ids=mturk_agent_ids
)
self.worker_manager = self.mturk_manager.worker_manager
self.worker_state_1 = self.worker_manager.worker_alive(TEST_WORKER_ID_1)
self.worker_state_2 = self.worker_manager.worker_alive(TEST_WORKER_ID_2)
self.worker_state_3 = self.worker_manager.worker_alive(TEST_WORKER_ID_3)
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser()
parser.add_argument('-n', '--num-examples', default=10)
opt = parser.parse_args()
# create repeat label agent and assign it to the specified task
agent = RepeatLabelAgent(opt)
world = create_task(opt, agent)
# Show some example dialogs.
with world:
for k in range(int(opt['num_examples'])):
world.parley()
print(world.display() + "\n~~")
if world.epoch_done():
print("EPOCH DONE")
break
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser()
parser.add_argument('-n', '--num-examples', default=10, type=int)
opt = parser.parse_args()
# create repeat label agent and assign it to the specified task
agent = RepeatLabelAgent(opt)
world = create_task(opt, agent)
# Show some example dialogs.
with world:
for _ in range(opt['num_examples']):
world.parley()
print(world.display() + '\n~~')
if world.epoch_done():
print('EPOCH DONE')
break
def _create_safety_model(self, custom_model_file, device):
from parlai.core.params import ParlaiParser
parser = ParlaiParser(False, False)
TransformerClassifierAgent.add_cmdline_args(parser, partial_opt=None)
parser.set_params(
model='transformer/classifier',
model_file=custom_model_file,
print_scores=True,
data_parallel=False,
)
safety_opt = parser.parse_args([])
safety_opt["override"]["no_cuda"] = False if "cuda" in device else True
if "cuda:" in device:
safety_opt["override"]["gpu"] = int(device.split(":")[1])
elif "cuda" in device:
safety_opt["override"]["gpu"] = 0
return create_agent(safety_opt, requireModelExists=True)
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser(True, True)
parser.add_argument('-n', '--num-examples', default=10)
opt = parser.parse_args()
# Create model and assign it to the specified task
agent = create_agent(opt)
world = create_task(opt, agent)
# Show some example dialogs.
with world:
for k in range(int(opt['num_examples'])):
world.parley()
print(world.display() + "\n~~")
if world.epoch_done():
print("EPOCH DONE")
break
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
human_agent_2_id = 'human_2'
# Create the MTurk agents
opt.update(task_config)
opt['conversation_id'] = str(int(time.time()))
opt['mturk_agent_ids'] = [mturk_agent_1_id, mturk_agent_2_id]
opt['all_agent_ids'] = [
human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id
]
opt['agent_id'] = mturk_agent_1_id
mturk_agent_1 = MTurkAgent(opt=opt)
opt['agent_id'] = mturk_agent_2_id
mturk_agent_2 = MTurkAgent(opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = human_agent_2_id
world = MultiAgentDialogWorld(
opt=opt,
agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
while not world.episode_done():
world.parley()
world.shutdown()
def _test_resize_embeddings(self, model):
with testing_utils.tempdir() as tmpdir:
model_file = os.path.join(tmpdir, 'model_file')
_, _ = testing_utils.train_model(
dict(
model=model,
task='integration_tests:short_fixed',
n_layers=1,
n_encoder_layers=2,
n_decoder_layers=4,
num_epochs=1,
dict_tokenizer='bytelevelbpe',
bpe_vocab=DEFAULT_BYTELEVEL_BPE_VOCAB,
bpe_merge=DEFAULT_BYTELEVEL_BPE_MERGE,
bpe_add_prefix_space=False,
model_file=model_file,
save_after_valid=True,
))
# now create agent with special tokens
parser = ParlaiParser()
parser.set_params(
model=model,
task='integration_tests:short_fixed',
n_layers=1,
n_encoder_layers=2,
n_decoder_layers=4,
dict_tokenizer='bytelevelbpe',
bpe_vocab=DEFAULT_BYTELEVEL_BPE_VOCAB,
bpe_merge=DEFAULT_BYTELEVEL_BPE_MERGE,
bpe_add_prefix_space=False,
model_file=model_file,
save_after_valid=True,
special_tok_lst='PARTY,PARROT',
)
opt = parser.parse_args([])
agent = create_agent(opt)
# assert that the embeddings were resized
assert agent.resized_embeddings
# assert model has special tokens
self.assertEqual(agent.special_toks, ['PARTY', 'PARROT'])
def get_bot_agents(args: DictConfig,
model_opts: Dict[str, str],
no_cuda=False) -> Dict[str, dict]:
"""
Return shared bot agents.
Pass in model opts with the `model_opts` arg, where `model_opts` is a dictionary
whose keys are model names and whose values are strings that specify model
params (i.e. `--model image_seq2seq`).
"""
# Set up overrides
model_overrides = {
'model_parallel': args.blueprint.task_model_parallel
}
if no_cuda:
# If we load many models at once, we have to keep it on CPU
model_overrides['no_cuda'] = no_cuda
else:
logging.warning(
'WARNING: MTurk task has no_cuda FALSE. Models will run on GPU. Will '
'not work if loading many models at once.')
# Convert opt strings to Opt objects
parser = ParlaiParser(True, True)
parser.set_params(**model_overrides)
processed_opts = {}
for name, opt_string in model_opts.items():
processed_opts[name] = parser.parse_args(opt_string.split())
# Load and share all model agents
logging.info(
f'Got {len(list(processed_opts.keys()))} models: {processed_opts.keys()}.'
)
shared_bot_agents = {}
for model_name, model_opt in processed_opts.items():
logging.info('\n\n--------------------------------')
logging.info(f'model_name: {model_name}, opt_dict: {model_opt}')
model_agent = create_agent(model_opt, requireModelExists=True)
shared_bot_agents[model_name] = model_agent.share()
return shared_bot_agents
def __init__(self, args=None, **kwargs):
"""Initialize the predictor, setting up opt automatically if needed.
Args is expected to be in the same format as sys.argv: e.g. a list in
the form ['--model', 'seq2seq', '-hs', 128, '-lr', 0.5].
kwargs is interpreted by appending '--' to it and replacing underscores
with hyphens, so 'dict_file=/tmp/dict.tsv' would be interpreted as
'--dict-file /tmp/dict.tsv'.
"""
from parlai.core.params import ParlaiParser
from parlai.core.agents import create_agent
if args is None:
args = []
for k, v in kwargs.items():
args.append('--' + str(k).replace('_', '-'))
args.append(str(v))
parser = ParlaiParser(True, True)
self.opt = parser.parse_args(args)
self.agent = create_agent(self.opt)
def __init__(self, args=None, **kwargs):
"""Initializes the predictor, setting up opt automatically if necessary.
Args is expected to be in the same format as sys.argv: e.g. a list in
the form ['--model', 'seq2seq', '-hs', 128, '-lr', 0.5].
kwargs is interpreted by appending '--' to it and replacing underscores
with hyphens, so 'dict_file=/tmp/dict.tsv' would be interpreted as
'--dict-file /tmp/dict.tsv'.
"""
from parlai.core.params import ParlaiParser
from parlai.core.agents import create_agent
if args is None:
args = []
for k, v in kwargs.items():
args.append('--' + str(k).replace('_', '-'))
args.append(str(v))
parser = ParlaiParser(True, True, model_argv=args)
self.opt = parser.parse_args(args)
self.agent = create_agent(self.opt)
def setup_args():
"""
Set up args, specifically for the port number.
:return: A parser that parses the port from commandline arguments.
"""
parser = ParlaiParser(False, False)
parser_grp = parser.add_argument_group('Terminal Chat')
parser_grp.add_argument('--port',
default=35496,
type=int,
help='Port to run the terminal chat server')
parser_grp.add_argument("--host",
default="localhost",
type=str,
help="Host to run the terminal chat server")
parser_grp.add_argument("--jsonl",
default="conv.jsonl",
type=str,
help="Where to store conversation")
return parser.parse_args()
def main():
parser = ParlaiParser(True, True)
parser.set_defaults(batchsize=10,
sample=True,
max_seq_len=256,
beam_size=3,
annealing_topk=None,
annealing=0.6,
length_penalty=0.7)
ConvAIWorld.add_cmdline_args(parser)
TransformerAgent.add_cmdline_args(parser)
opt = parser.parse_args()
agent = TransformerAgent(opt)
world = ConvAIWorld(opt, [agent])
while True:
try:
world.parley()
except Exception as e:
print('Exception: {}'.format(e))
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser(True, True)
parser.add_argument('-d', '--display-examples', type='bool', default=False)
opt = parser.parse_args()
opt['task'] = 'parlai.agents.local_human.local_human:LocalHumanAgent'
print(opt)
# Create model and assign it to the specified task
agent = create_agent(opt)
world = create_task(opt, agent)
# Show some example dialogs:
while True:
world.parley()
if opt['display_examples']:
print("---")
print(world.display() + "\n~~")
if world.epoch_done():
print("EPOCH DONE")
break
def run_actor(self):
MODEL_FILE = '/Volumes/Data/ParlAI/from_pretrained_wiki/wikiqa_tdifd'
DB_PATH = '/Volumes/Data/ParlAI/from_pretrained_wiki/wikiqa_tdifd.db'
TFIDF_PATH = '/Volumes/Data/ParlAI/from_pretrained_wiki/wikiqa_tdifd.tfidf'
# keep things quiet
logger.setLevel(ERROR)
parser = ParlaiParser(True, True)
parser.set_defaults(
model_file=MODEL_FILE,
interactive_mode=True,
)
opt = parser.parse_args([], print_args=False)
opt['interactive_mode'] = True
agent = create_agent(opt)
train_world = create_task(opt, agent)
# pass examples to dictionary
while not train_world.epoch_done():
train_world.parley()
agent.observe(obs)
reply = 'Hello'
return reply
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser(True, True)
parser.add_argument('-d', '--display-examples', type='bool', default=False)
opt = parser.parse_args()
opt['task'] = 'parlai.agents.local_human.local_human:LocalHumanAgent'
print(opt)
# Create model and assign it to the specified task
agent = create_agent(opt)
world = create_task(opt, agent)
# Show some example dialogs:
while True:
world.parley()
if opt['display_examples']:
print("---")
print(world.display() + "\n~~")
if world.epoch_done():
print("EPOCH DONE")
break
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser()
parser.add_argument(
'-n',
'--num-examples',
default=-1,
type=int,
help='Total number of exs to convert, -1 to convert all examples',
)
parser.add_argument(
'-of',
'--outfile',
default=None,
type=str,
help='Output file where to save, by default will be created in /tmp',
)
parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=2)
parser.set_defaults(datatype='train:evalmode')
opt = parser.parse_args()
build_cands(opt)
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser(True, True)
parser.add_argument('-n', '--num-examples', default=10)
# by default we want to display info about the validation set
parser.set_defaults(datatype='valid')
opt = parser.parse_args()
# Create model and assign it to the specified task
agent = create_agent(opt)
world = create_task(opt, agent)
# Show some example dialogs.
with world:
for k in range(int(opt['num_examples'])):
world.parley()
print(world.display() + "\n~~")
if world.epoch_done():
print("EPOCH DONE")
break
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
model_agent = IrBaselineAgent(opt=opt)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
# Create the MTurk agent which provides a chat interface to the Turker
opt.update(task_config)
mturk_agent_id = 'Worker'
opt['agent_id'] = mturk_agent_id
opt['mturk_agent_ids'] = [mturk_agent_id]
opt['all_agent_ids'] = [
ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id
]
opt['conversation_id'] = str(int(time.time()))
mturk_agent = MTurkAgent(opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def setUp(self):
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt.copy(),
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
self.mturk_manager.send_message = mock.MagicMock()
self.mturk_manager.send_state_change = mock.MagicMock()
self.mturk_manager.send_command = mock.MagicMock()
self.turk_agent = MTurkAgent(
self.opt.copy(),
self.mturk_manager,
TEST_HIT_ID_1,
TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1,
)
def test_fused_adam(self):
with self.assertRaises(ImportError):
# we should crash if the user tries not giving --opt adam
testing_utils.train_model(
dict(
model_file='zoo:unittest/apex_fused_adam/model',
task='integration_tests:nocandidate',
))
# no problem if we give the option
pp = ParlaiParser(True, True)
opt = pp.parse_args([
'--model-file',
'zoo:unittest/apex_fused_adam/model',
'--dict-file',
'zoo:unittest/apex_fused_adam/model.dict',
'--task',
'integration_tests:nocandidate',
'--optimizer',
'adam',
])
create_agent(opt, requireModelExists=True)
def main(args=None):
# Get command line arguments
parser = ParlaiParser(True, True)
train = parser.add_argument_group('Training Loop Arguments')
train.add_argument('-et', '--evaltask',
help=('task to use for valid/test (defaults to the ' +
'one used for training if not set)'))
train.add_argument('-d', '--display-examples',
type='bool', default=False)
train.add_argument('-e', '--num-epochs', type=float, default=-1)
train.add_argument('-ttim', '--max-train-time',
type=float, default=-1)
train.add_argument('-ltim', '--log-every-n-secs',
type=float, default=2)
train.add_argument('-le', '--log-every-n-epochs',
type=int, default=0)
train.add_argument('-vtim', '--validation-every-n-secs',
type=float, default=-1)
train.add_argument('-ve', '--validation-every-n-epochs',
type=int, default=0)
train.add_argument('-vme', '--validation-max-exs',
type=int, default=-1,
help='max examples to use during validation (default ' +
'-1 uses all)')
train.add_argument('-vp', '--validation-patience',
type=int, default=5,
help=('number of iterations of validation where result '
+ 'does not improve before we stop training'))
train.add_argument('-dbf', '--dict-build-first',
type='bool', default=True,
help='build dictionary first before training agent')
train.add_argument('--chosen-metric', default='accuracy',
help='metric with which to measure improvement')
opt = parser.parse_args(args=args)
if opt.get('cross_validation_splits_count', 0) > 1 and opt.get('cross_validation_model_index') is None:
train_cross_valid(opt)
else:
train_model(opt)
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_messenger_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt['password'] = '******' # If password is none anyone can chat
messenger_manager = MessengerManager(opt=opt)
messenger_manager.setup_server()
messenger_manager.init_new_state()
def get_overworld(opt, agent):
return MessengerOverworld(opt, agent)
onboard_functions = {name: worlds[0].run for (name, worlds)
in MessengerOverworld.DEMOS.items()}
messenger_manager.set_onboard_functions(onboard_functions)
task_functions = {name: worlds[1].run for (name, worlds)
in MessengerOverworld.DEMOS.items()}
assign_agent_roles = {name: worlds[1].assign_roles for (name, worlds)
in MessengerOverworld.DEMOS.items()}
agents_required = {name: worlds[1].MAX_AGENTS for (name, worlds)
in MessengerOverworld.DEMOS.items()}
messenger_manager.set_agents_required(agents_required)
messenger_manager.set_overworld_func(get_overworld)
messenger_manager.setup_socket()
try:
messenger_manager.start_new_run()
messenger_manager.start_task(
assign_role_functions=assign_agent_roles,
task_functions=task_functions,
)
except BaseException:
raise
finally:
messenger_manager.shutdown()
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser(True, True)
parser.add_argument('-n', '--num_examples', default=1000)
parser.add_argument('-d', '--display_examples', type='bool', default=False)
opt = parser.parse_args()
# Create model and assign it to the specified task
agent = create_agent(opt)
world = create_task(opt, agent)
# Show some example dialogs:
for k in range(int(opt['num_examples'])):
world.parley()
print("---")
if opt['display_examples']:
print(world.display() + "\n~~")
print(world.report())
if world.epoch_done():
print("EPOCH DONE")
break
world.shutdown()
def main():
# Get command line arguments
parser = ParlaiParser(True, False)
parser.set_defaults(datatype='train:ordered')
opt = parser.parse_args()
bsz = opt.get('batchsize', 1)
opt['no_cuda'] = False
opt['gpu'] = 0
opt['num_epochs'] = 1
# create repeat label agent and assign it to the specified task
agent = RepeatLabelAgent(opt)
world = create_task(opt, agent)
logger = ProgressLogger(should_humanize=False)
print("Beginning image extraction...")
exs_seen = 0
total_exs = world.num_examples()
while not world.epoch_done():
world.parley()
exs_seen += bsz
logger.log(exs_seen, total_exs)
print("Finished extracting images")
def parse_options():
parser = ParlaiParser()
pth_file = "world_best.pth"
# pth_file = 'checkpoints/world-07-Nov-2019-15:56:51/world_epoch_02900.pth'
parser.add_argument(
"--load-path",
type=str,
default=pth_file,
help="path to pth file of the world checkpoint",
)
parser.add_argument(
"--print-conv",
default=False,
action="store_true",
help="whether to print the conversation between bots or not",
)
parser.add_argument(
"--conv-save-path",
type=str,
default=None,
help="whether to print the conversation between bots or not",
)
return parser.parse_args()
def main():
random.seed(42)
# Get command line arguments
parser = ParlaiParser(True, True)
parser.add_argument('-n', '--num-examples', default=100000000)
parser.add_argument('-d', '--display-examples', type='bool', default=False)
parser.set_defaults(datatype='valid')
opt = parser.parse_args()
# Create model and assign it to the specified task
agent = create_agent(opt)
world = create_task(opt, agent)
# Show some example dialogs:
for k in range(int(opt['num_examples'])):
world.parley()
print("---")
if opt['display_examples']:
print(world.display() + "\n~~")
print(world.report())
if world.epoch_done():
print("EPOCH DONE")
break
world.shutdown()
def main():
# Get command line arguments
parser = ParlaiParser(True, True)
parser.add_argument('-d', '--display-examples', type='bool', default=False)
opt = parser.parse_args()
# Create model and assign it to the specified task
opt['model'] = 'ir_baseline'
agent = create_agent(opt)
opt['datatype'] = 'train:ordered'
world = create_task(opt, agent)
# Show some example dialogs:
for _ in range(len(world)):
world.parley()
if opt['display_examples']:
print(world.display() + "\n~~")
if world.epoch_done():
print("TRAIN EPOCH DONE")
break
world.shutdown()
if opt['model_file']:
agent.save(opt['model_file'])
# Now eval on validation data.
opt['datatype'] = 'valid'
valid_world = create_task(opt, agent)
for _ in range(len(valid_world)):
valid_world.parley()
print("---")
if opt['display_examples']:
print(valid_world.display() + "\n~~")
print(valid_world.report())
if valid_world.epoch_done():
print("VALIDATION DONE")
break
valid_world.shutdown()
def test_knowledge_retriever(self):
from parlai.core.params import ParlaiParser
parser = ParlaiParser(False, False)
KnowledgeRetrieverAgent.add_cmdline_args(parser)
parser.set_params(
model='projects:wizard_of_wikipedia:knowledge_retriever',
add_token_knowledge=True,
)
knowledge_opt = parser.parse_args([], print_args=False)
knowledge_agent = create_agent(knowledge_opt)
knowledge_agent.observe(
{
'text': 'what do you think of mountain dew?',
'chosen_topic': 'Mountain Dew',
'episode_done': False,
}
)
knowledge_act = knowledge_agent.act()
title = knowledge_act['title']
self.assertEqual(title, 'Mountain Dew', 'Did not save chosen topic correctly')
knowledge = knowledge_act['text']
self.assertIn(
TOKEN_KNOWLEDGE, knowledge, 'Knowledge token was not inserted correctly'
)
checked_sentence = knowledge_act['checked_sentence']
self.assertEqual(
checked_sentence,
'Mountain Dew (stylized as Mtn Dew) is a carbonated soft drink brand produced and owned by PepsiCo.',
'Did not correctly choose the checked sentence',
)
import os
import json
from parlai.core.params import ParlaiParser
from parlai.tasks.light_dialog.agents import DefaultTeacher
SPLIT = 'train'
BASE_DIR = '.'
parser = ParlaiParser(True, True, 'Evaluate a model')
DefaultTeacher.add_cmdline_args(parser)
args = ['-dt', SPLIT]
opt = parser.parse_args(args, print_args=False)
teacher = DefaultTeacher(opt)
def first_word(s):
l = s.split()
return '' if len(l) == 0 else l[0]
def extractTurn(t, step):
found_partner = False
for s in step['text'].split('\n'):
fw = first_word(s)
if fw.find('partner') != -1:
found_partner = True
if found_partner:
if fw == '_partner_say':
t['partner_say'] = s[len(fw):]
elif fw == '_self_say':
t['self_say'] = s[len(fw):]
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# Initialize a SQuAD teacher agent, which we will get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=[mturk_agent_id])
mturk_manager.setup_server()
def run_onboard(worker):
world = QADataCollectionOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=None)
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
mturk_manager.ready_to_accept_workers()
def check_workers_eligibility(workers):
return workers
eligibility_function = {
'func': check_workers_eligibility,
'multiple': True,
}
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
task = task_class(task_opt)
mturk_agent = workers[0]
world = QADataCollectionWorld(opt=opt,
task=task,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.start_task(eligibility_function=eligibility_function,
assign_role_function=assign_worker_roles,
task_function=run_conversation)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""
This task consists of one local human agent and two MTurk agents,
each MTurk agent will go through the onboarding step to provide
information about themselves, before being put into a conversation.
You can end the conversation by sending a message ending with
`[DONE]` from human_1.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id]
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = MTurkMultiAgentDialogOnboardWorld(opt=opt,
mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
# You can set onboard_function to None to skip onboarding
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
# Create mturk agents
mturk_agent_1 = workers[0]
mturk_agent_2 = workers[1]
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
world = MTurkMultiAgentDialogWorld(
opt=opt, agents=[human_agent_1, mturk_agent_1, mturk_agent_2])
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.start_task(eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""This task consists of one agent, model or MTurk worker, talking to an
MTurk worker to negotiate a deal.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('-min_t',
'--min_turns',
default=5,
type=int,
help='minimum number of turns')
argparser.add_argument('-mt',
'--max_turns',
default=10,
type=int,
help='maximal number of chat turns')
argparser.add_argument(
'-mx_rsp_time',
'--max_resp_time',
default=150,
type=int,
help='time limit for entering a dialog message',
)
argparser.add_argument(
'--ag_shutdown_time',
default=120,
type=int,
help='time limit for entering a dialog message',
)
argparser.add_argument(
'--persona-type',
default='both',
type=str,
choices=['both', 'self', 'other'],
help='Which personas to load from personachat',
)
opt = argparser.parse_args()
directory_path = os.path.dirname(os.path.abspath(__file__))
opt['task'] = os.path.basename(directory_path)
opt['extract_personas_path'] = os.path.join(opt['datapath'], opt['task'])
opt.update(task_config)
mturk_agent_ids = ['PERSON_1']
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
mturk_manager.setup_server(task_directory_path=directory_path)
personas_generator = PersonasGenerator(opt)
opt['personas_generator'] = personas_generator
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
if not opt['is_sandbox']:
# ADD BLOCKED WORKERS HERE
blocked_worker_list = []
for w in blocked_worker_list:
mturk_manager.block_worker(
w,
'We found that you have unexpected behaviors in our '
'previous HITs. For more questions please email us.',
)
def run_onboard(worker):
pass
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
worker = workers[0]
world = RephrasePersonaWorld(opt, worker)
while not world.episode_done():
world.parley()
world.save_data()
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation,
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
print('---Extracting and saving personas---')
teacher_name = 'personachat:{}'.format(opt.get('persona_type'))
teacher_name += 'Revised' if opt.get('revised') else 'Original'
opt['task'] = teacher_name
if 'personas_path' not in opt:
personas_path = '{}/data/wizard_of_perZOna/personas-{}/'.format(
os.getcwd(), opt['task']
)
opt['personas_path'] = personas_path
opt['datatype'] = 'train:ordered:stream'
opt['numthreads'] = 1
opt['batchsize'] = 1
personas = extract_and_save(opt)
return personas
if __name__ == '__main__':
parser = ParlaiParser()
parser.add_argument(
'--persona-type',
default='both',
type=str,
choices=['both', 'self', 'other'],
help='Which personas to load from personachat',
)
parser.add_argument(
'--revised', default=False, type='bool', help='Whether to use revised personas'
)
opt = parser.parse_args()
personas = main(opt)
def main():
"""
This task consists of one local human agent and two MTurk agents,
each MTurk agent will go through the onboarding step to provide
information about themselves, before being put into a conversation.
You can end the conversation by sending a message ending with
`[DONE]` from human_1.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id]
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=mturk_agent_ids
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = MTurkMultiAgentDialogOnboardWorld(
opt=opt,
mturk_agent=worker
)
while not world.episode_done():
world.parley()
world.shutdown()
# You can set onboard_function to None to skip onboarding
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
eligibility_function = {
'func': check_worker_eligibility,
'multiple': False,
}
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
# Create mturk agents
mturk_agent_1 = workers[0]
mturk_agent_2 = workers[1]
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
world = MTurkMultiAgentDialogWorld(
opt=opt,
agents=[human_agent_1, mturk_agent_1, mturk_agent_2]
)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.start_task(
eligibility_function=eligibility_function,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
argparser.add_argument('--num_machines', type=int, default=1)
argparser.add_argument('--job_timeout', type=float, default=3600*4)
argparser.add_argument('--split', action='store_true', default=False)
argparser.add_argument('--train', action='store_true', default=False)
argparser.add_argument('--eval', action='store_true', default=False)
argparser.add_argument('--seq2seq', action='store_true', default=False)
argparser.add_argument('--constrain', action='store_true', default=False)
argparser.add_argument('--rounds_breakdown', action='store_true', default=False)
argparser.add_argument('--data_breakdown', action='store_true', default=False)
argparser.add_argument('--ablation', action='store_true', default=False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
# ============ below copied from projects/graph_world2/train.py ============
opt['bidir'] = True
opt['action_type_emb_dim'] = 5
opt['counter_max'] = 3
opt['counter_emb_dim'] = 5
# ============ above copied from projects/graph_world2/train.py ============
if opt['split']:
overall_split(opt)
quit()
if opt['train']:
overall_run(opt, seq2seq=opt['seq2seq'])
quit()
if opt['eval']:
def main():
# Get command line arguments
parser = ParlaiParser(True, True)
train = parser.add_argument_group('Training Loop Arguments')
train.add_argument('-et', '--evaltask',
help=('task to use for valid/test (defaults to the '
'one used for training if not set)'))
train.add_argument('-d', '--display-examples',
type='bool', default=False)
train.add_argument('-e', '--num-epochs', type=float, default=-1)
train.add_argument('-ttim', '--max-train-time',
type=float, default=-1)
train.add_argument('-ltim', '--log-every-n-secs',
type=float, default=2)
train.add_argument('-vtim', '--validation-every-n-secs',
type=float, default=-1)
train.add_argument('-vme', '--validation-max-exs',
type=int, default=-1,
help='max examples to use during validation (default '
'-1 uses all)')
train.add_argument('-vp', '--validation-patience',
type=int, default=5,
help=('number of iterations of validation where result'
' does not improve before we stop training'))
train.add_argument('-vmt', '--validation-metric', default='accuracy',
help='key into report table for selecting best '
'validation')
train.add_argument('-dbf', '--dict-build-first',
type='bool', default=True,
help='build dictionary first before training agent')
opt = parser.parse_args()
# Possibly build a dictionary (not all models do this).
if opt['dict_build_first'] and 'dict_file' in opt:
if opt['dict_file'] is None and opt.get('model_file'):
opt['dict_file'] = opt['model_file'] + '.dict'
print("[ building dictionary first... ]")
build_dict.build_dict(opt)
# Create model and assign it to the specified task
agent = create_agent(opt)
world = create_task(opt, agent)
train_time = Timer()
validate_time = Timer()
log_time = Timer()
print('[ training... ]')
parleys = 0
total_exs = 0
max_exs = opt['num_epochs'] * len(world)
max_parleys = math.ceil(max_exs / opt['batchsize'])
best_valid = 0
impatience = 0
saved = False
valid_world = None
while True:
world.parley()
parleys += 1
if opt['num_epochs'] > 0 and parleys >= max_parleys:
print('[ num_epochs completed: {} ]'.format(opt['num_epochs']))
break
if opt['max_train_time'] > 0 and train_time.time() > opt['max_train_time']:
print('[ max_train_time elapsed: {} ]'.format(train_time.time()))
break
if opt['log_every_n_secs'] > 0 and log_time.time() > opt['log_every_n_secs']:
if opt['display_examples']:
print(world.display() + '\n~~')
logs = []
# time elapsed
logs.append('time:{}s'.format(math.floor(train_time.time())))
logs.append('parleys:{}'.format(parleys))
# get report and update total examples seen so far
if hasattr(agent, 'report'):
train_report = agent.report()
agent.reset_metrics()
else:
train_report = world.report()
world.reset_metrics()
if hasattr(train_report, 'get') and train_report.get('total'):
total_exs += train_report['total']
logs.append('total_exs:{}'.format(total_exs))
# check if we should log amount of time remaining
time_left = None
if opt['num_epochs'] > 0:
exs_per_sec = train_time.time() / total_exs
time_left = (max_exs - total_exs) * exs_per_sec
if opt['max_train_time'] > 0:
other_time_left = opt['max_train_time'] - train_time.time()
if time_left is not None:
time_left = min(time_left, other_time_left)
else:
time_left = other_time_left
if time_left is not None:
logs.append('time_left:{}s'.format(math.floor(time_left)))
# join log string and add full metrics report to end of log
log = '[ {} ] {}'.format(' '.join(logs), train_report)
print(log)
log_time.reset()
if (opt['validation_every_n_secs'] > 0 and
validate_time.time() > opt['validation_every_n_secs']):
valid_report, valid_world = run_eval(
agent, opt, 'valid', opt['validation_max_exs'],
valid_world=valid_world)
if valid_report[opt['validation_metric']] > best_valid:
best_valid = valid_report[opt['validation_metric']]
impatience = 0
print('[ new best {}: {} ]'.format(
opt['validation_metric'], best_valid))
world.save_agents()
saved = True
if opt['validation_metric'] == 'accuracy' and best_valid == 1:
print('[ task solved! stopping. ]')
break
else:
impatience += 1
print('[ did not beat best {}: {} impatience: {} ]'.format(
opt['validation_metric'], round(best_valid, 4),
impatience))
validate_time.reset()
if opt['validation_patience'] > 0 and impatience >= opt['validation_patience']:
print('[ ran out of patience! stopping training. ]')
break
world.shutdown()
if not saved:
world.save_agents()
else:
# reload best validation model
agent = create_agent(opt)
run_eval(agent, opt, 'valid', write_log=True)
run_eval(agent, opt, 'test', write_log=True)
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_id]
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = ModelEvaluatorOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
model_agent = IrBaselineAgent(opt=opt)
world = ModelEvaluatorWorld(
opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent
)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# Initialize a SQuAD teacher agent, which we will get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_id]
)
mturk_manager.setup_server()
def run_onboard(worker):
world = QADataCollectionOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=None)
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
mturk_manager.ready_to_accept_workers()
def check_workers_eligibility(workers):
return workers
eligibility_function = {
'func': check_workers_eligibility,
'multiple': True,
}
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
task = task_class(task_opt)
mturk_agent = workers[0]
world = QADataCollectionWorld(
opt=opt,
task=task,
mturk_agent=mturk_agent
)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=eligibility_function,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
# Get command line arguments
argparser = ParlaiParser()
DictionaryAgent.add_cmdline_args(argparser)
ParsedRemoteAgent.add_cmdline_args(argparser)
argparser.add_argument('--num-examples', default=1000, type=int)
argparser.add_argument('--num-its', default=100, type=int)
argparser.add_argument('--dict-max-exs', default=10000, type=int)
parlai_home = os.environ['PARLAI_HOME']
if '--remote-cmd' not in sys.argv:
if os.system('which luajit') != 0:
raise RuntimeError('Could not detect torch luajit installed: ' +
'please install torch from http://torch.ch ' +
'or manually set --remote-cmd for this example.')
sys.argv.append('--remote-cmd')
sys.argv.append('luajit {}/parlai/agents/'.format(parlai_home) +
'memnn_luatorch_cpu/memnn_zmq_parsed.lua')
if '--remote-args' not in sys.argv:
sys.argv.append('--remote-args')
sys.argv.append('{}/examples/'.format(parlai_home) +
'memnn_luatorch_cpu/params_default.lua')
opt = argparser.parse_args()
# set up dictionary
print('Setting up dictionary.')
dictionary = DictionaryAgent(opt)
if not opt.get('dict_file'):
# build dictionary since we didn't load it
ordered_opt = copy.deepcopy(opt)
ordered_opt['datatype'] = 'train:ordered'
ordered_opt['numthreads'] = 1
world_dict = create_task(ordered_opt, dictionary)
print('Dictionary building on training data.')
cnt = 0
# pass examples to dictionary
for _ in world_dict:
cnt += 1
if cnt > opt['dict_max_exs'] and opt['dict_max_exs'] > 0:
print('Processed {} exs, moving on.'.format(
opt['dict_max_exs']))
# don't wait too long...
break
world_dict.parley()
# we need to save the dictionary to load it in memnn (sort it by freq)
dictionary.sort()
dictionary.save('/tmp/dict.txt', sort=True)
print('Dictionary ready, moving on to training.')
opt['datatype'] = 'train'
agent = ParsedRemoteAgent(opt, {'dictionary_shared': dictionary.share()})
world_train = create_task(opt, agent)
opt['datatype'] = 'valid'
world_valid = create_task(opt, agent)
start = time.time()
with world_train:
for _ in range(opt['num_its']):
print('[ training ]')
for _ in range(opt['num_examples'] * opt.get('numthreads', 1)):
world_train.parley()
world_train.synchronize()
print('[ validating ]')
world_valid.reset()
for _ in world_valid: # check valid accuracy
world_valid.parley()
print('[ validation summary. ]')
report_valid = world_valid.report()
print(report_valid)
if report_valid['accuracy'] > 0.95:
break
# show some example dialogs after training:
world_valid = create_task(opt, agent)
for _k in range(3):
world_valid.parley()
print(world_valid.display())
print('finished in {} s'.format(round(time.time() - start, 2)))
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id]
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids = mturk_agent_ids
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = MTurkMultiAgentDialogOnboardWorld(
opt=opt,
mturk_agent=worker
)
while not world.episode_done():
world.parley()
world.shutdown()
# You can set onboard_function to None to skip onboarding
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
# Create mturk agents
mturk_agent_1 = workers[0]
mturk_agent_2 = workers[1]
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
world = MTurkMultiAgentDialogWorld(
opt=opt,
agents=[human_agent_1, mturk_agent_1, mturk_agent_2]
)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""This task consists of an MTurk agent evaluating a chit-chat model. They
are asked to chat to the model adopting a specific persona. After their
conversation, they are asked to evaluate their partner on several metrics.
"""
argparser = ParlaiParser(False, add_model_args=True)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('-mt', '--max-turns', default=10, type=int,
help='maximal number of chat turns')
argparser.add_argument('--max-resp-time', default=240,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('--max-persona-time', type=int,
default=300, help='time limit for turker'
'entering the persona')
argparser.add_argument('--ag-shutdown-time', default=120,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('--persona-type', default='both', type=str,
choices=['both', 'self', 'other'],
help='Which personas to load from personachat')
argparser.add_argument('--revised', default=False, type='bool',
help='Whether to use revised personas')
argparser.add_argument('-rt', '--range-turn', default='5,6',
help='sample range of number of turns')
argparser.add_argument('--auto-approve-delay', type=int,
default=3600 * 24 * 1,
help='how long to wait for auto approval')
argparser.add_argument('--only-masters', type='bool', default=False,
help='Set to True to use only master turks for this' +
' test eval, default is %(default)s')
# ADD MODEL ARGS HERE, UNCOMMENT TO USE KVMEMNN MODEL AS AN EXAMPLE
# argparser.set_defaults(
# model='projects.personachat.kvmemnn.kvmemnn:Kvmemnn',
# model_file='models:convai2/kvmemnn/model',
# )
opt = argparser.parse_args()
# add additional model args
opt['override'] = {
'no_cuda': True,
'interactive_mode': True,
'tensorboard_log': False
}
bot = create_agent(opt)
shared_bot_params = bot.share()
print(
'=== Actual bot opt === :\n {}'.format(
'\n'.join(["[{}] : {}".format(k, v) for k, v in bot.opt.items()])
)
)
folder_name = (
'master_{}_YOURCOMMENT__'.format(opt['only_masters']) +
'__'.join(['{}_{}'.format(k, v) for k, v in opt['override'].items()])
)
# this is mturk task, not convai2 task from ParlAI
opt['task'] = 'convai2:self'
if 'data_path' not in opt:
opt['data_path'] = os.getcwd() + '/data/' + folder_name
opt.update(task_config)
mturk_agent_ids = ['PERSON_1']
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=mturk_agent_ids
)
persona_generator = PersonasGenerator(opt)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
agent_qualifications = []
if opt['only_masters']:
if opt['is_sandbox']:
agent_qualifications.append(MASTER_QUALIF_SDBOX)
else:
agent_qualifications.append(MASTER_QUALIF)
mturk_manager.create_hits(qualifications=agent_qualifications)
if not opt['is_sandbox']:
# ADD SOFT-BLOCKED WORKERS HERE
# NOTE: blocking qual *must be* specified
blocked_worker_list = []
for w in blocked_worker_list:
print('Soft Blocking {}\n'.format(w))
mturk_manager.soft_block_worker(w)
time.sleep(0.1) # do the sleep to prevent amazon query drop
def run_onboard(worker):
worker.persona_generator = persona_generator
world = PersonaProfileWorld(opt, worker)
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
agents = workers[0]
conv_idx = mturk_manager.conversation_index
world = Convai2EvalWorld(
opt=opt,
agents=[agents],
range_turn=[int(s) for s in opt['range_turn'].split(',')],
max_turn=opt['max_turns'],
max_resp_time=opt['max_resp_time'],
model_agent_opt=shared_bot_params,
world_tag='conversation t_{}'.format(conv_idx),
agent_timeout_shutdown=opt['ag_shutdown_time'],
)
world.reset_random()
while not world.episode_done():
world.parley()
world.save_data()
world.shutdown()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
completed_workers = []
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_id]
)
mturk_manager.setup_server()
qual_name = 'ParlAIExcludeQual{}t{}'.format(
random.randint(10000, 99999), random.randint(10000, 99999))
qual_desc = (
'Qualification for a worker not correctly completing the '
'first iteration of a task. Used to filter to different task pools.'
)
qualification_id = \
mturk_utils.find_or_create_qualification(qual_name, qual_desc)
print('Created qualification: ', qualification_id)
def run_onboard(worker):
world = QualificationFlowOnboardWorld(opt, worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
try:
mturk_manager.start_new_run()
agent_qualifications = [{
'QualificationTypeId': qualification_id,
'Comparator': 'DoesNotExist',
'RequiredToPreview': True
}]
mturk_manager.create_hits(qualifications=agent_qualifications)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
world = QualificationFlowSoloWorld(
opt=opt,
mturk_agent=mturk_agent,
qualification_id=qualification_id,
firstTime=(mturk_agent.worker_id not in completed_workers),
)
while not world.episode_done():
world.parley()
completed_workers.append(mturk_agent.worker_id)
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_utils.delete_qualification(qualification_id)
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""This task consists of one agent, model or MTurk worker, talking to an
MTurk worker to negotiate a deal.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('--two_mturk_agents', dest='two_mturk_agents',
action='store_true', help='data collection mode '
'with converations between two MTurk agents')
opt = argparser.parse_args()
opt['task'] = 'dealnodeal'
opt['datatype'] = 'valid'
opt.update(task_config)
local_agent_1_id = 'local_1'
mturk_agent_ids = ['mturk_agent_1']
if opt['two_mturk_agents']:
mturk_agent_ids.append('mturk_agent_2')
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=mturk_agent_ids
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
mturk_manager.set_onboard_function(onboard_function=None)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
agents = workers[:]
# Create a local agent
if not opt['two_mturk_agents']:
if 'model' in opt:
local_agent = create_agent(opt)
else:
local_agent = LocalHumanAgent(opt=None)
local_agent.id = local_agent_1_id
agents.append(local_agent)
opt["batchindex"] = mturk_manager.started_conversations
world = MTurkDealNoDealDialogWorld(
opt=opt,
agents=agents
)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
# Get command line arguments
argparser = ParlaiParser()
DictionaryAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
build_dict(opt)
