def setup_socket(self):
"""Set up socket to start communicating to workers"""
shared_utils.print_and_log(logging.INFO,
'Local: Setting up SocketIO...',
should_print=True)
self.app_token = None
if self.opt.get('force_page_token'):
pass
else:
if not os.path.exists(os.path.expanduser('~/.parlai/')):
os.makedirs(os.path.expanduser('~/.parlai/'))
access_token_file_path = '~/.parlai/messenger_token'
expanded_file_path = os.path.expanduser(access_token_file_path)
if os.path.exists(expanded_file_path):
with open(expanded_file_path, 'r') as access_token_file:
self.app_token = access_token_file.read()
# cache the app token
if self.app_token is None:
self.app_token = input(
'Enter your page\'s access token from the developer page at'
'https://developers.facebook.com/apps/<YOUR APP ID>'
'/messenger/settings/ to continue setup:')
access_token_file_path = '~/.parlai/messenger_token'
expanded_file_path = os.path.expanduser(access_token_file_path)
with open(expanded_file_path, 'w+') as access_token_file:
access_token_file.write(self.app_token)
self.message_socket = MessageSocket(self.server_url, self.port,
self.app_token,
self._on_new_message)
def _confirm_message_delivery(self, event):
# By default we don't actually do anything when messages are marked as
# being delivered, but we expose the ability for others to
shared_utils.print_and_log(
logging.DEBUG,
"Messages {} marked as received.".format(event['delivery']['mids'])
)
def _done_callback(fut):
"""Log and raise exception of task world, if there is one.
Additionally, set active agent to overworld agent.
"""
e = fut.exception()
if e is not None:
shared_utils.print_and_log(
logging.ERROR,
'World {} had error {}'.format(world_type, repr(e)),
should_print=True,
)
traceback.print_exc(file=sys.stdout)
for agent in agents:
self.observe_message(
agent.id,
'Sorry, this world closed. Returning to overworld.')
else:
shared_utils.print_and_log(
logging.INFO,
'World {} had no error'.format(world_type),
should_print=True,
)
self.active_worlds[task_id] = None
for agent in agents:
self.after_agent_removed(agent.id)
agent_state = self.get_agent_state(agent.id)
agent_state.set_active_agent(agent_state.get_overworld_agent())
def _log_missing_agent(self, agent_id, assignment_id):
"""Log the occurence of a missing agent."""
shared_utils.print_and_log(
logging.WARN,
'Expected to have an agent for {}_{}, yet none was found'.format(
agent_id, assignment_id),
)
def _onboard_function(opt, agent_id, world_type, task_id):
"""Onboarding wrapper to set state to onboarding properly"""
agent_state = self.get_agent_state(agent_id)
data = None
if (world_type in self.onboard_functions and
self.onboard_functions[world_type] is not None):
# call onboarding function
agent = self._create_agent(task_id, agent_id)
agent_state.set_active_agent(agent)
agent_state.assign_agent_to_task(agent, task_id)
try:
data = \
self.onboard_functions[world_type](opt, agent, task_id)
agent_state.onboard_data = data
agent_state.set_active_agent(None)
except Exception as e:
shared_utils.print_and_log(
logging.ERROR,
'Onboard {} had error {}'.format(world_type, repr(e)),
should_print=True
)
traceback.print_exc(file=sys.stderr)
self.observe_message(
agent.id,
"Sorry, this world closed. Returning to overworld."
)
agent_state.set_active_agent(
agent_state.get_overworld_agent())
return
# once onboarding is done, move into a waiting world
self.add_agent_to_pool(agent_state, world_type)
def setup_socket(self):
"""Set up socket to start communicating to workers"""
shared_utils.print_and_log(logging.INFO,
'Local: Setting up SocketIO...',
should_print=True)
self.app_token = None
if self.opt.get('force_page_token'):
pass
else:
if not os.path.exists(os.path.expanduser('~/.parlai/')):
os.makedirs(os.path.expanduser('~/.parlai/'))
access_token_file_path = '~/.parlai/messenger_token'
expanded_file_path = os.path.expanduser(access_token_file_path)
if os.path.exists(expanded_file_path):
with open(expanded_file_path, 'r') as access_token_file:
self.app_token = access_token_file.read()
# cache the app token
if self.app_token is None:
self.app_token = input(
'Enter your page\'s access token from the developer page at'
'https://developers.facebook.com/apps/<YOUR APP ID>'
'/messenger/settings/ to continue setup:'
)
access_token_file_path = '~/.parlai/messenger_token'
expanded_file_path = os.path.expanduser(access_token_file_path)
with open(expanded_file_path, 'w+') as access_token_file:
access_token_file.write(self.app_token)
self.message_socket = MessageSocket(self.server_url, self.port,
self.app_token,
self._on_new_message)
def _log_missing_agent(self, agent_id, assignment_id):
"""Logs when an agent was expected to exist, yet for some reason it
didn't. If these happen often there is a problem"""
shared_utils.print_and_log(
logging.WARN,
'Expected to have an agent for {}_{}, yet none was found'.format(
agent_id, assignment_id))
def _task_function(opt, agents, conversation_id, world_type):
"""Run the task function for the given agents"""
shared_utils.print_and_log(
logging.INFO,
'Starting task {}...'.format(conversation_id)
)
try:
task_functions[world_type](self, opt, agents, conversation_id)
except Exception as e:
shared_utils.print_and_log(
logging.ERROR,
'World {} had error {}'.format(world_type, repr(e)),
should_print=True,
)
print("Exception in user code:")
traceback.print_exc(file=sys.stdout)
for agent in agents:
self.observe_message(
agent.id,
"Sorry, this world closed. Returning to overworld."
)
self.active_worlds[task_id] = None
for agent in agents:
self.after_agent_removed(agent.id)
agent_state = self.get_agent_state(agent.id)
agent_state.set_active_agent(agent_state.get_overworld_agent())
def add_agent_to_pool(self, agent, world_type='default'):
"""Add the agent to pool"""
with self.agent_pool_change_condition:
shared_utils.print_and_log(
logging.DEBUG,
"Adding agent {} to pool...".format(agent.messenger_id))
if world_type not in self.agent_pool:
self.agent_pool[world_type] = []
self.agent_pool[world_type].append(agent)
def add_agent_to_pool(self, agent, world_type='default'):
"""Add the agent to pool"""
with self.agent_pool_change_condition:
shared_utils.print_and_log(
logging.DEBUG,
"Adding agent {} to pool...".format(agent.messenger_id)
)
if world_type not in self.agent_pool:
self.agent_pool[world_type] = []
self.agent_pool[world_type].append(agent)
def _log_missing_agent(self, agent_id, assignment_id):
"""Logs when an agent was expected to exist, yet for some reason it
didn't. If these happen often there is a problem"""
shared_utils.print_and_log(
logging.WARN,
'Expected to have an agent for {}_{}, yet none was found'.format(
agent_id,
assignment_id
)
)
def add_agent_to_pool(self, agent, world_type='default'):
"""Add the agent to pool"""
with self.agent_pool_change_condition:
shared_utils.print_and_log(
logging.DEBUG,
"Adding agent {} to pool...".format(agent.messenger_id))
# time agent entered agent_pool
agent.time_in_pool.setdefault(world_type, time.time())
# add agent to pool
self.agent_pool.setdefault(world_type, []).append(agent)
def setup_server(self):
"""Prepare the Messenger server for handling messages"""
shared_utils.print_and_log(
logging.INFO,
'\nYou are going to allow people on Facebook to be agents in '
'ParlAI.\nDuring this process, Internet connection is required, '
'and you should turn off your computer\'s auto-sleep '
'feature.\n',
should_print=True)
input('Please press Enter to continue... ')
shared_utils.print_and_log(logging.NOTSET, '', True)
shared_utils.print_and_log(logging.INFO,
'Setting up Messenger webhook...',
should_print=True)
# Setup the server with a task name related to the current task
task_name = '{}-{}'.format('ParlAI-Messenger', self.opt['task'])
self.server_task_name = \
''.join(e for e in task_name.lower() if e.isalnum() or e == '-')
self.server_url = server_utils.setup_server(self.server_task_name)
shared_utils.print_and_log(logging.INFO,
'Webhook address: {}/webhook'.format(
self.server_url),
should_print=True)
def setup_server(self):
"""Prepare the Messenger server for handling messages"""
shared_utils.print_and_log(
logging.INFO,
'\nYou are going to allow people on Facebook to be agents in '
'ParlAI.\nDuring this process, Internet connection is required, '
'and you should turn off your computer\'s auto-sleep '
'feature.\n',
should_print=True
)
input('Please press Enter to continue... ')
shared_utils.print_and_log(logging.NOTSET, '', True)
shared_utils.print_and_log(logging.INFO,
'Setting up Messenger webhook...',
should_print=True)
# Setup the server with a task name related to the current task
task_name = '{}-{}'.format('ParlAI-Messenger', self.opt['task'])
self.server_task_name = \
''.join(e for e in task_name.lower() if e.isalnum() or e == '-')
self.server_url = server_utils.setup_server(self.server_task_name)
shared_utils.print_and_log(
logging.INFO,
'Webhook address: {}/webhook'.format(self.server_url),
should_print=True
)
def _handle_message_read(self, event):
# If the message was sent by another user (as in during a conversation)
# then we need to propogate the read back to that user.
shared_utils.print_and_log(
logging.DEBUG, "Messages {} marked as read.".format(event['read']))
reader = event['sender']['id']
agent_state = self.get_agent_state(reader)
if agent_state is None:
return
agent = agent_state.get_active_agent()
if agent is not None:
for partner in agent.message_partners:
# We don't know who sent the message that was seen, but we can
# send a message observed event to everyone else in the chat
self.message_sender.send_read(partner.id)
def setup_socket(self):
"""Set up socket to start communicating to workers"""
if not self.bypass_server_setup:
shared_utils.print_and_log(logging.INFO,
'Local: Setting up WebSocket...',
should_print=True)
self.app_token = self.get_app_token()
self.message_sender = MessageSender(self.app_token)
# Set up receive
if not self.bypass_server_setup:
socket_use_url = self.server_url
if (self.opt['local']): # skip some hops for local stuff
socket_use_url = "https://localhost"
self.message_socket = MessageSocket(socket_use_url, self.port,
self._handle_webhook_event)
def shutdown(self):
"""Handle any client shutdown cleanup."""
# Ensure all threads are cleaned and conversations are handled
try:
self.is_running = False
self.world_runner.shutdown()
if not self.bypass_server_setup:
self.message_socket.keep_running = False
self._expire_all_conversations()
except BaseException as e:
shared_utils.print_and_log(logging.ERROR,
f'world ended in error: {e}')
finally:
if not self.bypass_server_setup:
server_utils.delete_server(self.server_task_name,
self.opt['local'])
def remove_agent_from_pool(self, agent, world_type='default',
mark_removed=True):
"""Remove agent from the pool"""
with self.agent_pool_change_condition:
shared_utils.print_and_log(
logging.DEBUG,
"Removing agent {} from pool...".format(agent.messenger_id)
)
if world_type in self.agent_pool and \
agent in self.agent_pool[world_type]:
self.agent_pool[world_type].remove(agent)
# reset agent's time_in_pool
if world_type in agent.time_in_pool:
del agent.time_in_pool[world_type]
# maybe mark agent as removed
if mark_removed:
agent.stored_data['removed_from_pool'] = True
def setup_server(self):
"""Prepare the Messenger server for handling messages"""
if self.bypass_server_setup:
return
shared_utils.print_and_log(
logging.INFO,
'\nYou are going to allow people on Facebook to be agents in '
'ParlAI.\nDuring this process, Internet connection is required, '
'and you should turn off your computer\'s auto-sleep '
'feature.\n',
should_print=True
)
input('Please press Enter to continue... ')
shared_utils.print_and_log(logging.NOTSET, '', True)
if self.opt['local'] is True:
shared_utils.print_and_log(
logging.INFO,
"In order to run the server locally, you will need "
"to have a public HTTPS endpoint (SSL signed) running on "
"the server you are currently excecuting ParlAI on. Enter "
"that public URL hostname when prompted and ensure that the "
"port being used by ParlAI (usually 3000) has external "
"traffic routed to it.",
should_print=True,
)
input('Please press Enter to continue... ')
shared_utils.print_and_log(logging.INFO,
'Setting up Messenger webhook...',
should_print=True)
# Setup the server with a task name related to the current task
task_name = '{}-{}'.format('ParlAI-Messenger', self.opt['task'])
self.server_task_name = \
''.join(e for e in task_name.lower() if e.isalnum() or e == '-')
self.server_url = server_utils.setup_server(
self.server_task_name, local=self.opt['local'])
shared_utils.print_and_log(
logging.INFO,
'Webhook address: {}/webhook'.format(self.server_url),
should_print=True
)
def _task_function(opt, agents, conversation_id, world_type):
"""Run the task function for the given agents"""
shared_utils.print_and_log(
logging.INFO, 'Starting task {}...'.format(conversation_id))
try:
task_functions[world_type](self, opt, agents, conversation_id)
except Exception as e:
shared_utils.print_and_log(
logging.ERROR,
'Starting world {} had error {}'.format(world_type, e),
)
for agent in agents:
self.observe_message(
agent.id,
"Sorry, this world closed. Returning to overworld.")
self.active_worlds[task_id] = None
for agent in agents:
agent_state = self._get_agent_state(agent.id)
agent_state.set_active_agent(agent_state.get_overworld_agent())
def _task_function(opt, agents, conversation_id, world_type):
"""Run the task function for the given agents"""
shared_utils.print_and_log(
logging.INFO,
'Starting task {}...'.format(conversation_id)
)
try:
task_functions[world_type](self, opt, agents, conversation_id)
except Exception as e:
shared_utils.print_and_log(
logging.ERROR,
'Starting world {} had error {}'.format(world_type, e),
)
for agent in agents:
self.observe_message(
agent.id,
"Sorry, this world closed. Returning to overworld."
)
self.active_worlds[task_id] = None
for agent in agents:
agent_state = self._get_agent_state(agent.id)
agent_state.set_active_agent(agent_state.get_overworld_agent())
def _log_debug(self, text):
time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
shared_utils.print_and_log(logging.DEBUG,
f'{time}: {text}',
should_print=True)
def start_task(self):
"""Begin handling task.
Periodically check to see when enough agents are in the agent pool
to start an instance of the task. Continue doing this until the desired
number of conversations is had.
"""
def _done_callback(fut):
"""Log and raise exception of task world, if there is one.
Additionally, set active agent to overworld agent.
"""
e = fut.exception()
if e is not None:
shared_utils.print_and_log(
logging.ERROR,
'World {} had error {}'.format(world_type, repr(e)),
should_print=True,
)
traceback.print_exc(file=sys.stdout)
for agent in agents:
self.observe_message(
agent.id,
'Sorry, this world closed. Returning to overworld.')
else:
shared_utils.print_and_log(
logging.INFO,
'World {} had no error'.format(world_type),
should_print=True,
)
self.active_worlds[task_id] = None
for agent in agents:
self.after_agent_removed(agent.id)
agent_state = self.get_agent_state(agent.id)
agent_state.set_active_agent(agent_state.get_overworld_agent())
self.running = True
while self.running:
# Loop forever until the server is shut down
with self.agent_pool_change_condition:
valid_pools = self._get_unique_pool()
for world_type, agent_pool in valid_pools.items():
# check if agent has exceeded max time in pool
world_config = self.task_configs[world_type]
if world_config.max_time_in_pool is not None:
self.check_timeout_in_pool(
world_type, agent_pool,
world_config.max_time_in_pool)
needed_agents = self.max_agents_for[world_type]
if len(agent_pool) >= needed_agents:
shared_utils.print_and_log(logging.INFO,
'starting pool',
should_print=True)
# enough agents in pool to start new conversation
self.conversation_index += 1
task_id = 't_{}'.format(self.conversation_index)
# Add the required number of valid agents to the conv
agent_states = [w for w in agent_pool[:needed_agents]]
agents = []
for state in agent_states:
agent = self._create_agent(task_id, state.get_id())
agent.onboard_data = state.onboard_data
state.assign_agent_to_task(agent, task_id)
state.set_active_agent(agent)
agents.append(agent)
# reset wait message state
state.stored_data['seen_wait_message'] = False
assign_role_function = shared_utils.get_assign_roles_fn(
self.world_module, self.taskworld_map[world_type])
if assign_role_function is None:
assign_role_function = shared_utils.default_assign_roles_fn
assign_role_function(agents)
# Allow task creator to filter out workers and run
# versions of the task that require fewer agents
agents = [a for a in agents if a.disp_id is not None]
for a in agents:
# Remove selected workers from the agent pool
self.remove_agent_from_pool(
self.get_agent_state(a.id),
world_type=world_type,
mark_removed=False,
)
for a in agents:
partner_list = agents.copy()
partner_list.remove(a)
a.message_partners = partner_list
# launch task world.
future = self.world_runner.launch_task_world(
task_id, self.taskworld_map[world_type], agents)
future.add_done_callback(_done_callback)
self.active_worlds[task_id] = future
time.sleep(shared_utils.THREAD_MEDIUM_SLEEP)
