def get_primary(config):
primary_handler = PrimaryServer(
config["primary_url"],
config["primary_data_dir"])
channels = [node_channel_name(primary_handler.node.name),
all_nodes_channel_name()]
primary = Dispatcher(
redis_connection(),
primary_handler,
primary_queue_name(primary_handler.node.name),
channels)
return primary
def get_worker(config):
worker_handler = WorkerServer(
config["worker_url"],
config["worker_data_dir"],
config["model"],
config["command_dict"])
channels = [node_channel_name(worker_handler.node.name),
all_nodes_channel_name()]
worker = Dispatcher(redis_connection(),
worker_handler,
job_queue_name(config["model"]),
channels)
return worker
def refresh_node_status(self):
"""
Refresh the status of all nodes by
1. Deleting existing state
2. Publishing a request for all nodes to update
All listening nodes will update the Node hash with their state
"""
for node in Node.values():
del Node[node.name]
status_command = {"command": "UPDATE_STATUS"}
publish_command(
redis_connection(),
all_nodes_channel_name(),
status_command)
logger = logging.getLogger('modelrunner')
logger.info("modelrunner %s (Python %s)" %
(__version__,
'.'.join(map(str, sys.version_info[:3]))))
# so we can load config via cmd line args
parse_command_line()
parse_config_file(config.options.config_file)
# initialize the global application settings
initialize(config.options.redis_url)
# get the command_ keys
command_dict = config.options.group_dict("model_command")
primary_handler = PrimaryServer(
config.options.primary_url,
config.options.data_dir)
channels = [node_channel_name(primary_handler.node.name),
all_nodes_channel_name()]
primary = Dispatcher(
redis_connection(),
primary_handler,
primary_queue_name(primary_handler.node.name),
channels)
# continuously wait for jobs to complete and for status inquiries
Thread(target=primary.wait_for_queue_commands).start()
Thread(target=primary.wait_for_channel_commands).start()
# setup log
logger = logging.getLogger('modelrunner')
logger.info("modelrunner %s (Python %s)" %
(__version__, '.'.join(map(str, sys.version_info[:3]))))
# so we can load config via cmd line args
parse_command_line()
parse_config_file(config.options.config_file)
# initialize the global application settings
initialize(config.options.redis_url)
# get the command_ keys
command_dict = config.options.group_dict("model_command")
worker_handler = WorkerServer(config.options.worker_url,
config.options.data_dir, config.options.model,
command_dict)
channels = [
node_channel_name(worker_handler.node.name),
all_nodes_channel_name()
]
worker = Dispatcher(redis_connection(), worker_handler,
job_queue_name(config.options.model), channels)
# start listening for commands on queue and channels in bg
Thread(target=worker.wait_for_queue_commands).start()
Thread(target=worker.wait_for_channel_commands).start()