def get_primary(config): primary_handler = PrimaryServer( config["primary_url"], config["primary_data_dir"]) channels = [node_channel_name(primary_handler.node.name), all_nodes_channel_name()] primary = Dispatcher( redis_connection(), primary_handler, primary_queue_name(primary_handler.node.name), channels) return primary
def get_worker(config): worker_handler = WorkerServer( config["worker_url"], config["worker_data_dir"], config["model"], config["command_dict"]) channels = [node_channel_name(worker_handler.node.name), all_nodes_channel_name()] worker = Dispatcher(redis_connection(), worker_handler, job_queue_name(config["model"]), channels) return worker
def refresh_node_status(self): """ Refresh the status of all nodes by 1. Deleting existing state 2. Publishing a request for all nodes to update All listening nodes will update the Node hash with their state """ for node in Node.values(): del Node[node.name] status_command = {"command": "UPDATE_STATUS"} publish_command( redis_connection(), all_nodes_channel_name(), status_command)
logger = logging.getLogger('modelrunner') logger.info("modelrunner %s (Python %s)" % (__version__, '.'.join(map(str, sys.version_info[:3])))) # so we can load config via cmd line args parse_command_line() parse_config_file(config.options.config_file) # initialize the global application settings initialize(config.options.redis_url) # get the command_ keys command_dict = config.options.group_dict("model_command") primary_handler = PrimaryServer( config.options.primary_url, config.options.data_dir) channels = [node_channel_name(primary_handler.node.name), all_nodes_channel_name()] primary = Dispatcher( redis_connection(), primary_handler, primary_queue_name(primary_handler.node.name), channels) # continuously wait for jobs to complete and for status inquiries Thread(target=primary.wait_for_queue_commands).start() Thread(target=primary.wait_for_channel_commands).start()
# setup log logger = logging.getLogger('modelrunner') logger.info("modelrunner %s (Python %s)" % (__version__, '.'.join(map(str, sys.version_info[:3])))) # so we can load config via cmd line args parse_command_line() parse_config_file(config.options.config_file) # initialize the global application settings initialize(config.options.redis_url) # get the command_ keys command_dict = config.options.group_dict("model_command") worker_handler = WorkerServer(config.options.worker_url, config.options.data_dir, config.options.model, command_dict) channels = [ node_channel_name(worker_handler.node.name), all_nodes_channel_name() ] worker = Dispatcher(redis_connection(), worker_handler, job_queue_name(config.options.model), channels) # start listening for commands on queue and channels in bg Thread(target=worker.wait_for_queue_commands).start() Thread(target=worker.wait_for_channel_commands).start()