def __init__(self, configuration_path=CONFIG_PATH): log.info('Running Grid master, stop it with CTRL-C') # CTRL-C interception SignalManager() # Setup object configuration self._configure(configuration_path) # Team_dashboard web graphs self.dashboard = Dashboard() # Logs monitoring self.logio = LogIO(self.configuration['nodes']) # Nodes are physical machines of the cluster self.nodes = { ip: Node(ip, self.configuration['monitored'], self.configuration['restful']) for ip in self.configuration['nodes'] } self.processed_engines = []
def __init__(self, configuration_path=CONFIG_PATH): log.info('Running Grid master, stop it with CTRL-C') # CTRL-C interception SignalManager() # Setup object configuration self._configure(configuration_path) # Team_dashboard web graphs self.dashboard = Dashboard() # Logs monitoring self.logio = LogIO(self.configuration['nodes']) # Nodes are physical machines of the cluster self.nodes = {ip: Node(ip, self.configuration['monitored'], self.configuration['restful']) for ip in self.configuration['nodes']} self.processed_engines = []
class Grid(object): ''' Responsible to run QuanTrade runtime and communicate with drones It forks: - log.io for logs aggregation - dashboards for trading purpose And dynamically: - Remote rest_services for database wrappers - Drones to process remote calls - Glances servers and clients for ressources monitoring It basically waits for new tasks to pop (ie remote engines to appear), and fork trading processes on them according to their associated configuration. It can as well create by itself remote/local drones for classic cluster purpose. The object can be configured through ~/.quantrade/default.json. ''' def __init__(self, configuration_path=CONFIG_PATH): log.info('Running Grid master, stop it with CTRL-C') # CTRL-C interception SignalManager() # Setup object configuration self._configure(configuration_path) # Team_dashboard web graphs self.dashboard = Dashboard() # Logs monitoring self.logio = LogIO(self.configuration['nodes']) # Nodes are physical machines of the cluster self.nodes = { ip: Node(ip, self.configuration['monitored'], self.configuration['restful']) for ip in self.configuration['nodes'] } self.processed_engines = [] def _configure(self, configuration_path): ''' Read and set configuration ''' self.configuration = json.load(open(configuration_path, 'r'))['grid'] #http://docs.fabfile.org/en/1.4.3/usage/env.html#full-list-of-env-vars #env.forward_agent = True #env.key_filename = [""] env.user = self.configuration['name'] env.password = self.configuration['password'] env.hosts = self.configuration['nodes'] env.roledefs = { 'local': ['127.0.0.1'], 'controller': self.configuration['controller'], 'nodes': self.configuration['nodes'] } def deploy(self): ''' Set up local ipcontroller ''' log.info('Deploying grid trade-system') log.info('Activating local ipcontroller') execute(fab.activate_controller) # Main interface to drones self.engines = Client() def _is_idle(self, state): ''' Check if there is pending tasks to do ''' if 'queue' in state: return not state['queue'] # Else, no informations to answer return None def detect_drones(self): new_engines = [] engines_status = self.engines.queue_status() #NOTE what is the use of status['unassigned'] ? for key, state in engines_status.iteritems(): if key == 'unassigned': continue if (self._is_idle(state) and key not in self.processed_engines): self.processed_engines.append(key) new_engines.append(self.engines[key]) self._dispatch_engines(new_engines) return len(new_engines) def _dispatch_engines(self, engines): for engine in engines: ip = engine.apply_sync(get_local_ip) log.info('New engine detected on {}'.format(ip)) if ip not in self.nodes: log.info('New node connected') self.nodes[ip] = Node(ip, self.configuration['monitored'], self.configuration['restful']) self.nodes[ip].register_drone(engine.targets, engine) drone_name = self.nodes[ip].drones[engine.targets].name self.dashboard.add_description(remote_ip=ip, portfolio=drone_name) self.logio.add_description(drone_name, remote_ip=ip) log.info('Drone registered') def process(self, function, node_ip=None, drone_id=None): ''' Process pending tasks on available, and eventually provided, drones ''' processed_nodes = self.nodes.values() for node in processed_nodes: processed_drones = node.drones.values() #FIXME use self.engines.shutdown([1, 3]) insteand of #non-functionnal drone.shutdown node.inspect_armada() for drone in processed_drones: drone.run(function) def fireup_dashboards(self): if self.configuration['logserver']: self.logio.build() self.logio.run() log.notice('Log.io available at http://192.168.0.12:28778') if self.configuration['dashboard']: self.dashboard.build() self.dashboard.run(public_ip=False) log.notice('Dashboard available at http://192.168.0.12:4000')
class Grid(object): ''' Responsible to run QuanTrade runtime and communicate with drones It forks: - log.io for logs aggregation - dashboards for trading purpose And dynamically: - Remote rest_services for database wrappers - Drones to process remote calls - Glances servers and clients for ressources monitoring It basically waits for new tasks to pop (ie remote engines to appear), and fork trading processes on them according to their associated configuration. It can as well create by itself remote/local drones for classic cluster purpose. The object can be configured through ~/.quantrade/default.json. ''' def __init__(self, configuration_path=CONFIG_PATH): log.info('Running Grid master, stop it with CTRL-C') # CTRL-C interception SignalManager() # Setup object configuration self._configure(configuration_path) # Team_dashboard web graphs self.dashboard = Dashboard() # Logs monitoring self.logio = LogIO(self.configuration['nodes']) # Nodes are physical machines of the cluster self.nodes = {ip: Node(ip, self.configuration['monitored'], self.configuration['restful']) for ip in self.configuration['nodes']} self.processed_engines = [] def _configure(self, configuration_path): ''' Read and set configuration ''' self.configuration = json.load(open(configuration_path, 'r'))['grid'] #http://docs.fabfile.org/en/1.4.3/usage/env.html#full-list-of-env-vars #env.forward_agent = True #env.key_filename = [""] env.user = self.configuration['name'] env.password = self.configuration['password'] env.hosts = self.configuration['nodes'] env.roledefs = { 'local': ['127.0.0.1'], 'controller': self.configuration['controller'], 'nodes': self.configuration['nodes'] } def deploy(self): ''' Set up local ipcontroller ''' log.info('Deploying grid trade-system') log.info('Activating local ipcontroller') execute(fab.activate_controller) # Main interface to drones self.engines = Client() def _is_idle(self, state): ''' Check if there is pending tasks to do ''' if 'queue' in state: return not state['queue'] # Else, no informations to answer return None def detect_drones(self): new_engines = [] engines_status = self.engines.queue_status() #NOTE what is the use of status['unassigned'] ? for key, state in engines_status.iteritems(): if key == 'unassigned': continue if (self._is_idle(state) and key not in self.processed_engines): self.processed_engines.append(key) new_engines.append(self.engines[key]) self._dispatch_engines(new_engines) return len(new_engines) def _dispatch_engines(self, engines): for engine in engines: ip = engine.apply_sync(get_local_ip) log.info('New engine detected on {}'.format(ip)) if ip not in self.nodes: log.info('New node connected') self.nodes[ip] = Node(ip, self.configuration['monitored'], self.configuration['restful']) self.nodes[ip].register_drone(engine.targets, engine) drone_name = self.nodes[ip].drones[engine.targets].name self.dashboard.add_description(remote_ip=ip, portfolio=drone_name) self.logio.add_description(drone_name, remote_ip=ip) log.info('Drone registered') def process(self, function, node_ip=None, drone_id=None): ''' Process pending tasks on available, and eventually provided, drones ''' processed_nodes = self.nodes.values() for node in processed_nodes: processed_drones = node.drones.values() #FIXME use self.engines.shutdown([1, 3]) insteand of #non-functionnal drone.shutdown node.inspect_armada() for drone in processed_drones: drone.run(function) def fireup_dashboards(self): if self.configuration['logserver']: self.logio.build() self.logio.run() log.notice('Log.io available at http://192.168.0.12:28778') if self.configuration['dashboard']: self.dashboard.build() self.dashboard.run(public_ip=False) log.notice('Dashboard available at http://192.168.0.12:4000')