def handle(self, task): result = False if isinstance(task, Task): self.display( OUTPUT_DEBUG, 'task to handle %s' % task.id() ) result = True if task.state == NEW: task.worker = self._rank task.hostname = self._name if isinstance( task, ControlTask ): self.display( OUTPUT_DEBUG, 'running command task locally' ) task.execute() result = self.handle( task ) elif task.pid in self._procs: if (self._inuse + task.slots) <= self._poolsize: self.display( OUTPUT_DEBUG, 'running task with process pool' ) rp, lp = Pipe(False) lp.send( self._procs[ task.pid ] ) lp.send( task ) self._pipes[ task.key ] = ( rp, lp ) tmp = pickle.dumps(reduce_connection(rp)) h = self._pool.apply_async( handle_task, (tmp,) ) self._pollq.append( h ) self._inuse = self._inuse + task.slots self.display( OUTPUT_DEBUG, 'using: %d, task used %d' % ( self._inuse, task.slots) ) elif self._inuse == 0 and self._poolsize < task.slots: self.display( OUTPUT_DEBUG, 'running oversized task with process pool' ) rp, lp = Pipe(False) lp.send( self._procs[ task.pid ] ) lp.send( task ) self._pipes[ task.key ] = ( rp, lp ) tmp = pickle.dumps(reduce_connection(rp)) h = self._pool.apply_async( handle_task, (tmp,) ) self._pollq.append( h ) self._inuse = self._inuse + task.slots self.display( OUTPUT_DEBUG, 'using: %d, task used %d' % ( self._inuse, task.slots) ) else: self.display( OUTPUT_DEBUG, 'no processes available to run task' ) self.queueFirst( task ) result = False else: self.display( OUTPUT_DEBUG, 'process resources not available on worker yet' ) self.queueFirst( task ) result = False else: self.display( OUTPUT_DEBUG, 'returning task to manager' ) self.returnTask( task ) else: self.display( OUTPUT_DEBUG, 'got invalid task to handle' ) return result
def main(): _setproctitle('structurarium.graph') parser = argparse.ArgumentParser(description='Run Structurarium graph server') parser.add_argument('--version', '-v', action='version', version=__version__) parser.add_argument('host') parser.add_argument('port', type=int) parser.add_argument('path') parser.add_argument('--authkey', '-k', action='store') parser.add_argument( '--worker', '-w', action='store', type=int, help='default is set to the number of CPU' ) args = parser.parse_args() listener = Listener((args.host, args.port), family='AF_INET') database = Graph(args.path, authkey=args.authkey) pool = Pool(processes=args.worker) print 'Running on %s:%s' % (args.host, args.port) while True: connection = listener.accept() connection = reduce_connection(connection) # pool.apply_async(connect, [database, connection]) connect(database, connection)
def _ask(self, msg, args=(), kwargs={}): i, o = Pipe() reduced = reduce_connection(i) self.inbox.put([msg, args, kwargs, reduced[1]]) ret = o.recv() i.close() o.close() return ret
def pack(conn): """ pickle a connection for transport over connection :param conn: connection obj to pickle :returns: pickled connection """ try: return pickle.dumps(reduction.reduce_connection(conn)) except Exception as e: raise BrineException(e)
def main(): recv_1, send_1 = Pipe(duplex=False) recv_2, send_2 = Pipe(duplex=False) recv_3, send_3 = Pipe(duplex=False) recv_4, send_4 = Pipe(duplex=False) lines = {1: send_1, 2: send_2, 3: send_3} encoders = Queue() lines[1].send({'text': '3Radio', 'style': 'center'}) lines[2].send({'text': '------', 'style': 'center'}) lines[3].send({'text': '', 'style': 'center'}) send_4.send({'text': 'Loading...', 'style': 'center'}) # print 'RE#1' pe1 = rotary_encoder_proc.RE_runner(encoders, 1, 8, 9, 7) # print 'RE#2' pe2 = rotary_encoder_proc.RE_runner(encoders, 2, 0, 2, 3) # print 'RE#3' pe3 = rotary_encoder_proc.RE_runner(encoders, 3, 12, 13, 14) # print 'LCD' plcd = lcd_proc.LCD_runner(recv_1, recv_2, recv_3, recv_4) mc = pylibmc.Client(['127.0.0.1'], binary=True, behaviors={'tcp_nodelay': True, 'ketama': True}) mc['city'] = 'Moscow,ru' mc['TNWU_authkey'] = current_process().authkey mc['TNWU_reduced_connection'] = reduction.reduce_connection(send_4) mc['TNWU_running'] = False mc['TNWU_last_temp'] = '-666.666' amount = {1: 0, 2: 0, 3: 0} state = {1: False, 2: False, 3: False} while True: encoder_message = encoders.get(block=True) encoder_id = encoder_message['name'] if 'rot' in encoder_message: amount[encoder_id] += encoder_message['rot'] lines[encoder_id].send({'text': amount[encoder_id], 'style': 'center'}) elif 'button' in encoder_message: state = button(lines[encoder_id], state, encoder_message, encoder_id) if stop_all(state): break pe1.terminate() pe2.terminate() pe3.terminate() plcd.terminate()
def process_request(self, request, query_num, atomic): """Processes a feedback requests and returns its result. This call is used by queries for submitting individual requests. It is a blocking call that should be called from a separate execution thread. @param request: The feedback request to process. @param query_num: The unique query number. @param atomic: Whether subsequent request(s) are expected and should be processed without interruption. """ reply_pipe_send, reply_pipe_recv = multiprocessing.Pipe() reduced_reply_pipe_send = reduction.reduce_connection(reply_pipe_send) self._request_queue.put( ReqTuple(request, reduced_reply_pipe_send, query_num, atomic)) return reply_pipe_recv.recv()
def rpc(self, target, topic, message, timeout=50): leftPipe, rightPipe = multiprocessing.Pipe() reduced_Pipe = reduce_connection(rightPipe) message = { "reduced_return_pipe": reduced_Pipe, "actual_message": message } self.parent.bemoss_publish(target='bacnetagent', topic=topic, message=message) if leftPipe.poll(timeout): result = leftPipe.recv() if result == None: raise Exception('Received None. Device communication problem') return result else: raise self.RPCTimeout("Time Out")
def main(): _setproctitle('structurarium.taskqueue') parser = argparse.ArgumentParser( description='Run Structurarium graph server' ) parser.add_argument( '--version', '-v', action='version', version=__version__ ) parser.add_argument('host') parser.add_argument('port', type=int) parser.add_argument('path') parser.add_argument('--authkey', '-k') parser.add_argument( '--worker', '-w', action='store', type=int, help='default is set to the number of CPU' ) args = parser.parse_args() listener = Listener((args.host, args.port), family='AF_INET') database = TaskQueue(args.path, authkey=args.authkey) print 'Running on %s:%s' % (args.host, args.port) if args.worker > 1: pool = Pool(processes=args.worker) while True: pool = Pool(processes=args.worker) connection = listener.accept() connection = cPickle.dumps(reduce_connection(connection)) database.process(connection) pool.apply_async(process, [database, connection]) else: print 'monothread' database.replay() while True: connection = listener.accept() command = loads(connection.recv()) output = database.play(command) connection.send(dumps(output)) connection.close()
def _reduce_connection(conn): """Reduce a connection object so it can be pickled. WARNING this puts the current process' authentication key in the data to be pickled. Connections pickled with this function should not be sent over an untrusted network. HACK work around ``multiprocessing`` connection authentication because we are using ``subprocess.Popen`` instead of ``multiprocessing.Process`` to spawn new child processes. This will not be necessary when ``multiprocessing.Connection`` objects can be pickled. See http://bugs.python.org/issue4892 """ obj = reduce_connection(conn) assert obj[0] is rebuild_connection, obj assert len(obj) == 2, obj args = (bytes(current_process().authkey),) + obj[1] return (_rebuild_connection, args)
def dbrpc(self, command, args, kwargs, timeout=30): leftPipe, rightPipe = multiprocessing.Pipe() reduced_Pipe = reduce_connection(rightPipe) message = dict() message['command'] = command message['args'] = args message['kwargs'] = kwargs finalmessage = { "reduced_return_pipe": reduced_Pipe, "actual_message": message } self.publish(source=self.name, destinations=['metadataagent'], topic='dbrpc', message=finalmessage) if leftPipe.poll(timeout): result = leftPipe.recv() return result else: raise self.RPCTimeout("Time Out")
def pickle_connection(connection): return pickle.dumps(reduce_connection(connection))
def encode(self, connection): # reduces Connection() objects to pickleable objects reduced = reduce_connection(connection) # pickles the Connection() into a string return dumps(reduced)
def processCommand(command): global process_count def find_matches(pid): matches = [] l = len(pid) for process_name, process in processes_dict.items(): if str(process.id)[-l:] == str(pid): matches.append(process_name) return matches commands = command.split(' ') reply = '' if len(commands) == 2 and commands[0].lower() == "error": agent_name = commands[1] # assume the argument is agent_name try: process = processes_dict.get(agent_name) error = process.err except AttributeError: return "" return error elif commands[0].lower() == 'status': running_count = 0 stopped_count = 0 stop_requested_count = 0 for process_name, process in processes_dict.items(): if process.is_alive(): if process.config['stopFlag'].value == 0: status, code = "running", "" running_count += 1 else: status, code = "stop_requested", "" stop_requested_count += 1 else: status, code = "stopped", process.exitcode stopped_count += 1 if hasattr(process, 'pid'): pid = process.pid else: pid = os.getpid() reply += "{:<6} {:<10} {:<30} {:<6} {:<10}\n".format( process.id, pid, process.name, status, code) reply += "Total %s Agents. Running: %s. Stoped: %s. Stop_Requested: %s" % ( len(processes_dict.keys()), running_count, stopped_count, stop_requested_count) return reply # elif commands[0].lower() == 'tstatus': #thread_status # threads = threading.enumerate() # # replies = [] # for thread in threads: # if hasattr(thread, 'id'): # replies.append((thread.id,thread.name)) # else: # if hasattr(thread,'parent') and hasattr(thread.parent,'id'): # replies.append((thread.parent.id,thread.name)) # else: # replies.append((-1,thread.name)) # # replies = sorted(replies,key=lambda x: x[0]) # if len(commands) > 1: # id = commands[1] #filter only this # new_reply = [] # for rep in replies: # if str(rep[0]) == str(id): # new_reply.append(rep) # replies = new_reply # for id, name in replies: # reply +=("{:<6} {:<50}\n".format(id, name)) # # return reply elif len(commands) == 2 and commands[0].lower() == "tstatus": target = commands[1] leftPipe, rightPipe = multiprocessing.Pipe() reduced_Pipe = reduce_connection(rightPipe) finalmessage = {"reduced_return_pipe": reduced_Pipe} outQueue.put(("commandhandler", [target], 'tstatus', finalmessage)) if leftPipe.poll(20): result = leftPipe.recv() reply = "" count = 0 alive_count = 0 for id, name, status, watchdogtimer, msg in result: reply += "{:<6} {:<60} {:<6} {:<5} {:<10}\n".format( id, name, status, watchdogtimer, msg) count += 1 if status: alive_count += 1 reply += "Total %s threads. Running: %s. Stoped: %s." % ( count, alive_count, count - alive_count) return reply else: return "Timed out" elif len(commands) == 2 and commands[0].lower() in [ "start", "start_agent", "stop", "stop_agent" ]: agent_name = commands[1] #assume the argument is agent_name if not agent_name in processes_dict: #if agent_name is not present #assume it to be pid from the end matches = find_matches(pid=agent_name) if len(matches) == 1: agent_name = matches[0] if len(matches) == 0: return "invalid pid" elif len(matches) > 1: reply = "\n".join([ str(processes_dict[process_name].id) + " " + process_name for process_name in matches ]) return reply process_name = agent_name process = processes_dict[process_name] if commands[0].lower() in ["start", "start_agent"]: old_agent = process.agent old_config = process.config old_config['stopFlag'].value = 0 old_config['id'] = process_count p = BProcess(target=process.agent, name=process.name, kwargs=old_config) p.id = process_count process_count += 1 p.agent = old_agent p.config = old_config p.start() processes_dict[process_name] = p else: process.config['stopFlag'].value = 1 archived_process_list.append((datetime.datetime.utcnow(), process)) return reply elif len(commands) == 3 and commands[0].lower() in [ "launch_agent", "start" ]: agent_type = commands[1] agent_name = commands[2] agent = getAgent(agent_type) if agent: print "Launching " + agent.__name__ + " with name: " + agent_name inQ = Queue(AGENT_QUEUE_SIZE) inQueues_dict[agent_name] = inQ stopFlag = Value('i', 0) config = { 'name': agent_name, 'inQ': inQ, 'outQ': outQueue, 'logQ': logQueue, 'stopFlag': stopFlag, 'id': process_count } p = BProcess(target=agent, name=agent_name, kwargs=config) p.id = process_count process_count += 1 p.agent = agent p.config = config p.daemon = True p.start() if agent_name in processes_dict: # if there is existing process by that name, terminate it processes_dict[agent_name].config[ 'stopFlag'].value = 1 # make the old process stop archived_process_list.append( (datetime.datetime.utcnow(), processes_dict[agent_name]) ) #TODO kill processes in archived process if still running and clear it periodically processes_dict[agent_name] = p return "" else: return agent_type + " Agent not found" elif commands[0].lower() in ["remove_agent", "remove"]: if len(commands) == 2: agent_name = commands[1] #assume the argument is agent_name else: return "Invalid number of arguments" if not agent_name in processes_dict: #if agent_name is not present #assume it to be pid from the end matches = find_matches(pid=agent_name) if len(matches) == 1: agent_name = matches[0] elif len(matches) == 0: return "invalid pid" else: return " ".join([ processes_dict[process_name].id for process_name in matches ]) if agent_name in processes_dict or find_matches(pid=agent_name): if processes_dict[agent_name].is_alive(): if processes_dict[agent_name].config['stopFlag'].value == 1: return "Wait for it to stop first" else: return "Should stop first" processes_dict.pop(agent_name) #remove it from process-dict inQueues_dict.pop(agent_name) return "" else: return "Invalid command"
def new_websocket_client(self, path=None): """ Called after a new WebSocket connection has been established. """ # Checking for a token is done in validate_connection() _id = None msg = None try: if path != None and path != '/' and path.index("-") != -1: # remove first / and get mode(client or manager) mode = str(path[1:path.index("-")]) # Now extract id & pw tmp = str(path[path.index("-")+1:]) wanted_id = str(tmp[:tmp.index("-")]) wanted_pw = str(tmp[tmp.index("-")+1:]) # Clear tmp del tmp # Create a queue for any new client/manager manager = Manager() if mode == "manager": if wanted_pw == self.session[wanted_id]["pw"]: # Generate the id of the manager _id = id_gen() self.wanted_id = wanted_id if self.pipe == None: reduced = self.session[wanted_id]["pipe"] self.pipe = reduced[0](*reduced[1]) self.connected = True self.pipe.send('{ "vnc": "connect" }') # Init manager queue in the session self.session[_id]= { "mode": "manager", "id": _id, "wanted_id": wanted_id, "wanted_pw": wanted_pw, "authenticated": True } msg = "New manager id: %s wanted client id: %s" % (_id, wanted_id) else: _id = None self.session[_id] = None elif mode == "client": # Use client provided id _id = wanted_id # Since we are a client we dont have a wanted id self.wanted_id = None # Waiting for manager self.connected = False self.pipe, manager_conn = Pipe() try: # Init the client queue in the session self.session[wanted_id]= { "mode": "client", "id": wanted_id, "pw": wanted_pw, "authenticated": True, "pipe": reduction.reduce_connection(manager_conn) } except: print "pipe error" print sys.exc_type,sys.exc_value msg = "New client id: %s" % _id else: msg = "Error no client/manager mode detected !!!" self.send_close() if msg != None: # add log entry self.log_message(msg) except: print "Error in new_websocket_client()" print sys.exc_type, sys.exc_value del self.session[_id] try: # if exception occured the close socket self.send_close() except: pass # Start proxying try: if _id != None: self.vawrouter(_id) else: del self.session[_id] self.send_close() except: del self.session[_id] self.send_close() print "Error in vaw router" print sys.exc_type,sys.exc_value
def compress_pipe(p): pp = pickle.dumps(reduce_connection(p)) return (pp)
def reduce(self): self.urgent = reduce_connection(self.urgent) self.normal = reduce_connection(self.normal)
def create_task(self): return Task( wait_for_data_in_pipe, (pickle.dumps(reduce_connection(self.__child_pipe)),))