class SuggestionsThread(Thread): def __init__(self, suggestionsService): Thread.__init__(self) self.suggestionsService = suggestionsService self.value = None self.running = True self.condition = Condition() def run(self): while self.running: self.condition.acquire() if self.value is None: self.condition.wait() value = self.value self.value = None self.condition.release() if value is not None: self.suggestionsService.getSuggestions(value) def stop(self): self.running = False self.condition.acquire() self.condition.notify() self.condition.release() self.join() def getSuggestions(self, value): self.condition.acquire() self.value = value self.condition.notify() self.condition.release()
class DBPool(object): def __init__(self, dbconfig, maxconn): self.lock = ThreadLock() self.cond = ThreadCondition(self.lock) self.dbs = [] for _ in range(0, maxconn): master_conn = None slave_conn = None if 'master' in dbconfig: master_conn = Connection(dbconfig['master']) if 'slave' in dbconfig: slave_conn = Connection(dbconfig['slave']) if master_conn is None and slave_conn is None: master_conn = Connection(dbconfig) self.dbs.append(ConnectionProxy(master_conn, slave_conn)) def close_idle(self): pass def acquire(self): with self.lock: if len(self.dbs) == 0: self.cond.wait() return self.dbs.pop() def release(self, conn): with self.lock: old_cnt = len(self.dbs) self.dbs.append(conn) if not old_cnt: self.cond.notify()
class MultithreadConnection(sqlite3.Connection): def __init__(self, *args, **kwargs): super(MultithreadConnection, self).__init__( check_same_thread=False, *args, **kwargs) self.waiting_list = [] self.mutex = Lock() self.empty = Condition(self.mutex) def _write_execute(self, method, sql, parameters=()): self.waiting_list.append(None) with self.mutex: cur = method(sql, parameters) self.waiting_list.pop() if not self.waiting_list: try: self.empty.notify() except RuntimeError: pass return cur def commit(self): with self.empty: while self.waiting_list: self.empty.wait() super(MultithreadConnection, self).commit() def write_execute(self, sql, parameters=()): return self._write_execute(self.execute, sql, parameters) def write_executemany(self, sql, parameters=()): return self._write_execute(self.executemany, sql, parameters) def writeable_cursor(self): return WriteableCursor(self)
class GridGenerationUpdateThread(Thread): disconnect = False elecgen = None sleeper = None def __init__(self, parent): Thread.__init__(self) self.disconnect = False self.elecgen = parent self.ngdata = ElectricityGenerationDataSource() self.sleeper = Condition() def stopUpdates(self): self.disconnect = True self.sleeper.acquire() self.sleeper.notify() self.sleeper.release() def run(self): while self.disconnect == False: emxml = self.ngdata.DownloadRealtimeXML() try: self.elecgen.energyMix = self.ngdata.ParseRealtimeXML(emxml) except Exception, exc: trc.Error("failed to parse realtime xml") trc.Error(str(emxml)) trc.Error(repr(exc)) trc.Error(repr(exc.message)) # go to sleep for a while - the realtime data doesn't update very # often, so no need to download it constantly! self.sleeper.acquire() self.sleeper.wait(180) self.sleeper.release()
class FileTrigger(Thread): """Wait for a file. """ def __init__(self, filename): Thread.__init__(self) self.filename = filename self.loop = True self.cond = Condition() # self.start() self.exit_status = 0 def run(self): while self.loop: self.cond.acquire() # Polling is ugly, this should be replaced when possible by inotify. if os.path.exists(self.filename): # Could we use yield instead ? self.cond.release() return self.cond.wait(1) self.cond.release() def cancel(self): self.loop = False self.exit_status = 1 self.cond.acquire() self.cond.notify() self.cond.release() def __repr__(self): return "File trigger " + self.filename
class TimeTrigger(Thread): """Wait for a given time. """ def __init__(self, time): Thread.__init__(self) self.time = time self.loop = True self.cond = Condition() # self.start() self.exit_status = 0 def run(self): while self.loop and self.time > datetime.utcnow(): self.cond.acquire() now = datetime.utcnow() diff = (self.time - now).seconds + (self.time - now).microseconds / 1000000.0 self.cond.wait(diff) self.cond.release() def cancel(self): self.loop = False self.exit_status = 1 self.cond.acquire() self.cond.notify() self.cond.release() def __repr__(self): return "Time trigger " + str(self.time)
class AutoFetcher(Thread): _interval = 60 * 60 # = 1 hour _inactive_block = None def __init__(self): super(AutoFetcher, self).__init__() self._inactive_block = Condition() self.daemon = True def trigger(self): with self._inactive_block: self._inactive_block.notify() def run(self): while True: self.launch_task() time.sleep(self._interval) if not center.main_win.win.isActiveWindow(): logging.debug('AutoFetcher paused.') with self._inactive_block: self._inactive_block.wait() logging.debug('AutoFetcher resumed.') @qt.run_in_qt def launch_task(self): tasks.run_task(tasks.FetchTask()) if center.settings['update_notify'] and '-dev' not in center.VERSION: tasks.run_task(tasks.CheckUpdateTask())
class Semaphore(object): def __init__(self, count=1): if count < 1: raise ValueError('Semaphore count should be at least 1.') self._count = count self._cond = Condition(Lock()) def P(self): '''Acquire a lock. Block until acquired successfully. ''' with self._cond: while self._count <= 0: self._cond.wait() # block and wait for the lock. else: self._count = self._count - 1 def V(self): '''Release a lock.''' with self._cond: self._count = self._count + 1 self._cond.notify() # wake up a waiting thread.
class MasterSlaveLock: def __init__(self): self.__master_lock = Lock() self.__slave_locks_counter_condition = Condition() self.__slave_locks_count = 0 def acquire_master(self): self.__master_lock.acquire() # wait for all slavelocks to unlock self.__slave_locks_counter_condition.acquire() while self.__slave_locks_count != 0: self.__slave_locks_counter_condition.wait() self.__slave_locks_counter_condition.release() def release_master(self): self.__master_lock.release() def acquire_slave(self): # stop further slave locks here, when master-process is waiting for a full stop of slave processes self.__master_lock.acquire() self.__master_lock.release() self.__slave_locks_counter_condition.acquire() self.__slave_locks_count += 1 self.__slave_locks_counter_condition.release() def release_slave(self): self.__slave_locks_counter_condition.acquire() self.__slave_locks_count -= 1 if not self.__slave_locks_count: # we released the last lock, notify (maybe) waiting master-process self.__slave_locks_counter_condition.notify() self.__slave_locks_counter_condition.release()
class _Timer(object): def __init__(self, event_engine, seconds=1): # 计时器,用于触发计时器事件 self._timer = Thread(target = self._run_timer) self._timer.daemon = True self._timer_active = False self._timer_sleep = seconds self._timer_pause_condition = Condition(Lock()) self._event_engine = event_engine def set_timer(self, seconds): self._timer_sleep = seconds def start_timer(self): self._timer_active = True self._timer.start() def pause_timer(self): self._timer_active = False self._timer_pause_condition.acquire() def resume_timer(self): self._timer_active = True self._timer_pause_condition.notify() self._timer_pause_condition.release() def _run_timer(self): event = Event(route=EVENT_TIMER) while True: with self._timer_pause_condition: if not self._timer_active: self._timer_pause_condition.wait() self._event_engine.emit(event) # 等待 sleep(self._timer_sleep)
class Club: def __init__(self): self.clubLock = Lock() self.redditCond = Condition(self.clubLock) self.fourchannerCond = Condition(self.clubLock) self.fourchannerCount = 0 self.redditorCount = 0 def redditor_enter(self): with self.clubLock: while self.fourchannerCount > 0 : self.redditCond.wait() self.redditorCount = self.redditorCount + 1 def redditor_exit(self): with self.clubLock: self.redditorCount = self.redditorCount - 1 if self.redditorCount == 0 : self.fourchannerCond.notify() def fourchanner_enter(self): with self.clubLock: while self.redditorCount > 0 : self.fourchannerCond.wait() self.fourchannerCount = self.fourchannerCount + 1 def fourchanner_exit(self): with self.clubLock: self.fourchannerCount = self.fourchannerCount - 1 if self.fourchannerCount == 0: self.redditCond.notify()
class Rtp_proxy_client_local(object): is_local = True wi_available = None wi = None nworkers = None workers = None def __init__(self, global_config, address = '/var/run/rtpproxy.sock', \ bind_address = None, nworkers = 1): self.address = address self.is_local = True self.wi_available = Condition() self.wi = [] self.nworkers = nworkers self.workers = [] for i in range(0, self.nworkers): self.workers.append(_RTPPLWorker(self)) def send_command(self, command, result_callback = None, *callback_parameters): self.wi_available.acquire() self.wi.append((command, result_callback, callback_parameters)) self.wi_available.notify() self.wi_available.release() def reconnect(self, address, bind_address = None): self.shutdown() self.address = address self.workers = [] for i in range(0, self.nworkers): self.workers.append(_RTPPLWorker(self)) def shutdown(self): for rworker in self.workers: rworker.shutdown() self.workers = None
class _InterruptableLock(object): def __init__(self, lock): """Creates a new lock.""" self.__lock = lock self.__condition = Condition(Lock()) def __enter__(self): return self.acquire() def __exit__(self, exc_type, exc_value, traceback): self.release() def acquire(self, blocking=True, timeout=-1): """Acquire a lock, blocking or non-blocking. Blocks until timeout, if timeout a positive float and blocking=True. A timeout value of -1 blocks indefinitely, unless blocking=False.""" if not isinstance(timeout, (int, float)): raise TypeError("a float is required") if blocking: # blocking indefinite if timeout == -1: with self.__condition: while not self.__lock.acquire(False): # condition with timeout is interruptable self.__condition.wait(60) return True # same as non-blocking elif timeout == 0: return self.__lock.acquire(False) elif timeout < 0: raise ValueError("timeout value must be strictly positive (or -1)") # blocking finite else: start = time() waited_time = 0 with self.__condition: while waited_time < timeout: if self.__lock.acquire(False): return True else: self.__condition.wait(timeout - waited_time) waited_time = time() - start return False elif timeout != -1: raise ValueError("can't specify a timeout for a non-blocking call") else: # non-blocking return self.__lock.acquire(False) def release(self): """Release a lock.""" self.__lock.release() with self.__condition: self.__condition.notify()
class Servermonitor: def __init__(self): self.serverlock = Lock() self.numberofwaitingthreads = 0 self.numberofwaitingclients = 0 self.waitingthreadcond = Condition(self.serverlock) def thread_start_serving(self,id): with self.serverlock: self.numberofwaitingthreads +=1 while self.numberofwaitingclients == 0: self.waitingthreadcond.wait(); self.numberofwaitingclients -=1 #User by server to find if it can serve request or not def checkifcanserveclient(self): retval = False with self.serverlock: if(self.numberofwaitingthreads > 0): retval = True return retval #This should only be called if there are waiting threads on the condition variable def wakeupthread(self): with self.serverlock: self.numberofwaitingclients += 1 self.numberofwaitingthreads -=1 if(self.numberofwaitingthreads <0): print "SHITsssssssssssssssssssssssssssssssssssssss" self.waitingthreadcond.notify()
class Future(object): def __init__(self, func, *args, **kwargs): self.__done = False self.__result = None self.__cond = Condition() self.__func = func self.__args = args self.__kwargs = kwargs self.__except = None def __call__(self): with self.__cond: try: self.__result = self.__func(*self.__args, **self.__kwargs) except: self.__result = None self.__except = sys.exc_info() self.__done = True self.__cond.notify() def result(self): with self.__cond: while not self.__done: self.__cond.wait() if self.__except: exc = self.__except reraise(exc[0], exc[1], exc[2]) result = self.__result return copy.deepcopy(result)
class MessageQueue(object): """The message queue used internally by Gossip.""" def __init__(self): self._queue = deque() self._condition = Condition() def pop(self): self._condition.acquire() try: while len(self._queue) < 1: self._condition.wait() return self._queue.pop() finally: self._condition.release() def __len__(self): return len(self._queue) def __deepcopy__(self, memo): newmq = MessageQueue() newmq._queue = copy.deepcopy(self._queue, memo) return newmq def appendleft(self, msg): self._condition.acquire() try: self._queue.appendleft(msg) self._condition.notify() finally: self._condition.release()
class DataQueue ( Queue ): def __init__ ( self, count ): Queue.__init__(self) self._count = count self._cond = Condition() def get ( self, block = True, timeout = None ): self._cond.acquire() if self.empty(): if not self._count: raise Empty() elif block: self._cond.wait(timeout) if self.empty() and not self._count: self._cond.release() raise Empty() self._cond.release() return Queue.get(self, block, timeout) def put ( self, data ): self._cond.acquire() self._count = self._count - 1 Queue.put(self, data) if self._count: self._cond.notify() else: self._cond.notifyAll() self._cond.release() def remain ( self ): return self._count + self.unfinished_tasks
class ZEventLoopThread(object): def __init__(self, cb=None): self._event_loop = None self._thr = Thread(target=self._thread_func) self._guard = Condition(Lock()) self._thr_callback = cb def start_loop(self): self._thr.start() self._guard.acquire() while self._event_loop is None: self._guard.wait() return self._event_loop def _thread_func(self): # Note: create event loop in the new thread # instead in the constructor event_loop = ZEventLoop() if self._thr_callback: self._thr_callback(event_loop) with self._guard: self._event_loop = event_loop self._guard.notify() event_loop.loop()
class External_command(object): work_available = None work = None def __init__(self, command, max_workers = _MAX_WORKERS): self.work_available = Condition() self.work = [] for i in range(0, max_workers): _Worker(command, self) def process_command(self, data, result_callback, *callback_parameters): wi = Work_item(tuple(data), result_callback, callback_parameters) self.work_available.acquire() self.work.append(wi) self.work_available.notify() self.work_available.release() return wi def shutdown(self): self.work_available.acquire() self.work.append(None) self.work_available.notify() self.work_available.release() def process_result(self, result_callback, result, *callback_parameters): try: result_callback(result, *callback_parameters) except Exception as ex: if isinstance(ex, SystemExit): raise dump_exception('External_command: unhandled exception in external command results callback')
class SharedCell(object): """Shared data for the producer/consumer problem.""" def __init__(self): self._data = -1 self._writeable = True self._condition = Condition() def setData(self, data): """Producer's method to write to shared data.""" self._condition.acquire() while not self._writeable: self._condition.wait() print "%s setting data to %d" % \ (currentThread().getName(), data) self._data = data self._writeable = False self._condition.notify() self._condition.release() def getData(self): """Consumer's method to read from shared data.""" self._condition.acquire() while self._writeable: self._condition.wait() print "%s accessing data %d" % \ (currentThread().getName(), self._data) self._writeable = True self._condition.notify() self._condition.release() return self._data
class External_command(object): work_available = None work = None def __init__(self, command, max_workers = _MAX_WORKERS): self.work_available = Condition() self.work = [] for i in range(0, max_workers): _Worker(command, self) def process_command(self, data, result_callback, *callback_parameters): wi = Work_item(tuple(data), result_callback, callback_parameters) self.work_available.acquire() self.work.append(wi) self.work_available.notify() self.work_available.release() return wi def shutdown(self): self.work_available.acquire() self.work.append(None) self.work_available.notify() self.work_available.release() def process_result(self, result_callback, result, *callback_parameters): try: result_callback(result, *callback_parameters) except: print datetime.now(), 'External_command: unhandled exception in external command results callback' print '-' * 70 print_exc(file = stdout) print '-' * 70 stdout.flush()
class Queue(object): def __init__(self): self.lock = Lock() self.cond = Condition(self.lock) self.data = [] self.live = True def put(self, values): self.cond.acquire() self.data.extend(values) self.cond.notify() self.cond.release() def get(self): if not self.cond.acquire(False): return [] self.cond.wait() result = self.data self.data = [] self.cond.release() return result def close(self): self.cond.acquire() self.live = False self.cond.notify() self.cond.release()
class RepceJob(object): """class representing message status we can use for waiting on reply""" def __init__(self, cbk): """ - .rid: (process-wise) unique id - .cbk: what we do upon receiving reply """ self.rid = (os.getpid(), thread.get_ident(), time.time()) self.cbk = cbk self.lever = Condition() self.done = False def __repr__(self): return ':'.join([str(x) for x in self.rid]) def wait(self): self.lever.acquire() if not self.done: self.lever.wait() self.lever.release() return self.result def wakeup(self, data): self.result = data self.lever.acquire() self.done = True self.lever.notify() self.lever.release()
class ConnectorPool: """ A thread-safe connection pool. TODO: Make this an actual queue, not a stack. Nomenclature is imporant sometimes. """ def __init__(self, Connector, count=20, user=None, password=None, db=None): self.pool = [ Connector(user, password, db) for _ in range(count) ] self.C_pool = Condition() def dequeue(self): """ Get connector. :rtype: seaice.SeaIceConnector.SeaIceConnector """ self.C_pool.acquire() while len(self.pool) == 0: self.C_pool.wait() db_con = self.pool.pop() self.C_pool.release() return db_con def enqueue(self, db_con): """ Release connector. :param db_con: The connector. :type db_con: seaice.SeaIceConnector.SeaIceConnector """ self.C_pool.acquire() self.pool.append(db_con) self.C_pool.notify() self.C_pool.release()
class NotificationQueue(Queue): """ A queue class that notifies waiting threads when the queue is available for reading.""" def __init__(self): """ Constructor.""" Queue.__init__(self) self.lock = Condition() def get(self): """ Get an item from the queue. This method is thread-safe. Method must be called on a non-empty queue. Return : the first item from the queue Raises: a EmptyQueue exeception """ return Queue.get(self, False) def put(self, item): """ Add a new item to the queue. This method is thread-safe. Threads that are waiting for get an item are notified. item: new item to add to the queue. Return : None """ self.lock.acquire() Queue.put(self, item) self.lock.notify() self.lock.release() def acquire(self): self.lock.acquire() def release(self): self.lock.release() def wait(self): self.lock.wait()
class SSHProtocol(Protocol): def __init__(self,proxy): print "SShProtocol %s"%self.rules Protocol.__init__(self,self.rules) self.proxy=proxy self.cv=Condition() def init(self): print "OK initialized" self.state=2 def handle(self,msg): print "handle message %s"%msg if msg['type'] == "send": thread.start_new_thread(self.send, (msg,)) def send(self,message): print "Sending Message %s"%message self.release() def release(self): self.cv.acquire() self.cv.notify() self.cv.release() def wait(self): self.cv.acquire() self.cv.wait() self.cv.release() rules={1:init,2:handle}
class TmThread(Thread): def __init__(self, target, args): # name, event_for_wait, event_for_set Thread.__init__(self) # , target=target, args=args) self.__target = target self.__args = args self.__kwargs = None self.priority = 0 self.time = 0 #flag to Finish thread self.Finishd = False # Explicitly using Lock over RLock since the use of self.Finishd # break reentrancy anyway, and I believe using Lock could allow # one thread to Finish the worker, while another resumes; haven't # checked if Condition imposes additional limitations that would # prevent that. In Python 2, use of Lock instead of RLock also # boosts performance. self.Finish_cond = Condition(Lock()) def Finish(self): self.Finishd = True # If in sleep, we acquire immediately, otherwise we wait for thread # to release condition. In race, worker will still see self.Finishd # and begin waiting until it's set back to False self.Finish_cond.acquire() #should just resume the thread def resume(self): self.Finishd = False # Notify so thread will wake after lock released self.Finish_cond.notify() # Now release the lock self.Finish_cond.release()
class DirtyBit: def __init__(self): self._mon = RLock() self._rc = Condition(self._mon) self._dirty = 0 def clear(self): self._mon.acquire() self._dirty = 0 #self._rc.notify() only interested in knowing when we are dirty self._mon.release() def set(self): self._mon.acquire() self._dirty = 1 self._rc.notify() self._mon.release() def value(self): return self._dirty def wait(self): self._mon.acquire() self._rc.wait() self._mon.release()
class Future: def __init__(self, correlation_id, request_type=None): self.correlation_id = correlation_id self._result = None self._condition = Condition() self._request_type = request_type def done(self): return self._result is not None def result(self, timeout=None): with self._condition: if self._result is None: if not self._condition.wait(timeout): message_type = validator_pb2.Message.MessageType.Name( self._request_type) if self._request_type else None raise FutureTimeoutError( 'Future timed out waiting for response to {}'.format( message_type)) return self._result def set_result(self, result): with self._condition: self._result = result self._condition.notify()
class Mailboxmonitor: def __init__(self): self.mailboxlock = Lock() self.mailidentifier = 0 self.fileHandle = open("mailbox", "w") self.backupthreadactive = 0 self.backupthreadswaiting = 0 self.waitingforbackuptocomplete = Condition(self.mailboxlock) self.backupneededcond = Condition(self.mailboxlock) def delivermail(self, clientname, mailsender, mailreceiver, mailcontent): with self.mailboxlock: #if back up is happening stop writes everywhere while self.backupthreadactive == 1: self.waitingforbackuptocomplete.wait() mail = "Received from " + str(clientname) + " by pm489 (CS4410MP3)\n" self.mailidentifier += 1 mail += "Number: " + str(self.mailidentifier) + "\n" mail += "From: " + str(mailsender) + "\n" for i in range(0, len(mailreceiver)): mail += "To: " + str(mailreceiver[i]) + "\n" mail += "\n" mail += mailcontent + "\n" mail += "\n" while self.backupthreadactive == 1: #if back up is happening stop writes everywhere print "worker sleeping" self.waitingforbackuptocomplete.wait() self.fileHandle.write(mail) self.fileHandle.flush() if self.mailidentifier%32 == 0 and self.backupthreadswaiting > 0: self.backupthreadactive = 1 self.backupthreadswaiting = 0 self.backupneededcond.notify() return self.mailidentifier def startbackupthread(self): with self.mailboxlock: while self.backupthreadactive==0: #print "backup slept" self.backupthreadswaiting = 1 self.backupneededcond.wait() def initiatebackup(self, lastbackedmessage): with self.mailboxlock: if( self.mailidentifier%32 != 0): print "Something went very horribly wrong" newfilename = "mailbox." + str(lastbackedmessage+1)+"-"+ str(lastbackedmessage+32) self.fileHandle.close() os.rename("mailbox", newfilename) self.fileHandle = open("mailbox", "w") lastbackedmessage += 32 self.backupthreadactive = 0 self.waitingforbackuptocomplete.notifyAll() #wake up all waiting threads to write return lastbackedmessage
class Runner: def __init__(self): logger.info('Starting performances node') self.robot_name = rospy.get_param('/robot_name') self.running = False self.paused = False self.pause_time = 0 self.start_time = 0 self.start_timestamp = 0 self.lock = Lock() self.run_condition = Condition() self.queue = Queue.Queue() self.ids = [] self.running_nodes = [] # References to event subscribing node callbacks self.observers = {} self.worker = Thread(target=self.worker) self.worker.setDaemon(True) rospy.init_node('performances') self.services = { 'head_pau_mux': rospy.ServiceProxy('/' + self.robot_name + '/head_pau_mux/select', MuxSelect), 'neck_pau_mux': rospy.ServiceProxy('/' + self.robot_name + '/neck_pau_mux/select', MuxSelect) } self.topics = { 'look_at': rospy.Publisher('/blender_api/set_face_target', Target, queue_size=1), 'gaze_at': rospy.Publisher('/blender_api/set_gaze_target', Target, queue_size=1), 'head_rotation': rospy.Publisher('/blender_api/set_head_rotation', Float32, queue_size=1), 'emotion': rospy.Publisher('/blender_api/set_emotion_state', EmotionState, queue_size=3), 'gesture': rospy.Publisher('/blender_api/set_gesture', SetGesture, queue_size=3), 'expression': rospy.Publisher('/' + self.robot_name + '/make_face_expr', MakeFaceExpr, queue_size=3), 'kfanimation': rospy.Publisher('/' + self.robot_name + '/play_animation', PlayAnimation, queue_size=3), 'interaction': rospy.Publisher('/behavior_switch', String, queue_size=1), 'bt_control': rospy.Publisher('/behavior_control', Int32, queue_size=1), 'events': rospy.Publisher('~events', Event, queue_size=1), 'chatbot': rospy.Publisher('/' + self.robot_name + '/speech', ChatMessage, queue_size=1), 'speech_events': rospy.Publisher('/' + self.robot_name + '/speech_events', String, queue_size=1), 'soma_state': rospy.Publisher("/blender_api/set_soma_state", SomaState, queue_size=2), 'tts': { 'en': rospy.Publisher('/' + self.robot_name + '/tts_en', String, queue_size=1), 'zh': rospy.Publisher('/' + self.robot_name + '/tts_zh', String, queue_size=1), 'default': rospy.Publisher('/' + self.robot_name + '/tts', String, queue_size=1), } } rospy.Service('~load', srv.Load, self.load_callback) rospy.Service('~load_nodes', srv.LoadNodes, self.load_nodes_callback) rospy.Service('~load_sequence', srv.LoadSequence, self.load_sequence_callback) rospy.Service('~run', srv.Run, self.run_callback) rospy.Service('~run_by_name', srv.RunByName, self.run_by_name_callback) rospy.Service('~resume', srv.Resume, self.resume_callback) rospy.Service('~pause', srv.Pause, self.pause_callback) rospy.Service('~stop', srv.Stop, self.stop) rospy.Service('~current', srv.Current, self.current_callback) # Shared subscribers for nodes rospy.Subscriber('/' + self.robot_name + '/speech_events', String, lambda msg: self.notify('speech_events', msg)) self.worker.start() rospy.spin() def load_callback(self, request): return srv.LoadResponse(success=True, nodes=json.dumps( self.load_sequence([request.id]))) def load_nodes_callback(self, request): self.load_nodes(json.loads(request.nodes)) return srv.LoadNodesResponse(True) def load_sequence_callback(self, request): return srv.LoadSequenceResponse(success=True, nodes=json.dumps( self.load_sequence(request.ids))) def run_by_name_callback(self, request): self.stop() nodes = self.load_sequence([request.id]) if not nodes: return srv.RunByNameResponse(False) return srv.RunByNameResponse(self.run(0.0)) def load_sequence(self, ids): nodes = [] offset = 0 for id in ids: robot_name = rospy.get_param('/robot_name') path = os.path.join(rospack.get_path('robots_config'), robot_name, 'performances', id + '.yaml') duration = 0 if os.path.isfile(path): with open(path, 'r') as f: data = yaml.load(f.read()) if 'nodes' in data and isinstance(data['nodes'], list): for node in data['nodes']: if not 'start_time' in node: node['start_time'] = 0 if node['name'] == 'pause': node['duration'] = 0.1 duration = max( duration, (node['duration'] if 'duration' in node else 0) + node['start_time']) node['start_time'] += offset offset += duration nodes += data['nodes'] self.load_nodes(nodes, ids) return nodes def load_nodes(self, nodes, ids=None): self.stop() if ids is None: ids = [] for node in nodes: node.pop('id', None) with self.lock: self.ids = ids self.running_nodes = nodes def run_callback(self, request): return srv.RunResponse(self.run(request.startTime)) def run(self, start_time): self.stop() # Wait for worker to stop performance and enter waiting before proceeding self.run_condition.acquire() with self.lock: rospy.logerr(start_time) if len(self.running_nodes) > 0: self.running = True self.start_time = start_time self.start_timestamp = time.time() # notify worker thread self.run_condition.notify() self.run_condition.release() return True else: return False def resume_callback(self, request): success = self.resume() with self.lock: run_time = self.get_run_time() return srv.ResumeResponse(success, run_time) def resume(self): success = False with self.lock: if self.running and self.paused: run_time = self.get_run_time() self.paused = False self.start_timestamp = time.time() - run_time self.start_time = 0 self.topics['events'].publish(Event('resume', run_time)) success = True return success def stop(self, request=None): stop_time = 0 with self.lock: if self.running: stop_time = self.get_run_time() self.running = False self.paused = False return srv.StopResponse(True, stop_time) def pause_callback(self, request): if self.pause(): with self.lock: return srv.PauseResponse(True, self.get_run_time()) else: return srv.PauseResponse(False, 0) # Pauses current def pause(self): with self.lock: if self.running and not self.paused: self.pause_time = time.time() self.paused = True self.topics['events'].publish( Event('paused', self.get_run_time())) return True else: return False # Returns current performance def current_callback(self, request): with self.lock: current_time = self.get_run_time() running = self.running and not self.paused return srv.CurrentResponse(ids=self.ids, nodes=json.dumps(self.running_nodes), current_time=current_time, running=running) def worker(self): self.run_condition.acquire() while True: with self.lock: self.paused = False self.topics['events'].publish(Event('idle', 0)) self.run_condition.wait() with self.lock: nodes = [ Node.createNode(node, self, self.start_time) for node in self.running_nodes ] self.topics['events'].publish(Event('running', self.start_time)) if len(nodes) == 0: continue running = True while running: with self.lock: run_time = self.get_run_time() if not self.running: self.topics['events'].publish( Event('finished', run_time)) break elif self.paused: continue running = False # checks if any nodes still running for node in nodes: running = node.run(run_time) or running self.run_condition.release() def get_run_time(self): """ Must acquire self.lock in order to safely use this method :return: """ run_time = 0 if self.running: run_time = self.start_time if self.paused: run_time += self.pause_time - self.start_timestamp else: run_time += time.time() - self.start_timestamp return run_time # Notifies register nodes on the events from ROS. def notify(self, event, msg): if event not in self.observers.keys(): return for i in xrange(len(self.observers[event]) - 1, -1, -1): try: self.observers[event][i](msg) except TypeError: # Remove dead methods del self.observers[event][i] # Registers callbacks for specific events. Uses weak reference to allow nodes cleanup after finish. def register(self, event, cb): if not event in self.observers: self.observers[event] = [] m = WeakMethod(cb) self.observers[event].append(m) return m # Allows nodes to unsubscribe from events def unregister(self, event, ref): if event in self.observers: if ref in self.observers[event]: self.observers[event].remove(ref)
class PooledDB: """Pool for DB-API 2 connections. After you have created the connection pool, you can use connection() to get pooled, steady DB-API 2 connections. """ version = __version__ def __init__(self, creator, mincached=0, maxcached=0, maxshared=0, maxconnections=0, blocking=False, maxusage=None, setsession=None, reset=True, failures=None, ping=1, *args, **kwargs): """Set up the DB-API 2 connection pool. creator: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module mincached: initial number of idle connections in the pool (0 means no connections are made at startup) maxcached: maximum number of idle connections in the pool (0 or None means unlimited pool size) maxshared: maximum number of shared connections (0 or None means all connections are dedicated) When this maximum number is reached, connections are shared if they have been requested as shareable. maxconnections: maximum number of connections generally allowed (0 or None means an arbitrary number of connections) blocking: determines behavior when exceeding the maximum (if this is set to true, block and wait until the number of connections decreases, otherwise an error will be reported) maxusage: maximum number of reuses of a single connection (0 or None means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). setsession: optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to ...", "set time zone ..."] reset: how connections should be reset when returned to the pool (False or None to rollback transcations started with begin(), True to always issue a rollback for safety's sake) failures: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InternalError) is not adequate ping: determines when the connection should be checked with ping() (0 = None = never, 1 = default = whenever fetched from the pool, 2 = when a cursor is created, 4 = when a query is executed, 7 = always, and all other bit combinations of these values) args, kwargs: the parameters that shall be passed to the creator function or the connection constructor of the DB-API 2 module """ try: threadsafety = creator.threadsafety except AttributeError: try: if not callable(creator.connect): raise AttributeError except AttributeError: threadsafety = 2 else: threadsafety = 0 if not threadsafety: raise NotSupportedError("Database module is not thread-safe.") self._creator = creator self._args, self._kwargs = args, kwargs self._maxusage = maxusage self._setsession = setsession self._reset = reset self._failures = failures self._ping = ping if mincached is None: mincached = 0 if maxcached is None: maxcached = 0 if maxconnections is None: maxconnections = 0 if maxcached: if maxcached < mincached: maxcached = mincached self._maxcached = maxcached else: self._maxcached = 0 if threadsafety > 1 and maxshared: self._maxshared = maxshared self._shared_cache = [] # the cache for shared connections else: self._maxshared = 0 if maxconnections: if maxconnections < maxcached: maxconnections = maxcached if maxconnections < maxshared: maxconnections = maxshared self._maxconnections = maxconnections else: self._maxconnections = 0 self._idle_cache = [] # the actual pool of idle connections self._condition = Condition() if not blocking: def wait(): raise TooManyConnections self._condition.wait = wait self._connections = 0 # Establish an initial number of idle database connections: idle = [self.dedicated_connection() for i in range(mincached)] while idle: idle.pop().close() def steady_connection(self): """Get a steady, unpooled DB-API 2 connection.""" return connect(self._creator, self._maxusage, self._setsession, self._failures, self._ping, True, *self._args, **self._kwargs) def connection(self, shareable=True): """Get a steady, cached DB-API 2 connection from the pool. If shareable is set and the underlying DB-API 2 allows it, then the connection may be shared with other threads. """ if shareable and self._maxshared: self._condition.acquire() try: while (not self._shared_cache and self._maxconnections and self._connections >= self._maxconnections): self._condition.wait() if len(self._shared_cache) < self._maxshared: # shared cache is not full, get a dedicated connection try: # first try to get it from the idle cache con = self._idle_cache.pop(0) except IndexError: # else get a fresh connection con = self.steady_connection() else: con._ping_check() # check this connection con = SharedDBConnection(con) self._connections += 1 else: # shared cache full or no more connections allowed self._shared_cache.sort() # least shared connection first con = self._shared_cache.pop(0) # get it while con.con._transaction: # do not share connections which are in a transaction self._shared_cache.insert(0, con) self._condition.wait() self._shared_cache.sort() con = self._shared_cache.pop(0) con.con._ping_check() # check the underlying connection con.share() # increase share of this connection # put the connection (back) into the shared cache self._shared_cache.append(con) self._condition.notify() finally: self._condition.release() con = PooledSharedDBConnection(self, con) else: # try to get a dedicated connection self._condition.acquire() try: while (self._maxconnections and self._connections >= self._maxconnections): self._condition.wait() # connection limit not reached, get a dedicated connection try: # first try to get it from the idle cache con = self._idle_cache.pop(0) except IndexError: # else get a fresh connection con = self.steady_connection() else: con._ping_check() # check connection con = PooledDedicatedDBConnection(self, con) self._connections += 1 finally: self._condition.release() return con def dedicated_connection(self): """Alias for connection(shareable=False).""" return self.connection(False) def unshare(self, con): """Decrease the share of a connection in the shared cache.""" self._condition.acquire() try: con.unshare() shared = con.shared if not shared: # connection is idle, try: # so try to remove it self._shared_cache.remove(con) # from shared cache except ValueError: pass # pool has already been closed finally: self._condition.release() if not shared: # connection has become idle, self.cache(con.con) # so add it to the idle cache def cache(self, con): """Put a dedicated connection back into the idle cache.""" self._condition.acquire() try: if not self._maxcached or len(self._idle_cache) < self._maxcached: con._reset(force=self._reset) # rollback possible transaction # the idle cache is not full, so put it there self._idle_cache.append(con) # append it to the idle cache else: # if the idle cache is already full, con.close() # then close the connection self._connections -= 1 self._condition.notify() finally: self._condition.release() def close(self): """Close all connections in the pool.""" self._condition.acquire() try: while self._idle_cache: # close all idle connections con = self._idle_cache.pop(0) try: con.close() except Exception: pass if self._maxshared: # close all shared connections while self._shared_cache: con = self._shared_cache.pop(0).con try: con.close() except Exception: pass self._connections -= 1 self._condition.notifyAll() finally: self._condition.release() def __del__(self): """Delete the pool.""" try: self.close() except Exception: pass
class CairnsmoreHotplugWorker(BaseWorker): version = "theseven.cairnsmore hotplug manager v0.1.0" default_name = "Cairnsmore hotplug manager" can_autodetect = True settings = dict( BaseWorker.settings, **{ "baudrate": { "title": "Baud rate", "type": "int", "position": 1100 }, "jobinterval": { "title": "Job interval", "type": "float", "position": 1200 }, "initialspeed": { "title": "Initial clock frequency", "type": "int", "position": 2000 }, "maximumspeed": { "title": "Maximum clock frequency", "type": "int", "position": 2100 }, "invalidwarning": { "title": "Warning invalids", "type": "int", "position": 3200 }, "invalidcritical": { "title": "Critical invalids", "type": "int", "position": 3300 }, "warmupstepshares": { "title": "Shares per warmup step", "type": "int", "position": 3400 }, "speedupthreshold": { "title": "Speedup threshold", "type": "int", "position": 3500 }, "scaninterval": { "title": "Bus scan interval", "type": "float", "position": 4000 }, }) @classmethod def autodetect(self, core): try: import serial found = False for port in glob( "/dev/serial/by-id/usb-FTDI_Cairnsmore1_*-if0?-port0"): try: handle = serial.Serial(port, 115200, serial.EIGHTBITS, serial.PARITY_NONE, serial.STOPBITS_ONE, 1, False, False, 5, False, None) handle.close() found = True break except: pass if found: core.add_worker(self(core)) except: pass # Constructor, gets passed a reference to the miner core and the saved worker state, if present def __init__(self, core, state=None): # Initialize bus scanner wakeup event self.wakeup = Condition() # Let our superclass do some basic initialization and restore the state if neccessary super(CairnsmoreHotplugWorker, self).__init__(core, state) # Validate settings, filling them with default values if neccessary. # Called from the constructor and after every settings change. def apply_settings(self): # Let our superclass handle everything that isn't specific to this worker module super(CairnsmoreHotplugWorker, self).apply_settings() if not "scaninterval" in self.settings or not self.settings.scaninterval: self.settings.scaninterval = 10 if not "baudrate" in self.settings or not self.settings.baudrate: self.settings.baudrate = 115200 if not "jobinterval" in self.settings or not self.settings.jobinterval: self.settings.jobinterval = 20 if not "initialspeed" in self.settings: self.settings.initialspeed = 150 self.settings.initialspeed = min(max(self.settings.initialspeed, 4), 250) if not "maximumspeed" in self.settings: self.settings.maximumspeed = 200 self.settings.maximumspeed = min(max(self.settings.maximumspeed, 4), 300) if not "invalidwarning" in self.settings: self.settings.invalidwarning = 2 self.settings.invalidwarning = min( max(self.settings.invalidwarning, 1), 10) if not "invalidcritical" in self.settings: self.settings.invalidcritical = 10 self.settings.invalidcritical = min( max(self.settings.invalidcritical, 1), 50) if not "warmupstepshares" in self.settings: self.settings.warmupstepshares = 5 self.settings.warmupstepshares = min( max(self.settings.warmupstepshares, 1), 10000) if not "speedupthreshold" in self.settings: self.settings.speedupthreshold = 100 self.settings.speedupthreshold = min( max(self.settings.speedupthreshold, 50), 10000) # Push our settings down to our children fields = [ "baudrate", "jobinterval", "initialspeed", "maximumspeed", "invalidwarning", "invalidcritical", "warmupstepshares", "speedupthreshold" ] for child in self.children: for field in fields: child.settings[field] = self.settings[field] child.apply_settings() # Rescan the bus immediately to apply the new settings with self.wakeup: self.wakeup.notify() # Reset our state. Called both from the constructor and from self.start(). def _reset(self): # Let our superclass handle everything that isn't specific to this worker module super(CairnsmoreHotplugWorker, self)._reset() # These need to be set here in order to make the equality check in apply_settings() happy, # when it is run before starting the module for the first time. (It is called from the constructor.) # Start up the worker module. This is protected against multiple calls and concurrency by a wrapper. def _start(self): # Let our superclass handle everything that isn't specific to this worker module super(CairnsmoreHotplugWorker, self)._start() # Initialize child map self.childmap = {} # Reset the shutdown flag for our threads self.shutdown = False # Start up the main thread, which handles pushing work to the device. self.mainthread = Thread(None, self.main, self.settings.name + "_main") self.mainthread.daemon = True self.mainthread.start() # Shut down the worker module. This is protected against multiple calls and concurrency by a wrapper. def _stop(self): # Let our superclass handle everything that isn't specific to this worker module super(CairnsmoreHotplugWorker, self)._stop() # Set the shutdown flag for our threads, making them terminate ASAP. self.shutdown = True # Trigger the main thread's wakeup flag, to make it actually look at the shutdown flag. with self.wakeup: self.wakeup.notify() # Wait for the main thread to terminate. self.mainthread.join(10) # Shut down child workers while self.children: child = self.children.pop(0) try: self.core.log( self, "Shutting down worker %s...\n" % (child.settings.name), 800) child.stop() except Exception as e: self.core.log( self, "Could not stop worker %s: %s\n" % (child.settings.name, traceback.format_exc()), 100, "rB") # Main thread entry point # This thread is responsible for scanning for boards and spawning worker modules for them def main(self): import serial # Loop until we are shut down while not self.shutdown: try: boards = {} for port in glob( "/dev/serial/by-id/usb-FTDI_Cairnsmore1_*-if0?-port0"): available = False try: handle = serial.Serial(port, 115200, serial.EIGHTBITS, serial.PARITY_NONE, serial.STOPBITS_ONE, 1, False, False, 5, False, None) handle.close() available = True except: pass boards[port] = available kill = [] for port, child in self.childmap.items(): if not port in boards: kill.append((port, child)) for port, child in kill: try: self.core.log( self, "Shutting down worker %s...\n" % (child.settings.name), 800) child.stop() except Exception as e: self.core.log( self, "Could not stop worker %s: %s\n" % (child.settings.name, traceback.format_exc()), 100, "rB") childstats = child.get_statistics() fields = [ "ghashes", "jobsaccepted", "jobscanceled", "sharesaccepted", "sharesrejected", "sharesinvalid" ] for field in fields: self.stats[field] += childstats[field] try: self.child.destroy() except: pass del self.childmap[port] try: self.children.remove(child) except: pass for port, available in boards.items(): if port in self.childmap or not available: continue child = CairnsmoreWorker(self.core) child.settings.name = "Cairnsmore1 board %s FPGA%s" % ( re.match( "/dev/serial/by-id/usb-FTDI_Cairnsmore1_([0-9A-Z]+)-if0([0-3])-port0", port).group(1, 2)) child.settings.port = port fields = [ "jobinterval", "baudrate", "initialspeed", "maximumspeed", "invalidwarning", "invalidcritical", "warmupstepshares", "speedupthreshold" ] for field in fields: child.settings[field] = self.settings[field] child.apply_settings() self.childmap[port] = child self.children.append(child) try: self.core.log( self, "Starting up worker %s...\n" % (child.settings.name), 800) child.start() except Exception as e: self.core.log( self, "Could not start worker %s: %s\n" % (child.settings.name, traceback.format_exc()), 100, "rB") except: self.core.log( self, "Caught exception: %s\n" % traceback.format_exc(), 100, "rB") with self.wakeup: self.wakeup.wait(self.settings.scaninterval)
class GAThread(Thread): """ separate thread to run Genetic Algorithm while not blocking the main thread. """ def __init__(self): Thread.__init__(self) # flag if start method has been called self.start_triggered = False # thread pause condition self.pause_cond = Condition(Lock()) # flag to pause thread self.__pause_now = False # flag to state thread state self.paused = True # flag to stop thread self.__stop_now = False def run(self): # initializing the population pop = Population(g_pop_size, g_genes_num) # fittest fitness of the previous generation, used to send deviation value prv_fitness = pop.fittest().fitness() # started signal to the renderer process to_json({ "started": True, "genesNum": pop.genes_num, "fitness": prv_fitness, "first-step": self.__pause_now, }) # first generated solutions (generation 0) to_json({ "generation": pop.generation, "genes": pop.fittest().genes, "fitness": prv_fitness, }) while g_max_gen is False or g_max_gen < 0 or pop.generation < g_max_gen: Evolve.evolve_population(pop) # takes the current generation fitness cur_fitness = pop.fittest().fitness() # if g_del_rate is 0 or False than just ignore it if g_del_rate: sleep(g_del_rate) if g_pause_gen is not False and pop.generation == g_pause_gen + 1: self.pause() to_json({'forced-pause': True}) # pause check, moved down to avoid another iteration if stop event # was triggered after a pause event self.__pause_check() # stopped event, separating finished naturally (if there is valid # solution) from being forcefully stopped if self.__stop_now: to_json({"stopped": True}) return # moved down, so when GA is heavy (slow), user might stop it before the point is added # the point must not be added, so ga stops before executing below code to_json({ "prv-fitness": prv_fitness, "fitness": cur_fitness, "generation": pop.generation, "genes": pop.fittest().genes, }) # update prv_fitness prv_fitness = cur_fitness # finished event to_json({"finished": True}) def __pause_check(self): """ pause if pause() is called """ if self.__pause_now: # halt self.pause_cond.acquire() self.__pause_now = False self.paused = True with self.pause_cond: while self.paused: self.pause_cond.wait() # should just resume the thread def __resume(self): """ resume thread if paused """ # thread should be paused to resume if self.paused: # Notify so thread will wake after lock released self.pause_cond.notify() # Now release the lock self.pause_cond.release() self.paused = False # notify app to_json({"resumed": True}) # user triggered pause (through play button) through GUI and self.paused is still false means # GA is too slow on generating the next generation, than when the user clicked play (for resume) # it just turns self.__pause_now to false to prevent GA from pausing. elif self.__pause_now: self.__pause_now = False def start(self): """ starts thread activity if start method was not called before on this thread """ if not self.start_triggered: self.start_triggered = True self.paused = False Thread.start(self) else: self.__resume() def pause(self): """ pause thread if running """ # thread should be running to pause if not self.paused: self.__pause_now = True # notify app of the pause to_json({"paused": True}) def step_forward(self): """ move one iteration forward """ # start it if not started yet if not self.start_triggered: self.start_triggered = True self.__pause_now = True # fixes the blocking that happens when user clicks step_f multiple times on a heavy GA self.paused = False Thread.start(self) return # release if paused, it will lock automatically after one generation # because __pause_now is set to True elif not self.paused: self.__pause_now = True to_json({"paused": True}) elif not self.__pause_now: # Notify so thread will wake after lock released self.pause_cond.notify() # Now release the lock self.pause_cond.release() # pause now self.paused = False self.__pause_now = True def stop(self): """ if thread is alive, terminate it """ if self.is_alive(): self.__stop_now = True # resume if paused to break out of running loop if self.paused: # Notify so thread will wake after lock released self.pause_cond.notify() # Now release the lock self.pause_cond.release() self.paused = False # in case is going to pause but user pressed stop, GA should pass the pause check test to stop else: self.__pause_now = False self.join(10)
class Explore(object): def __init__(self, height): self.motion_height = height # Height of the motion to the ground # Create trajectory server self.exploration_server = SimpleActionServer('assessment_server', ExecuteAssesstAction, self.exploreCallback, False) self.server_feedback = ExecuteAssesstFeedback() self.server_result = ExecuteAssesstResult() # Get client from trajectory server self.trajectory_client = SimpleActionClient( "approach_server", ExecuteDroneApproachAction) self.trajectory_client.wait_for_server() self.next_point = ExecuteDroneApproachGoal( ) # Message to define next position to look for victims #Planning scene client self.frontiers_client = rospy.ServiceProxy('frontiers_server/find', Frontiers) self.frontiers_client.wait_for_service() self.frontiers_req = FrontiersRequest() #Frontiers request message # Variables self.sonar_me = Condition() self.odometry_me = Condition() self.current_height = None self.odometry = None # Subscribe to sonar_height rospy.Subscriber("sonar_height", Range, self.sonar_callback, queue_size=10) # Subscribe to ground_truth to monitor the current pose of the robot rospy.Subscriber("ground_truth/state", Odometry, self.poseCallback) self.scene_req = GetPlanningSceneRequest() # Start trajectory server self.exploration_server.start() def sonar_callback(self, msg): ''' Function to update drone height ''' self.sonar_me.acquire() self.current_height = msg.range self.sonar_me.release() def poseCallback(self, odometry): ''' Monitor the current position of the robot ''' self.odometry_me.acquire() self.odometry = odometry.pose.pose self.odometry_me.notify() self.odometry_me.release() def trajectory_feed(self, msg): ''' Verifies preemption requisitions ''' print("\n\n\nASSESSMENT FEEDBACK") if self.exploration_server.is_preempt_requested(): self.trajectory_client.cancel_goal() def exploreCallback(self, pose): ''' Execute a loop looking for frontiers and moving to points unvisited into the defined area ''' # Wait till the robot pose is received self.odometry_me.acquire() while self.odometry == None: self.odometry_me.wait() self.next_point.goal = self.odometry self.odometry_me.release() self.frontiers_req.x_min = 0.0 self.frontiers_req.x_max = 50.0 self.frontiers_req.y_min = 0.0 self.frontiers_req.y_max = 50.0 trials = 0 # v_search trials while not rospy.is_shutdown(): self.odometry_me.acquire() self.server_result.last_pose = self.odometry self.server_feedback.current_pose = self.odometry self.odometry_me.release() if self.exploration_server.is_preempt_requested(): self.exploration_server.set_preempted(self.server_result) return self.exploration_server.publish_feedback(self.server_feedback) self.sonar_me.acquire() # print("Current height from ground:\n\n{}".format(self.current_height)) # Current distance from ground h_error = self.motion_height - self.current_height self.sonar_me.release() self.odometry_me.acquire() self.next_point.goal.position.z = self.odometry.position.z + h_error # Desired z position self.odometry_me.release() self.trajectory_client.send_goal(self.next_point, feedback_cb=self.trajectory_feed) self.trajectory_client.wait_for_result() # Wait for the result result = self.trajectory_client.get_state( ) # Get the state of the action # print(result) if result == GoalStatus.SUCCEEDED: p = Pose() self.odometry_me.acquire() p = self.odometry self.frontiers_req.explored.append(p) self.odometry_me.release() # Verify if all the area have been explored and find next frontier point if needed # if 'all area explored': # self.odometry_me.acquire() # self.server_result.last_pose = self.odometry # self.odometry_me.release() # self.exploration_server.set_succeeded(self.server_result) status = self.findFrontiers() if not status: self.exploration_server.set_succeeded(self.server_result) return self.odometry_me.acquire() self.server_result.last_pose = self.odometry self.odometry_me.release() # self.next_point.goal.position.y = # self.next_point.goal.position.x = # theta = # Convert desired angle # q = quaternion_from_euler(0,0,theta,'ryxz') # self.next_point.goal.orientation.x = q[0] # self.next_point.goal.orientation.y = q[1] # self.next_point.goal.orientation.z = q[2] # self.next_point.goal.orientation.w = q[3] elif result == GoalStatus.ABORTED: trials += 1 if trials == 2: self.exploration_server.set_aborted(self.server_result) return def findFrontiers(self): ''' Return points not visited into the specified frontier ''' rospy.loginfo("Looking for frontiers!") frontiers = self.frontiers_client.call(self.frontiers_req) if frontiers.frontiers: self.next_point.goal.position.x = frontiers.frontiers[0].x self.next_point.goal.position.y = frontiers.frontiers[0].y self.next_point.goal.position.x = self.motion_height return True else: return False
class SocketIORunnerClientNamespace(ClientNamespace): def __init__( self, socketio_runner: SocketIORunner, interactions: List[Interaction], substitutes: dict = {}, ): super().__init__() self.socketio_runner = socketio_runner self._interaction_stack: List[Tuple[ bool, dict]] = _create_interaction_stack( socketio_runner.interaction_loader, interactions, substitutes) self._timeout_condition = Condition() self._failed_interaction: Optional[FailedInteraction] = None self._current_user_input: dict = {} def session_request(self) -> None: self.emit("session_request") def on_bot_uttered(self, data: Any) -> None: self._timeout_notify() is_user_message, message = self._pop_interaction_stack() while is_user_message: self._send_user_input(message) is_user_message, message = self._pop_interaction_stack() json_diff = self.socketio_runner.comparator.compare(message, data) if not json_diff.identical and self._failed_interaction is None: self._failed_interaction = FailedInteraction( self._current_user_input, message, data, json_diff, ) if self._next_is_user_message(): _, message = self._pop_interaction_stack() self._send_user_input(message) def _next_is_user_message(self) -> bool: if len(self._interaction_stack) > 0: is_user_message, _ = self._interaction_stack[0] return is_user_message return False def _pop_interaction_stack(self) -> Tuple[bool, dict]: return (self._interaction_stack.pop(0) if len(self._interaction_stack) > 0 else (False, {})) def _get_remaining_bot_messages(self) -> List[dict]: return [ message for is_user_input, message in self._interaction_stack if not is_user_input ] def _timeout_notify(self): with self._timeout_condition: self._timeout_condition.notify() def _timeout_await(self) -> bool: with self._timeout_condition: return self._timeout_condition.wait(BOT_RESPONSE_TIMEOUT) def _send_user_input(self, message: dict) -> None: self._timeout_notify() self._current_user_input = {SESSION_ID_KEY: self.client.sid} self._current_user_input.update(message) self.emit(EVENT_USER_UTTERED, self._current_user_input) def run(self) -> Optional[FailedInteraction]: self.session_request() is_user_input, message = self._pop_interaction_stack() if is_user_input: self._send_user_input(message) while self._failed_interaction is None and self._timeout_await(): pass remaining_messages = self._get_remaining_bot_messages() json_diff = self.socketio_runner.comparator.compare({}, remaining_messages) if not json_diff.identical and self._failed_interaction is None: return FailedInteraction( self._current_user_input, {}, remaining_messages, json_diff, ) return self._failed_interaction
class MappingThread(Mapping): def __init__(self, graph, params): super().__init__(graph, params) self._requests_cv = Condition() self._requests = [False, False ] # requests: [LOCKWINDOW_REQUEST, PROCESS_REQUEST] self._lock = Lock() self.locked_window = set() self.status = defaultdict(bool) self._queue = Queue() self.maintenance_thread = Thread(target=self.maintenance) self.maintenance_thread.start() def add_keyframe(self, keyframe, measurements): self.graph.add_keyframe(keyframe) self.create_points(keyframe) for m in measurements: self.graph.add_measurement(keyframe, m.mappoint, m) self._queue.put(keyframe) with self._requests_cv: self._requests_cv.notify() def maintenance(self): stopped = False while not stopped: while not self._queue.empty(): keyframe = self._queue.get() if keyframe is None: stopped = True self._requests[1] = True break else: self.local_keyframes.append(keyframe) if len(self.local_keyframes) >= 5: self._requests[1] = True break with self._requests_cv: if self._requests.count(True) == 0: self._requests_cv.wait() while not self._queue.empty(): keyframe = self._queue.get() if keyframe is None: stopped = True self._requests[1] = True break else: self.local_keyframes.append(keyframe) if len(self.local_keyframes) >= 5: self._requests[1] = True requests = self._requests[:] self._requests[0] = False self._requests[1] = False self.status['processing'] = True if requests[1] and len(self.local_keyframes) > 0: self.fill(self.local_keyframes, self.local_keyframes[-1]) if requests[0]: with self._lock: for kf in self.local_keyframes: self.locked_window.add(kf) for ck, n in kf.covisibility_keyframes().items(): if n > 0: self.locked_window.add(ck) self.status['window_locked'] = True if requests[1] and len(self.local_keyframes) > 0: completed = self.bundle_adjust(self.local_keyframes) if completed: self.points_culling(self.local_keyframes) self.local_keyframes.clear() self.status['processing'] = False def stop(self): with self._requests_cv: self._requests_cv.notify() while not self._queue.empty(): time.sleep(1e-4) self._queue.put(None) # sentinel value self.maintenance_thread.join() print('mapping stopped') def is_safe(self, keyframe): with self._lock: return not self.is_window_locked( ) or keyframe in self.locked_window def is_processing(self): return self.status['processing'] def lock_window(self): with self._lock: self.status['window_locked'] = False self.locked_window.clear() with self._requests_cv: self._requests[0] = True self._requests_cv.notify() while not self.is_window_locked(): time.sleep(1e-4) return self.locked_window def free_window(self): with self._lock: self.status['window_locked'] = False self.locked_window.clear() def is_window_locked(self): return self.status['window_locked'] def wait_until_empty_queue(self): while not self._queue.empty(): time.sleep(1e-4) def interrupt_ba(self): self.optimizer.abort()
class Popen(object): _next_id = 0 _scheduler = None _redirector = None _lock = RLock() def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=None, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, cpus=None, mem=None, gpus=None): kw = dict(list(locals().items())) a = (args, ) kw.pop('self') kw.pop('args') close_fds = kw.pop('close_fds') if close_fds is False: logger.warning('Can not support `close_fds=False`, ' 'no fds will be inherrited.') stdin = kw.pop('stdin', None) stdout = kw.pop('stdout', None) stderr = kw.pop('stderr', None) cpus = kw.pop('cpus', None) mem = kw.pop('mem', None) gpus = kw.pop('gpus', None) kw['cwd'] = kw.get('cwd') or os.getcwd() kw['env'] = kw.get('env') or dict(list(os.environ.items())) self.id = self._new_id() self.cpus = float(cpus or CONFIG.get('default_cpus', 1.0)) self.mem = float(mem or CONFIG.get('default_mem', 1024.0)) self.gpus = int(gpus or CONFIG.get('default_gpus', 0)) self.pid = None self.returncode = None self._returncode = None self._a = a self._kw = kw self._exc = None self._state = _STARTING self._io_waiting = True self._cond = Condition() self._prepare_handlers(stdin, stdout, stderr) self._submit() with self._cond: deadline = time.time() + _STARTING_TIMEOUT while True: if self._state != _STARTING: break delta = deadline - time.time() if deadline <= 0: raise RuntimeError('Too long to start!') self._cond.wait(delta) if self._exc: raise self._exc @classmethod def _new_id(cls): cls._next_id += 1 return cls._next_id def _submit(self): cls = self.__class__ with cls._lock: if not cls._scheduler: cls._scheduler = ProcScheduler() cls._scheduler.start() atexit.register(cls._scheduler.stop) cls._scheduler.submit(self) def _prepare_handlers(self, stdin, stdout, stderr): def _dup_file(f, *a, **kw): fd = f if isinstance(f, int) else f.fileno() return os.fdopen(os.dup(fd), *a, **kw) cls = self.__class__ if not cls._redirector: cls._redirector = Redirector() atexit.register(cls._redirector.stop) if stdin == PIPE: r, w = os.pipe() _in = os.fdopen(r, 'rb', 0) self.stdin = os.fdopen(w, 'wb', 0) else: self.stdin = None if stdin is None: _in = _dup_file(sys.stdin, 'rb', 0) else: _in = _dup_file(stdin, 'rb', 0) if stdout == PIPE: r, w = os.pipe() _out = os.fdopen(w, 'wb', 0) self.stdout = os.fdopen(r, 'rb', 0) else: self.stdout = None if stdout is None: _out = _dup_file(sys.stdout, 'wb', 0) else: _out = _dup_file(stdout, 'wb', 0) if stderr == PIPE: r, w = os.pipe() _err = os.fdopen(w, 'wb', 0) self.stderr = os.fdopen(r, 'rb', 0) else: self.stderr = None if stderr is None: _err = _dup_file(sys.stderr, 'wb', 0) elif stderr == STDOUT: _err = _dup_file(_out, 'wb', 0) else: _err = _dup_file(sys.stderr, 'wb', 0) self._handlers = cls._redirector.register(self.id, _in, _out, _err, self._io_complete) def _io_complete(self): with self._cond: self._io_waiting = False self._cond.notify() def _kill(self): cls = self.__class__ cls._redirector.unregister(self.id) def __repr__(self): args = self._a[0] if isinstance(args, string_types): cmd = args else: cmd = ' '.join(args) return '%s: %s' % (self.__class__.__name__, cmd) @property def params(self): return dict( a=self._a, kw=self._kw, cpus=self.cpus, mem=self.mem, gpus=self.gpus, handlers=self._handlers, hostname=socket.gethostname(), ) def _started(self): logger.info('Started') with self._cond: self._state = _RUNNING self._cond.notify() def _finished(self, success, message, data): logger.info('Sucess:%s message:%s', success, message) if success: self._returncode, self._exc = data else: self._returncode, self._exc = data or (UNKNOWN_ERROR, None) self._kill() with self._cond: self._state = _STOPPED self._cond.notify() def poll(self): with self._cond: if self._state == _STOPPED and not self._io_waiting: if self.stdin and not self.stdin.closed: self.stdin.close() self.stdin = None if self.stdout and not self.stdout.closed: self.stdout.close() self.stdout = None if self.stderr and not self.stderr.closed: self.stderr.close() self.stderr = None self.returncode = self._returncode return self.returncode def wait(self): with self._cond: while self.poll() is None: self._cond.wait() return self.returncode def communicate(self, input=None): BUFFER_SIZE = PIPE_BUF buf = input and input[:] out = None err = None to_write = [_f for _f in [self.stdin] if _f] to_read = [_f for _f in [self.stdout, self.stderr] if _f] while True: can_wait = True readable, writeable, _ = select.select(to_read, to_write, [], 0) if writeable: if buf: can_wait = False size = os.write(self.stdin.fileno(), buf[:BUFFER_SIZE]) buf = buf[size:] else: if self.stdin and not self.stdin.closed: self.stdin.close() to_write = [] if readable: can_wait = False if self.stdout in readable: if out is None: out = self.stdout.read(BUFFER_SIZE) else: out += self.stdout.read(BUFFER_SIZE) if self.stderr in readable: if err is None: err = self.stderr.read(BUFFER_SIZE) else: err += self.stderr.read(BUFFER_SIZE) with self._cond: if self.poll() is None: if can_wait: self._cond.wait(0.1) else: return (out, err) def send_signal(self, signal): cls = self.__class__ cls._scheduler.send_data(self.id, _TYPE_SIGNAL, signal) def terminate(self): self.send_signal(signal.SIGTERM) def kill(self): self.send_signal(signal.SIGKILL) def cancel(self): cls = self.__class__ cls._scheduler.cancel(self) self._kill() self.returncode = UNKNOWN_ERROR
class RPC: def __init__(self, infile, outfile, onRequest, onNotification, onError): self.infile = infile self.outfile = outfile self.onRequest = onRequest self.onNotification = onNotification self.onError = onError self.mid = 0 self.queue = {} self.cv = Condition() self.result = None def incMid(self) -> int: mid = self.mid self.mid += 1 return mid def message(self, contentDict: Dict[str, Any]) -> None: content = json.dumps(contentDict) message = ("Content-Length: {}\r\n\r\n" "{}".format(len(content), content)) logger.debug(' => ' + content) self.outfile.write(message) self.outfile.flush() def call(self, method: str, params: Dict[str, Any], cb=None): """ @param cb: func. Callback to handle result. If None, turn to sync call. """ mid = self.incMid() if cb is not None: self.queue[mid] = cb contentDict = { "jsonrpc": "2.0", "method": method, "params": params, "id": mid, } # type: Dict[str, Any] self.message(contentDict) if cb is not None: return with self.cv: if not self.cv.wait_for(lambda: self.result is not None, 3): return None result = self.result self.result = None return result def notify(self, method: str, params: Dict[str, Any]) -> None: contentDict = { "jsonrpc": "2.0", "method": method, "params": params, } # type: Dict[str, Any] self.message(contentDict) def serve(self): contentLength = 0 while not self.infile.closed: try: line = self.infile.readline().strip() if line: header, value = line.split(":") if header == "Content-Length": contentLength = int(value) else: content = self.infile.read(contentLength) logger.debug(' <= ' + content) self.handle(json.loads(content)) except Exception as ex: msg = "Error handling server output." self.onError(msg) logger.exception(msg) break def handle(self, message: Dict[str, Any]): if "error" in message: # error if "id" in message: mid = message["id"] del self.queue[mid] self.onError(message["error"]) elif "result" in message: # result mid = message["id"] if isinstance(mid, str): mid = int(mid) result = message["result"] if mid in self.queue: # async call cb = self.queue[mid] del self.queue[mid] cb(result) else: # sync call with self.cv: self.result = result self.cv.notify() elif "method" in message: # request/notification if "id" in message: # request self.onRequest(message) else: self.onNotification(message) else: logger.error('Unknown message.')
class Bouncer: def __init__(self, max_capacity): self.bouncerLock = Lock() self.totalGreeksCapacity = max_capacity self.totalGreeksInside = 0 self.nkegsInside = 0 self.ndogsInside = 0 self.ngirlsInside = 0 self.nkegsWaiting = 0 self.ndogsWaiting = 0 self.ngirlssWaiting = 0 self.kegsCond = Condition(self.bouncerLock) self.dogsCond = Condition(self.bouncerLock) self.girlsCond = Condition(self.bouncerLock) def keg_enter(self): with self.bouncerLock: while (self.totalGreeksInside >= self.totalGreeksCapacity) \ or (self.totalGreeksInside > 0 and ((1.0*self.ndogsInside)/self.totalGreeksInside) > .75): self.nkegsWaiting = self.nkegsWaiting + 1 self.kegsCond.wait() self.nkegsWaiting = self.nkegsWaiting - 1 self.nkegsInside = self.nkegsInside + 1 self.totalGreeksInside = self.totalGreeksInside + 1 def keg_exit(self): with self.bouncerLock: self.nkegsInside = self.nkegsInside - 1 self.totalGreeksInside = self.totalGreeksInside - 1 if self.ndogsWaiting > 0 and (self.totalGreeksInside==0 \ or(((1.0*self.ndogsInside)/self.totalGreeksInside) > .75)): self.dogsCond.notify() elif self.ngirlssWaiting > 0: self.girlsCond.notify() elif self.nkegsWaiting > 0: self.kegsCond.notify() def dog_enter(self): with self.bouncerLock: while (self.totalGreeksInside >= self.totalGreeksCapacity) \ or (self.totalGreeksInside>0 and ((1.0*self.nkegsInside)/self.totalGreeksInside) > .50): self.ndogsWaiting = self.ndogsWaiting + 1 self.dogsCond.wait() self.ndogsWaiting = self.ndogsWaiting - 1 self.ndogsInside = self.ndogsInside + 1 self.totalGreeksInside = self.totalGreeksInside + 1 def dog_exit(self): with self.bouncerLock: self.ndogsInside = self.ndogsInside - 1 self.totalGreeksInside = self.totalGreeksInside - 1 if self.nkegsWaiting > 0 and (self.totalGreeksInside==0 \ or (((1.0*self.ndogsInside)/self.totalGreeksInside) > .75)): self.kegsCond.notify() elif self.ngirlssWaiting > 0: self.girlsCond.notify() elif self.ndogsWaiting > 0: self.dogsCond.notify() def girl_enter(self): with self.bouncerLock: while (self.totalGreeksInside >= self.totalGreeksCapacity) \ or (((1.0*self.totalGreeksInside)/self.totalGreeksCapacity)<.10): self.ngirlssWaiting = self.ngirlssWaiting + 1 self.girlsCond.wait() self.ngirlssWaiting = self.ngirlssWaiting - 1 self.ngirlsInside = self.ngirlsInside + 1 self.totalGreeksInside = self.totalGreeksInside + 1 def girl_exit(self): with self.bouncerLock: self.ngirlsInside = self.ngirlsInside - 1 self.totalGreeksInside = self.totalGreeksInside - 1 if self.nkegsWaiting > 0: self.kegsCond.notify() elif self.ndogsWaiting > 0: self.dogsCond.notify() elif self.ngirlssWaiting > 0: self.girlsCond.notify()
class EVNotify: """ Interface to EVNotify. """ def __init__(self, config, car): self._log = logging.getLogger("EVNotiPi/EVNotify") self._log.info("Initializing EVNotify") self._car = car self._config = config self._poll_interval = config['interval'] self._running = False self._thread = None self._data = [] self._gps_data = [] self._data_lock = Condition() def start(self): """ Start submit thread. """ self._running = True self._thread = Thread(target=self.submit_data, name="EVNotiPi/EVNotify") self._thread.start() self._car.register_data(self.data_callback) def stop(self): """ Stop submit thread. """ self._car.unregister_data(self.data_callback) self._running = False with self._data_lock: self._data_lock.notify() self._thread.join() def data_callback(self, data): """ Callback to be called from 'car'. """ self._log.debug("Enqeue...") with self._data_lock: self._data.append(data) self._data_lock.notify() def submit_data(self): """ Thread that submits data to EVNotify in regular intervals. """ log = self._log evn = EVNotifyAPI.EVNotify(self._config['akey'], self._config['token']) abort_notification = ARMED charging_start_soc = 0 last_charging = time() last_charging_soc = 0 last_evn_settings_poll = 0 is_charging = 0 is_connected = 0 settings = None soc_notification = ARMED soc_threshold = self._config.get('soc_threshold', 100) log.info("Get settings from backend") while self._running and settings is None: try: settings = evn.getSettings() except EVNotifyAPI.CommunicationError as err: log.info("Waiting for network connectivity (%s)", err) sleep(3) while self._running: with self._data_lock: log.debug('Waiting...') self._data_lock.wait(max(10, self._poll_interval)) now = time() # Detect aborted charge if ((now - last_charging > ABORT_NOTIFICATION_INTERVAL and charging_start_soc > 0 and 0 < last_charging_soc < soc_threshold and abort_notification is ARMED) or abort_notification is PENDING): log.info( "Aborted charge detected, send abort notification now-last_charging(%i) charging_start_soc(%i) last_charging_soc(%i) soc_threshold(%i) abort_notification(%i)", now - last_charging, charging_start_soc, last_charging_soc, soc_threshold, abort_notification) try: evn.sendNotification(True) abort_notification = SENT except EVNotifyAPI.CommunicationError as err: log.error("Communication Error: %s", err) abort_notification = PENDING if len(self._data) == 0: continue new_data = self._data.copy() self._data.clear() log.debug("Transmit...") avgs = { 'dcBatteryCurrent': [], 'dcBatteryPower': [], 'dcBatteryVoltage': [], 'speed': [], 'latitude': [], 'longitude': [], 'altitude': [], } for data in new_data: for key, values in avgs.items(): if data.get(key, None) is not None: values.append(data[key]) # Need to copy data here because we update it later data = new_data[-1] data.update( {k: sum(v) / len(v) for k, v in avgs.items() if len(v) > 0}) try: if (data['SOC_DISPLAY'] is not None or data['SOC_BMS'] is not None): current_soc = data['SOC_DISPLAY'] or data['SOC_BMS'] is_charging = bool(data['charging']) is_connected = bool(data['normalChargePort'] or data['rapidChargePort']) if is_charging: last_charging = now last_charging_soc = current_soc evn.setSOC(data['SOC_DISPLAY'], data['SOC_BMS']) extended_data = { a: round(data[a], EXTENDED_FIELDS[a]) for a in EXTENDED_FIELDS if data[a] is not None } log.debug(extended_data) evn.setExtended(extended_data) if data['fix_mode'] > 1 and not is_charging and not is_connected: location = { a: data[a] for a in ('latitude', 'longitude', 'speed') } evn.setLocation({'location': location}) # Notification handling from here on if is_charging and now - last_evn_settings_poll > EVN_SETTINGS_INTERVAL: try: settings = evn.getSettings() last_evn_settings_poll = now if 'soc' in settings: new_soc = int(settings['soc']) if new_soc != soc_threshold: soc_threshold = new_soc log.info("New notification threshold: %i", soc_threshold) except EVNotifyAPI.CommunicationError as err: log.error("Communication error occured %s", err) # track charging started if is_charging and charging_start_soc == 0: charging_start_soc = current_soc or 0 elif not is_connected: # Rearm abort notification charging_start_soc = 0 abort_notification = ARMED # SoC threshold notification if ((is_charging and 0 < last_charging_soc < soc_threshold <= current_soc) or soc_notification is PENDING): log.info("Notification threshold(%i) reached: %i", soc_threshold, current_soc) try: evn.sendNotification() soc_notification = ARMED except EVNotifyAPI.CommunicationError as err: log.info("Communication Error: %s", err) soc_notification = PENDING except EVNotifyAPI.CommunicationError as err: log.info("Communication Error: %s", err) # Prime next loop iteration if self._running: interval = self._poll_interval - (time() - now) sleep(max(0, interval)) def check_thread(self): """ Return running state of thread. """ return self._thread.is_alive()
class SrSender: def __init__(self, addr): self.__sock = socket(AF_INET, SOCK_DGRAM) self.__sock.connect(addr) self.__cache = [] self.__timer = [] self.__timer_lock = [] self.__wait_send = [] self.__win_size = 10 self.__interval = 4. self.__notify_sender = Condition() # 用于控制对缓存的访问 self.__lock = Lock() self.__seq = 0 self.__min_seq = 0 self.__SEQ_SIZE = 255 t = Thread(target=self.__ack_thread) t.start() t = Thread(target=self.__send_thread) t.start() def __get_seq(self): self.__seq += 1 if self.__seq == self.__SEQ_SIZE: self.__seq = 0 return self.__seq def __find_index(self, seq): """ 通过seq查找index,在查找过程中会获得缓存的锁。 :param seq: seq :return: 如果查找到了返回True,否则返回False """ self.__lock.acquire() for i in range(len(self.__cache)): if self.__cache[i][1] == seq: self.__lock.release() return i self.__lock.release() return None def __send_thread(self): """ 包发送线程,被发送方法send唤醒之后发送待发送缓存中数据(不一定是所有),并将数据发乳到窗口中。 :return: """ self.__notify_sender.acquire() while True: while len(self.__wait_send) == 0: self.__notify_sender.wait() num_send = min(len(self.__wait_send), self.__win_size - len(self.__cache)) for i in range(num_send): # 构造分组 seq = self.__get_seq() data = bytes([0, seq]) + len(self.__wait_send[i]).to_bytes(4, 'big') + self.__wait_send[i] # 计算校验和 data += (lib.checksum(data) ^ 0xffff).to_bytes(2, 'big') assert lib.checksum(data) == 0xffff self.__sock.send(data) self.__lock.acquire() self.__cache.append(data) t = Timer(4, self.__timer_thread, args=(seq,)) self.__timer.append(t) self.__timer_lock.append(Lock()) self.__lock.release() t.start() for i in range(num_send): self.__wait_send.pop(0) def send(self, data): """ 向用户提供的API接口,用于发送数据 :param data: 需要发送的数据,应该是bytes类型的; :return: """ if len(data) % 2 == 1: data += b'\x00' self.__wait_send.append(data) self.__notify_sender.acquire() self.__notify_sender.notify() self.__notify_sender.release() def __timer_thread(self, seq): """ 每一个数据包的定时器线程,在函数执行的过程中会获取数据包的锁。 :param seq: 数据包的seq :return: 无 """ index = self.__find_index(seq) if index is None: return lock = self.__timer_lock[index] lock.acquire() self.__sock.send(self.__cache[index]) timer = Timer(self.__interval, self.__timer_thread, args=(seq, )) timer.start() self.__timer[index] = timer lock.release() def __ack_thread(self): while True: data = self.__sock.recv(1024) seq = data[1] index = self.__find_index(seq) if index is None: continue lock = self.__timer_lock[index] lock.acquire() timer = self.__timer[index] # timer 是 None 说明其ack在之前已经收到了 if timer is None: continue timer.cancel() self.__timer[index] = None lock.release() self.__lock.acquire() while len(self.__timer) > 0 and self.__timer[0] is None: self.__timer.pop(0) self.__cache.pop(0) self.__timer_lock.pop(0) self.__lock.release()
class worker(object): """ serialize main thread load/unload of PLC shared objects """ def __init__(self): # Only one job at a time self._finish = False self._threadID = None self.mutex = Lock() self.todo = Condition(self.mutex) self.done = Condition(self.mutex) self.free = Condition(self.mutex) self.job = None self.enabled = False def reraise(self, job): """ reraise exception happend in a job @param job: job where original exception happend """ exc_type = job.exc_info[0] exc_value = job.exc_info[1] exc_traceback = job.exc_info[2] six.reraise(exc_type, exc_value, exc_traceback) def runloop(self, *args, **kwargs): """ meant to be called by worker thread (blocking) """ self._threadID = _thread.get_ident() self.mutex.acquire() self.enabled = True if args or kwargs: _job = job(*args, **kwargs) _job.do() # _job.success can't be None after do() if not _job.success: self.reraise(_job) while not self._finish: self.todo.wait() if self.job is not None: self.job.do() self.done.notify() else: break self.mutex.release() def call(self, *args, **kwargs): """ creates a job, execute it in worker thread, and deliver result. if job execution raise exception, re-raise same exception meant to be called by non-worker threads, but this is accepted. blocking until job done """ _job = job(*args, **kwargs) if self._threadID == _thread.get_ident(): # if caller is worker thread execute immediately _job.do() else: # otherwise notify and wait for completion self.mutex.acquire() if not self.enabled: self.mutex.release() raise EOFError("Worker is disabled") while self.job is not None: self.free.wait() self.job = _job self.todo.notify() self.done.wait() self.job = None self.free.notify() self.mutex.release() if _job.success is None: raise EOFError("Worker job was interrupted") if _job.success: return _job.result else: self.reraise(_job) def quit(self): """ unblocks main thread, and terminate execution of runloop() """ # mark queue self._finish = True self.mutex.acquire() self.enabled = False self.job = None self.todo.notify() self.done.notify() self.mutex.release()
class RndUploader: #Esta clase tiene un único atributo que es enable y #se inicializa como True. Miestras tenga este valor #se seguiran subiendo datos cada 2 minutos. #cuando se cambie su valor a Flase parara. def __init__(self, flaskApp, handSQL=None, handBee=None, handMongo=None,\ tiempoSleep = 120, debug = False,handSSE=None): #self.__enable -> __enable es privado gracias a '__' #para acceder al valor de enable utilizaremos #self.__enable.value self.__enable = Value('b', True) #modo debug self.__debug = True #tiempo a esperar entre inserciones self.__tiempo = tiempoSleep #Creo instancias PRIVADAS de las clases a utilizar #para obtener los numeros aleatorios self.__RndGen = web_fetcher.rnd_fetcher.Rnd_fetcher() #manejar BBDD #self.__SQLHand = sql_rnd.SQLHandler(flaskApp) #self.__BeeHand = beebotte_rnd.BeeHandler() if handSQL: self.__SQLHand = handSQL else: logging.info("SQLHandler - Creando instancia Propia") self.__SQLHand = sql_rnd.SQLHandler(flaskApp) if handBee: self.__BeeHand = handBee else: logging.info("BeeHandler - Creando instancia Propia") self.__BeeHand = beebotte_rnd.BeeHandler() #MongoClient opened before fork. Create MongoClient only after forking. #See PyMongo's documentation for details: #http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe #No es seguro que esta clase reciba la instancia de MongoDB del main #Por lo que creare en esta clase mi propia instancia de MongoHandler #Esto no debería importar ya que aunque sean dos instancias distintas #leerán de la misma base de datos. #La instancia en esta clase se dedicará principalmente a escribir y la #que está en el main a leer. # #Debo crearla una vez dentro del subproceso creado, es decir, #dentro de la función upload que es la que ejecuta el proceso. #UPDATE: ya no es necesario, ya que ahora rnd_uploader crea un hilo #en vez de un proceso, por lo que comparten memoria y no hace fork() #de esta instancia recibida de MongoHandler. #self.__MongoHand= handMongo if handMongo: self.__MongoHand = handMongo else: logging.info("MongoHandler - Creando instancia Propia") self.__MongoHand = mongo_rnd.MongoHandler() #self.__MongoHand= mongo_rnd.MongoHandler() #Manejador de SSE self.__SSEHand = handSSE if handSSE: self.__SSEHand = handSSE else: logging.info("SSEHandler - Creando instancia Propia") self.__SSEHand = SSEHandler() #inicio proceso para subir los datos a las BBDD self.lanzar() #Devuelve el manejador de MySQL def getSQLHandler(self): return self.__SQLHand #Devuelve el manejador de Beebotte def getBeeHandler(self): return self.__BeeHand #Devuelve el manejador de MongoDB def getMongoHandler(self): return self.__MongoHand #Subira un número aleatorio cada 2 min def upload(self): #MongoClient opened before fork. Create MongoClient only after forking. #See PyMongo's documentation for details: #http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe #No es seguro que esta clase reciba la instancia de MongoDB del main #Por lo que creare en esta clase mi propia instancia de MongoHandler #Esto no debería importar ya que aunque sean dos instancias distintas #leerán de la misma base de datos. #La instancia en esta clase se dedicará principalmente a escribir y la #que está en el main a leer. # #Debo crearla una vez dentro del subproceso creado, es decir, #dentro de la función upload que es la que ejecuta el proceso. #UPDATE: Ya no es necesario, porque la funcion upload() se #lanza mediante un hilo y no un proceso, por lo que ya no #se realiza ningún fork() que es sobre lo que va este aviso. #self.__MongoHand= mongo_rnd.MongoHandler() #Obtengo la condición de forma que este hilo pueda utlizar #cond.wait() para esperar. self.cond.acquire() while self.__enable.value: #obtenemos numero aleatorio a insertar rnd = self.__RndGen.get_web_rnd() if self.__debug: logging.debug("num aleatorio a escribir: " + str(rnd)) """ #BORRA ESTO!----------------------------- #Este trozo de codigo sirve para que esta #clase no suba numeros. logging.warning("SUBIDA DE NUMEROS DESACTIVADA!!!") self.__enable.value = False self.__SQLHand.readDataDB() self.__BeeHand.readRandom() self.__MongoHand.readRandom() #BORRA ESTO!----------------------------- """ """ if self.__debug: logging.debug("rnd_uploader - Las listas en rnd_uploader: ") logging.debug("BeeHandler : " + str(self.__BeeHand.listaGlobalNumero)) logging.debug("SQLHandler : " + str(self.__SQLHand.listaGlobalNumero)) logging.debug("MongoHandler : " + str(self.__MongoHand.listaGlobalNumero)) """ #Escribir if (self.__enable.value): #La funcion de obtener numeros aleatorios #devuelve -1 en caso de no haber podido conectar #con la web de donde obtenemos los numeros aleatorios. #En este caso, rnd sera = a -1, por lo que no guardaremos #este valor en las BDs. if rnd > -1: #escribo en Beebotte. 0 si bien. 1 si mal. resBee = self.__BeeHand.writeRandom(rnd, self.__debug) #solo necesario para la BD local, ya que Beebotte #almacena automaticamente la fecha fecha = str(date_handler.getDatetimeMs()) #escribo en MongoDB. 0 si bien. 1 si mal. resMongo = self.__MongoHand.writeRandom(rnd, fecha) #escribo en MySQL. 0 si bien. 1 si mal. resSQL = self.__SQLHand.writeDataDB( rnd, fecha, self.__debug) #--- #envio SSE (notificación a los clientes con #el número obtenido) #FORMATO SSE (parseado mediante js en el cliente) #NUM,FECHA#BD1,BD2,BD3, msg = str(rnd) + "," + str(fecha) + "#" if resBee == 0: msg += "Beebotte," if resMongo == 0: msg += "MongoDB," if resSQL == 0: msg += "MySQL," res = self.__SSEHand.createSSE(str(msg)) #if self.__debug: if True: logging.info("ENVIANDO SSE: " + str(msg)) logging.debug(res) """ #ACTUALIZO LOS DATOS EN LAS LISTAS LOCALES #DE LOS MANEJADORES self.__SQLHand.readDataDB() self.__BeeHand.readRandom() self.__MongoHand.readRandom() """ """ if self.__debug: logging.debug("Tablas MySQL:") logging.debug(self.__SQLHand.listaGlobalFecha) logging.debug(self.__SQLHand.listaGlobalNumero) logging.debug("Tablas Bee:") logging.debug(self.__BeeHand.listaGlobalFecha) logging.debug(self.__BeeHand.listaGlobalNumero) logging.debug("Tablas Mongo:") logging.debug(self.__MongoHand.listaGlobalFecha) logging.debug(self.__MongoHand.listaGlobalNumero) """ #esperar entre escrituras try: #time.sleep(self.__tiempo) #Utilizo la condicion para esperar #el tiempo entre escrituras. #Si la funcion finalizar() llama #al método cond.notify() mientras #hago wait(), la espera se #interrumpe, al contrario que con #time.sleep. self.cond.wait(self.__tiempo) except: if self.__debug: logging.debug("Uploader.upload(): sleep interrumpido!") #Libero la condicion antes de salir. self.cond.release() logging.info("Uploader.upload(): saliendo...") #Función que genera un proceso que ejecuta la función #upload() de esta misma clase en segundo plano. def lanzar(self): #proceso que será el uploader. #si solo pasamos un parametro como argumento (args), #tendremos que poner una coma detras de el, que es la #forma de decir que es una tupla de un solo elemento. #Sin esta coma (',') la creación del proceso falla #self.proceso = Process(target=self.upload, args=(self.__debug,) ) #self.proceso = Process(target=self.upload) #Lo utilizara el proceso para esperar en vez de time.sleep() self.cond = Condition() self.proceso = Thread(target=self.upload) #inicio proceso logging.info("Soy un hilo (RndUploader.upload()).") self.proceso.start() #Marca y espera que los procesos en segundo plano terminen su ejecución. def finalizar(self): estavivo = self.proceso.isAlive() logging.info("PROCESO VIVO: " + str(estavivo)) if estavivo: logging.debug("PARANDO") #Obtengo la condicion de forma que #pueda utilizar el método notify() self.cond.acquire() #Hago notify de forma que si el hilo #de lanzar() esta esperando con wait(), #interrumpa su espera para terminar self.cond.notify() #Libero la condicion self.cond.release() self.__enable.value = False if self.__debug: logging.info("Bandera activada para finalizar: " + str(self.__enable.value)) #Si el proceso no esta vivo no espero #pues bloquerá el programa if estavivo: logging.info("Esperando para acabar - thread.join()") self.proceso.join() if self.__debug: logging.info("el proceso ya ha acabado")
class HostConnectionPool(object): """ Used to pool connections to a host for v1 and v2 native protocol. """ host = None host_distance = None is_shutdown = False open_count = 0 _scheduled_for_creation = 0 _next_trash_allowed_at = 0 _keyspace = None def __init__(self, host, host_distance, session): self.host = host self.host_distance = host_distance self._session = weakref.proxy(session) self._lock = RLock() self._conn_available_condition = Condition() log.debug("Initializing new connection pool for host %s", self.host) core_conns = session.cluster.get_core_connections_per_host( host_distance) self._connections = [ session.cluster.connection_factory(host.endpoint) for i in range(core_conns) ] self._keyspace = session.keyspace if self._keyspace: for conn in self._connections: conn.set_keyspace_blocking(self._keyspace) self._trash = set() self._next_trash_allowed_at = time.time() self.open_count = core_conns log.debug("Finished initializing new connection pool for host %s", self.host) def borrow_connection(self, timeout): if self.is_shutdown: raise ConnectionException( "Pool for %s is shutdown" % (self.host, ), self.host) conns = self._connections if not conns: # handled specially just for simpler code log.debug("Detected empty pool, opening core conns to %s", self.host) core_conns = self._session.cluster.get_core_connections_per_host( self.host_distance) with self._lock: # we check the length of self._connections again # along with self._scheduled_for_creation while holding the lock # in case multiple threads hit this condition at the same time to_create = core_conns - (len(self._connections) + self._scheduled_for_creation) for i in range(to_create): self._scheduled_for_creation += 1 self._session.submit(self._create_new_connection) # in_flight is incremented by wait_for_conn conn = self._wait_for_conn(timeout) return conn else: # note: it would be nice to push changes to these config settings # to pools instead of doing a new lookup on every # borrow_connection() call max_reqs = self._session.cluster.get_max_requests_per_connection( self.host_distance) max_conns = self._session.cluster.get_max_connections_per_host( self.host_distance) least_busy = min(conns, key=lambda c: c.in_flight) request_id = None # to avoid another thread closing this connection while # trashing it (through the return_connection process), hold # the connection lock from this point until we've incremented # its in_flight count need_to_wait = False with least_busy.lock: if least_busy.in_flight < least_busy.max_request_id: least_busy.in_flight += 1 request_id = least_busy.get_request_id() else: # once we release the lock, wait for another connection need_to_wait = True if need_to_wait: # wait_for_conn will increment in_flight on the conn least_busy, request_id = self._wait_for_conn(timeout) # if we have too many requests on this connection but we still # have space to open a new connection against this host, go ahead # and schedule the creation of a new connection if least_busy.in_flight >= max_reqs and len( self._connections) < max_conns: self._maybe_spawn_new_connection() return least_busy, request_id def _maybe_spawn_new_connection(self): with self._lock: if self._scheduled_for_creation >= _MAX_SIMULTANEOUS_CREATION: return if self.open_count >= self._session.cluster.get_max_connections_per_host( self.host_distance): return self._scheduled_for_creation += 1 log.debug("Submitting task for creation of new Connection to %s", self.host) self._session.submit(self._create_new_connection) def _create_new_connection(self): try: self._add_conn_if_under_max() except (ConnectionException, socket.error) as exc: log.warning("Failed to create new connection to %s: %s", self.host, exc) except Exception: log.exception("Unexpectedly failed to create new connection") finally: with self._lock: self._scheduled_for_creation -= 1 def _add_conn_if_under_max(self): max_conns = self._session.cluster.get_max_connections_per_host( self.host_distance) with self._lock: if self.is_shutdown: return True if self.open_count >= max_conns: return True self.open_count += 1 log.debug("Going to open new connection to host %s", self.host) try: conn = self._session.cluster.connection_factory(self.host.endpoint) if self._keyspace: conn.set_keyspace_blocking(self._session.keyspace) self._next_trash_allowed_at = time.time() + _MIN_TRASH_INTERVAL with self._lock: new_connections = self._connections[:] + [conn] self._connections = new_connections log.debug( "Added new connection (%s) to pool for host %s, signaling availablility", id(conn), self.host) self._signal_available_conn() return True except (ConnectionException, socket.error) as exc: log.warning("Failed to add new connection to pool for host %s: %s", self.host, exc) with self._lock: self.open_count -= 1 if self._session.cluster.signal_connection_failure( self.host, exc, is_host_addition=False): self.shutdown() return False except AuthenticationFailed: with self._lock: self.open_count -= 1 return False def _await_available_conn(self, timeout): with self._conn_available_condition: self._conn_available_condition.wait(timeout) def _signal_available_conn(self): with self._conn_available_condition: self._conn_available_condition.notify() def _signal_all_available_conn(self): with self._conn_available_condition: self._conn_available_condition.notify_all() def _wait_for_conn(self, timeout): start = time.time() remaining = timeout while remaining > 0: # wait on our condition for the possibility that a connection # is useable self._await_available_conn(remaining) # self.shutdown() may trigger the above Condition if self.is_shutdown: raise ConnectionException("Pool is shutdown") conns = self._connections if conns: least_busy = min(conns, key=lambda c: c.in_flight) with least_busy.lock: if least_busy.in_flight < least_busy.max_request_id: least_busy.in_flight += 1 return least_busy, least_busy.get_request_id() remaining = timeout - (time.time() - start) raise NoConnectionsAvailable() def return_connection(self, connection): with connection.lock: connection.in_flight -= 1 in_flight = connection.in_flight if connection.is_defunct or connection.is_closed: if not connection.signaled_error: log.debug( "Defunct or closed connection (%s) returned to pool, potentially " "marking host %s as down", id(connection), self.host) is_down = self._session.cluster.signal_connection_failure( self.host, connection.last_error, is_host_addition=False) connection.signaled_error = True if is_down: self.shutdown() else: self._replace(connection) else: if connection in self._trash: with connection.lock: if connection.in_flight == 0: with self._lock: if connection in self._trash: self._trash.remove(connection) log.debug("Closing trashed connection (%s) to %s", id(connection), self.host) connection.close() return core_conns = self._session.cluster.get_core_connections_per_host( self.host_distance) min_reqs = self._session.cluster.get_min_requests_per_connection( self.host_distance) # we can use in_flight here without holding the connection lock # because the fact that in_flight dipped below the min at some # point is enough to start the trashing procedure if len(self._connections) > core_conns and in_flight <= min_reqs and \ time.time() >= self._next_trash_allowed_at: self._maybe_trash_connection(connection) else: self._signal_available_conn() def _maybe_trash_connection(self, connection): core_conns = self._session.cluster.get_core_connections_per_host( self.host_distance) did_trash = False with self._lock: if connection not in self._connections: return if self.open_count > core_conns: did_trash = True self.open_count -= 1 new_connections = self._connections[:] new_connections.remove(connection) self._connections = new_connections with connection.lock: if connection.in_flight == 0: log.debug( "Skipping trash and closing unused connection (%s) to %s", id(connection), self.host) connection.close() # skip adding it to the trash if we're already closing it return self._trash.add(connection) if did_trash: self._next_trash_allowed_at = time.time() + _MIN_TRASH_INTERVAL log.debug("Trashed connection (%s) to %s", id(connection), self.host) def _replace(self, connection): should_replace = False with self._lock: if connection in self._connections: new_connections = self._connections[:] new_connections.remove(connection) self._connections = new_connections self.open_count -= 1 should_replace = True if should_replace: log.debug("Replacing connection (%s) to %s", id(connection), self.host) connection.close() self._session.submit(self._retrying_replace) else: log.debug("Closing connection (%s) to %s", id(connection), self.host) connection.close() def _retrying_replace(self): replaced = False try: replaced = self._add_conn_if_under_max() except Exception: log.exception("Failed replacing connection to %s", self.host) if not replaced: log.debug("Failed replacing connection to %s. Retrying.", self.host) self._session.submit(self._retrying_replace) def shutdown(self): with self._lock: if self.is_shutdown: return else: self.is_shutdown = True self._signal_all_available_conn() for conn in self._connections: conn.close() self.open_count -= 1 for conn in self._trash: conn.close() def ensure_core_connections(self): if self.is_shutdown: return core_conns = self._session.cluster.get_core_connections_per_host( self.host_distance) with self._lock: to_create = core_conns - (len(self._connections) + self._scheduled_for_creation) for i in range(to_create): self._scheduled_for_creation += 1 self._session.submit(self._create_new_connection) def _set_keyspace_for_all_conns(self, keyspace, callback): """ Asynchronously sets the keyspace for all connections. When all connections have been set, `callback` will be called with two arguments: this pool, and a list of any errors that occurred. """ remaining_callbacks = set(self._connections) errors = [] if not remaining_callbacks: callback(self, errors) return def connection_finished_setting_keyspace(conn, error): self.return_connection(conn) remaining_callbacks.remove(conn) if error: errors.append(error) if not remaining_callbacks: callback(self, errors) self._keyspace = keyspace for conn in self._connections: conn.set_keyspace_async(keyspace, connection_finished_setting_keyspace) def get_connections(self): return self._connections def get_state(self): in_flights = [c.in_flight for c in self._connections] return { 'shutdown': self.is_shutdown, 'open_count': self.open_count, 'in_flights': in_flights }
class Dispatcher(InstrumentedThread): def __init__(self, timeout=10): super().__init__(name='Dispatcher') self._timeout = timeout self._msg_type_handlers = {} self._in_queue = queue.PriorityQueue() self._send_message = {} self._send_last_message = {} self._message_information = {} self._condition = Condition() self._dispatch_timers = {} self._priority = {} def _get_dispatch_timer(self, tag): if tag not in self._dispatch_timers: self._dispatch_timers[tag] = COLLECTOR.timer( 'dispatch_execution_time', tags={"handler": tag}, instance=self) return self._dispatch_timers[tag] def add_send_message(self, connection, send_message): """Adds a send_message function to the Dispatcher's dictionary of functions indexed by connection. Args: connection (str): A locally unique identifier provided by the receiver of messages. send_message (fn): The method that should be called by the dispatcher to respond to messages which arrive via connection. """ self._send_message[connection] = send_message LOGGER.debug("Added send_message function " "for connection %s", connection) def add_send_last_message(self, connection, send_last_message): """Adds a send_last_message function to the Dispatcher's dictionary of functions indexed by connection. Args: connection (str): A locally unique identifier provided by the receiver of messages. send_last_message (fn): The method that should be called by the dispatcher to respond to messages which arrive via connection, when the connection should be closed after the message has been sent. """ self._send_last_message[connection] = send_last_message LOGGER.debug("Added send_last_message function " "for connection %s", connection) def remove_send_message(self, connection): """Removes a send_message function previously registered with the Dispatcher. Args: connection (str): A locally unique identifier provided by the receiver of messages. """ if connection in self._send_message: del self._send_message[connection] LOGGER.debug("Removed send_message function " "for connection %s", connection) else: LOGGER.warning( "Attempted to remove send_message " "function for connection %s, but no " "send_message function was registered", connection) def remove_send_last_message(self, connection): """Removes a send_last_message function previously registered with the Dispatcher. Args: connection (str): A locally unique identifier provided by the receiver of messages. """ if connection in self._send_last_message: del self._send_last_message[connection] LOGGER.debug( "Removed send_last_message function " "for connection %s", connection) else: LOGGER.warning( "Attempted to remove send_last_message " "function for connection %s, but no " "send_last_message function was registered", connection) def dispatch(self, connection, message, connection_id): if message.message_type in self._msg_type_handlers: priority = self._priority.get(message.message_type, Priority.LOW) message_id = _gen_message_id() self._message_information[message_id] = ( connection, connection_id, message, _ManagerCollection( self._msg_type_handlers[message.message_type])) self._in_queue.put_nowait((priority, message_id)) queue_size = self._in_queue.qsize() if queue_size > 10: LOGGER.debug("Dispatch incoming queue size: %s", queue_size) else: LOGGER.info( "received a message of type %s " "from %s but have no handler for that type", get_enum_name(message.message_type), connection_id) def add_handler(self, message_type, handler, executor, priority=None): if not isinstance(handler, Handler): raise TypeError("%s is not a Handler subclass" % handler) if message_type not in self._msg_type_handlers: self._msg_type_handlers[message_type] = [ _HandlerManager(executor, handler) ] else: self._msg_type_handlers[message_type].append( _HandlerManager(executor, handler)) if priority is not None: self._priority[message_type] = priority def set_message_priority(self, message_type, priority): self._priority[message_type] = priority def _process(self, message_id): _, connection_id, \ message, collection = self._message_information[message_id] try: handler_manager = next(collection) except IndexError: # IndexError is raised if done with handlers del self._message_information[message_id] return timer_tag = type(handler_manager.handler).__name__ timer_ctx = self._get_dispatch_timer(timer_tag).time() def do_next(result): timer_ctx.stop() try: self._determine_next(message_id, result) except Exception: # pylint: disable=broad-except LOGGER.exception("Unhandled exception while determining next") handler_manager.execute(connection_id, message.content, do_next) def _determine_next(self, message_id, result): if result is None: LOGGER.debug('Ignoring None handler result, likely due to an ' 'unhandled error while executing the handler') return if result.status == HandlerStatus.DROP: del self._message_information[message_id] elif result.status == HandlerStatus.PASS: self._process(message_id) elif result.status == HandlerStatus.RETURN_AND_PASS: connection, connection_id, \ original_message, _ = self._message_information[message_id] if result.message_out and result.message_type: message = validator_pb2.Message( content=result.message_out.SerializeToString(), correlation_id=original_message.correlation_id, message_type=result.message_type) try: self._send_message[connection](msg=message, connection_id=connection_id) except KeyError: LOGGER.warning( "Can't send message %s back to " "%s because connection %s not in dispatcher", get_enum_name(message.message_type), connection_id, connection) self._process(message_id) else: LOGGER.error("HandlerResult with status of RETURN_AND_PASS " "is missing message_out or message_type") elif result.status == HandlerStatus.RETURN: connection, connection_id, \ original_message, _ = self._message_information[message_id] del self._message_information[message_id] if result.message_out and result.message_type: message = validator_pb2.Message( content=result.message_out.SerializeToString(), correlation_id=original_message.correlation_id, message_type=result.message_type) try: self._send_message[connection](msg=message, connection_id=connection_id) except KeyError: LOGGER.warning( "Can't send message %s back to " "%s because connection %s not in dispatcher", get_enum_name(message.message_type), connection_id, connection) else: LOGGER.error("HandlerResult with status of RETURN " "is missing message_out or message_type") elif result.status == HandlerStatus.RETURN_AND_CLOSE: connection, connection_id, \ original_message, _ = self._message_information[message_id] del self._message_information[message_id] if result.message_out and result.message_type: message = validator_pb2.Message( content=result.message_out.SerializeToString(), correlation_id=original_message.correlation_id, message_type=result.message_type) try: LOGGER.warning( "Sending hang-up in reply to %s to connection %s", get_enum_name(original_message.message_type), connection_id) self._send_last_message[connection]( msg=message, connection_id=connection_id) except KeyError: LOGGER.warning( "Can't send last message %s back to " "%s because connection %s not in dispatcher", get_enum_name(message.message_type), connection_id, connection) else: LOGGER.error("HandlerResult with status of RETURN_AND_CLOSE " "is missing message_out or message_type") with self._condition: if not self._message_information: self._condition.notify() def run(self): while True: try: _, msg_id = self._in_queue.get() if msg_id == -1: break self._process(msg_id) except Exception: # pylint: disable=broad-except LOGGER.exception("Unhandled exception while dispatching") def stop(self): self._in_queue.put_nowait((Priority.HIGH, -1)) def block_until_complete(self): """Blocks until no more messages are in flight, useful for unit tests. """ with self._condition: if self._message_information: self._condition.wait()
class NailgunConnection(object): """Stateful object holding the connection to the Nailgun server.""" def __init__( self, server_name, server_port=None, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, cwd=None, heartbeat_interval_sec=DEFAULT_HEARTBEAT_INTERVAL_SEC, ): self.transport = make_nailgun_transport(server_name, server_port, cwd) self.stdin = stdin self.stdout = stdout self.stderr = stderr self.recv_flags = 0 self.send_flags = 0 self.header_buf = ctypes.create_string_buffer(CHUNK_HEADER_LEN) self.buf = ctypes.create_string_buffer(BUFSIZE) self.exit_code = None self.shutdown_event = Event() self.error_lock = RLock() self.error = None self.error_traceback = None self.stdin_condition = Condition() self.stdin_thread = Thread(target=stdin_thread_main, args=(self, )) self.stdin_thread.daemon = True self.send_queue = Queue.Queue() self.send_condition = Condition() self.send_thread = Thread(target=send_thread_main, args=(self, )) self.send_thread.daemon = True self.heartbeat_interval_sec = heartbeat_interval_sec self.heartbeat_condition = Condition() self.heartbeat_thread = None if heartbeat_interval_sec > 0: self.heartbeat_thread = Thread(target=heartbeat_thread_main, args=(self, )) self.heartbeat_thread.daemon = True def send_command(self, cmd, cmd_args=[], filearg=None, env=os.environ, cwd=os.getcwd()): """ Sends the command and environment to the nailgun server, then loops forever reading the response until the server sends an exit chunk. Returns the exit value, or raises NailgunException on error. """ try: return self._send_command_and_read_response( cmd, cmd_args, filearg, env, cwd) except socket.error as e: re_raise( NailgunException( "Server disconnected unexpectedly: {0}".format(e), NailgunException.CONNECTION_BROKEN, )) def _send_command_and_read_response(self, cmd, cmd_args, filearg, env, cwd): self.stdin_thread.start() self.send_thread.start() try: if filearg: self._send_file_arg(filearg) for cmd_arg in cmd_args: self._send_chunk(cmd_arg, CHUNKTYPE_ARG) self._send_env_var("NAILGUN_FILESEPARATOR", os.sep) self._send_env_var("NAILGUN_PATHSEPARATOR", os.pathsep) self._send_tty_format(self.stdin) self._send_tty_format(self.stdout) self._send_tty_format(self.stderr) for k, v in env.items(): self._send_env_var(k, v) self._send_chunk(cwd, CHUNKTYPE_DIR) self._send_chunk(cmd, CHUNKTYPE_CMD) if self.heartbeat_thread is not None: self.heartbeat_thread.start() while self.exit_code is None: self._process_next_chunk() finally: self.shutdown_event.set() with self.stdin_condition: self.stdin_condition.notify() with self.send_condition: self.send_condition.notify() if self.heartbeat_thread is not None: with self.heartbeat_condition: self.heartbeat_condition.notify() self.heartbeat_thread.join(THREAD_TERMINATION_TIMEOUT_SEC) self.stdin_thread.join(THREAD_TERMINATION_TIMEOUT_SEC) self.send_thread.join(THREAD_TERMINATION_TIMEOUT_SEC) return self.exit_code def _process_next_chunk(self): """ Processes the next chunk from the nailgun server. """ readable, exceptional = self.transport.select( SELECT_MAX_BLOCK_TIME_SEC) if readable: self._process_nailgun_stream() if exceptional: raise NailgunException("Server disconnected in select", NailgunException.CONNECTION_BROKEN) # if daemon thread threw, rethrow here if self.shutdown_event.is_set(): e = None e_tb = None with self.error_lock: e = self.error e_tb = self.error_traceback if e is not None: re_raise(e, e_tb) def _send_chunk(self, buf, chunk_type): """ Send chunk to the server asynchronously """ self.send_queue.put((chunk_type, buf)) with self.send_condition: self.send_condition.notify() def _send_env_var(self, name, value): """ Sends an environment variable in KEY=VALUE format. """ self._send_chunk("=".join((name, value)), CHUNKTYPE_ENV) def _send_tty_format(self, f): """ Sends a NAILGUN_TTY_# environment variable. """ if not f or not hasattr(f, "fileno"): return try: fileno = f.fileno() isatty = os.isatty(fileno) self._send_env_var("NAILGUN_TTY_" + str(fileno), str(int(isatty))) except UnsupportedOperation: return def _send_file_arg(self, filename): """ Sends the contents of a file to the server. """ with open(filename) as f: while True: num_bytes = f.readinto(self.buf) if not num_bytes: break self._send_chunk(self.buf.raw[:num_bytes], CHUNKTYPE_LONGARG) def _recv_to_fd(self, dest_file, num_bytes): """ Receives num_bytes bytes from the nailgun socket and copies them to the specified file object. Used to route data to stdout or stderr on the client. """ bytes_read = 0 while bytes_read < num_bytes: bytes_to_read = min(len(self.buf), num_bytes - bytes_read) bytes_received = self.transport.recv_into(self.buf, bytes_to_read) if dest_file: dest_file.write(bytes_to_str(self.buf[:bytes_received])) bytes_read += bytes_received def _recv_to_buffer(self, num_bytes, buf): """ Receives num_bytes from the nailgun socket and writes them into the specified buffer. """ # We'd love to use socket.recv_into() everywhere to avoid # unnecessary copies, but we need to support Python 2.6. The # only way to provide an offset to recv_into() is to use # memoryview(), which doesn't exist until Python 2.7. if HAS_MEMORYVIEW: self._recv_into_memoryview(num_bytes, compat_memoryview(buf)) else: self._recv_to_buffer_with_copy(num_bytes, buf) def _recv_into_memoryview(self, num_bytes, buf_view): """ Receives num_bytes from the nailgun socket and writes them into the specified memoryview to avoid an extra copy. """ bytes_read = 0 while bytes_read < num_bytes: bytes_received = self.transport.recv_into(buf_view[bytes_read:], num_bytes - bytes_read) if not bytes_received: raise NailgunException( "Server unexpectedly disconnected in recv_into()", NailgunException.CONNECTION_BROKEN, ) bytes_read += bytes_received def _recv_to_buffer_with_copy(self, num_bytes, buf): """ Receives num_bytes from the nailgun socket and writes them into the specified buffer. """ bytes_read = 0 while bytes_read < num_bytes: recv_buf = self.transport.recv(num_bytes - bytes_read) if not len(recv_buf): raise NailgunException( "Server unexpectedly disconnected in recv()", NailgunException.CONNECTION_BROKEN, ) buf[bytes_read:bytes_read + len(recv_buf)] = recv_buf bytes_read += len(recv_buf) def _process_exit(self, exit_len): """ Receives an exit code from the nailgun server and sets nailgun_connection.exit_code to indicate the client should exit. """ num_bytes = min(len(self.buf), exit_len) self._recv_to_buffer(num_bytes, self.buf) self.exit_code = int(self.buf.raw[:num_bytes]) def _send_heartbeat(self): """ Sends a heartbeat to the nailgun server to indicate the client is still alive. """ self._send_chunk("", CHUNKTYPE_HEARTBEAT) def _process_nailgun_stream(self): """ Processes a single chunk from the nailgun server. """ self._recv_to_buffer(len(self.header_buf), self.header_buf) (chunk_len, chunk_type) = struct.unpack_from(">ic", self.header_buf.raw) if chunk_type == CHUNKTYPE_STDOUT: self._recv_to_fd(self.stdout, chunk_len) elif chunk_type == CHUNKTYPE_STDERR: self._recv_to_fd(self.stderr, chunk_len) elif chunk_type == CHUNKTYPE_EXIT: self._process_exit(chunk_len) elif chunk_type == CHUNKTYPE_SENDINPUT: # signal stdin thread to get and send more data with self.stdin_condition: self.stdin_condition.notify() else: raise NailgunException( "Unexpected chunk type: {0}".format(chunk_type), NailgunException.UNEXPECTED_CHUNKTYPE, ) def wait_termination(self, timeout): """ Wait for shutdown event to be signalled within specified interval Return True if termination was signalled, False otherwise """ wait_time = timeout start = monotonic_time_nanos() with self.send_condition: while True: if self.shutdown_event.is_set(): return True self.send_condition.wait(wait_time) elapsed = (monotonic_time_nanos() - start) * 1.0 / NSEC_PER_SEC wait_time = timeout - elapsed if wait_time <= 0: return False return False def __enter__(self): return self def __exit__(self, type, value, traceback): try: self.transport.close() except socket.error: pass
class HostConnection(object): """ When using v3 of the native protocol, this is used instead of a connection pool per host (HostConnectionPool) due to the increased in-flight capacity of individual connections. """ host = None host_distance = None is_shutdown = False shutdown_on_error = False _session = None _connection = None _lock = None _keyspace = None def __init__(self, host, host_distance, session): self.host = host self.host_distance = host_distance self._session = weakref.proxy(session) self._lock = Lock() # this is used in conjunction with the connection streams. Not using the connection lock because the connection can be replaced in the lifetime of the pool. self._stream_available_condition = Condition(self._lock) self._is_replacing = False if host_distance == HostDistance.IGNORED: log.debug("Not opening connection to ignored host %s", self.host) return elif host_distance == HostDistance.REMOTE and not session.cluster.connect_to_remote_hosts: log.debug("Not opening connection to remote host %s", self.host) return log.debug("Initializing connection for host %s", self.host) self._connection = session.cluster.connection_factory(host.endpoint) self._keyspace = session.keyspace if self._keyspace: self._connection.set_keyspace_blocking(self._keyspace) log.debug("Finished initializing connection for host %s", self.host) def borrow_connection(self, timeout): if self.is_shutdown: raise ConnectionException( "Pool for %s is shutdown" % (self.host, ), self.host) conn = self._connection if not conn: raise NoConnectionsAvailable() start = time.time() remaining = timeout while True: with conn.lock: if conn.in_flight <= conn.max_request_id: conn.in_flight += 1 return conn, conn.get_request_id() if timeout is not None: remaining = timeout - time.time() + start if remaining < 0: break with self._stream_available_condition: self._stream_available_condition.wait(remaining) raise NoConnectionsAvailable("All request IDs are currently in use") def return_connection(self, connection): with connection.lock: connection.in_flight -= 1 with self._stream_available_condition: self._stream_available_condition.notify() if connection.is_defunct or connection.is_closed: if connection.signaled_error and not self.shutdown_on_error: return is_down = False if not connection.signaled_error: log.debug( "Defunct or closed connection (%s) returned to pool, potentially " "marking host %s as down", id(connection), self.host) is_down = self._session.cluster.signal_connection_failure( self.host, connection.last_error, is_host_addition=False) connection.signaled_error = True if self.shutdown_on_error and not is_down: is_down = True self._session.cluster.on_down(self.host, is_host_addition=False) if is_down: self.shutdown() else: self._connection = None with self._lock: if self._is_replacing: return self._is_replacing = True self._session.submit(self._replace, connection) def _replace(self, connection): with self._lock: if self.is_shutdown: return log.debug("Replacing connection (%s) to %s", id(connection), self.host) try: conn = self._session.cluster.connection_factory(self.host.endpoint) if self._keyspace: conn.set_keyspace_blocking(self._keyspace) self._connection = conn except Exception: log.warning("Failed reconnecting %s. Retrying." % (self.host.endpoint, )) self._session.submit(self._replace, connection) else: with self._lock: self._is_replacing = False self._stream_available_condition.notify() def shutdown(self): with self._lock: if self.is_shutdown: return else: self.is_shutdown = True self._stream_available_condition.notify_all() if self._connection: self._connection.close() self._connection = None def _set_keyspace_for_all_conns(self, keyspace, callback): if self.is_shutdown or not self._connection: return def connection_finished_setting_keyspace(conn, error): self.return_connection(conn) errors = [] if not error else [error] callback(self, errors) self._keyspace = keyspace self._connection.set_keyspace_async( keyspace, connection_finished_setting_keyspace) def get_connections(self): c = self._connection return [c] if c else [] def get_state(self): connection = self._connection open_count = 1 if connection and not (connection.is_closed or connection.is_defunct) else 0 in_flights = [connection.in_flight] if connection else [] return { 'shutdown': self.is_shutdown, 'open_count': open_count, 'in_flights': in_flights } @property def open_count(self): connection = self._connection return 1 if connection and not (connection.is_closed or connection.is_defunct) else 0
class Queue: """This class implements multi-producer, multi-consumer FIFO queue. The main objective of Queue is to create a list of tasks for asynchronous processing """ def __init__(self, maxsize) -> None: self._maxsize = maxsize self._queue = OrderedDict() self._lock = RLock() self._not_full = Condition(self._lock) self._not_empty = Condition(self._lock) def put(self, id: str, content: Any, timeout: int = 10) -> None: """Put an message into the queue. The 'id' and 'content' values are used to create a new QueueMessage and into the Queue. If the queue is full it will try to put the message for 'timeout' seconds before raise a Full Exception. """ if timeout < 0: raise ValueError if id in self._queue: raise RepeatedMessage with self._not_full: if not self.full() or self._not_full.wait(timeout): queue_message = QueueMessage(id, content) self._queue[id] = queue_message self._not_empty.notify() return raise Full def qsize(self) -> int: """Return the size of the queue.""" with self._lock: return len(self._queue) def full(self) -> bool: """Return True if the queue is full.""" return 0 < self._maxsize <= self.qsize() def get(self, timeout: int = 10, acquire_timeout: int = 10) -> QueueMessage: """Return an message from the queue. This message is 'hidden' from the queue until its deletion or 'acquire_timeout' is reached. 'timeout' arg inform how long the Queue object should try to get the message if the queue is empty before raising the `Empty` error. 'acquire_timeout' arg value inform how long the thread wants to have this message for itself, if the message is not deleted it should be available in the queue after this time. """ if timeout < 0 or acquire_timeout < 0: raise ValueError with self._not_empty: max_timeout = min_timeout = time.monotonic() + timeout while max_timeout > time.monotonic(): if self.qsize() > 0: for id in self._queue: if (self._queue[id].timeout is None or self._queue[id].timeout < time.monotonic()): self._queue[id].timeout = time.monotonic( ) + acquire_timeout return self._queue[id] else: if self._queue[id].timeout < min_timeout: min_timeout = self._queue[id].timeout self._not_empty.wait(min_timeout - time.monotonic()) raise Empty def delete(self, id: str) -> None: """Removes the message from the queue""" with self._lock: if id in self._queue: del self._queue[id] self._not_full.notify() return raise DeleteAttemptToUnknownMessage
class ZtexBoardProxy(Process): def __init__(self, rxconn, txconn, serial, takeover, firmware, pollinterval): super(ZtexBoardProxy, self).__init__() self.rxconn = rxconn self.txconn = txconn self.serial = serial self.takeover = takeover self.firmware = firmware self.pollinterval = pollinterval def run(self): signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) self.lock = RLock() self.wakeup = Condition() self.error = None self.pollingthread = None self.shutdown = False self.job = None self.checklockout = 0 self.lastnonce = 0 self.multiplier = 0 try: # Listen for setup commands while True: data = self.rxconn.recv() if data[0] == "connect": break else: raise Exception("Unknown setup message: %s" % str(data)) # Connect to board and upload firmware if neccessary self.device = ZtexDevice(self, self.serial, self.takeover, self.firmware) # Configure clock self._set_multiplier(self.device.default_multiplier) # Start polling thread self.pollingthread = Thread(None, self.polling_thread, "polling_thread") self.pollingthread.daemon = True self.pollingthread.start() self.send("started_up") # Listen for commands while True: if self.error: raise self.error data = self.rxconn.recv() if data[0] == "shutdown": break elif data[0] == "ping": self.send("pong") elif data[0] == "pong": pass elif data[0] == "set_pollinterval": self.pollinterval = data[1] with self.wakeup: self.wakeup.notify() elif data[0] == "send_job": self.checklockout = time.time() + 1 self.job = data[1] with self.wakeup: start = time.time() self.device.send_job(data[1][64:76] + data[2]) end = time.time() self.lastnonce = 0 self.checklockout = end + 0.5 self.respond(start, end) else: raise Exception("Unknown message: %s" % str(data)) except: self.log("Exception caught: %s" % traceback.format_exc(), 100, "r") finally: self.shutdown = True with self.wakeup: self.wakeup.notify() try: self.pollingthread.join(2) except: pass self.send("dying") def send(self, *args): with self.lock: self.txconn.send(args) def respond(self, *args): self.send("response", *args) def log(self, message, loglevel, format=""): self.send("log", message, loglevel, format) def polling_thread(self): try: lastshares = [] errorcount = [0] * (self.device.maximum_multiplier + 1) errorweight = [0] * (self.device.maximum_multiplier + 1) maxerrorrate = [0] * (self.device.maximum_multiplier + 1) errorlimit = 0.05 errorhysteresis = 0.1 counter = 0 while not self.shutdown: counter += 1 # Poll for nonces now = time.time() nonces = self.device.read_nonces() exhausted = False with self.wakeup: if nonces[0][1] < self.lastnonce: self.lastnonce = nonces[0][1] exhausted = True if exhausted: self.send("keyspace_exhausted") for nonce in nonces: if nonce[0] != -self.device.nonce_offset and not nonce[ 0] in lastshares: if self.job: self.send("nonce_found", time.time(), struct.pack("<I", nonce[0])) lastshares.append(nonce[0]) while len(lastshares) > len(nonces): lastshares.pop(0) # Verify proper operation and adjust clocking if neccessary if now > self.checklockout and self.job: errorcount[self.multiplier] *= 0.995 errorweight[ self. multiplier] = errorweight[self.multiplier] * 0.995 + 1 for nonce in nonces: invalid = True for offset in (0, 1, -1, 2, -2): hash = Job.calculate_hash( self.job[:76] + struct.pack("<I", nonce[1] + offset)) if struct.unpack( "!I", hash[-4:])[0] == (nonce[2] + 0x5be0cd19) & 0xffffffff: invalid = False break if invalid: errorcount[self.multiplier] += 1. / len(nonces) certainty = min(1, errorweight[self.multiplier] / 100) errorrate = errorcount[self.multiplier] / errorweight[ self.multiplier] maxerrorrate[self.multiplier] = max( maxerrorrate[self.multiplier], errorrate * certainty) for i in range(len(maxerrorrate) - 1): if maxerrorrate[i + 1] * i < maxerrorrate[i] * (i + 20): maxerrorrate[i + 1] = maxerrorrate[i] * (1 + 20.0 / i) limit = 0 while limit < self.device.default_multiplier and maxerrorrate[ limit + 1] < errorlimit: limit += 1 while limit < self.device.maximum_multiplier and errorweight[ limit] > 150 and maxerrorrate[limit + 1] < errorlimit: limit += 1 multiplier = 0 best = 0 for i in range(limit + 1): effective = (i + 1 + (errorhysteresis if i == self.multiplier else 0)) * (1 - maxerrorrate[i]) if effective > best: best = effective multiplier = i self._set_multiplier(multiplier) if counter >= 10: counter = 0 try: self.send( "error_rate", errorcount[self.multiplier] / errorweight[self.multiplier]) except: pass with self.wakeup: self.wakeup.wait(self.pollinterval) except Exception as e: self.log("Exception caught: %s" % traceback.format_exc(), 100, "r") self.error = e # Unblock main thread self.send("ping") def _set_multiplier(self, multiplier): multiplier = min(max(multiplier, 1), self.device.maximum_multiplier) if multiplier == self.multiplier: return self.device.set_multiplier(multiplier) self.multiplier = multiplier self.checklockout = time.time() + 2 self.send("speed_changed", (multiplier + 1) * self.device.base_frequency * self.device.hashes_per_clock)
class TimedTaskQueue: __single = None def __init__(self, nameprefix="TimedTaskQueue"): self.cond = Condition() self.queue = [] self.thread = Thread(target=self.run) self.thread.setDaemon(True) self.thread.setName(nameprefix + self.thread.getName()) self.thread.start() def add_task(self, task, t, id=None): """ t parameter is now usable, unlike before. If id is given, all the existing tasks with the same id will be removed before inserting this task """ if task is None: print_stack() self.cond.acquire() when = time() + t if DEBUG: print >> sys.stderr, "ttqueue: ADD EVENT", t, task if id != None: # remove all redundant tasks self.queue = filter(lambda item: item[2] != id, self.queue) self.queue.append((when, task, id)) self.cond.notify() self.cond.release() def run(self): """ Run by server thread """ while True: task = None timeout = None flag = False self.cond.acquire() while True: while len(self.queue) == 0 or flag: flag = False if timeout is None: # Wait until something is queued self.cond.wait() else: # Wait till first event is due self.cond.wait(timeout) # A new event was added or an event is due self.queue.sort() (when, task, id) = self.queue[0] if DEBUG: print >> sys.stderr, "ttqueue: EVENT IN QUEUE", when, task now = time() if now < when: # Event not due, wait some more if DEBUG: print >> sys.stderr, "ttqueue: EVENT NOT TILL", when - now timeout = when - now flag = True else: # Event due, execute if DEBUG: print >> sys.stderr, "ttqueue: EVENT DUE" self.queue.pop(0) break self.cond.release() # Execute task outside lock try: task() except: print_exc()
class AntColony: def __init__(self, graph, num_ants, num_iterations): self.graph = graph self.num_ants = num_ants self.num_iterations = num_iterations self.Alpha = 0.1 # condition var self.cv = Condition() self.reset() def reset(self): self.best_path_cost = sys.maxint self.best_path_vec = None self.best_path_mat = None self.last_best_path_iteration = 0 def start(self): self.ants = self.create_ants() self.iter_counter = 0 while self.iter_counter < self.num_iterations: self.iteration() self.cv.acquire() # wait until update calls notify() self.cv.wait() lock = self.graph.lock lock.acquire() self.global_updating_rule() lock.release() self.cv.release() # one iteration involves spawning a number of ant threads def iteration(self): self.avg_path_cost = 0 self.ant_counter = 0 self.iter_counter += 1 print("iter_counter = %s" % (self.iter_counter, )) for ant in self.ants: print("starting ant = %s" % (ant.ID)) ant.start() def num_ants(self): return len(self.ants) def num_iterations(self): return self.num_iterations def iteration_counter(self): return self.iter_counter # called by individual ants def update(self, ant): lock = Lock() lock.acquire() #outfile = open("results.dat", "a") print("Update called by %s" % (ant.ID, )) self.ant_counter += 1 self.avg_path_cost += ant.path_cost # book-keeping if ant.path_cost < self.best_path_cost: self.best_path_cost = ant.path_cost self.best_path_mat = ant.path_mat self.best_path_vec = ant.path_vec self.last_best_path_iteration = self.iter_counter if self.ant_counter == len(self.ants): self.avg_path_cost /= len(self.ants) print("Best: %s, %s, %s, %s" % ( self.best_path_vec, self.best_path_cost, self.iter_counter, self.avg_path_cost, )) #outfile.write("\n%s\t%s\t%s" % (self.iter_counter, self.avg_path_cost, self.best_path_cost,)) self.cv.acquire() self.cv.notify() self.cv.release() #outfile.close() lock.release() def done(self): return self.iter_counter == self.num_iterations # assign each ant a random start-node def create_ants(self): self.reset() ants = [] for i in range(0, self.num_ants): ant = Ant(i, random.randint(0, self.graph.num_nodes - 1), self) ants.append(ant) return ants # changes the tau matrix based on evaporation/deposition def global_updating_rule(self): evaporation = 0 deposition = 0 for r in range(0, self.graph.num_nodes): for s in range(0, self.graph.num_nodes): if r != s: delt_tau = self.best_path_mat[r][s] / self.best_path_cost evaporation = (1 - self.Alpha) * self.graph.tau(r, s) deposition = self.Alpha * delt_tau self.graph.update_tau(r, s, evaporation + deposition)
class Sampler(object): """ Sampler used to play, stop and mix multiple sounds. .. warning:: A single sampler instance should be used at a time. """ def __init__(self, sr=22050, backend='sounddevice', timeout=1): """ :param int sr: samplerate used - all sounds added to the sampler will automatically be resampled if needed (- his can be a CPU consumming task, try to use sound with all identical sampling rate if possible. :param str backend: backend used for playing sound. Can be either 'sounddevice' or 'dummy'. """ self.sr = sr self.sounds = [] self.chunks = Queue(1) self.chunk_available = Condition() self.is_done = Event() # new event to prevent play to be called again before the sound is actually played self.timeout = timeout # timeout value for graceful exit of the BackendStream if backend == 'dummy': from .dummy_stream import DummyStream self.BackendStream = DummyStream elif backend == 'sounddevice': from sounddevice import OutputStream self.BackendStream = OutputStream else: raise ValueError("Backend can either be 'sounddevice' or 'dummy'") # TODO: use a process instead? self.play_thread = Thread(target=self.run) self.play_thread.daemon = True self.play_thread.start() def __enter__(self): return self def __exit__(self, *args): self.play_thread.join() def play(self, sound): """ Adds and plays a new Sound to the Sampler. :param sound: sound to play .. note:: If the sound is already playing, it will restart from the beginning. """ self.is_done.clear() # hold is_done until the sound is played if self.sr != sound.sr: raise ValueError('You can only play sound with a samplerate of {} (here {}). Use the Sound.resample method for instance.', self.sr, sound.sr) if sound in self.sounds: self.remove(sound) with self.chunk_available: self.sounds.append(sound) sound.playing = True self.chunk_available.notify() self.is_done.wait() # wait for the sound to be entirely played def remove(self, sound): """ Remove a currently played sound. """ with self.chunk_available: sound.playing = False self.sounds.remove(sound) # Play loop def next_chunks(self): """ Gets a new chunk from all played sound and mix them together. """ with self.chunk_available: while True: playing_sounds = [s for s in self.sounds if s.playing] chunks = [] for s in playing_sounds: try: chunks.append(next(s.chunks)) except StopIteration: s.playing = False self.sounds.remove(s) self.is_done.set() # sound was played, release is_done to end the wait in play if chunks: break self.chunk_available.wait() return numpy.mean(chunks, axis=0) def run(self): """ Play loop, i.e. send all sound chunk by chunk to the soundcard. """ self.running = True def chunks_producer(): while self.running: self.chunks.put(self.next_chunks()) t = Thread(target=chunks_producer) t.start() with self.BackendStream(samplerate=self.sr, channels=1) as stream: while self.running: try: stream.write(self.chunks.get(timeout=self.timeout)) # timeout so stream.write() thread can exit except Empty: self.running = False # let play_thread exit
class NtTestBase(NetworkTablesInstance): """ Object for managing a live pair of NT server/client """ _wait_lock = None _testing_verbose_logging = True def shutdown(self): logger.info("shutting down %s", self.__class__.__name__) NetworkTablesInstance.shutdown(self) if self._wait_lock is not None: self._wait_init_listener() def disconnect(self): self._api.dispatcher.stop() def _init_common(self, proto_rev): # This resets the instance to be independent self.shutdown() self._api.dispatcher.setDefaultProtoRev(proto_rev) self.proto_rev = proto_rev if self._testing_verbose_logging: self.enableVerboseLogging() # self._wait_init() def _init_server(self, proto_rev, server_port=0): self._init_common(proto_rev) self.port = server_port def _init_client(self, proto_rev): self._init_common(proto_rev) def _wait_init(self): self._wait_lock = Condition() self._wait = 0 self._wait_init_listener() def _wait_init_listener(self): self._api.addEntryListener( "", self._wait_cb, NetworkTablesInstance.NotifyFlags.NEW | NetworkTablesInstance.NotifyFlags.UPDATE | NetworkTablesInstance.NotifyFlags.DELETE | NetworkTablesInstance.NotifyFlags.FLAGS, ) def _wait_cb(self, *args): with self._wait_lock: self._wait += 1 # logger.info('Wait callback, got: %s', args) self._wait_lock.notify() @contextmanager def expect_changes(self, count): """Use this on the *other* instance that you're making changes on, to wait for the changes to propagate to the other instance""" if self._wait_lock is None: self._wait_init() with self._wait_lock: self._wait = 0 logger.info("Begin actions") yield logger.info("Waiting for %s changes", count) with self._wait_lock: result, msg = ( self._wait_lock.wait_for(lambda: self._wait == count, 4), "Timeout waiting for %s changes (got %s)" % (count, self._wait), ) logger.info("expect_changes: %s %s", result, msg) assert result, msg
class IbApi(EWrapper): """""" data_filename = "ib_contract_data.db" data_filepath = str(get_file_path(data_filename)) local_tz = get_localzone() def __init__(self, gateway: BaseGateway): """""" super().__init__() self.gateway = gateway self.gateway_name = gateway.gateway_name self.status = False self.reqid = 0 self.orderid = 0 self.clientid = 0 self.account = "" self.ticks = {} self.orders = {} self.accounts = {} self.contracts = {} self.tick_exchange = {} self.history_req = None self.history_condition = Condition() self.history_buf = [] self.client = IbClient(self) self.thread = Thread(target=self.client.run) def connectAck(self): # pylint: disable=invalid-name """ Callback when connection is established. """ self.status = True self.gateway.write_log("IB TWS连接成功") self.load_contract_data() def connectionClosed(self): # pylint: disable=invalid-name """ Callback when connection is closed. """ self.status = False self.gateway.write_log("IB TWS连接断开") def nextValidId(self, orderId: int): # pylint: disable=invalid-name """ Callback of next valid orderid. """ super().nextValidId(orderId) if not self.orderid: self.orderid = orderId def currentTime(self, time: int): # pylint: disable=invalid-name """ Callback of current server time of IB. """ super().currentTime(time) dt = datetime.fromtimestamp(time) time_string = dt.strftime("%Y-%m-%d %H:%M:%S.%f") msg = f"服务器时间: {time_string}" self.gateway.write_log(msg) def error(self, reqId: TickerId, errorCode: int, errorString: str): # pylint: disable=invalid-name """ Callback of error caused by specific request. """ super().error(reqId, errorCode, errorString) msg = f"信息通知,代码:{errorCode},内容: {errorString}" self.gateway.write_log(msg) def tickPrice( # pylint: disable=invalid-name self, reqId: TickerId, tickType: TickType, price: float, attrib: TickAttrib): """ Callback of tick price update. """ super().tickPrice(reqId, tickType, price, attrib) if tickType not in TICKFIELD_IB2VT: return tick = self.ticks[reqId] name = TICKFIELD_IB2VT[tickType] setattr(tick, name, price) # Update name into tick data. contract = self.contracts.get(tick.vt_symbol, None) if contract: tick.name = contract.name # Forex and spot product of IDEALPRO has no tick time and last price. # We need to calculate locally. exchange = self.tick_exchange[reqId] if exchange is Exchange.IDEALPRO: tick.last_price = (tick.bid_price_1 + tick.ask_price_1) / 2 tick.datetime = datetime.now(self.local_tz) self.gateway.on_tick(copy(tick)) def tickSize(self, reqId: TickerId, tickType: TickType, size: int): # pylint: disable=invalid-name """ Callback of tick volume update. """ super().tickSize(reqId, tickType, size) if tickType not in TICKFIELD_IB2VT: return tick = self.ticks[reqId] name = TICKFIELD_IB2VT[tickType] setattr(tick, name, size) self.gateway.on_tick(copy(tick)) def tickString(self, reqId: TickerId, tickType: TickType, value: str): # pylint: disable=invalid-name """ Callback of tick string update. """ super().tickString(reqId, tickType, value) if tickType != TickTypeEnum.LAST_TIMESTAMP: return tick = self.ticks[reqId] dt = datetime.fromtimestamp(int(value)) tick.datetime = dt.replace(tzinfo=self.local_tz) self.gateway.on_tick(copy(tick)) def orderStatus( # pylint: disable=invalid-name self, orderId: OrderId, status: str, filled: float, remaining: float, avgFillPrice: float, permId: int, parentId: int, lastFillPrice: float, clientId: int, whyHeld: str, mktCapPrice: float, ): """ Callback of order status update. """ super().orderStatus( orderId, status, filled, remaining, avgFillPrice, permId, parentId, lastFillPrice, clientId, whyHeld, mktCapPrice, ) orderid = str(orderId) order = self.orders.get(orderid, None) order.traded = filled # To filter PendingCancel status order_status = STATUS_IB2VT.get(status, None) if order_status: order.status = order_status self.gateway.on_order(copy(order)) def openOrder( # pylint: disable=invalid-name self, orderId: OrderId, ib_contract: Contract, ib_order: Order, orderState: OrderState, ): """ Callback when opening new order. """ super().openOrder(orderId, ib_contract, ib_order, orderState) orderid = str(orderId) order = OrderData( symbol=ib_contract.conId, exchange=EXCHANGE_IB2VT.get(ib_contract.exchange, ib_contract.exchange), type=ORDERTYPE_IB2VT[ib_order.orderType], orderid=orderid, direction=DIRECTION_IB2VT[ib_order.action], volume=ib_order.totalQuantity, gateway_name=self.gateway_name, ) if order.type == OrderType.LIMIT: order.price = ib_order.lmtPrice elif order.type == OrderType.STOP: order.price = ib_order.auxPrice self.orders[orderid] = order self.gateway.on_order(copy(order)) def updateAccountValue( # pylint: disable=invalid-name self, key: str, val: str, currency: str, accountName: str): """ Callback of account update. """ super().updateAccountValue(key, val, currency, accountName) if not currency or key not in ACCOUNTFIELD_IB2VT: return accountid = f"{accountName}.{currency}" account = self.accounts.get(accountid, None) if not account: account = AccountData(accountid=accountid, gateway_name=self.gateway_name) self.accounts[accountid] = account name = ACCOUNTFIELD_IB2VT[key] setattr(account, name, float(val)) def updatePortfolio( # pylint: disable=invalid-name self, contract: Contract, position: float, marketPrice: float, marketValue: float, averageCost: float, unrealizedPNL: float, realizedPNL: float, accountName: str, ): """ Callback of position update. """ super().updatePortfolio( contract, position, marketPrice, marketValue, averageCost, unrealizedPNL, realizedPNL, accountName, ) if contract.exchange: exchange = EXCHANGE_IB2VT.get(contract.exchange, None) elif contract.primaryExchange: exchange = EXCHANGE_IB2VT.get(contract.primaryExchange, None) else: exchange = Exchange.SMART # Use smart routing for default if not exchange: msg = f"存在不支持的交易所持仓{contract.conId} {contract.exchange} {contract.primaryExchange}" self.gateway.write_log(msg) return try: ib_size = int(contract.multiplier) except ValueError: ib_size = 1 price = averageCost / ib_size pos = PositionData( symbol=generate_symbol(contract), exchange=exchange, direction=Direction.NET, volume=position, price=price, pnl=unrealizedPNL, gateway_name=self.gateway_name, ) self.gateway.on_position(pos) def updateAccountTime(self, timeStamp: str): # pylint: disable=invalid-name """ Callback of account update time. """ super().updateAccountTime(timeStamp) for account in self.accounts.values(): self.gateway.on_account(copy(account)) def contractDetails(self, reqId: int, contractDetails: ContractDetails): # pylint: disable=invalid-name """ Callback of contract data update. """ super().contractDetails(reqId, contractDetails) # Generate symbol from ib contract details ib_contract = contractDetails.contract if not ib_contract.multiplier: ib_contract.multiplier = 1 symbol = generate_symbol(ib_contract) # Generate contract contract = ContractData( symbol=symbol, exchange=EXCHANGE_IB2VT[ib_contract.exchange], name=contractDetails.longName, product=PRODUCT_IB2VT[ib_contract.secType], size=ib_contract.multiplier, pricetick=contractDetails.minTick, net_position=True, history_data=True, stop_supported=True, gateway_name=self.gateway_name, ) if contract.vt_symbol not in self.contracts: self.gateway.on_contract(contract) self.contracts[contract.vt_symbol] = contract self.save_contract_data() def execDetails(self, reqId: int, contract: Contract, execution: Execution): # pylint: disable=invalid-name """ Callback of trade data update. """ super().execDetails(reqId, contract, execution) dt = datetime.strptime(execution.time, "%Y%m%d %H:%M:%S") dt = dt.replace(tzinfo=self.local_tz) trade = TradeData( symbol=contract.conId, exchange=EXCHANGE_IB2VT.get(contract.exchange, contract.exchange), orderid=str(execution.orderId), tradeid=str(execution.execId), direction=DIRECTION_IB2VT[execution.side], price=execution.price, volume=execution.shares, datetime=dt, gateway_name=self.gateway_name, ) self.gateway.on_trade(trade) def managedAccounts(self, accountsList: str): """ Callback of all sub accountid. """ super().managedAccounts(accountsList) if not self.account: for account_code in accountsList.split(","): self.account = account_code self.gateway.write_log(f"当前使用的交易账号为{self.account}") self.client.reqAccountUpdates(True, self.account) def historicalData(self, reqId: int, ib_bar: IbBarData): """ Callback of history data update. """ dt = datetime.strptime(ib_bar.date, "%Y%m%d %H:%M:%S") dt = dt.replace(tzinfo=self.local_tz) bar = BarData(symbol=self.history_req.symbol, exchange=self.history_req.exchange, datetime=dt, interval=self.history_req.interval, volume=ib_bar.volume, open_price=ib_bar.open, high_price=ib_bar.high, low_price=ib_bar.low, close_price=ib_bar.close, gateway_name=self.gateway_name) self.history_buf.append(bar) def historicalDataEnd(self, reqId: int, start: str, end: str): """ Callback of history data finished. """ self.history_condition.acquire() self.history_condition.notify() self.history_condition.release() def connect(self, host: str, port: int, clientid: int, account: str): """ Connect to TWS. """ if self.status: return self.clientid = clientid self.account = account self.client.connect(host, port, clientid) self.thread.start() self.client.reqCurrentTime() def close(self): """ Disconnect to TWS. """ if not self.status: return self.status = False self.client.disconnect() def subscribe(self, req: SubscribeRequest): """ Subscribe tick data update. """ if not self.status: return if req.exchange not in EXCHANGE_VT2IB: self.gateway.write_log(f"不支持的交易所{req.exchange}") return # Extract ib contract detail ib_contract = generate_ib_contract(req.symbol, req.exchange) if not ib_contract: self.gateway.write_log("代码解析失败,请检查格式是否正确") return # Get contract data from TWS. self.reqid += 1 self.client.reqContractDetails(self.reqid, ib_contract) # Subscribe tick data and create tick object buffer. self.reqid += 1 self.client.reqMktData(self.reqid, ib_contract, "", False, False, []) tick = TickData( symbol=req.symbol, exchange=req.exchange, datetime=datetime.now(self.local_tz), gateway_name=self.gateway_name, ) self.ticks[self.reqid] = tick self.tick_exchange[self.reqid] = req.exchange def send_order(self, req: OrderRequest): """ Send a new order. """ if not self.status: return "" if req.exchange not in EXCHANGE_VT2IB: self.gateway.write_log(f"不支持的交易所:{req.exchange}") return "" if req.type not in ORDERTYPE_VT2IB: self.gateway.write_log(f"不支持的价格类型:{req.type}") return "" self.orderid += 1 ib_contract = generate_ib_contract(req.symbol, req.exchange) if not ib_contract: return "" ib_order = Order() ib_order.orderId = self.orderid ib_order.clientId = self.clientid ib_order.action = DIRECTION_VT2IB[req.direction] ib_order.orderType = ORDERTYPE_VT2IB[req.type] ib_order.totalQuantity = req.volume ib_order.account = self.account if req.type == OrderType.LIMIT: ib_order.lmtPrice = req.price elif req.type == OrderType.STOP: ib_order.auxPrice = req.price self.client.placeOrder(self.orderid, ib_contract, ib_order) self.client.reqIds(1) order = req.create_order_data(str(self.orderid), self.gateway_name) self.gateway.on_order(order) return order.vt_orderid def cancel_order(self, req: CancelRequest): """ Cancel an existing order. """ if not self.status: return self.client.cancelOrder(int(req.orderid)) def query_history(self, req: HistoryRequest): """""" self.history_req = req self.reqid += 1 ib_contract = generate_ib_contract(req.symbol, req.exchange) if req.end: end = req.end end_str = end.strftime("%Y%m%d %H:%M:%S") else: end = datetime.now() end_str = "" delta = end - req.start days = min(delta.days, 180) # IB only provides 6-month data duration = f"{days} D" bar_size = INTERVAL_VT2IB[req.interval] if req.exchange == Exchange.IDEALPRO: bar_type = "MIDPOINT" else: bar_type = "TRADES" self.client.reqHistoricalData(self.reqid, ib_contract, end_str, duration, bar_size, bar_type, 1, 1, False, []) self.history_condition.acquire() # Wait for async data return self.history_condition.wait() self.history_condition.release() history = self.history_buf self.history_buf = [] # Create new buffer list self.history_req = None return history def load_contract_data(self): """""" f = shelve.open(self.data_filepath) self.contracts = f.get("contracts", {}) f.close() for contract in self.contracts.values(): self.gateway.on_contract(contract) self.gateway.write_log("本地缓存合约信息加载成功") def save_contract_data(self): """""" f = shelve.open(self.data_filepath) f["contracts"] = self.contracts f.close()
class InstanceRegistry: """Keeps track of running instances. The InstanceRegistry is a simple in-memory database that stores information about running instances of compute elements. """ def __init__(self, expected_instances: List[str]) -> None: """Construct an empty InstanceRegistry. Args: expected_instances: List of instance names expected to register. """ self.__status = dict() # type: Dict[Reference, _InstanceStatus] self.__deregistered_one = Condition() # doubles as lock for __status self.__locations = dict() # type: Dict[Reference, List[str]] self.__ports = dict() # type: Dict[Reference, List[Port]] for instance_name in expected_instances: self.__status[Reference(instance_name)] = _InstanceStatus.EXPECTED def add(self, name: Reference, locations: List[str], ports: List[Port]) -> None: """Add an instance to the registry. Args: name: Name of the instance. locations: Network locations where it can be reached. ports: List of ports of this instance. Raises: ValueError: If an instance with this name has already been registered. """ if name in self.__locations or name in self.__ports: raise ValueError('Instance already registered') with self.__deregistered_one: self.__status[name] = _InstanceStatus.REGISTERED self.__locations[name] = locations self.__ports[name] = ports def get_locations(self, name: Reference) -> List[str]: """Retrieves the locations of a registered instance. Args: name: The name of the instance to get the location of. Raises: KeyError: If no instance with this name was registered. """ return self.__locations[name] def get_ports(self, name: Reference) -> List[Port]: """Retrieves the ports of a registered instance. Args: name: The name of the instance whose ports to retrieve. Raises: KeyError: If no instance with this name was registered. """ return self.__ports[name] def remove(self, name: Reference) -> None: """Remove an instance from the registry. Args: name: Name of the instance to remove. Raises: KeyError: If the instance does not exist. """ del (self.__locations[name]) del (self.__ports[name]) with self.__deregistered_one: self.__status[name] = _InstanceStatus.DEREGISTERED self.__deregistered_one.notify() def wait(self) -> None: """Waits until all instance are deregistered. This function blocks, and returns after each expected instance has been registered and deregistered again, signalling the end of the simulation run. """ # this is called from a different thread than add/remove def all_deregistered() -> bool: return all( map(lambda x: x == _InstanceStatus.DEREGISTERED, self.__status.values())) with self.__deregistered_one: while not all_deregistered(): self.__deregistered_one.wait()
class RLUQueue: def __init__(self, size): self.size = size self.prev = [None] * size self.next = [None] * size self.prev.append(size) self.next.append(size) self.cv = Condition() ## interface def put(self, idx): assert 0 <= idx < self.size self.cv.acquire() if self.in_list(idx): self.remove(idx) self.append(idx) self.cv.notify() self.cv.release() def get(self, block=True): self.cv.acquire() if not block and self._empty(): self.cv.release() return None while self._empty(): self.cv.wait() ret = self.remove(self.next[-1]) self.cv.release() return ret def unget(self, idx): assert 0 <= idx < self.size with self.cv: assert not self.in_list(idx) self.prepend(idx) def pop(self, idx): assert 0 <= idx < self.size with self.cv: return self.remove(idx) if self.in_list(idx) else None def empty(self): with self.cv: return self._empty() ## helper def _empty(self): return self.prev[-1] == self.size def in_list(self, idx): return self.prev[idx] is not None def remove(self, idx): self.next[self.prev[idx]] = self.next[idx] self.prev[self.next[idx]] = self.prev[idx] self.prev[idx] = None self.next[idx] = None return idx def append(self, idx): self.next[idx] = self.size self.prev[idx] = self.prev[-1] self.prev[self.next[idx]] = idx self.next[self.prev[idx]] = idx def prepend(self, idx): self.next[idx] = self.next[-1] self.prev[idx] = self.size self.prev[self.next[idx]] = idx self.next[self.prev[idx]] = idx
class OSXCentralManager(NSObject, Central): """ CentralManager is the host handle for performing scan, connect, disconnect to peripheral(s). After a peripheral is connected, a peripheral handle would be returned. CBCentralManager Class Reference: https://developer.apple.com/librarY/mac/documentation/CoreBluetooth/Reference/CBCentralManager_Class/translated_content/CBCentralManager.html CBCentralManagerDelegate Protocol Reference: https://developer.apple.com/librarY/mac/documentation/CoreBluetooth/Reference/CBCentralManagerDelegate_Protocol/translated_content/CBCentralManagerDelegate.html """ def init(self): try: super().__init__() except: super(OSXCentralManager, self).__init__() # enable trace self.trace.traceInstance(self) # initialize manager with delegate self.logger.info("Initialize CBCentralManager") self.manager = CBCentralManager.alloc().initWithDelegate_queue_( self, nil) self.scanedList = [] self.connectedList = [] self.BLEReady_callback = None self.BLEAvailableList_callback = None self.BLEConnectedList_callback = None self.cv = Condition() self.ready = False self.wait4Startup() self._stop = Event() return self # decorators for condition variables def _waitResp(func): @wraps(func) def wrapper(self, *args, **kwargs): with self.cv: self.ready = False ret = func(self, *args, **kwargs) while True: self.cv.wait(0) NSRunLoop.currentRunLoop().runMode_beforeDate_( NSDefaultRunLoopMode, NSDate.distantPast()) if self.ready: break return ret return wrapper def _notifyResp(func): @wraps(func) def wrapper(self, *args, **kwargs): ret = func(self, *args, **kwargs) self.ready = True with self.cv: try: self.cv.notify() except Exception as e: print e return ret return wrapper @_waitResp def wait4Startup(self): self.logger.debug("Waiting CentralManager to be ready ...") @python_method def setBLEReadyCallback(self, func): self.BLEReady_callback = func @python_method def setBLEAvailableListCallback(self, func): self.BLEAvailableList_callback = func @python_method def setBLEConnectedListCallback(self, func): self.BLEConnectedList_callback = func @python_method def updateAvailableList(self): if self.BLEAvailableList_callback: try: self.BLEAvailableList_callback(self.scanedList) except: pass @python_method def updateConnectedList(self): if self.BLEConnectedList_callback: try: self.BLEConnectedList_callback(self.connectedList) except: pass @python_method def loop(self, duration=0): if duration == 0: self.logger.info("Stop the loop by Ctrl+C ...") startTime = datetime.now() while True: try: NSRunLoop.currentRunLoop().runMode_beforeDate_( NSDefaultRunLoopMode, NSDate.distantPast()) if duration > 0 and datetime.now() - startTime > timedelta( seconds=duration): break if self._stop.isSet(): break except KeyboardInterrupt: print "\nEnd loop ..." break @python_method def stop(self): self._stop.set() @python_method def startScan(self, withServices=[], timeout=5, numOfPeripherals=1, allowDuplicates=False): self.logger.debug("Start Scan") if len(self.scanedList): del self.scanedList self.scanedList = [] if allowDuplicates: value = NSNumber.numberWithBool_(YES) else: value = NSNumber.numberWithBool_(NO) options = NSDictionary.dictionaryWithObject_forKey_( value, CBCentralManagerScanOptionAllowDuplicatesKey) self.manager.scanForPeripheralsWithServices_options_(nil, options) startTime = datetime.now() with self.cv: while True: self.cv.wait(0.1) NSRunLoop.currentRunLoop().runMode_beforeDate_( NSDefaultRunLoopMode, NSDate.distantPast()) if timeout > 0 and datetime.now() - startTime > timedelta( seconds=timeout): self.stopScan() raise BLETimeoutError("Scan timeout after %s seconds!!" % timeout) if len(withServices): if len(self.scanedList): tmpList = self.scanedList[:] self.scanedList = [] for service in withServices: for p in tmpList: if service in p.advServiceUUIDs: self.scanedList.append(p) if numOfPeripherals > 0 and len( self.scanedList) >= numOfPeripherals: self.logger.info("Found %s peripherals." % len(self.scanedList)) break self.stopScan() # return the first found peripheral if len(self.scanedList): return self.scanedList[0] return None @python_method def stopScan(self): self.logger.debug("Stop Scan") self.manager.stopScan() @python_method def getScanedList(self): return self.scanedList @python_method def getConnectedList(self): return self.connectedList @python_method @_waitResp def retrieveConnectedPeripherals(self): self.logger.info("Deprecated in OSX v10.9.") return # self.manager.retrieveConnectedPeripherals() @python_method def retrieveConnectedPeripheralsWithServices(self, services): if not isinstance(services, list) and isinstance(services, str): services = list(services.split()) known_services = NSMutableArray.alloc().init() for service in services: UUID = CBUUID.UUIDWithString_(service) if UUID is not nil: known_services.addObject_(UUID) peripherals = self.manager.retrieveConnectedPeripheralsWithServices_( known_services) for p in peripherals: if not self.findPeripheralFromList(p, self.scanedList): temp = OSXPeripheral.alloc().init() temp.instance = p temp.UUID = uuid.UUID(p._.identifier.UUIDString()) temp.name = p._.name self.scanedList.append(temp) # update lists self.updateAvailableList() @python_method def retrievePeriphersWithIdentifiers(self, identifiers): if not isinstance(identifiers, list) and isinstance(identifiers, str): identifiers = list(identifiers.split()) known_identifiers = NSMutableArray.alloc().init() for identifier in identifiers: UUID = NSUUID.alloc().UUIDWithString_(identifer) if UUID is not nil: known_identifiers.addObject_(UUID) peripherals = self.manager.retrievePeripheralsWithIdentifiers_( known_identifiers) for p in peripherals: if not self.findPeripheralFromList(p, self.scanedList): temp = OSXPeripheral.alloc().init() temp.instance = p temp.UUID = uuid.UUID(p._.identifier.UUIDString()) temp.name = p._.name self.scanedList.append(temp) # update lists self.updateAvailableList() @python_method def connectPeripheral(self, peripheral): with self.cv: self.ready = False self.logger.debug("Connecting to Peripheral: " + str(peripheral)) options = NSDictionary.dictionaryWithObject_forKey_( NSNumber.numberWithBool_(YES), CBConnectPeripheralOptionNotifyOnDisconnectionKey) peripheral.state = Peripheral.CONNECTING self.manager.connectPeripheral_options_(peripheral.instance, options) while True: self.cv.wait(0) NSRunLoop.currentRunLoop().runMode_beforeDate_( NSDefaultRunLoopMode, NSDate.distantPast()) if self.ready == True: break # return the newly added peripheral if peripheral in self.connectedList: return peripheral return None @python_method def disconnectAllPeripherals(self): self.logger.debug("Disconnecting all Peripherals") for p in self.connectedList: self.disconnectPeripheral(p) @python_method @_waitResp def disconnectPeripheral(self, peripheral): self.logger.debug("Disconnecting Peripheral: " + str(peripheral)) self.manager.cancelPeripheralConnection_(peripheral.instance) @python_method @staticmethod def findPeripheralFromList(peripheral, peripherals): name = peripheral._.name UUID = uuid.UUID(peripheral._.identifier.UUIDString()) for p in peripherals: if not isinstance(p, Peripheral): return None if name == p.name and UUID == p.UUID: return p return None # CBCentralManager delegate methods # Monitoring Connections with Peripherals def centralManager_didConnectPeripheral_(self, central, peripheral): self.didConnectPeripheral(central, peripheral) @python_method @_notifyResp def didConnectPeripheral(self, central, peripheral): self.logger.info( "Peripheral %s (%s) is connected" % (peripheral._.name, peripheral._.identifier.UUIDString())) p = self.findPeripheralFromList(peripheral, self.scanedList) if p: if p not in self.connectedList: self.connectedList.append(p) self.scanedList.remove(p) # update peripheral state p.state = Peripheral.CONNECTED # update lists self.updateAvailableList() self.updateConnectedList() def centralManager_didDisconnectPeripheral_error_(self, central, peripheral, error): self.didDisconnectPeripheral(central, peripheral, error) @python_method @_notifyResp def didDisconnectPeripheral(self, central, peripheral, error): self.logger.info("Peripheral %s is disconnected" % peripheral._.name) p = self.findPeripheralFromList(peripheral, self.connectedList) if p: p.state = Peripheral.DISCONNECTED self.connectedList.remove(p) if p not in self.scanedList: self.scanedList.append(p) # update lists self.updateConnectedList() def centralManager_didFailToConnectPeripheral_error_( self, central, peripheral, error): self.didFailtoConnectPeripheral(central, peripheral, error) @python_method @_notifyResp def didFailtoConnectPeripheral(self, central, peripheral, error): self.logger.debug("Fail to connect Peripheral %s" % peripheral._.name) p = self.findPeripheralFromList(peripheral, self.scanedList) if p: p.state = Peripheral.DISCONNECTED # update lists self.updateAvailableList() # Discovering and Retrieving Peripherals def centralManager_didDiscoverPeripheral_advertisementData_RSSI_( self, central, peripheral, advertisementData, rssi): self.didDiscoverPeripheral(central, peripheral, advertisementData, rssi) @python_method @_notifyResp def didDiscoverPeripheral(self, central, peripheral, advertisementData, rssi): temp = OSXPeripheral.alloc().init() idx = -1 p = None try: idx = self.scanedList.index(temp) except ValueError: idx = -1 except Exception as e: self.logger.error(str(e)) if idx >= 0: p = self.scanedList[idx] else: p = temp p.instance = peripheral p.UUID = uuid.UUID(peripheral._.identifier.UUIDString()) p.name = peripheral._.name # handle advertisement data # local name if CBAdvertisementDataLocalNameKey in advertisementData: p.advLocalName = advertisementData[CBAdvertisementDataLocalNameKey] # manufacturer data if CBAdvertisementDataManufacturerDataKey in advertisementData: p.advManufacturerData = advertisementData[ CBAdvertisementDataManufacturerDataKey] # provided services UUIDs if CBAdvertisementDataServiceUUIDsKey in advertisementData: p.advServiceUUIDS = [] UUIDs = advertisementData[CBAdvertisementDataServiceUUIDsKey] for UUID in UUIDs: p.advServiceUUIDs.append(CBUUID2String(UUID._.data)) # Tx Power Level if CBAdvertisementDataTxPowerLevelKey in advertisementData: p.advTxPowerLevel = int( advertisementData[CBAdvertisementDataTxPowerLevelKey]) # ServiceData if CBAdvertisementDataServiceDataKey in advertisementData: p.advServiceData = advertisementData[ CBAdvertisementDataServiceDataKey] # OverflowServiceUUIDs if CBAdvertisementDataOverflowServiceUUIDsKey in advertisementData: p.advOverflowServiceUUIDs = [] UUIDs = advertisementData[ CBAdvertisementDataOverflowServiceUUIDsKey] for UUID in UUIDs: p.advOverflowServiceUUIDs.append(CBUUID2String(UUID._.data)) # IsConnectable if CBAdvertisementDataIsConnectable in advertisementData: p.advIsConnectable = advertisementData[ CBAdvertisementDataIsConnectable] # SolicitedServiceUUIDs if CBAdvertisementDataSolicitedServiceUUIDsKey in advertisementData: p.advSolicitedServiceUUIDs = [] UUIDs = advertisementData[ CBAdvertisementDataSolicitedServiceUUIDsKey] for UUID in UUIDs: p.advSolicitedServiceUUIDs.append(CBUUID2String(UUID._.data)) # RSSI p.rssi = rssi if p not in self.scanedList: self.scanedList.append(p) self.logger.info("Found Peripheral %s", peripheral._.name) self.logger.info("RSSI: %d", rssi) self.logger.info("UUID: %s", peripheral._.identifier.UUIDString()) self.logger.info("State: %s", peripheral._.state) # update lists self.updateAvailableList() def centralManager_didRetrieveConnectedPeripherals_( self, central, peripherals): self.didRetrieveConnectedPeripherals(central, peripherals) @python_method @_notifyResp def didRetrieveConnectedPeripherals(self, central, peripherals): self.logger.info("Deprecated in OSX v10.9.") return self.logger.info("didRetrieveConnectedPeripherals") for p in peripherals: print p def centralManager_didRetrievePeripherals_(self, central, peripherals): self.logger.info("didRetrievePeripherals") # Monitoring Changes to the Central Manager's State def centralManagerDidUpdateState_(self, central): self.didUpdateState(central) @python_method @_notifyResp def didUpdateState(self, central): ble_state = central._.state if ble_state == CBCentralManagerStateUnkown: self.logger.debug("CentralManager State: Unkown") elif ble_state == CBCentralManagerStateResetting: self.logger.debug("CentralManager State: Resetting") elif ble_state == CBCentralManagerStateUnsupported: self.logger.debug("CentralManager State: Unsupported") elif ble_state == CBCentralManagerStateUnauthorized: self.logger.debug("CentralManager State: Unauthorized") elif ble_state == CBCentralManagerStatePoweredOff: self.logger.debug("CentralManager State: PoweredOff") elif ble_state == CBCentralManagerStatePoweredOn: self.logger.debug("CentralManager State: PoweredOn") self.logger.info("BLE is ready!!") if self.BLEReady_callback: try: self.BLEReady_callback() except Exception as e: self.logger.error("BLEReady_callback error") self.logger.error(str(e)) else: self.logger.info("Cannot get CBCentralManager's state!!") raise BLECentralManagerStateError def centralManager_willRestoreState_(self, central, dict): pass
class EarsGPIO(Ears): ENCODERS_CHANNELS = [24, 23] MOTOR_CHANNELS = [[12, 11], [10, 9]] ENABLE_CHANNELS = [5, 6] HOLES = Ears.STEPS FORWARD_INCREMENT = 1 BACKWARD_INCREMENT = -1 def __init__(self): self.running = [False, False] self.targets = [0, 0] self.encoder_cv = Condition() self.positions = [0, 0] self.directions = [1, 1] self.executor = ThreadPoolExecutor(max_workers=1) self.lock = asyncio.Lock() GPIO.setwarnings(True) GPIO.setmode(GPIO.BCM) for channel in EarsGPIO.ENCODERS_CHANNELS: GPIO.setup(channel, GPIO.IN) try: GPIO.add_event_detect(channel, GPIO.RISING, callback=self._encoder_cb) except RuntimeError: print('Could not set edge detection (please reboot ?)') sys.exit(1) for pairs in EarsGPIO.MOTOR_CHANNELS: for channel in pairs: GPIO.setup(channel, GPIO.OUT) GPIO.output(channel, GPIO.LOW) for channel in EarsGPIO.ENABLE_CHANNELS: GPIO.setup(channel, GPIO.OUT) GPIO.output(channel, GPIO.HIGH) def _encoder_cb(self, channel): """ Callback from GPIO. Thread: Rpi.GPIO event thread """ if channel == EarsGPIO.ENCODERS_CHANNELS[0]: ear = 0 elif channel == EarsGPIO.ENCODERS_CHANNELS[1]: ear = 1 with self.encoder_cv: direction = self.directions[ear] if direction == 0: self.positions[ear] = None (loop, callback) = self.callback loop.call_soon_threadsafe(lambda ear=ear: callback(ear)) else: self.positions[ear] = (self.positions[ear] + direction) % EarsGPIO.HOLES if self.targets[ear] == None: # reset mode self.encoder_cv.notify() else: if self.positions[ear] == self.targets[ear]: self._stop_motor(ear) self.encoder_cv.notify() elif self.positions[ear] == (self.targets[ear] % EarsGPIO.HOLES): if self.targets[ear] >= EarsGPIO.HOLES: self.targets[ ear] = self.targets[ear] - EarsGPIO.HOLES elif self.targets[ear] < 0: self.targets[ ear] = self.targets[ear] + EarsGPIO.HOLES def _stop_motor(self, ear): """ Stop motor by changing the channels GPIOs. Thread: RPi.GPIO event """ for channel in EarsGPIO.MOTOR_CHANNELS[ear]: GPIO.output(channel, GPIO.LOW) self.running[ear] = False self.directions[ear] = 0 def _start_motor(self, ear, direction): """ Start motor for given ear. ear = 0 or 1 direction = 1 or -1 Threads: main loop or executor """ dir_ix = int((1 - direction) / 2) GPIO.output(EarsGPIO.MOTOR_CHANNELS[ear][1 - dir_ix], GPIO.LOW) GPIO.output(EarsGPIO.MOTOR_CHANNELS[ear][dir_ix], GPIO.HIGH) self.running[ear] = True self.directions[ear] = direction def on_move(self, loop, callback): self.callback = (loop, callback) async def reset_ears(self, target_left, target_right): async with self.lock: await asyncio.get_event_loop().run_in_executor( self.executor, self._do_reset_ears, target_left, target_right) def _do_reset_ears(self, target_left, target_right): """ Reset ears by running a detection and ignoring the result. Thread: executor """ self.positions = [None, None] self._run_detection(target_left, target_right) def _run_detection(self, target_left, target_right): """ Run detection of any ear in unknown position. Thread: executor """ for ear in [0, 1]: if self.positions[ear] == None: self.positions[ear] = 0 self.targets[ear] = None self._start_motor(ear, EarsGPIO.FORWARD_INCREMENT) start = time.time() previous_risings = [start, start] with self.encoder_cv: current_positions = self.positions.copy() while self.running[0] or self.running[1]: if self.encoder_cv.wait(0.3): # Got a signal now = time.time() for ear in range(2): if self.targets[ear] == None and self.positions[ ear] != current_positions[ear]: delta = now - previous_risings[ear] if delta > 0.4: # passed the missing hole if target_left != None and ear == Ears.LEFT_EAR: self.targets[ear] = target_left elif target_right != None and ear == Ears.RIGHT_EAR: self.targets[ear] = target_right else: self.targets[ear] = ( self.directions[ear] - self.positions[ear]) % EarsGPIO.HOLES self.positions[ear] = self.directions[ear] current_positions[ear] = self.positions[ear] previous_risings[ear] = now else: # Got no signal. now = time.time() for ear in range(2): if self.targets[ear] == None: delta = now - previous_risings[ear] if delta > 0.4: # At missing hole if target_left != None and ear == Ears.LEFT_EAR: self.targets[ear] = target_left elif target_right != None and ear == Ears.RIGHT_EAR: self.targets[ear] = target_right else: self.targets[ear] = ( -self.positions[ear]) % EarsGPIO.HOLES self.positions[ear] = 0 return self.positions.copy() async def move(self, motor, delta, direction): await self.go(motor, self.targets[motor] + delta, direction) async def wait_while_running(self): await asyncio.get_event_loop().run_in_executor( self.executor, self._do_wait_while_running) def _do_wait_while_running(self): """ Wait until motors are no longer running, using a condition variable. Thread: executor """ with self.encoder_cv: while self.running[0] or self.running[1]: self.encoder_cv.wait() async def detect_positions(self): """ Get the position of the ears, running a detection if required. """ async with self.lock: if self.positions[0] == None or self.positions[1] == None: return await asyncio.get_event_loop().run_in_executor( self.executor, self._run_detection, None, None) return (self.positions[0], self.positions[1]) async def go(self, ear, position, direction): """ Go to a specific position. If direction is 0, turn forward, otherwise, turn backward If position is not within 0-16, it represents additional turns. For example, 17 means to position the ear at 0 after at least a complete turn. """ async with self.lock: # Return ears to a known state if self.positions[0] == None or self.positions[1] == None: await asyncio.get_event_loop().run_in_executor( self.executor, self._run_detection, 0, 0) self.targets[ear] = position if direction: dir = EarsGPIO.BACKWARD_INCREMENT else: dir = EarsGPIO.FORWARD_INCREMENT if self.positions[ear] == self.targets[ear] % EarsGPIO.HOLES: if self.targets[ear] >= EarsGPIO.HOLES: self.targets[ear] = self.targets[ear] - EarsGPIO.HOLES elif self.targets[ear] < 0: self.targets[ear] = self.targets[ear] + EarsGPIO.HOLES else: return # we already are at requested position self._start_motor(ear, dir)
class TileServer(object): '''Base implementation for a tile provider. Check GoogleTileServer and YahooTileServer if you intend to use more ''' provider_name = 'unknown' providers = dict() @staticmethod def register(cls): TileServer.providers[cls.provider_name] = cls def __init__(self, poolsize=TILESERVER_POOLSIZE): self.cache_path = join(dirname(__file__), 'cache', self.provider_name) if not isdir(self.cache_path): makedirs(self.cache_path) black = Loader.image(join('documents', 'black.png')) #Loader._loading_image = black self.q_in = deque() self.q_out = deque() self.q_count = 0 self.c_in = Condition() self.workers = [] self.poolsize = poolsize self.uniqid = 1 self.want_close = False self.available_maptype = dict(roadmap='Roadmap') self.hcsvnt = Loader.image(join('documents', 'hcsvnt.png')) def start(self): '''Start all the workers ''' for i in xrange(self.poolsize): self.create_worker() def create_worker(self): '''Create a new worker, and append to the list of current workers ''' thread = Thread(target=self._worker_run, args=(self.c_in, self.q_in, self.q_out)) thread.daemon = True thread.start() self.workers.append(thread) def stop(self, wait=False): '''Stop all workers ''' self.want_close = True if wait: for x in self.workers: x.join() def post_download(self, filename): '''Callback called after the download append. You can use it for doing some image processing, like cropping .. warning:: This function is called inside a worker Thread. ''' pass def to_filename(self, nx, ny, zoom, maptype, format): fid = self.to_id(nx, ny, zoom, maptype, format) hash = fid[0:2] return join(self.cache_path, hash, fid) def to_id(self, nx, ny, zoom, maptype, format): return '%d_%d_%d_%s.%s' % (nx, ny, zoom, maptype, format) def exist(self, nx, ny, zoom, maptype, format='png'): filename = self.to_filename(nx, ny, zoom, maptype, format) img = Cache.get('tileserver.tiles', filename) return bool(img) def get(self, nx, ny, zoom, maptype, format='png'): '''Get a tile ''' filename = self.to_filename(nx, ny, zoom, maptype, format) img = Cache.get('tileserver.tiles', filename) # check if the tile is already being loaded if img is False: return None # check if the tile exist in the cache if img is not None: return img # no tile, ask to workers to download Cache.append('tileserver.tiles', filename, False) self.q_count += 1 self.q_in.append((nx, ny, zoom, maptype, format)) self.c_in.acquire() self.c_in.notify() self.c_in.release() return None def update(self): '''Must be called to get pull image from the workers queue ''' pop = self.q_out.pop while True: try: filename, image = pop() self.q_count -= 1 except: return Cache.append('tileserver.tiles', filename, image) def _worker_run(self, c_in, q_in, q_out): '''Internal. Main function for every worker ''' do = self._worker_run_once while not self.want_close: try: do(c_in, q_in, q_out) except: Logger.exception( 'TileServerWorker: Unknown exception, stop the worker') return def _worker_run_once(self, c_in, q_in, q_out): '''Internal. Load one image, process, and push. ''' # get one tile to process try: nx, ny, zoom, maptype, format = q_in.pop() except: c_in.acquire() c_in.wait() c_in.release() return # check if the tile already have been downloaded filename = self.to_filename(nx, ny, zoom, maptype, format) loaded = True if not isfile(filename): loaded = False # calculate the good tile index tz = pow(2, zoom) lx, ly = unit_to_latlon(2.0 * (nx + 0.5) / tz - 1, 1 - 2.0 * (ny + 0.5) / tz) lx, ly = map(fix180, (lx, ly)) # get url for this specific tile url = self.geturl(nx=nx, ny=ny, lx=lx, ly=ly, tilew=256, tileh=256, zoom=zoom, format=format, maptype=maptype) for i in xrange(1, 3): try: conn = urlopen("http://%s%s" % (self.provider_host, url)) except Exception, ex: print "ERROR IN %s/%s \n%s" % (self.provider_host, url, str(ex)) continue try: data = conn.read() conn.close() except Exception, e: print 'Exception %s' % (url) Logger.error('TileServer: "%s": %s' % (str(e), filename)) Logger.error('TileServer: "%s": URL=%s' % (str(e), url)) continue # discard error messages if data[:5] == "<?xml": msg = "" try: msg = data[data.index("<ServiceException>") + 18:data.index("</ServiceException")] except: pass Logger.error('Tileserver: Received error fetching %s: %s' % (url, msg)) continue # write data on disk try: directory = sep.join(filename.split(sep)[:-1]) if not isdir(directory): try: mkdir(directory) except: pass # that was probably just a concurrency error - if dir is missing, all threads report it with open(filename, 'wb') as fd: fd.write(data) except: Logger.exception('Tileserver: Unable to write %s' % filename) continue # post processing self.post_download(filename) loaded = True break