class MWPixelClock(object): def __init__(self, conduitName): self.conduit = Conduit(conduitName) self.conduit.initialize() self.conduit.register_local_event_code(0,'#stimDisplayUpdate') self.conduit.register_callback_for_name('#stimDisplayUpdate', self.receive_event) self.codes = [] self.cond = Condition() self.maxCodes = 100 def receive_event(self, event): for s in event.data: if s is None: continue if s.has_key('bit_code'): self.cond.acquire() self.codes.append((s['bit_code'],event.time/1000000.)) # if len(self.codes) > 2: # #logging.debug('MW bit_code = %i' % s['bit_code']) # #print s['bit_code'] # #logging.debug("MW Delta: %s" % delta_code(self.codes[-1][0], self.codes[-2][0])) while len(self.codes) > self.maxCodes: self.codes.pop(0) self.cond.notifyAll() self.cond.release()
class _CoreScheduleThread(Thread): def __init__(self,threadpool): self.scheduletasks = []; self.tasklock = Lock(); self.condition = Condition(Lock()) self.threadpool = threadpool Thread.__init__(self) def run(self): while True: self.condition.acquire() if len(self.scheduletasks) == 0: self.condition.wait(); else: task = self.scheduletasks.pop(0) if dates.current_timestamps()>=task.nexttime: self.threadpool.execute(task.function,*task.args,**task.kwargs) task.nexttime = dates.current_timestamps()+task.period; else: self.condition.wait(task.nexttime-dates.current_timestamps()) self.addtask(task) self.condition.release() def addtask(self,task): # copy on write self.tasklock.acquire() tasks = [ t for t in self.scheduletasks ] tasks.append(task) tasks.sort(key=lambda task:task.nexttime) self.scheduletasks = tasks self.tasklock.release()
class Node(object): STATE_INIT = 0 STATE_STARTING = 1 STATE_RUNNING = 2 STATE_STOPPED = 3 STATE_PARTITIONED = 4 STATE_FAILED = 5 state_str = {STATE_INIT: "Initial", STATE_STARTING: "Starting", STATE_RUNNING: "Running", STATE_STOPPED: "Stopped", STATE_FAILED: "Failed"} def __init__(self, node_id): self.node_id = node_id self.state = Node.STATE_INIT self.cv_lock = Lock() self.cv = Condition(self.cv_lock) def wait_for_state(self, state): self.cv.acquire() while self.state not in (state, Node.STATE_FAILED): self.cv.wait() self.cv.release() if self.state != state: raise ChistributedException("Node entered failed state while waiting for state %s" % Node.state_str[state]) def set_state(self, state): self.cv.acquire() self.state = state self.cv.notify_all() self.cv.release()
class SimpleRemoteCall(object): returncode = None returned = False def signal(self): self.cond.acquire() self.returned = True self.cond.notify() self.cond.release() def errorHandler(self, error): self.returncode = error self.signal() def successHandler(self, object): self.returncode = object self.signal() def __call__(*arg, **kwargs): pass def __init__(self, remoteReference, *arg, **kwargs): self.cond = Condition() deref = remoteReference.callRemote(*arg, **kwargs) deref.addCallbacks(self.successHandler, self.errorHandler) def wait(self, timeout = None): self.cond.acquire() if not self.returned: self.cond.wait(timeout) self.cond.release() if not self.returned: raise TimeExpired('timeout')
def process_sequence(self, sequence): """ @param sequence: the movement sequence to execute @return: dictionary of sensor readings """ self.log.debug('process sequence') # parse the json message jseq = json.loads(sequence) moveseq = jseq['sequence'] for moves in moveseq: self.log.debug('processing move %s' % moves) condition = Condition() thread_count = 0 for move in moves: thread_count += 1 servo_instruction = self.process_move(move) move_servo_thread = ServoThread(servo_instruction, self.pwm_queue, condition, thread_count, self.update_current_pulse) move_servo_thread.setDaemon(False) move_servo_thread.setName('Servo %d' % servo_instruction['channel']) move_servo_thread.start() condition.acquire() if thread_count > 0: condition.wait() # wait for all threads to finish before doing next loop condition.release()
class DataQueue ( Queue ): def __init__ ( self, count ): Queue.__init__(self) self._count = count self._cond = Condition() def get ( self, block = True, timeout = None ): self._cond.acquire() if self.empty(): if not self._count: raise Empty() elif block: self._cond.wait(timeout) if self.empty() and not self._count: self._cond.release() raise Empty() self._cond.release() return Queue.get(self, block, timeout) def put ( self, data ): self._cond.acquire() self._count = self._count - 1 Queue.put(self, data) if self._count: self._cond.notify() else: self._cond.notifyAll() self._cond.release() def remain ( self ): return self._count + self.unfinished_tasks
class MessageQueue(object): """The message queue used internally by Gossip.""" def __init__(self): self._queue = deque() self._condition = Condition() def pop(self): self._condition.acquire() try: while len(self._queue) < 1: self._condition.wait() return self._queue.pop() finally: self._condition.release() def __len__(self): return len(self._queue) def __deepcopy__(self, memo): newmq = MessageQueue() newmq._queue = copy.deepcopy(self._queue, memo) return newmq def appendleft(self, msg): self._condition.acquire() try: self._queue.appendleft(msg) self._condition.notify() finally: self._condition.release()
class External_command(object): work_available = None work = None def __init__(self, command, max_workers = _MAX_WORKERS): self.work_available = Condition() self.work = [] for i in range(0, max_workers): _Worker(command, self) def process_command(self, data, result_callback, *callback_parameters): wi = Work_item(tuple(data), result_callback, callback_parameters) self.work_available.acquire() self.work.append(wi) self.work_available.notify() self.work_available.release() return wi def shutdown(self): self.work_available.acquire() self.work.append(None) self.work_available.notify() self.work_available.release() def process_result(self, result_callback, result, *callback_parameters): try: result_callback(result, *callback_parameters) except Exception as ex: if isinstance(ex, SystemExit): raise dump_exception('External_command: unhandled exception in external command results callback')
class ZumyROS: def __init__(self): self.zumy = Zumy() rospy.init_node("zumy_ros") self.cmd = (0, 0) rospy.Subscriber("cmd_vel", Twist, self.cmd_callback, queue_size=1) self.lock = Condition() self.rate = rospy.Rate(30.0) self.name = socket.gethostname() self.heartBeat = rospy.Publisher("/" + self.name + "/heartBeat", String, queue_size=5) self.imu_pub = rospy.Publisher("/" + self.name + "/imu", Imu, queue_size=1) self.imu_count = 0 self.timestamp = time.time() self.publisher = rospy.Publisher("from_zumy", Float64, queue_size=1) self.msg = None def cmd_callback(self, msg): self.msg = msg self.timestamp = time.time() lv = 0.6 la = 0.4 v = msg.linear.x print v a = msg.angular.z print a r = lv * v + la * a l = lv * v - la * a self.lock.acquire() self.cmd = (l, r) self.lock.release() def run(self): while not rospy.is_shutdown(): time_now = time.time() if time_now - self.timestamp > 0.5: self.cmd = (0, 0) self.lock.acquire() self.zumy.cmd(*self.cmd) imu_data = self.zumy.read_imu() self.lock.release() imu_msg = Imu() imu_msg.header = Header(self.imu_count, rospy.Time.now(), self.name) imu_msg.linear_acceleration.x = 9.81 * imu_data[0] imu_msg.linear_acceleration.y = 9.81 * imu_data[1] imu_msg.linear_acceleration.z = 9.81 * imu_data[2] imu_msg.angular_velocity.x = 3.14 / 180.0 * imu_data[3] imu_msg.angular_velocity.y = 3.14 / 180.0 * imu_data[4] imu_msg.angular_velocity.z = 3.14 / 180.0 * imu_data[5] self.imu_pub.publish(imu_msg) self.heartBeat.publish("I am alive from Glew") if self.msg != None: self.publisher.publish(self.msg.linear.y) self.rate.sleep() # If shutdown, turn off motors self.zumy.cmd(0, 0)
class External_command(object): work_available = None work = None def __init__(self, command, max_workers = _MAX_WORKERS): self.work_available = Condition() self.work = [] for i in range(0, max_workers): _Worker(command, self) def process_command(self, data, result_callback, *callback_parameters): wi = Work_item(tuple(data), result_callback, callback_parameters) self.work_available.acquire() self.work.append(wi) self.work_available.notify() self.work_available.release() return wi def shutdown(self): self.work_available.acquire() self.work.append(None) self.work_available.notify() self.work_available.release() def process_result(self, result_callback, result, *callback_parameters): try: result_callback(result, *callback_parameters) except: print datetime.now(), 'External_command: unhandled exception in external command results callback' print '-' * 70 print_exc(file = stdout) print '-' * 70 stdout.flush()
class ZEventLoopThread(object): def __init__(self, cb=None): self._event_loop = None self._thr = Thread(target=self._thread_func) self._guard = Condition(Lock()) self._thr_callback = cb def start_loop(self): self._thr.start() self._guard.acquire() while self._event_loop is None: self._guard.wait() return self._event_loop def _thread_func(self): # Note: create event loop in the new thread # instead in the constructor event_loop = ZEventLoop() if self._thr_callback: self._thr_callback(event_loop) with self._guard: self._event_loop = event_loop self._guard.notify() event_loop.loop()
class RepceJob(object): """class representing message status we can use for waiting on reply""" def __init__(self, cbk): """ - .rid: (process-wise) unique id - .cbk: what we do upon receiving reply """ self.rid = (os.getpid(), thread.get_ident(), time.time()) self.cbk = cbk self.lever = Condition() self.done = False def __repr__(self): return ':'.join([str(x) for x in self.rid]) def wait(self): self.lever.acquire() if not self.done: self.lever.wait() self.lever.release() return self.result def wakeup(self, data): self.result = data self.lever.acquire() self.done = True self.lever.notify() self.lever.release()
class SharedCell(object): """Shared data for the producer/consumer problem.""" def __init__(self): self._data = -1 self._writeable = True self._condition = Condition() def setData(self, data): """Producer's method to write to shared data.""" self._condition.acquire() while not self._writeable: self._condition.wait() print "%s setting data to %d" % \ (currentThread().getName(), data) self._data = data self._writeable = False self._condition.notify() self._condition.release() def getData(self): """Consumer's method to read from shared data.""" self._condition.acquire() while self._writeable: self._condition.wait() print "%s accessing data %d" % \ (currentThread().getName(), self._data) self._writeable = True self._condition.notify() self._condition.release() return self._data
class LockedDirectories(object): """ Class that keeps a list of locked directories """ def __init__(self): self.dirs = set() self.cv = Condition() def run_in(self, dir_): """ Start running in the directory and lock it """ self.cv.acquire() while dir_ in self.dirs: self.cv.wait() self.dirs.add(dir_) self.cv.release() def done(self, dir_): """ Finished with the directory, unlock it """ self.cv.acquire() self.dirs.remove(dir_) self.cv.notify_all() self.cv.release()
class Queue(object): def __init__(self): self.lock = Lock() self.cond = Condition(self.lock) self.data = [] self.live = True def put(self, values): self.cond.acquire() self.data.extend(values) self.cond.notify() self.cond.release() def get(self): if not self.cond.acquire(False): return [] self.cond.wait() result = self.data self.data = [] self.cond.release() return result def close(self): self.cond.acquire() self.live = False self.cond.notify() self.cond.release()
class NotificationQueue(Queue): """ A queue class that notifies waiting threads when the queue is available for reading.""" def __init__(self): """ Constructor.""" Queue.__init__(self) self.lock = Condition() def get(self): """ Get an item from the queue. This method is thread-safe. Method must be called on a non-empty queue. Return : the first item from the queue Raises: a EmptyQueue exeception """ return Queue.get(self, False) def put(self, item): """ Add a new item to the queue. This method is thread-safe. Threads that are waiting for get an item are notified. item: new item to add to the queue. Return : None """ self.lock.acquire() Queue.put(self, item) self.lock.notify() self.lock.release() def acquire(self): self.lock.acquire() def release(self): self.lock.release() def wait(self): self.lock.wait()
def wait_statement(self, wtime, N=5): ''' ''' start_time = time.time() if self.debug: time.sleep(1) else: if isinstance(wtime, str): wtime = float(self._get_interpolation_value(wtime)) if wtime > N: c = Condition() c.acquire() wd = WaitDialog(wtime=wtime, condition=c, parent=self) do_later(wd.edit_traits, kind='livemodal') c.wait() c.release() do_later(wd.close) else: time.sleep(wtime) msg = 'actual wait time %0.3f s' % (time.time() - start_time) self.log_statement(msg)
class ConnectorPool: """ A thread-safe connection pool. TODO: Make this an actual queue, not a stack. Nomenclature is imporant sometimes. """ def __init__(self, Connector, count=20, user=None, password=None, db=None): self.pool = [ Connector(user, password, db) for _ in range(count) ] self.C_pool = Condition() def dequeue(self): """ Get connector. :rtype: seaice.SeaIceConnector.SeaIceConnector """ self.C_pool.acquire() while len(self.pool) == 0: self.C_pool.wait() db_con = self.pool.pop() self.C_pool.release() return db_con def enqueue(self, db_con): """ Release connector. :param db_con: The connector. :type db_con: seaice.SeaIceConnector.SeaIceConnector """ self.C_pool.acquire() self.pool.append(db_con) self.C_pool.notify() self.C_pool.release()
class ZumyROS: def __init__(self): rospy.init_node('zumy_ros') self.cmd = (0,0) rospy.Subscriber('cmd_vel', Twist, self.cmd_callback) self.lock = Condition() self.rate = rospy.Rate(30.0) self.zumy = Zumy() self.name = socket.gethostname() self.heartBeat = rospy.Publisher('/' + self.name + '/heartBeat', String, queue_size=5) def cmd_callback(self, msg): lv = 0.6 la = 0.4 v = msg.linear.x a = msg.angular.z r = lv*v + la*a l = lv*v - la*a self.lock.acquire() self.cmd = (l,r) self.lock.release() def run(self): while not rospy.is_shutdown(): self.lock.acquire() self.zumy.cmd(*self.cmd) self.heartBeat.publish("I am alive") self.lock.release() self.rate.sleep() self.zumy.cmd(0,0)
class TmThread(Thread): def __init__(self, target, args): # name, event_for_wait, event_for_set Thread.__init__(self) # , target=target, args=args) self.__target = target self.__args = args self.__kwargs = None self.priority = 0 self.time = 0 #flag to Finish thread self.Finishd = False # Explicitly using Lock over RLock since the use of self.Finishd # break reentrancy anyway, and I believe using Lock could allow # one thread to Finish the worker, while another resumes; haven't # checked if Condition imposes additional limitations that would # prevent that. In Python 2, use of Lock instead of RLock also # boosts performance. self.Finish_cond = Condition(Lock()) def Finish(self): self.Finishd = True # If in sleep, we acquire immediately, otherwise we wait for thread # to release condition. In race, worker will still see self.Finishd # and begin waiting until it's set back to False self.Finish_cond.acquire() #should just resume the thread def resume(self): self.Finishd = False # Notify so thread will wake after lock released self.Finish_cond.notify() # Now release the lock self.Finish_cond.release()
class PeriodicWorker(Thread): def __init__(self, func, config, duration=600, logger=cherrypy.log, logDB=None): # use default RLock from condition # Lock wan't be shared between the instance used only for wait # func : function or callable object pointer self.wakeUp = Condition() self.stopFlag = False self.taskFunc = func self.config = config self.duration = duration self.logger = logger self.logDB = logDB try: name = func.__name__ print(name) except AttributeError: name = func.__class__.__name__ print(name) Thread.__init__(self, name=name) cherrypy.engine.subscribe('start', self.start, priority=100) cherrypy.engine.subscribe('stop', self.stop, priority=100) def stop(self): self.wakeUp.acquire() self.stopFlag = True self.wakeUp.notifyAll() self.wakeUp.release() def run(self): while not self.stopFlag: self.wakeUp.acquire() try: self.taskFunc(self.config) self.heartBeatInfoToLogDB() except Exception as e: self.logger.error("Periodic Thread ERROR %s.%s %s" % (getattr(e, "__module__", "__builtins__"), e.__class__.__name__, str(e))) traceMsg = traceback.format_exc() for line in traceMsg.rstrip().split("\n"): self.logger.error(" " + line) self.heartBeatErrorToLogDB(traceMsg) self.wakeUp.wait(self.duration) self.wakeUp.release() def heartBeatInfoToLogDB(self): if self.logDB: self.logDB.delete(mtype="error", this_thread=True) self.logDB.post(mtype="info") return def heartBeatErrorToLogDB(self, msg): if self.logDB: self.logDB.post(msg=msg, mtype="error") return
class SSHProtocol(Protocol): def __init__(self,proxy): print "SShProtocol %s"%self.rules Protocol.__init__(self,self.rules) self.proxy=proxy self.cv=Condition() def init(self): print "OK initialized" self.state=2 def handle(self,msg): print "handle message %s"%msg if msg['type'] == "send": thread.start_new_thread(self.send, (msg,)) def send(self,message): print "Sending Message %s"%message self.release() def release(self): self.cv.acquire() self.cv.notify() self.cv.release() def wait(self): self.cv.acquire() self.cv.wait() self.cv.release() rules={1:init,2:handle}
class Rtp_proxy_client_local(object): is_local = True wi_available = None wi = None nworkers = None workers = None def __init__(self, global_config, address = '/var/run/rtpproxy.sock', \ bind_address = None, nworkers = 1): self.address = address self.is_local = True self.wi_available = Condition() self.wi = [] self.nworkers = nworkers self.workers = [] for i in range(0, self.nworkers): self.workers.append(_RTPPLWorker(self)) def send_command(self, command, result_callback = None, *callback_parameters): self.wi_available.acquire() self.wi.append((command, result_callback, callback_parameters)) self.wi_available.notify() self.wi_available.release() def reconnect(self, address, bind_address = None): self.shutdown() self.address = address self.workers = [] for i in range(0, self.nworkers): self.workers.append(_RTPPLWorker(self)) def shutdown(self): for rworker in self.workers: rworker.shutdown() self.workers = None
def test_call_self(self): m = Mock() cond = Condition() self._recv_call_id = None def event_cb(_, evt): if evt.type == EventType.call_invite: m() self._recv_call_id = evt.request.call_id # condition of event callback returning cond.acquire() cond.notify() cond.release() self.ctx.event_callback = event_cb # start call: # build and send invitation message cond.acquire() with self.ctx.lock: msg = call.InitInvite( self.ctx, to_url='sip:{0[0]}:{0[1]}'.format(self.listen_address), from_url='sip:{0[0]}:{0[1]}'.format(self.listen_address), ) send_call_id = msg.call_id self.ctx.call_send_init_invite(msg) # wait event result! cond.wait(1) cond.release() # assertion self.assertEqual(m.call_count, 1) self.assertEqual(self._recv_call_id, send_call_id)
class ServerLock: """ Class to be used for locking entry and exit to the venue server. Mostly just a wrapper around a normal lock, but adds logging support. """ verbose = 0 def __init__(self, name = ""): if self.verbose: log.debug("Create server lock %s", name) self.lock = Condition(Lock()) self.name = name def acquire(self): if self.verbose: c = (traceback.extract_stack())[-2] fileName = c[0] line = c[1] log.debug("Try to acquire server lock %s... %s:%s", self.name, fileName, line) self.lock.acquire() if self.verbose: log.debug("Try to acquire server lock %s...done %s:%s", self.name, fileName, line) def release(self): if self.verbose: c = (traceback.extract_stack())[-2] fileName = c[0] line = c[1] log.debug("Releasing server lock %s %s:%s", self.name, fileName, line) self.lock.release()
class SafeLoopArray(object): def __init__(self , data, lengh = 10): self._lengh = lengh self._data = [] self._index = 0; self._lock = Condition(Lock()) for i in range(lengh): self._data.append(data) def add(self , data): self._lock.acquire() try: i = self._index%self._lengh #print "self._lengh = {0},self._index={1},i={2}".format(self._lengh,self._index, i) self._data[i] = data self._index = self._index+1 finally: self._lock.release() def get(self , pos): if (pos >= self._lengh): print("out of range") return None self._lock.acquire() try: return self._data[pos] finally: self._lock.release()
class GridGenerationUpdateThread(Thread): disconnect = False elecgen = None sleeper = None def __init__(self, parent): Thread.__init__(self) self.disconnect = False self.elecgen = parent self.ngdata = ElectricityGenerationDataSource() self.sleeper = Condition() def stopUpdates(self): self.disconnect = True self.sleeper.acquire() self.sleeper.notify() self.sleeper.release() def run(self): while self.disconnect == False: emxml = self.ngdata.DownloadRealtimeXML() try: self.elecgen.energyMix = self.ngdata.ParseRealtimeXML(emxml) except Exception, exc: trc.Error("failed to parse realtime xml") trc.Error(str(emxml)) trc.Error(repr(exc)) trc.Error(repr(exc.message)) # go to sleep for a while - the realtime data doesn't update very # often, so no need to download it constantly! self.sleeper.acquire() self.sleeper.wait(180) self.sleeper.release()
class MultiProcBuilder: def __init__(self, makeFile, param): makeFileList = [] index = 1 while os.path.isfile(os.getcwd() + '\\' + makeFile.replace('.mak', 'C' + str(index) + '.mak')): makeFileList.append(makeFile.replace('.mak', 'C' + str(index) + '.mak')) index = index + 1 #TODO externalizar a extensoes para parametros configuraveis makeFileLinker = makeFile.replace('.mak', 'L.mak') self.condition = Condition() self.output = [] self.group = MakeGroup(makeFileList, makeFileLinker, self.output, self.condition, param) def start(self): """Dispara compilacao multiprocessada""" self.group.start() def readMsg(self): self.condition.acquire() out = "" if len(self.output) == 0 and self.group.isAlive(): self.condition.wait() while len(self.output) > 0: out += self.output.pop(0) self.condition.release() return out def getResult(self): return self.group.status
class Barrier: def __init__(self, num_threads): self.num_threads = num_threads self.count = num_threads self.iter = 0 self.cond = Condition() def wait(self): self.cond.acquire() i = self.iter self.count -= 1 if self.count == 0: self.iter += 1 self.count = self.num_threads self.cond.notify_all() self.cond.release() return True # Release lock and block this thread until notified/woken. # Allow for spurious wake-ups. while 1: self.cond.wait(None) if i != self.iter: break self.cond.release() return False
def callFunctionInQtThread(self, functionToCall, blocking, *arguments): """ Calls the given function in the Qt thread. If this is used from the Qt main thread the given function is directly called. If C{blocking} is C{True} and the method is not called from the Qt main thread it will be ensured that the python thread is blocked until the function is executed. @param functionToCall: Pointer to the function that has to be executed. @param arguments: Arguments required for the function. @param blocking: Blocks the calling thread. @type blocking: C{boolean} """ if get_ident() == self.__mainThreadIdentifier: # call directly self._performCall(functionToCall, None, arguments) else: # post event in the main Qt thread event = _MyCustomEvent(functionToCall, arguments) if blocking: conditionVar = Condition(Lock()) conditionVar.acquire() def _callback(): conditionVar.acquire() conditionVar.notify() conditionVar.release() event.callback = _callback qApp.postEvent(self, event) conditionVar.wait() conditionVar.release() else: qApp.postEvent(self, event)
class _SubprocessThread(Thread): def __init__(self, executable_name, args, shell, cwd, verbose, stdout, stderr, stdin_input): Thread.__init__(self, name="Subprocess %s" % executable_name) self._args = args self._shell = shell self._cwd = cwd self._verbose = verbose self._stdout = stdout self._stderr = stderr self._stdin_input = stdin_input self._pid = None self._started_cv = Condition() self.stdout_result = None self.stderr_result = None self.returncode = None self._exception = None @property def exception(self): return self._exception def run(self): try: self._started_cv.acquire() stdin = PIPE if self._stdin_input else None proc = Popen(self._args, shell=self._shell, cwd=self._cwd, stdin=stdin, stdout=self._stdout, stderr=self._stderr) self._pid = proc.pid self._started_cv.notify() self._started_cv.release() if self._stdin_input: proc.stdin.write(self._stdin_input) proc.stdin.flush() self.process_output(proc) self.returncode = proc.returncode except Exception as err: # pylint: disable=broad-except self._exception = err def get_pid(self): self._started_cv.acquire() while self._pid is None: self._started_cv.wait() self._started_cv.release() return self._pid def process_output(self, proc): if self._verbose and self._stdout == PIPE and (self._stderr == PIPE or self._stderr == STDOUT): self.stdout_result = "" self.stderr_result = "" while True: reads = [proc.stdout.fileno()] if self._stderr == PIPE: reads.append(proc.stderr.fileno()) ret = select(reads, [], []) for file_no in ret[0]: if file_no == proc.stdout.fileno(): read = output_as_str(proc.stdout.readline()) sys.stdout.write(read) self.stdout_result += read if self._stderr == PIPE and file_no == proc.stderr.fileno( ): read = output_as_str(proc.stderr.readline()) sys.stderr.write(read) self.stderr_result += read if proc.poll() is not None: break else: stdout_r, stderr_r = proc.communicate() self.stdout_result = output_as_str(stdout_r) self.stderr_result = output_as_str(stderr_r)
class SharedCell(object): """Synchronizes readers and writers around shared data, to support concurrent reading and safe writing.""" def __init__(self, data): """Sets up the conditions and count of active readers.""" self.data = data self.writing = False self.readerCount = 0 self.okToRead = Condition() self.okToWrite = Condition() def beginRead(self): """Waits until a writer is not writing or the writers condition queue is empty. Then increments the reader count and notifies the next waiting reader.""" self.okToRead.acquire() self.okToWrite.acquire() while self.writing or len(self.okToWrite._waiters) > 0: self.okToRead.wait() self.readerCount += 1 self.okToRead.notify() def endRead(self): """Notifies a waiting writer if there are no active readers.""" self.readerCount -= 1 if self.readerCount == 0: self.okToWrite.notify() self.okToWrite.release() self.okToRead.release() def beginWrite(self): """Can write only when someone else is not writing and there are no readers are ready.""" self.okToWrite.acquire() self.okToRead.acquire() while self.writing or self.readerCount != 0: self.okToWrite.wait() self.writing = True def endWrite(self): """Notify the next waiting writer if the readers condition queue is empty. Otherwise, notify the next waiting reader.""" self.writing = False if len(self.okToRead._waiters) > 0: self.okToRead.notify() else: self.okToWrite.notify() self.okToRead.release() self.okToWrite.release() def read(self, readerFunction): """Observe the data in the shared cell.""" self.beginRead() # Enter reader's critical section result = readerFunction(self.data) # Exit reader's critical section self.endRead() return result def write(self, writerFunction): """Modify the data in the shared cell.""" self.beginWrite() # Enter writer's critical section result = writerFunction(self.data) # Exit writer's critical section self.endWrite() return result
class EventsPriorityQueue(PriorityQueue): def __init__(self): PriorityQueue.__init__(self) self.first_element_changed = Condition(self.mutex) def put_and_notify(self, item, block=True, timeout=None): """ Add event into events priority queue and notify thread waiting to get. :param item: the event to be added :param block: whether to block until the event is added :param timeout: if not block, how long wait at most to add event :return: """ log.debug("Adding Event:" + str(item)) self.not_full.acquire() try: first_element_before_insertion = None if self._qsize() > 0: first_element_before_insertion = self.queue[0] if self.maxsize > 0: if not block: if self._qsize() == self.maxsize: raise Full elif timeout is None: while self._qsize() == self.maxsize: self.not_full.wait() elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = _time() + timeout while self._qsize() == self.maxsize: remaining = endtime - _time() if remaining <= 0.0: raise Full self.not_full.wait(remaining) self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() first_element_after_insertion = self.queue[0] if first_element_before_insertion != first_element_after_insertion: self.first_element_changed.notify() finally: self.not_full.release() def get_next_element_when_ready(self): """ Get next event from events priority queue when it is ready: for SystemExit event and dead event, pop from queue immediately; for other active event, wait until next run time, then pop from queue. :return: """ self.first_element_changed.acquire() try: isNotReady = True while isNotReady: if self._qsize() > 0: first_element = self.queue[0] if isinstance(first_element, SystemExit): first_element = self._get() break if not first_element.flag_alive: log.debug("Early termination of dead metric") first_element = self._get() break timeout = (first_element.get_next_run_time() - getUTCmillis()) / 1000.0 log.debug( "Waiting on acquired first_element_changed LOCK " + "for: %.2f" % timeout) self.first_element_changed.wait(timeout) else: self.first_element_changed.wait() first_element = self.queue[0] if isinstance(first_element, SystemExit): first_element = self._get() break if (first_element.get_next_run_time() - getUTCmillis()) <= 0 \ or not first_element.flag_alive: isNotReady = False first_element = self._get() return first_element finally: self.first_element_changed.release()
class ArduinoConnectionDialog(QDialog, QObject): signal_update = pyqtSignal() def __init__(self, parent=None): super(ArduinoConnectionDialog, self).__init__(parent) self.ui = Ui_ArduinoConnectionDialog() self.ui.setupUi(self) self._dialog_started = Condition() self._started = False # Init status images self.__ok_tick = QPixmap() self.__ok_tick.load(create_local_full_path("images", "ok_b.png")) self.__ok_tick = self.__ok_tick.scaled(100, 100) self.__error_tick = QPixmap() self.__error_tick.load(create_local_full_path("images", "error.png")) self.__error_tick = self.__error_tick.scaled(100, 100) # Init loading spinner self.__spinner = QtWaitingSpinner(self.ui.widget) self.ui.verticalLayout_2.addWidget(self.__spinner) self.__label = QLabel() self.__spinner.start() # self.__spinner.setFileName(create_local_full_path("images", "connection.gif")) # self.__spinner = QtWaitingSpinner(self) # self.ui.loadingPanel.setMovie(self.movie) self.resize(150, 150) self.setFixedSize(self.size()) # Avoid Dialog resize self.__update_callback = self.__no_change self.signal_update.connect(self.handle_update) # self.set_ok() def __remove_spinner(self): self.ui.widget.layout().removeWidget(self.__spinner) # self.ui.verticalLayout_2.removeWidget(self.__spinner) if self.__spinner is not None: self.__spinner.deleteLater() self.__spinner = None self.ui.widget.layout().removeWidget(self.__label) def exec_(self): self._dialog_started.acquire() self._started = True self._dialog_started.notify_all() self._dialog_started.release() # Race condition here :/ QDialog.exec_(self) def __no_change(self): return def __config_ok_tick(self): self.__remove_spinner() self.__label = QLabel(self.ui.widget) self.ui.verticalLayout_2.addWidget(self.__label) self.__label.setPixmap(self.__ok_tick) self.ui.status_label.setText("Succesfully connected!") self.ui.buttonBox.setStandardButtons(QDialogButtonBox.Ok) def __config_error_tick(self): self.__remove_spinner() self.__label = QLabel(self.ui.widget) self.ui.verticalLayout_2.addWidget(self.__label) self.__label.setPixmap(self.__error_tick) self.ui.status_label.setText(self.__error) self.ui.buttonBox.setStandardButtons(QDialogButtonBox.Ok) def set_ok(self): self._dialog_started.acquire() if not self._started: self._dialog_started.wait() self.__update_callback = self.__config_ok_tick self.signal_update.emit() def set_error(self, msg): self._dialog_started.acquire() if not self._started: self._dialog_started.wait() self.__update_callback = self.__config_error_tick self.__error = msg self.signal_update.emit() def handle_update(self): self.__update_callback()
def read(self, offset, length, callback=None): assert 0 <= offset < offset + length <= self.total_size idxl = offset // self.block_size idxr = (offset + length - 1) // self.block_size cv = Condition() state = [idxr + 1 - idxl, None] data_list = [None] * state[0] for idx in xrange(idxl, idxr + 1): rngl = max(offset, idx * self.block_size) rngr = min(offset + length, (idx + 1) * self.block_size) shift = rngl % self.block_size to_read = rngr - rngl def gcb(idx, shift, to_read): def cb(err, obj, data): with cv: if state[1] is not None: return False if err: state[1] = err if callback: callback(err, None) else: cv.notify() return False if to_read == self.block_size: data_list[idx - idxl] = data else: with self.cache_lock: self.cache.seek( self.calc_offset(obj) + shift, os.SEEK_SET) data_list[idx - idxl] = self.cache.read(to_read) state[0] = state[0] - 1 if state[0] == 0: if callback is None: cv.notify() else: callback(None, ''.join(data_list)) return False return cb self.pull(idx, read_data=(to_read == self.block_size), callback=gcb(idx, shift, to_read)) if callback is None: cv.acquire() while state[0] != 0: cv.wait() cv.release() if state[1]: raise state[1] else: assert all(x is not None for x in data_list) return ''.join(data_list)
class Sender(object): """ qpid sender """ def __init__(self, broker, address): self.broker = broker self.address = address self.address_self = str(uuid.uuid1()) + ''';{ create: always , delete: always , node: {type:queue , x-declare : {auto-delete: true}} }''' self.ret_msg_queue = Queue() self.cond = Condition() self.send_callback = {} self.send_response = {} self.connection = None self.session = None self.sender = None def init(self): self.connection = messaging.Connection(self.broker) try: self.connection.open() self.session = self.connection.session() self.sender = self.session.sender(self.address) except messaging.MessagingError: print traceback.format_exc() return False thread = Thread(target=self.ret_msg_processor) thread.daemon = True thread.start() message_receiver = _MessageReceiver(self.broker, self.address_self) message_receiver.start(self.ret_msg_queue, daemon=True) return True def destroy(self): self.connection.close() def send(self, message, callback=None): if self.sender is None: return False message_id = uuid.uuid1().hex message.reply_to = self.address_self message.properties[message_id_tag] = message_id if callback: self.send_callback[message_id] = callback try: self.sender.send(message) self.session.acknowledge() except messaging.MessagingError: print traceback.format_exc() return False return True def sync_send(self, message, timeout=1): if self.sender is None: return None message_id = uuid.uuid1().hex message.reply_to = self.address_self message.properties[message_id_tag] = message_id try: self.send_response[message_id] = None self.sender.send(message) self.session.acknowledge() except messaging.MessagingError: print traceback.format_exc() return None self.cond.acquire() request_time = time.time() while self.send_response[ message_id] is None and request_time + timeout > time.time(): self.cond.wait(0.1) self.cond.release() response = self.send_response.pop(message_id, None) return response def ret_msg_processor(self): while 1: message = self.ret_msg_queue.get() if not isinstance(message, Message): continue message_id = message.properties.get(message_id_tag, None) if message_id: if message_id in self.send_response.keys(): self.cond.acquire() self.send_response[message_id] = message self.cond.notifyAll() self.cond.release() if message_id in self.send_callback.keys(): callback = self.send_callback.get(message_id, None) if callback and hasattr(callback, '__call__'): ret = callback(message) if ret is None or ret: self.send_callback.pop(message_id, None) else: self.send_callback.pop(message_id, None)
class DataPlane: """ Class defining access primitives to the data plane Controls a list of DataPlanePort objects """ def __init__(self): self.port_list = {} # pkt_sync serves double duty as a regular top level lock and # as a condition variable self.pkt_sync = Condition() # These are used to signal async pkt arrival for polling self.want_pkt = False self.want_pkt_port = None # What port required (or None) self.got_pkt_port = None # On what port received? self.packets_pending = 0 # Total pkts in all port queues self.logger = logging.getLogger("dataplane") self.pkt_handler = None def port_add(self, interface_name, port_number): """ Add a port to the dataplane TBD: Max packets for queue? @param interface_name The name of the physical interface like eth1 @param port_number The port number used to refer to the port """ self.port_list[port_number] = DataPlanePort(interface_name, port_number, self) self.port_list[port_number].start() if self.pkt_handler is not None: self.port_list[port_number].register(self.pkt_handler) def register(self, pkt_handler): """ Register pkt_handler for all ports """ self.pkt_handler = pkt_handler for port_number in self.port_list.keys(): self.port_list[port_number].register(pkt_handler) def send(self, port_number, packet, queue_id=0): """ Send a packet to the given port @param port_number The port to send the data to @param packet Raw packet data to send to port @param queue_id The queue to send to (to be implemented) """ #@todo Verify port_number is in keys of port_list self.logger.debug("Sending %d bytes to port %d" % (len(packet), port_number)) bytes = self.port_list[port_number].send(packet, queue_id=queue_id) if bytes != len(packet): self.logger.error( "Unhandled send error, length mismatch %d != %d" % (bytes, len(packet))) return bytes def flood(self, packet): """ Send a packet to all ports @param packet Raw packet data to send to port """ for port_number in self.port_list.keys(): bytes = self.port_list[port_number].send(packet) if bytes != len(packet): self.logger.error("Unhandled send error" + ", port %d, length mismatch %d != %d" % (port_number, bytes, len(packet))) def _oldest_packet_find(self): # Find port with oldest packet min_time = 0 min_port = -1 for port_number in self.port_list.keys(): ptime = self.port_list[port_number].timestamp_head() if ptime: if (min_port == -1) or (ptime < min_time): min_time = ptime min_port = port_number oft_assert(min_port != -1, "Could not find port when pkts pending") return min_port def poll(self, port_number=None, timeout=None): """ Poll one or all dataplane ports for a packet If port_number is given, get the oldest packet from that port. Otherwise, find the port with the oldest packet and return that packet. @param port_number If set, get packet from this port @param timeout If positive and no packet is available, block until a packet is received or for this many seconds @return The triple port_number, packet, pkt_time where packet is received from port_number at time pkt_time. If a timeout occurs, return None, None, None """ self.pkt_sync.acquire() # Check if requested specific port and it has a packet if port_number and len(self.port_list[port_number].packets) != 0: pkt, time = self.port_list[port_number].dequeue(use_lock=False) self.pkt_sync.release() oft_assert(pkt, "Poll: packet not found on port " + str(port_number)) return port_number, pkt, time # Check if requested any port and some packet pending if not port_number and self.packets_pending != 0: port = self._oldest_packet_find() pkt, time = self.port_list[port].dequeue(use_lock=False) self.pkt_sync.release() oft_assert(pkt, "Poll: oldest packet not found") return port, pkt, time # No packet pending; blocking call requested? if not timeout: self.pkt_sync.release() return None, None, None # Desired packet isn't available and timeout is specified # Already holding pkt_sync; wait on pkt_sync variable self.want_pkt = True self.want_pkt_port = port_number self.got_pkt_port = None self.pkt_sync.wait(timeout) self.want_pkt = False if self.got_pkt_port: pkt, time = \ self.port_list[self.got_pkt_port].dequeue(use_lock=False) self.pkt_sync.release() oft_assert( pkt, "Poll: pkt reported, but not found at " + str(self.got_pkt_port)) return self.got_pkt_port, pkt, time self.pkt_sync.release() self.logger.debug("Poll time out, no packet from " + str(port_number)) return None, None, None def kill(self, join_threads=True): """ Close all sockets for dataplane @param join_threads If True call join on each thread """ for port_number in self.port_list.keys(): self.port_list[port_number].kill() if join_threads: self.logger.debug("Joining " + str(port_number)) self.port_list[port_number].join() self.logger.info("DataPlane shutdown") def show(self, prefix=''): print prefix + "Dataplane Controller" print prefix + "Packets pending" + str(self.packets_pending) for pnum, port in self.port_list.items(): print prefix + "OpenFlow Port Number " + str(pnum) port.show(prefix + ' ')
class Obstacle_Avoidance: """ A class for performing obstacle avoidance by running a sonar thread and a motor control thread in parallel. Attributes: max_speed (int): The maximum speed value for the motor driver. decel_rate (float): How quickly (from 0 to 1) the rover should decelerate. turn_time (float): How long (in seconds) the rover should turn left or right. reverse_time (float): How long (in seconds) the rover should reverse. turn_dist (int): How close an obstacle must be (in mm) before the rover turns. reverse_dist (int): How close an obstacle must be (in mm) before the rover reverses. offset (float): Since the wheels aren't straight, one wheel will turn slower. """ class Direction(Enum): """This enumerates the possible directions the rover can move/turn.""" FORWARD = 0 BACKWARD = 1 RIGHT = 2 LEFT = 3 def __init__(self, threshold=0.1, buffer_size=5, max_speed=480, decel_rate=0.05, turn_time=1, reverse_time=1, turn_dist=1000, reverse_dist=500, offset=0.8, debug=False): """ Constructs an Obstacle_Avoidance object using a series of overridable default parameters. Keyword Arguments: threshold (float): What threshold value to pass to the Sonar object. buffer_size (int): What buffer_size value to pass to the Sonar object. max_speed (int): Refer to the max_speed class attribute. decel_rate (float): Refer to the decel_rate class attribute. turn_time (float): Refer to the turn_time class attribute. reverse_time (float): Refer to the reverse_time class attribute. turn_dist (int): Refer to the turn_dist class attribute. reverse_dist (int): Refer to the reverse_dist class attribute. offset (float): Refer to the offset class attribute. debug (bool): Flag that controls whether or not to allow remote debugging. """ # If debug is enabled, wait for the remote debugger to attach before continuing. if debug: ptvsd.enable_attach("rover_senpai") print("Waiting for debugger to attach...", file=stderr) ptvsd.wait_for_attach() print("Debugger attached, continuing...", file=stderr) dual_mc33926.io_init_motor_drive() # Initializes the motor driver. self.max_speed = max_speed self.decel_rate = decel_rate self.turn_time = turn_time self.reverse_time = reverse_time self.turn_dist = turn_dist self.reverse_dist = reverse_dist self.offset = offset self._sonar_data = ( 0, b'') # Distance and the sonar indicator (b'L' or b'R'). self._stop_flag = False # Causes the threads to stop when set to true. self._condition = Condition( ) # Condition object protects _sonar_data and _stop_flag. self._sonar = Sonar(threshold=threshold, buffer_size=buffer_size) self._motor = dual_mc33926.MotorDriver() self._sonar_thread = Thread(target=self._run_sonar, daemon=True) self._motor_thread = Thread(target=self._run_motor, daemon=True) def _run_sonar(self): """Continously runs the sonars and notifies the motor thread whenever an object is close.""" while True: # Gets a measurement from the sonars. dist, dir = self._sonar.measure() # If the measured sonar distance is less than our turning threshold... if dist < self.turn_dist: self._condition.acquire() # If the stop flag has been set by stop(), release the lock and break out. if self._stop_flag: self._condition.release() break # Otherwise, publish the distance and which sonar detected it. self._sonar_data = (dist, dir) self._condition.notify_all() self._condition.release() def _run_motor(self): """Continuously runs the motors and turns away from any obstacles deteced by the sonars.""" self._motor.enable() def move(direction=Direction.FORWARD, duration=None): """ Helper method which simply moves/turns the rover in a given direction. Keyword Arguments: direction (Direction): The direction the rover should turn/move. duration (float): How long the rover should move in this direction. """ # First, gradually stop whatever the motors were doing before. for i in range(1 - self.decel_rate, 0, -self.decel_rate): self._motor.set_speeds(self._spd1 * i, self._dir1, self._spd2 * i, self._dir2) sleep(0.05) # Determine speed and direction of motors based on the desired direction. if direction == Direction.FORWARD: self._spd1 = self.max_speed self._spd2 = self.max_speed * self.offset self._dir1 = 1 self._dir2 = 1 elif direction == Direction.BACKWARD: self._spd1 = self.max_speed self._spd2 = self.max_speed * self.offset self._dir1 = 0 self._dir2 = 0 elif direction == Direction.RIGHT: self._spd1 = 0 self._spd2 = self.max_speed * self.offset self._dir1 = 1 self._dir2 = 1 else: self._spd1 = self.max_speed self._spd2 = 0 self._dir1 = 1 self._dir2 = 1 # Start moving. self._motor.set_speeds(self._spd1, self._dir1, self._spd2, self._dir2) # Wait for a duration (if specified) before accepting another move command. if (type(duration) in [int, float]) and (duration > 0): sleep(duration) # The rover continuously moves forward, reversing and turning away from detected obstacles. while True: # Move forward until sonar publishes data about a detected obstacle. move() self._condition.acquire() self._condition.wait() # If the stop flag has been set by stop(), release the lock and break out. if self._stop_flag: self._condition.release() break # Otherwise, save the distance from the obstacle (in mm) and which sonar detected it. dist, dir = self._sonar_data.deepcopy() self._condition.release() # If the object is very close, first reverse from it. if dist < self.reverse_dist: move(direction=Direction.BACKWARD, duration=self.reverse_time) # Turn right away from objects detected by the left sonar. if dir == b'R': move(direction=Direction.LEFT, duration=self.turn_time) # Turn left away from objects detected by the right sonar. elif dir == b'L': move(direction=Direction.RIGHT, duration=self.turn_time) # Raise an exception if we see an unrecognized sonar label. else: raise ValueError("Invalid sonar label") # Disable the motors once we break out of the loop. self._motor.disable() def start(self): """This method starts the obstacle avoidance algorithm.""" if not (self._motor_thread.is_alive() and self._sonar_thread.is_alive()): self._stop_flag = False self._sonar_thread.start() self._motor_thread.start() print("Obstacle avoidance is now running.", file=stderr) else: print("Obstacle avoidance is already running.", file=stderr) def stop(self): """This method stops the obstacle avoidance algorithm.""" if self._motor_thread.is_alive() and self._sonar_thread.is_alive(): self._condition.acquire() self._stop_flag = True self._condition.notify_all() self._condition.release() self._sonar_thread.join() self._motor_thread.join() print("Obstacle avoidance has stopped.", file=stderr) else: print("Obstacle avoidance is not currently running.", file=stderr)
class RDMTestThread(Thread): """The RDMResponder tests are closely coupled to the Wrapper (yuck!). So we need to run this all in a separate thread. This is all a bit of a hack and you'll get into trouble if multiple things are running at once... """ RUNNING, COMPLETED, ERROR = range(3) TESTS, COLLECTOR = range(2) def __init__(self, pid_store, logs_directory): super(RDMTestThread, self).__init__() self._pid_store = pid_store self._logs_directory = logs_directory self._terminate = False self._request = None # guards _terminate and _request self._cv = Condition() self._wrapper = None self._test_state_lock = Lock() # guards _test_state self._test_state = {} def Stop(self): self._cv.acquire() self._terminate = True self._cv.notify() self._cv.release() def ScheduleTests(self, universe, uid, test_filter, broadcast_write_delay, inter_test_delay, dmx_frame_rate, slot_count): """Schedule the tests to be run. Callable from any thread. Callbable by any thread. Returns: An error message, or None if the tests were scheduled. """ if not self._CheckIfConnected(): return 'Lost connection to OLAD' self._cv.acquire() if self._request is not None: self._cv.release() return 'Existing request pending' self._request = lambda: self._RunTests( universe, uid, test_filter, broadcast_write_delay, inter_test_delay, dmx_frame_rate, slot_count) self._cv.notify() self._cv.release() return None def ScheduleCollector(self, universe, skip_queued_messages): """Schedule the collector to run on a universe. Callable by any thread. Returns: An error message, or None if the collection was scheduled. """ if not self._CheckIfConnected(): return 'Lost connection to OLAD' self._cv.acquire() if self._request is not None: self._cv.release() return 'Existing request pending' self._request = lambda: self._RunCollector(universe, skip_queued_messages) self._cv.notify() self._cv.release() return None def Stat(self): """Check the state of the tests. Callable by any thread. Returns: The status of the tests. """ self._test_state_lock.acquire() state = dict(self._test_state) self._test_state_lock.release() return state def run(self): self._wrapper = ClientWrapper() self._collector = ModelCollector(self._wrapper, self._pid_store) while True: self._cv.acquire() if self._terminate: logging.info('quitting test thread') self._cv.release() return if self._request is not None: request = self._request self._request = None self._cv.release() request() continue # nothing to do, go into the wait self._cv.wait() self._cv.release() def _UpdateStats(self, tests_completed, total_tests): self._test_state_lock.acquire() self._test_state['tests_completed'] = tests_completed self._test_state['total_tests'] = total_tests self._test_state_lock.release() def _RunTests(self, universe, uid, test_filter, broadcast_write_delay, inter_test_delay, dmx_frame_rate, slot_count): self._test_state_lock.acquire() self._test_state = { 'action': self.TESTS, 'tests_completed': 0, 'total_tests': None, 'state': self.RUNNING, 'duration': 0, } start_time = datetime.now() self._test_state_lock.release() runner = TestRunner.TestRunner(universe, uid, broadcast_write_delay, inter_test_delay, self._pid_store, self._wrapper) for test in TestRunner.GetTestClasses(TestDefinitions): runner.RegisterTest(test) dmx_sender = None if dmx_frame_rate > 0 and slot_count > 0: logging.info( 'Starting DMXSender with slot count %d and FPS of %d' % (slot_count, dmx_frame_rate)) dmx_sender = DMXSender(self._wrapper, universe, dmx_frame_rate, slot_count) try: tests, device = runner.RunTests(test_filter, False, self._UpdateStats) except Exception as e: self._test_state_lock.acquire() self._test_state['state'] = self.ERROR self._test_state['exception'] = str(e) self._test_state['traceback'] = traceback.format_exc() self._test_state_lock.release() return finally: if dmx_sender is not None: dmx_sender.Stop() timestamp = int(time()) end_time = datetime.now() test_parameters = { 'broadcast_write_delay': broadcast_write_delay, 'inter_test_delay': inter_test_delay, 'dmx_frame_rate': dmx_frame_rate, 'dmx_slot_count': slot_count, } log_saver = TestLogger.TestLogger(self._logs_directory) logs_saved = True try: log_saver.SaveLog(uid, timestamp, end_time, tests, device, test_parameters) except TestLogger.TestLoggerException: logs_saved = False self._test_state_lock.acquire() # we can't use total_seconds() since it requires Python 2.7 time_delta = end_time - start_time self._test_state['duration'] = (time_delta.seconds + time_delta.days * 24 * 3600) self._test_state['state'] = self.COMPLETED self._test_state['tests'] = tests self._test_state['logs_saved'] = logs_saved self._test_state['timestamp'] = timestamp self._test_state['uid'] = uid self._test_state_lock.release() def _RunCollector(self, universe, skip_queued_messages): """Run the device model collector for a universe.""" logging.info('Collecting for %d' % universe) self._test_state_lock.acquire() self._test_state = { 'action': self.COLLECTOR, 'state': self.RUNNING, } self._test_state_lock.release() try: output = self._collector.Run(universe, skip_queued_messages) except Exception as e: self._test_state_lock.acquire() self._test_state['state'] = self.ERROR self._test_state['exception'] = str(e) self._test_state['traceback'] = traceback.format_exc() self._test_state_lock.release() return self._test_state_lock.acquire() self._test_state['state'] = self.COMPLETED self._test_state['output'] = output self._test_state_lock.release() def _CheckIfConnected(self): """Check if the client is connected to olad. Returns: True if connected, False otherwise. """ # TODO(simon): add this check, remember it needs locking. return True
class ZumyROS: def __init__(self): self.zumy = Zumy() rospy.init_node('zumy_ros') self.cmd = (0, 0) rospy.Subscriber('cmd_vel', Twist, self.cmd_callback, queue_size=1) self.lock = Condition() self.rate = rospy.Rate(30.0) self.name = socket.gethostname() self.heartBeat = rospy.Publisher('heartBeat', String, queue_size=1) self.imu_pub = rospy.Publisher('imu', Imu, queue_size=5) self.r_enc_pub = rospy.Publisher('r_enc', Int32, queue_size=1) self.l_enc_pub = rospy.Publisher('l_enc', Int32, queue_size=1) self.volt_pub = rospy.Publisher('voltage', Float32, queue_size=1) self.imu_count = 0 def cmd_callback(self, msg): lv = 0.6 la = 0.4 v = msg.linear.x a = msg.angular.z r = lv * v + la * a l = lv * v - la * a self.lock.acquire() self.cmd = (l, r) self.lock.release() def run(self): while not rospy.is_shutdown(): self.lock.acquire() self.zumy.cmd(*self.cmd) imu_data = self.zumy.read_imu() enc_data = self.zumy.read_enc() volt_data = self.zumy.read_voltage() self.lock.release() imu_msg = Imu() imu_msg.header = Header(self.imu_count, rospy.Time.now(), self.name) imu_msg.linear_acceleration.x = 9.81 * imu_data[0] imu_msg.linear_acceleration.y = 9.81 * imu_data[1] imu_msg.linear_acceleration.z = 9.81 * imu_data[2] imu_msg.angular_velocity.x = 3.14 / 180.0 * imu_data[3] imu_msg.angular_velocity.y = 3.14 / 180.0 * imu_data[4] imu_msg.angular_velocity.z = 3.14 / 180.0 * imu_data[5] self.imu_pub.publish(imu_msg) enc_msg = Int32() enc_msg.data = enc_data[0] self.r_enc_pub.publish(enc_msg) enc_msg = Int32() enc_msg.data = enc_data[1] self.l_enc_pub.publish(enc_msg) volt_msg = Float32() volt_msg.data = volt_data self.volt_pub.publish(volt_msg) #self.heartBeat.publish("I am alive") self.rate.sleep() # If shutdown, turn off motors self.zumy.cmd(0, 0)
class TimedTaskQueue: __single = None def __init__(self, nameprefix='TimedTaskQueue', isDaemon=True, debug=False): self.debug = debug self.cond = Condition() self.queue = [] self.count = 0.0 self.thread = Thread(target=self.run) self.thread.setDaemon(isDaemon) self.thread.setName(nameprefix + self.thread.getName()) self.thread.start() if DEBUG_STACK: self.callstack = {} def add_task(self, task, t=0, id=None, pos=None): if task is None: print_stack() self.cond.acquire() when = time() + t if DEBUG_STACK: self.callstack[self.count] = format_stack() if id != None: self.queue = filter(lambda item: item[3] != id, self.queue) item = (when, self.count, task, id) if pos is None: self.queue.append(item) else: self.queue.insert(pos, item) self.count += 1.0 self.cond.notify() self.cond.release() if DEBUG or self.debug: log('ttqueue:add_task: t', t, 'task', task, 'id', id, 'len(queue)', len(self.queue)) def run(self): while True: task = None timeout = None flag = False self.cond.acquire() while True: while len(self.queue) == 0 or flag: flag = False if timeout is None: self.cond.wait() else: self.cond.wait(timeout) self.queue.sort() when, count, task, id = self.queue[0] now = time() if now < when: timeout = when - now if DEBUG or self.debug: log('ttqueue::run: event not due: timeout', timeout, 'task', task) flag = True else: self.queue.pop(0) if DEBUG or self.debug: log('ttqueue::run: event due: task', task, 'len(queue)', len(self.queue)) if DEBUG_STACK: stack = self.callstack.pop(count) break self.cond.release() try: if task == 'stop': break elif task == 'quit': if len(self.queue) == 0: break else: when, count, task, id = self.queue[-1] t = when - time() + 0.001 self.add_task('quit', t) else: t = time() task() if DEBUG or self.debug: log('ttqueue::run: task finished: time', time() - t, 'task', task) except: log_exc() if DEBUG_STACK: print >> sys.stderr, '<<<<<<<<<<<<<<<<' print >> sys.stderr, 'TASK QUEUED FROM' print >> sys.stderr, ''.join(stack) print >> sys.stderr, '>>>>>>>>>>>>>>>>' if DEBUG: log('ttqueue::run: exit loop')
class StreamChannel(Channel): ''' The StreamChannel should provide access methods to remote streams (windows like). Beside pure data transfer (read/write), the channel needs to be able to communicate control data to the peer (for example: close, flush etc.). There are multiple options to achieve this: 1) Keep the channel bidirectional and mix pure data with control data, distinguished by some header bytes + additional logic is kept in channel - complex implemetation (priority of control data over already enqueued channel data has to be assured, channel control data has to have priority over other data from different channels), adjustments to multiple communication layers are needed anyway 2) Keep the channels standard functionality (read/write) and use the global control channel to communicate data + simpler implementation - the channel itself needs access to the Client object, which again is able to send and receive control messages - a control message and its remote handler have to be implemented for every possible "channel control communication type" + the client is able to create control messages, to push them to the server 3) The channel could hold a second communictaion queue with outbound control messages, the outer client implementation sends these control requests and receives the answers. Instead of using a own CTRL_MSG for every type of possible control communication, a single remote method could be used (CTRL_RUN_METHOD). Complexity could be hidden in a "big" remote method with a "big" local response handler. + remote methods are handled by control messages (RUN_METHOD + RUN_METHOD_RESPONSE) and thus are already given priority (at least as long as the other layers are able to assure that global control channel data is priorized) + the handler for the "channel_control_method" call (ipmlemented on client level, as all the other handlers), could pass in channel control method responses to the correct channel - this doesn't allow "push notifications" from the other peer, as remote methodes have to be requested from server to client Test approach is method 1: ''' BUFFER_SIZE = 3000 CHANNEL_CONTROL_REQUEST_STATE = 1 CHANNEL_CONTROL_REQUEST_READ = 2 CHANNEL_CONTROL_REQUEST_FLUSH = 3 CHANNEL_CONTROL_REQUEST_CLOSE = 4 # this means to close the stream, not the channel CHANNEL_CONTROL_REQUEST_POSITION = 5 CHANNEL_CONTROL_REQUEST_LENGTH = 6 CHANNEL_CONTROL_REQUEST_READ_TIMEOUT = 7 CHANNEL_CONTROL_REQUEST_WRITE_TIMEOUT = 8 CHANNEL_CONTROL_REQUEST_SEEK = 9 CHANNEL_CONTROL_REQUEST_WRITE = 10 CHANNEL_CONTROL_INFORM_REMOTEBUFFER_LIMIT = 1001 CHANNEL_CONTROL_INFORM_REMOTEBUFFER_SIZE = 1002 CHANNEL_CONTROL_INFORM_WRITE_SUCCEEDED = 1003 CHANNEL_CONTROL_INFORM_READ_SUCCEEDED = 1004 CHANNEL_CONTROL_INFORM_WRITE_FAILED = 1005 CHANNEL_CONTROL_INFORM_READ_FAILED = 1006 CHANNEL_CONTROL_INFORM_FLUSH_SUCCEEDED = 1007 CHANNEL_CONTROL_INFORM_FLUSH_FAILED = 1008 def __init__(self, channel_id, stream_id, passthrough = True): # stream attributes self.__can_read = False self.__can_seek = False self.__can_timeout = False self.__can_write = False self.__length = 0 self.__position = 0 self.__read_timeout = 0 self.__write_timeout = 0 self.__passthrough = passthrough if not passthrough: self.__write_condition = Condition() self.__read_condition = Condition() self.__flush_condition = Condition() self.__write_succeeded = False self.__read_succeeded = False self.__flush_succeeded = False self.__write_size = 0 self.__read_size = 0 self.__read_data = "" # internal attributes self.__stream_id = stream_id # id (hasCode) of the stream bound to this channel, if any self.__control_in_queue = Queue.Queue() # we start with a id of '-1' to indicate that the channel isn't bound to a remote stream # th channel is bidirectional, to allow control data to be sent (even it is a read only or write only stream) # encoding is BYTE (raw) super(StreamChannel, self).__init__(channel_id, Channel.TYPE_BIDIRECTIONAL, Channel.ENCODING_BYTEARRAY) @property def CanRead(self): return self.__can_read @property def CanSeek(self): return self.__can_seek @property def CanTimeout(self): return self.__can_timeout @property def CanWrite(self): return self.__can_write @property def Length(self): return self.__length @property def Position(self): return self.__position @Position.setter def Position(self, value): self.__position = value @property def ReadTimeout(): return self.__read_timeout @ReadTimeout.setter def ReadTimeout(self, value): self.__read_timeout = value @property def WriteTimeout(): return self.__write_timeout @WriteTimeout.setter def WriteTimeout(self, value): self.__write_timeout = value def Close(self): control_msg = struct.pack("!I", StreamChannel.CHANNEL_CONTROL_REQUEST_CLOSE) self.__sendControlMessage(control_msg) def Dispose(self): pass def Flush(self): control_msg = struct.pack("!I", StreamChannel.CHANNEL_CONTROL_REQUEST_FLUSH) self.__sendControlMessage(control_msg) if not self.__passthrough: self.__flush_condition.acquire() self.__flush_condition.wait(timeout=None) succeeded = self.__flush_succeeded self.__flush_condition.release() return succeeded def Read(self, count, timeout=0): # on demand read: #control_msg = struct.pack("!Iii", StreamChannel.CHANNEL_CONTROL_REQUEST_READ, count, timeout) #self.__sendControlMessage(control_msg) if self.__passthrough: if not self.hasInput(): return "" return self.readInput() else: # if passthrough is disable, data is written as control message and an answer is expected control_msg = struct.pack("!Iii", StreamChannel.CHANNEL_CONTROL_REQUEST_READ, count, timeout) self.__sendControlMessage(control_msg) # we wait till an answer is received (based on a condition) self.__read_condition.acquire() self.__read_condition.wait(timeout=None) #check if write succeeded succeeded = self.__read_succeeded read_size = self.__read_size read_data = self.__read_data self.__read_condition.release() if succeeded: return read_data else: raise ChannelException("Error reading from StreamChannel {0}".format(self.id)) def ReadByte(self): pass def Seek(self, offset, origin): control_msg = struct.pack("!Iii", StreamChannel.CHANNEL_CONTROL_REQUEST_SEEK, offset, origin) self.__sendControlMessage(control_msg) def Write(self, data): res = self.__writeData(data) if not self.__passthrough: return res def WriteByte(self, byte): res = self.__writeData(byte) if not self.__passthrough: return res def __writeData(self, data): if self.__passthrough: # in passthrough mode data is written as dedicated data message header = struct.pack("!B", 0) # header byte 0 indicates that data is written super(StreamChannel, self).writeOutput(header + data) else: # if passthrough is disable, data is written as control message and an answer is expected control_msg = struct.pack("!Ii", StreamChannel.CHANNEL_CONTROL_REQUEST_WRITE, len(data)) control_msg += data self.__sendControlMessage(control_msg) # we wait till an answer is received (based on a condition) self.__write_condition.acquire() self.__write_condition.wait(timeout=None) #check if write succeeded succeeded = self.__write_succeeded write_size = self.__write_size self.__write_condition.release() if succeeded: return write_size else: return -1 # indicate write error (alternatively a ChannelException could be raised) def __sendControlMessage(self, control_data): header = struct.pack("!B", 1) # header byte 1 indicates that control data # naive, if the queue is full, that message is pending as it hasn't been priorized super(StreamChannel, self).writeOutput(header + control_data) def writeOutput(self, data): res = self.__writeData(data) if not self.__passthrough: return res #def readInput(self): #print "StreamChannel error: writeOutput shouldn't be called, use Write instead" def enqueueInput(self, data): data_type = struct.unpack("!B", data[0])[0] data = data[1:] if data_type == 0: self._Channel__in_queue.put(data) # normal data else: self.__dispatchControlMessage(data) # control data def __dispatchControlMessage(self, control_data): #print "Channel with id '{0}' received control data: {1}".format(self.id, repr(control_data)) # grab control_msg id control_msg = struct.unpack("!I", control_data[:4])[0] data = control_data[4:] if control_msg == self.CHANNEL_CONTROL_INFORM_WRITE_FAILED: # set event for write operation response self.__write_condition.acquire() self.__write_succeeded = False self.__write_condition.notifyAll() self.__write_condition.release() elif control_msg == self.CHANNEL_CONTROL_INFORM_WRITE_SUCCEEDED: # read how many bytes have been written size_written = struct.unpack("!i", data[:4])[0] # set event for write operation response self.__write_condition.acquire() self.__write_size = size_written self.__write_succeeded = True self.__write_condition.notifyAll() self.__write_condition.release() elif control_msg == self.CHANNEL_CONTROL_INFORM_READ_SUCCEEDED: # read how many bytes have been read size_read = struct.unpack("!i", data[:4])[0] data_read = data[4:] # set event for read operation response self.__read_condition.acquire() self.__read_size = size_read self.__read_data = data_read self.__read_succeeded = True self.__read_condition.notifyAll() self.__read_condition.release() elif control_msg == self.CHANNEL_CONTROL_INFORM_READ_FAILED: # set event for read operation response self.__read_condition.acquire() self.__read_succeeded = False self.__read_condition.notifyAll() self.__read_condition.release() elif control_msg == self.CHANNEL_CONTROL_INFORM_FLUSH_SUCCEEDED: self.__flush_condition.acquire() self.__flush_succeeded = True self.__flush_condition.notifyAll() self.__flush_condition.release() elif control_msg == self.CHANNEL_CONTROL_INFORM_FLUSH_FAILED: self.__flush_condition.acquire() self.__flush_succeeded = False self.__flush_condition.notifyAll() self.__flush_condition.release() def onClose(self): if not self.__passthrough: # set error for pending writes self.__write_condition.acquire() self.__write_succeeded = False self.__write_condition.notifyAll() self.__write_condition.release() # set error for pending reads self.__read_condition.acquire() self.__read_succeeded = False self.__read_condition.notifyAll() self.__read_condition.release() super(StreamChannel, self).onClose()
class UcsCentralEventHandle(object): """This class provides api to add and remove event handler.""" def __init__(self, handle): self.__handle = handle self.__lock_object = None self.__wbs = [] self.__wbs_lock = Lock() self.__enqueue_thread = None self.__condition = Condition() self.__event_chan_resp = None self.__dequeue_thread = None self.__lowest_timeout = None self.__wb_to_remove = [] def __get_mo_elem(self, xml_str): """ Internal method to extract mo elements from xml string """ root = xc.extract_root_elem(xml_str) mo_elems = [] if root.tag == "methodVessel": for in_stimuli in root: for cmce in in_stimuli: for in_config in cmce: for mo_elem in in_config: mo_elems.append( (mo_elem, cmce.attrib.get('inEid'))) elif root.tag == "configMoChangeEvent": for in_config in root: for mo_elem in in_config: mo_elems.append(mo_elem) return mo_elems def __enqueue_function(self): """ Internal method used by add_event_handler. Provides functionality of enqueue/dequeue of the events and triggering callbacks. """ try: xml_query = '<eventSubscribe cookie="%s"/>' % self.__handle.cookie self.__event_chan_resp = self.__handle.post_xml( xml_str=xml_query.encode(), read=False) except Exception: raise try: while self.__event_chan_resp and len(self.__wbs): if self.__handle.cookie is None or \ self.__event_chan_resp is None: break resp = self.__event_chan_resp.readline() resp = self.__event_chan_resp.read(int(resp)) for mo_elem in self.__get_mo_elem(resp): gmo = ucscentralmo.generic_mo_from_xml_elem(mo_elem[0]) mce = MoChangeEvent(event_id=mo_elem[1], mo=gmo.to_mo(), change_list=gmo.properties.keys()) for watch_block in self.__wbs: if watch_block.fmce(mce): watch_block.enqueue(mce) with self.__condition: self.__condition.notify() if len(self.__wbs) == 0: self.__condition.acquire() self.__condition.notify() self.__condition.release() except: raise def __thread_enqueue_start(self): """ Internal method to start the enqueue thread which adds the events in an internal queue. """ self.__enqueue_thread = Thread(name="enqueue_thread", target=self.__enqueue_function) self.__enqueue_thread.daemon = True self.__enqueue_thread.start() def __time_left(self, watch_block): timeout_sec = watch_block.params["timeout_sec"] start_time = watch_block.params["start_time"] time_diff = datetime.datetime.now() - start_time if time_diff.seconds < timeout_sec: return timeout_sec - time_diff.seconds else: return 0 # return 2147483647 def __dequeue_mce(self, time_left, watch_block): if time_left and time_left > 0: if self.__lowest_timeout is None or \ self.__lowest_timeout > time_left: self.__lowest_timeout = time_left mce = watch_block.dequeue(time_left) else: mce = watch_block.dequeue(2147483647) return mce def __prop_val_exist(self, mo, prop, success_value, failure_value, transient_value, change_list=None): if isinstance(mo, ucscentralmo.GenericMo): n_prop = prop n_prop_val = mo.properties[n_prop] elif prop not in mo.prop_meta: n_prop = prop n_prop_val = getattr(mo, n_prop) else: n_prop = mo.prop_meta[prop].xml_attribute n_prop_val = getattr(mo, n_prop) if change_list and n_prop not in change_list: return False if (len(success_value) > 0 and n_prop_val in success_value) or \ (len(failure_value) > 0 and n_prop_val in failure_value) or \ (len(transient_value) > 0 and n_prop_val in transient_value): return True return False def __dequeue_mo_prop_poll(self, mo, prop, poll_sec, watch_block, timeout_sec=None, time_left=None): success_value = watch_block.params["success_value"] failure_value = watch_block.params["failure_value"] transient_value = watch_block.params["transient_value"] if not success_value or len(success_value) < 1: raise ValueError("success_value is missing.") pmo = self.__handle.query_dn(mo.dn) if pmo is None: UcsCentralWarning('Mo ' + pmo.dn + ' not found.') return if timeout_sec is not None and time_left is not None and time_left > 0: if time_left < poll_sec: poll_sec = timeout_sec - time_left if self.__lowest_timeout is None or self.__lowest_timeout > poll_sec: self.__lowest_timeout = poll_sec if self.__prop_val_exist(pmo, prop, success_value, failure_value, transient_value): log.info("Successful") self.__wb_to_remove.append(watch_block) def __dequeue_mo_prop_event(self, prop, watch_block, time_left=None): success_value = watch_block.params["success_value"] failure_value = watch_block.params["failure_value"] transient_value = watch_block.params["transient_value"] if not success_value or len(success_value) < 1: raise ValueError("success_value is missing.") # dequeue mce mce = self.__dequeue_mce(time_left, watch_block) if mce is None: return # checks if prop value exist in success or failure or transient values attributes = mce.change_list if self.__prop_val_exist(mce.mo, prop, success_value, failure_value, transient_value, attributes): if watch_block.callback: ctxt = watch_block.params['context'] ctxt["done"] = True watch_block.callback(mce) self.__wb_to_remove.append(watch_block) def __dequeue_mo_until_removed(self, watch_block, time_left=None): # dequeue mce mce = self.__dequeue_mce(time_left, watch_block) if mce is None: return if watch_block.callback is not None: watch_block.callback(mce) # watch mo until gets deleted if mce.mo.status == "deleted": self.__wb_to_remove.append(watch_block) def __dequeue_all_class_id(self, watch_block, time_left=None): # dequeue mce mce = self.__dequeue_mce(time_left, watch_block) if mce is not None and watch_block.callback is not None: watch_block.callback(mce) def __dequeue_function(self): """ Internal method to dequeue to events. """ while len(self.__wbs): self.__lowest_timeout = None self.__wb_to_remove = [] try: for watch_block in self.__wbs: mo = watch_block.params["managed_object"] prop = watch_block.params["prop"] poll_sec = watch_block.params["poll_sec"] timeout_sec = watch_block.params["timeout_sec"] # checks if watch_block is not timed out, else remove time_left = None if timeout_sec is not None: time_left = self.__time_left(watch_block) if time_left <= 0: self.__wb_to_remove.append(watch_block) continue # poll for mo. Not to monitor event. if poll_sec is not None and mo is not None: self.__dequeue_mo_prop_poll(mo, prop, poll_sec, watch_block, timeout_sec, time_left) elif mo is not None: # watch mo until prop_val changed to desired value if prop is not None: self.__dequeue_mo_prop_event( prop, watch_block, time_left) # watch mo until it is removed else: self.__dequeue_mo_until_removed( watch_block, time_left) elif mo is None: # watch all event or specific to class_id self.__dequeue_all_class_id(watch_block, time_left) except Exception as e: log.info(str(e)) self.__wb_to_remove.append(watch_block) # removing watch_block if len(self.__wb_to_remove): self.__wbs_lock.acquire() for wb in self.__wb_to_remove: if "context" in wb.params: ctxt = wb.params['context'] ctxt["done"] = True self.watch_block_remove(wb) self.__wb_to_remove = [] self.__wbs_lock.release() # wait for more events only if watch_block exists if len(self.__wbs): with self.__condition: self.__condition.wait(self.__lowest_timeout) return def __thread_dequeue_start(self): """ Internal method to start dequeue thread. """ self.__dequeue_thread = Thread(name="dequeue_thread", target=self.__dequeue_function) self.__dequeue_thread.daemon = True self.__dequeue_thread.start() def watch_block_add(self, params, filter_callback, capacity=500, callback=None): """ Internal method to add a watch block for starting event monitoring. """ if self.__handle.cookie is None: return None self.__wbs_lock.acquire() watch_block = WatchBlock(params, filter_callback, capacity, callback) # Add a List of Watchers if watch_block is not None and watch_block.callback is None: watch_block.callback = watch_block.dequeue_default_callback self.__wbs.append(watch_block) self.__wbs_lock.release() return watch_block def watch_block_remove(self, watch_block): """ Internal method to remove a watch block for stopping event monitoring. """ if watch_block in self.__wbs: self.__wbs.remove(watch_block) def _add_class_id_watch(self, class_id): if ucscentralcoreutils.find_class_id_in_mo_meta_ignore_case( class_id) is None: raise UcsCentralValidationException( "Invalid ClassId %s specified." % class_id) def watch__type_filter(mce): """ Callback method to work on events with a specific class_id. """ if mce.mo.get_class_id().lower() == class_id.lower(): return True return False return watch__type_filter def _add_mo_watch(self, managed_object, prop=None, success_value=[], poll_sec=None): if ucscentralcoreutils.find_class_id_in_mo_meta_ignore_case( managed_object.get_class_id()) is None: raise UcsCentralValidationException( "Unknown ClassId %s provided." % managed_object.get_class_id()) if prop is not None: mo_property_meta = ucscentralcoreutils.get_mo_property_meta( managed_object.get_class_id(), prop) if mo_property_meta is None: raise UcsCentralValidationException( "Unknown Property %s provided." % prop) if not success_value: raise UcsCentralValidationException( "success_value parameter is not provided.") if poll_sec is None: def watch_mo_filter(mce): """ Callback method to work on events specific to respective managed object. """ if mce.mo.dn == managed_object.dn: return True return False return watch_mo_filter else: def watch_none_filter(mce): """ Callback method to ignore all events. """ return False return watch_none_filter def add(self, class_id=None, managed_object=None, prop=None, success_value=[], failure_value=[], transient_value=[], poll_sec=None, timeout_sec=None, call_back=None, context=None): """ Adds an event handler. An event handler can be added using this method where an user can subscribe for the event channel from UCSCENTRAL and can monitor those events for any specific success value or failure value for a managed object. Args: class_id (str): managed object class id managed_object (ManagedObject) prop (str) - property of the managed object to monitor success_value (list) - success values of a prop failure_value (list) - failure values of a prop transient_value (list) - transient values of a prop poll_sec - specifies the time in seconds for polling event. timeout_sec - time after which method should stop monitoring. call_back - call back method """ if class_id is not None and managed_object is not None: raise UcsCentralValidationException( "Specify either class_id or managedObject, not both") if class_id is not None: filter_callback = self._add_class_id_watch(class_id) elif managed_object is not None: filter_callback = self._add_mo_watch(managed_object, prop, success_value, poll_sec) else: def watch_all_filter(mce): """ Callback method to work on all events. """ return True filter_callback = watch_all_filter param_dict = { 'class_id': class_id, 'managed_object': managed_object, 'prop': prop, 'success_value': success_value, 'failure_value': failure_value, 'transient_value': transient_value, 'poll_sec': poll_sec, 'timeout_sec': timeout_sec, 'call_back': call_back, 'start_time': datetime.datetime.now(), 'context': context } if filter_callback is None: raise UcsCentralValidationException("Error adding WatchBlock...") watch_block = self.watch_block_add(params=param_dict, filter_callback=filter_callback, callback=call_back) if watch_block is not None and len(self.__wbs) == 1: if poll_sec is None: self.__thread_enqueue_start() self.__thread_dequeue_start() return watch_block def remove(self, watch_block): """ Removes an event handler. """ self.__wbs_lock.acquire() if watch_block in self.__wbs: self.watch_block_remove(watch_block) else: UcsCentralWarning("Event handler not found") self.__wbs_lock.release() def clean(self): """ Removes all the watch blocks from the event handler """ self.__wbs_lock.acquire() for each in self.__wbs: self.watch_block_remove(each) self.__wbs_lock.release() def get(self): """ Returns the list of event handlers. """ return self.__wbs
class Perception(RobotPart): def __init__(self, robot_name, tf_listener, image_topic=None, projection_srv=None, camera_base_ns=''): super(Perception, self).__init__(robot_name=robot_name, tf_listener=tf_listener) if image_topic is None: self.image_topic = "/" + self.robot_name + "/top_kinect/rgb/image" else: self.image_topic = image_topic if projection_srv is None: projection_srv_name = '/' + robot_name + '/top_kinect/project_2d_to_3d' else: projection_srv_name = projection_srv self._camera_base_ns = camera_base_ns self._camera_lazy_sub = None self._camera_cv = Condition() self._camera_last_image = None self._annotate_srv = self.create_service_client( '/' + robot_name + '/people_recognition/face_recognition/annotate', Annotate) self._recognize_srv = self.create_service_client( '/' + robot_name + '/people_recognition/face_recognition/recognize', Recognize) self._clear_srv = self.create_service_client( '/' + robot_name + '/people_recognition/face_recognition/clear', Empty) self._image_data = (None, None, None) self._face_properties_srv = self.create_service_client( '/' + robot_name + '/people_recognition/face_recognition/get_face_properties', GetFaceProperties) self._projection_srv = self.create_service_client(projection_srv_name, Project2DTo3D) self._person_recognition_3d_srv = \ self.create_service_client('/' + robot_name + '/people_recognition/detect_people_3d', RecognizePeople3D) def close(self): pass def _image_cb(self, image): self._camera_cv.acquire() self._camera_last_image = image self._camera_cv.notify() self._camera_cv.release() def get_image(self, timeout=5): # lazy subscribe to the rgb(d) camera if not self._camera_lazy_sub: rospy.loginfo("Creating subscriber") self._camera_lazy_sub = rospy.Subscriber(self.image_topic, Image, self._image_cb) rospy.loginfo('lazy subscribe to %s', self._camera_lazy_sub.name) rospy.loginfo("getting one image...") self._camera_cv.acquire() self._camera_last_image = None for i in range(timeout): if self._camera_last_image: rospy.loginfo("len(self._camera_last_image): {}".format(len(self._camera_last_image.data))) break else: rospy.loginfo("self._camera_last_image: {}".format(self._camera_last_image)) if rospy.is_shutdown(): return self._camera_cv.wait(timeout=1) else: raise Exception('no image received from %s' % self._camera_lazy_sub.name) image = self._camera_last_image self._camera_cv.release() rospy.loginfo("got %d bytes of image data", len(image.data)) return image def project_roi(self, roi, frame_id=None): """ Projects a region of interest of a depth image to a 3D Point. Hereto, a service is used :param roi: sensor_msgs/RegionOfInterest :param frame_id: if specified, the result is transformed into this frame id :return: VectorStamped object """ response = self.project_rois(rois=[roi]).points[0] # Convert to VectorStamped result = VectorStamped(x=response.point.x, y=response.point.y, z=response.point.z, frame_id=response.header.frame_id) # If necessary, transform the point if frame_id is not None: rospy.loginfo("Transforming roi to {}".format(frame_id)) result = result.projectToFrame(frame_id=frame_id, tf_listener=self.tf_listener) # Return the result return result def project_rois(self, rois): # Call the service with the provided Region of Interest try: points = self._projection_srv(rois=rois) except rospy.ServiceException as e: raise ValueError('project_roi failed', e) else: rospy.loginfo('project_rois response: %s', points) return points # OpenFace def _get_faces(self, image=None): if not image: image = self.get_image() try: r = self._recognize_srv(image=image) rospy.loginfo('Found %d face(s) in the image', len(r.recognitions)) except rospy.ServiceException as e: rospy.logerr("Can't connect to face recognition service: {}".format(e)) r = RecognizeResponse() except Exception as e: rospy.logerr("Can't detect faces: {}".format(e)) return r def learn_person(self, name='operator'): HEIGHT_TRESHOLD = 88 WIDTH_TRESHOLD = 88 try: image = self.get_image() except Exception as e: rospy.logerr("Can't get image: {}".format(e)) return False raw_recognitions = self._get_faces(image).recognitions recognitions = [r for r in raw_recognitions if r.roi.height > HEIGHT_TRESHOLD and r.roi.width > WIDTH_TRESHOLD] rospy.loginfo('Found %d valid face(s)', len(recognitions)) if len(recognitions) != 1: rospy.loginfo("Too many faces: {}".format(len(recognitions))) return False recognition = recognitions[0] rospy.loginfo('annotating that face as %s', name) try: self._annotate_srv(image=image, annotations=[Annotation(label=name, roi=recognition.roi)]) except rospy.ServiceException as e: rospy.logerr("Can't connect to person learning service: {}".format(e)) return False except Exception as e: rospy.logerr("Can't learn a person: {}".format(e)) return False return True def detect_faces(self, image=None, stamp=False): """ Snap an image with the camera and return the recognized faces. :param image: image to use for recognition :type image: sensor_msgs/Image :param stamp: Return recognitions and stamp :type stamp: bool :return: recognitions of the faces :rtype: list[image_recognition_msgs/Recognition] """ if not image: image = self.get_image() if stamp: return self._get_faces(image).recognitions, image.header.stamp else: return self._get_faces(image).recognitions @staticmethod def get_best_face_recognition(recognitions, desired_label, probability_threshold=4.0): """ Returns the Recognition with the highest probability of having the desired_label. Assumes that the probability distributions in Recognition are already sorted by probability (descending, highest first) :param recognitions: The recognitions to select the best one with desired_label from :type recognitions: list[image_recognition_msgs/Recognition] :param desired_label: what label to look for in the recognitions :type desired_label: str :param probability_threshold: only accept recognitions with probability higher than threshold :type probability_threshold: double :return: the best recognition matching the given desired_label :rtype image_recognition_msgs/Recognition """ rospy.logdebug("get_best_face_recognition: recognitions = {}".format(recognitions)) # Only take detections with operator # detections = [] # The old implementation took, for each recognition, the (label, prob) pairs where label==desired_label. # Other pairs in the same distribution may have higher probability. # When the best_recognition is picked, it picked the recognition where the probability for the desired_label is hhighest comapared to other recognitions. BUT: a recognitions highest probability may be for a different label # because the selection only compares matching labels, not looking at the probability of non-matching pairs. # For example: we have 2 recognitions. # in recognition 1, A has 50%, desired_label has 30%, B has 20%. # in recognition 2, B has 60%, desired_label has 35%, A has 5%. # Then, recognition 2 has the highest probability for the desired_label and is thus picked. # Because we take the [0]'th index of the distribution, that name is B # # Solution: because the probability distributions are sorted, just take the probability distribution where the desired label has the highest probability. #for recog in recognitions: # for cp in recog.categorical_distribution.probabilities: # if cp.label == desired_label: # detections.append((recog, cp.probability)) # Sort based on probability #if detections: # sorted_detections = sorted(detections, key=lambda det: det[1]) # best_detection = sorted_detections[0][0] # A CategoricalDistribution in a Recognition is already ordered, max prob is at [0] #else: # best_detection = None rospy.loginfo("Probability threshold %.2f", probability_threshold) for index, recog in enumerate(recognitions): rospy.loginfo("{index}: {dist}".format(index=index, dist=[(cp.label, "{:.2f}".format(cp.probability)) for cp in recog.categorical_distribution.probabilities])) matching_recognitions = [recog for recog in recognitions if \ recog.categorical_distribution.probabilities and \ recog.categorical_distribution.probabilities[0].label == desired_label] if matching_recognitions: best_recognition = max(matching_recognitions, key=lambda recog: recog.categorical_distribution.probabilities[0].probability) return best_recognition if best_recognition.categorical_distribution.probabilities[0].probability > probability_threshold else None else: return None # TODO: Maybe so something smart with selecting a recognition where the desired_label is not the most probable for a recognition? def clear_face(self): """ clearing all faces from the OpenFace node. :return: no return """ rospy.loginfo('clearing all learned faces') self._clear_srv() # Skybiometry def get_face_properties(self, faces=None, image=None): """ Get the face properties of all faces or in an image. If faces is provided, image is ignored. If both aren't provided, an image is collected. :param faces: images of all faces :type faces: list[sensor_msgs/Image] :param image: image containing the faces :type image: sensor_msgs/Image :return: list of face properties :rtype: list[image_recognition_msgs/FaceProperties] """ if not faces: if not image: image = self.get_image() face_recognitions = self.detect_faces(image=image) rois = img_recognitions_to_rois(face_recognitions) faces = img_cutout(image, rois) face_properties = [] try: face_properties_response = self._face_properties_srv(faces) face_properties = face_properties_response.properties_array except Exception as e: rospy.logerr(e) return [None] * len(faces) face_log = '\n - '.join([''] + [repr(s) for s in face_properties]) rospy.loginfo('face_properties:%s', face_log) return face_properties def get_rgb_depth_caminfo(self, timeout=5): """ Get an rgb image and and depth image, along with camera info for the depth camera. The returned tuple can serve as input for world_model_ed.ED.detect_people. :param timeout: How long to wait until the images are all collected. :return: tuple(rgb, depth, depth_info) or a None if no images could be gathered. """ event = Event() def callback(rgb, depth, depth_info): rospy.loginfo('Received rgb, depth, cam_info') self._image_data = (rgb, depth, depth_info) event.set() # camera topics depth_info_sub = message_filters.Subscriber('{}/depth_registered/camera_info'.format(self._camera_base_ns), CameraInfo) depth_sub = message_filters.Subscriber('{}/depth_registered/image'.format(self._camera_base_ns), Image) rgb_sub = message_filters.Subscriber('{}/rgb/image_raw'.format(self._camera_base_ns), Image) ts = message_filters.ApproximateTimeSynchronizer([rgb_sub, depth_sub, depth_info_sub], queue_size=1, slop=10) ts.registerCallback(callback) event.wait(timeout) ts.callbacks.clear() del ts, depth_info_sub, depth_sub, rgb_sub, callback if any(self._image_data): return self._image_data else: return None def detect_person_3d(self, rgb, depth, depth_info): return self._person_recognition_3d_srv(image_rgb=rgb, image_depth=depth, camera_info_depth=depth_info).people
def recv_ack(s, host, port, window, lock, lasted_ack, packages_time): # Variaveis compartilhadas global max_packages global aux_time global aux_lock aux_lock = Condition() global print_lock print_lock = Condition() rtt = 2.0 while True: try: # Tempo que o cliente espera pela confirmação do ack s.settimeout(rtt) # Recebe o pacote e o endereço do servidor data, add = s.recvfrom(4096) # Adiciona o tempo de envio dos pacotes aux_lock.acquire() packages_time.append(time() - aux_time) aux_time = time() aux_lock.release() # Transforma o objeto de bytes no pacotes original packet = pickle.loads(data) print_lock.acquire() print("Ack com número de sequência {} recebido".format(packet.ack)) print_lock.release() # Flag para ver se o ack recebido é o esperado na janela ack_awaited = False # Percorre a janela de transmissão for i in range(lasted_ack, max_packages + 1): # Armazena o elemento (pacote, seqnumber) na variavel lock.acquire() index = window[i % 7] lock.release() # Verifica se o ack recebido é igual ao numero de sequencia do pacote armazenado na janela if (packet.ack == index[1]): # Ack recebido é o esperado ack_awaited = True lock.acquire() # Percorre a janela transformando em None os pacotes confirmados for j in range(lasted_ack, i + 1): window[j % 7] = (None, None) # Atualiza o ultimo ack recebido lasted_ack += 1 max_packages += 1 lock.notify() lock.release() # Ack não é o esperado if (ack_awaited == False): print_lock.acquire() print("Ack duplicado") print_lock.release() # Como foram recebidos ack duplicados a janela é reenviada resend_window(s, host, port, window, lock, lasted_ack) except socket.error as error: print_lock.acquire() print("Timeout") print_lock.release() # Se o houver um timeout a janela será reenviada resend_window(s, host, port, window, lock, lasted_ack) # Variavel para verificar se todos os pacotes da janela foram confirmados empty_window = True # Percorre a janela verificando se existi pacotes não confirmados for i in range(lasted_ack, max_packages + 1): if window[i % 7] != (None, None): # Se houver atualiza a variavel para falso empty_window = False # Verifica se a janela está vazia e se o servidor parou de enviar pacotes if end == True and empty_window == True: break
class TimerClock(Clock): """ :py:class:`pyctrl.block.clock.TimerClock` provides a clock that reads the current time periodically. :param float period: period in seconds """ def __init__(self, **kwargs): self.period = kwargs.pop('period', 0.01) super().__init__(**kwargs) self.condition = Condition() self.timer = None self.running = False if self.enabled: self.enabled = False self.set_enabled(True) def set(self, exclude=(), **kwargs): """ Set properties of :py:class:`pyctrl.block.clock.TimerClock`. :param tuple exclude: attributes to exclude :param float period: clock period :param kwargs kwargs: other keyword arguments :raise: :py:class:`pyctrl.block.BlockException` if any of the :py:attr:`kwargs` is left unprocessed """ if 'period' in kwargs: self.period = kwargs.pop('period') # call super return super().set(exclude, **kwargs) def get(self, *keys, exclude=()): """ Get properties of :py:class:`pyctrl.block.clock.TimerClock`. Available attributes are those from :py:meth:`pyctrl.block.clock.Clock.get` and: 1. :py:attr:`period` The elapsed time since initialization or last reset can be obtained using the method :py:meth:`pyctrl.block.clock.TimerClock.read`. :param keys: string or tuple of strings with property names :param tuple exclude: keys never to be returned (Default ()) """ # call super excluding time and last return super().get(*keys, exclude=exclude + ('condition', 'timer', 'running', 'thread')) def tick(self): # Acquire lock self.condition.acquire() # Got a tick time = perf_counter() #print('> TICK: {}, {}'.format(time, self)) dwell = time - self.time if (dwell < 0.9 * self.period): # need more time #print('> MORE TIME') # Setup new timer self.timer = Timer(self.period - dwell, self.tick) self.timer.start() # Release lock self.condition.release() # and return return # Close to period self.time = time # Add to count self.count += 1 # Notify lock self.condition.notify_all() # Release lock self.condition.release() def run(self): #print('> run') self.running = True while self.enabled and self.running: # Acquire condition self.condition.acquire() #print('> WILL TICK') # Setup timer self.timer = Timer(self.period, self.tick) self.timer.start() #print('> WAITING') # Wait self.condition.wait() # and release self.condition.release() self.running = False # print('> END OF RUN!') def set_enabled(self, enabled=True): """ Set :py:class:`pyctrl.block.clock.TimerClock` :py:attr:`enabled` state. :param bool enabled: True or False (default True) """ # quick return if enabled == self.enabled: return # enable if enabled: # print('> Enabling TimerClock') # set enabled super().set_enabled(enabled) # Start thread self.thread = Thread(target=self.run) self.thread.start() # disable else: # Acquire condition self.condition.acquire() # print('> Disabling TimerClock') # Prepare to stop self.running = False # Notify lock self.condition.notify_all() # set enabled super().set_enabled(enabled) # and release self.condition.release() def read(self): """ Read from :py:class:`pyctrl.block.clock.TimerClock`. :return: tuple with elapsed time since initialization or last reset """ #print('> read') if self.enabled: # Acquire condition self.condition.acquire() # wait self.condition.wait() # and release self.condition.release() return (self.time - self.time_origin, )
class CommunicationTokenCredential(object): """Credential type used for authenticating to an Azure Communication service. :param str token: The token used to authenticate to an Azure Communication service :keyword token_refresher: The token refresher to provide capacity to fetch fresh token :raises: TypeError """ _ON_DEMAND_REFRESHING_INTERVAL_MINUTES = 2 def __init__(self, token, # type: str **kwargs ): token_refresher = kwargs.pop('token_refresher', None) communication_token_refresh_options = CommunicationTokenRefreshOptions(token=token, token_refresher=token_refresher) self._token = communication_token_refresh_options.get_token() self._token_refresher = communication_token_refresh_options.get_token_refresher() self._lock = Condition(Lock()) self._some_thread_refreshing = False def get_token(self): # type () -> ~azure.core.credentials.AccessToken """The value of the configured token. :rtype: ~azure.core.credentials.AccessToken """ if not self._token_refresher or not self._token_expiring(): return self._token should_this_thread_refresh = False with self._lock: while self._token_expiring(): if self._some_thread_refreshing: if self._is_currenttoken_valid(): return self._token self._wait_till_inprogress_thread_finish_refreshing() else: should_this_thread_refresh = True self._some_thread_refreshing = True break if should_this_thread_refresh: try: newtoken = self._token_refresher() # pylint:disable=not-callable with self._lock: self._token = newtoken self._some_thread_refreshing = False self._lock.notify_all() except: with self._lock: self._some_thread_refreshing = False self._lock.notify_all() raise return self._token def _wait_till_inprogress_thread_finish_refreshing(self): self._lock.release() self._lock.acquire() def _token_expiring(self): return self._token.expires_on - self._get_utc_now() <\ timedelta(minutes=self._ON_DEMAND_REFRESHING_INTERVAL_MINUTES) def _is_currenttoken_valid(self): return self._get_utc_now() < self._token.expires_on @classmethod def _get_utc_now(cls): return datetime.now().replace(tzinfo=TZ_UTC)
class Serial: """ A Serial object offers a way to send and data using a HDLC-like formating. """ HDLC_FLAG_BYTE = 0x7e HDLC_CTLESC_BYTE = 0x7d TOS_SERIAL_ACTIVE_MESSAGE_ID = 0 TOS_SERIAL_CC1000_ID = 1 TOS_SERIAL_802_15_4_ID = 2 TOS_SERIAL_UNKNOWN_ID = 255 SERIAL_PROTO_ACK = 67 SERIAL_PROTO_PACKET_ACK = 68 SERIAL_PROTO_PACKET_NOACK = 69 SERIAL_PROTO_PACKET_UNKNOWN = 255 def __init__(self, port, baudrate, flush=False, debug=False, qsize=10): self._debug = debug self._in_queue = Queue.Queue(qsize) self._out_lock = Lock() self._out_ack = Condition() self._seqno = 0 self._ack = None self._write_counter = 0 self._write_counter_failures = 0 self._read_counter = 0 self._ts = None self._s = serial.Serial(port, baudrate, rtscts=0, timeout=0.5) self._s.flushInput() start = time.time() if flush: print >> sys.stdout, "Flushing the serial port", while time.time() - start < 1: p = self._read() sys.stdout.write(".") if not self._debug: sys.stdout.write("\n") self._s.close() self._s = serial.Serial(port, baudrate, rtscts=0, timeout=None) thread.start_new_thread(self.run, ()) def run(self): while True: p = self._read() self._read_counter += 1 if self._debug: print "Serial:run: got a packet(%d): %s" % (self._read_counter, p) ack = AckFrame(p.data) if ack.protocol == self.SERIAL_PROTO_ACK: if not self._ack: self._ack = ack if self._debug: print "Serial:run: got an ack:", ack self._ack = ack # Wake up the writer self._out_ack.acquire() self._out_ack.notify() self._out_ack.release() else: ampkt = ActiveMessage(NoAckDataFrame(p.data).data) if ampkt.type == 100: for t in "".join([chr(i) for i in ampkt.data ]).strip('\n\0').split('\n'): print "PRINTF:", t.strip('\n') else: if self._in_queue.full(): print "Warning: Buffer overflow" self._in_queue.get() self._in_queue.put(p, block=False) # Returns the next incoming serial packet def _read(self): """Wait for a packet and return it as a RawPacket.""" try: d = self._get_byte() ts = time.time() while d != self.HDLC_FLAG_BYTE: d = self._get_byte() ts = time.time() packet = [d] d = self._get_byte() if d == self.HDLC_FLAG_BYTE: d = self._get_byte() ts = time.time() else: packet.append(d) while d != self.HDLC_FLAG_BYTE: d = self._get_byte() packet.append(d) if self._debug == True: print "Serial:_read: unescaped", packet packet = self._unescape(packet) crc = self._crc16(0, packet[1:-3]) packet_crc = self._decode(packet[-3:-1]) if crc != packet_crc: print "Warning: wrong CRC! %x != %x %s" % ( crc, packet_crc, ["%2x" % i for i in packet]) if self._debug: if self._ts == None: self._ts = ts else: print "Serial:_read: %.4f (%.4f) Recv:" % ( ts, ts - self._ts), self._format_packet(packet[1:-3]) self._ts = ts return RawPacket(ts, packet[1:-3], crc == packet_crc) except socket.timeout: return None def read(self, timeout=None): start = time.time() done = False while not done: p = None while p == None: if timeout == 0 or time.time() - start < timeout: try: p = self._in_queue.get(True, timeout) except Queue.Empty: return None else: return None if p.crc: done = True else: p = None # In the current TinyOS the packets from the mote are always NoAckDataFrame return NoAckDataFrame(p.data) def write(self, payload): """ Write a packet. If the payload argument is a list, it is assumed to be exactly the payload. Otherwise the payload is assume to be a Packet and the real payload is obtain by calling the .payload(). """ if type(payload) != type([]): # Assume this will be derived from Packet payload = payload.payload() self._out_lock.acquire() self._seqno = (self._seqno + 1) % 100 packet = DataFrame() packet.protocol = self.SERIAL_PROTO_PACKET_ACK packet.seqno = self._seqno packet.dispatch = 0 packet.data = payload packet = packet.payload() crc = self._crc16(0, packet) packet.append(crc & 0xff) packet.append((crc >> 8) & 0xff) packet = [self.HDLC_FLAG_BYTE ] + self._escape(packet) + [self.HDLC_FLAG_BYTE] while True: self._put_bytes(packet) self._write_counter += 1 if self._debug == True: print "Send(%d/%d): %s" % ( self._write_counter, self._write_counter_failures, packet) print "Wait for ack %d ..." % (self._seqno) self._out_ack.acquire() self._out_ack.wait(0.2) if self._debug: print "Wait for ack %d done. Latest ack:" % ( self._seqno), self._ack self._out_ack.release() if self._ack and self._ack.seqno == self._seqno: if self._debug: print "The packet was acked." self._out_lock.release() if self._debug: print "Returning from Serial.write..." return True else: self._write_counter_failures += 1 if self._debug: print "The packet was not acked. Try again." # break # make only one sending attempt self._out_lock.release() return False def _format_packet(self, payload): f = NoAckDataFrame(payload) if f.protocol == self.SERIAL_PROTO_ACK: rpacket = AckFrame(payload) return "Ack seqno: %d" % (rpacket.seqno) else: rpacket = ActiveMessage(f.data) return "D: %04x S: %04x L: %02x G: %02x T: %02x | %s" % \ (rpacket.destination, rpacket.source, rpacket.length, rpacket.group, rpacket.type, list2hex(rpacket.data)) def _crc16(self, base_crc, frame_data): crc = base_crc for b in frame_data: crc = crc ^ (b << 8) for i in range(0, 8): if crc & 0x8000 == 0x8000: crc = (crc << 1) ^ 0x1021 else: crc = crc << 1 crc = crc & 0xffff return crc def _encode(self, val, dim): output = [] for i in range(dim): output.append(val & 0xFF) val = val >> 8 return output def _decode(self, v): r = long(0) for i in v[::-1]: r = (r << 8) + i return r def _get_byte(self): try: r = struct.unpack("B", self._s.read())[0] return r except struct.error: # Serial port read timeout raise socket.timeout def _put_bytes(self, data): #print "DEBUG: _put_bytes:", data for b in data: self._s.write(struct.pack('B', b)) def _unescape(self, packet): r = [] esc = False for b in packet: if esc: r.append(b ^ 0x20) esc = False elif b == self.HDLC_CTLESC_BYTE: esc = True else: r.append(b) return r def _escape(self, packet): r = [] for b in packet: if b == self.HDLC_FLAG_BYTE or b == self.HDLC_CTLESC_BYTE: r.append(self.HDLC_CTLESC_BYTE) r.append(b ^ 0x20) else: r.append(b) return r def debug(self, debug): self._debug = debug
class ReadWriteLock(object): """Read-Write lock class. A read-write lock differs from a standard threading.RLock() by allowing multiple threads to simultaneously hold a read lock, while allowing only a single thread to hold a write lock at the same point of time. When a read lock is requested while a write lock is held, the reader is blocked; when a write lock is requested while another write lock is held or there are read locks, the writer is blocked. Writers are always preferred by this implementation: if there are blocked threads waiting for a write lock, current readers may request more read locks (which they eventually should free, as they starve the waiting writers otherwise), but a new thread requesting a read lock will not be granted one, and block. This might mean starvation for readers if two writer threads interweave their calls to acquireWrite() without leaving a window only for readers. In case a current reader requests a write lock, this can and will be satisfied without giving up the read locks first, but, only one thread may perform this kind of lock upgrade, as a deadlock would otherwise occur. After the write lock has been granted, the thread will hold a full write lock, and not be downgraded after the upgrading call to acquireWrite() has been match by a corresponding release(). """ def __init__(self): """Initialize this read-write lock.""" # Condition variable, used to signal waiters of a change in object # state. self.__condition = Condition(Lock()) # Initialize with no writers. self.__writer = None self.__upgradewritercount = 0 self.__pendingwriters = [] # Initialize with no readers. self.__readers = {} def acquireRead(self, timeout=None): """Acquire a read lock for the current thread, waiting at most timeout seconds or doing a non-blocking check in case timeout is <= 0. In case timeout is None, the call to acquireRead blocks until the lock request can be serviced. In case the timeout expires before the lock could be serviced, a RuntimeError is thrown.""" if timeout is not None: endtime = time() + timeout me = currentThread() self.__condition.acquire() try: if self.__writer is me: # If we are the writer, grant a new read lock, always. self.__writercount += 1 return while True: if self.__writer is None: # Only test anything if there is no current writer. if self.__upgradewritercount or self.__pendingwriters: if me in self.__readers: # Only grant a read lock if we already have one # in case writers are waiting for their turn. # This means that writers can't easily get starved # (but see below, readers can). self.__readers[me] += 1 return # No, we aren't a reader (yet), wait for our turn. else: # Grant a new read lock, always, in case there are # no pending writers (and no writer). self.__readers[me] = self.__readers.get(me, 0) + 1 return if timeout is not None: remaining = endtime - time() if remaining <= 0: # Timeout has expired, signal caller of this. raise RuntimeError("Acquiring read lock timed out") self.__condition.wait(remaining) else: self.__condition.wait() finally: self.__condition.release() def acquireWrite(self, timeout=None): """Acquire a write lock for the current thread, waiting at most timeout seconds or doing a non-blocking check in case timeout is <= 0. In case the write lock cannot be serviced due to the deadlock condition mentioned above, a ValueError is raised. In case timeout is None, the call to acquireWrite blocks until the lock request can be serviced. In case the timeout expires before the lock could be serviced, a RuntimeError is thrown.""" if timeout is not None: endtime = time() + timeout me, upgradewriter = currentThread(), False self.__condition.acquire() try: if self.__writer is me: # If we are the writer, grant a new write lock, always. self.__writercount += 1 return elif me in self.__readers: # If we are a reader, no need to add us to pendingwriters, # we get the upgradewriter slot. if self.__upgradewritercount: # If we are a reader and want to upgrade, and someone # else also wants to upgrade, there is no way we can do # this except if one of us releases all his read locks. # Signal this to user. raise ValueError( "Inevitable dead lock, denying write lock") upgradewriter = True self.__upgradewritercount = self.__readers.pop(me) else: # We aren't a reader, so add us to the pending writers queue # for synchronization with the readers. self.__pendingwriters.append(me) while True: if not self.__readers and self.__writer is None: # Only test anything if there are no readers and writers. if self.__upgradewritercount: if upgradewriter: # There is a writer to upgrade, and it's us. Take # the write lock. self.__writer = me self.__writercount = self.__upgradewritercount + 1 self.__upgradewritercount = 0 return # There is a writer to upgrade, but it's not us. # Always leave the upgrade writer the advance slot, # because he presumes he'll get a write lock directly # from a previously held read lock. elif self.__pendingwriters[0] is me: # If there are no readers and writers, it's always # fine for us to take the writer slot, removing us # from the pending writers queue. # This might mean starvation for readers, though. self.__writer = me self.__writercount = 1 self.__pendingwriters = self.__pendingwriters[1:] return if timeout is not None: remaining = endtime - time() if remaining <= 0: # Timeout has expired, signal caller of this. if upgradewriter: # Put us back on the reader queue. No need to # signal anyone of this change, because no other # writer could've taken our spot before we got # here (because of remaining readers), as the test # for proper conditions is at the start of the # loop, not at the end. self.__readers[me] = self.__upgradewritercount self.__upgradewritercount = 0 else: # We were a simple pending writer, just remove us # from the FIFO list. self.__pendingwriters.remove(me) raise RuntimeError("Acquiring write lock timed out") self.__condition.wait(remaining) else: self.__condition.wait() finally: self.__condition.release() def release(self): """Release the currently held lock. In case the current thread holds no lock, a ValueError is thrown.""" me = currentThread() self.__condition.acquire() try: if self.__writer is me: # We are the writer, take one nesting depth away. self.__writercount -= 1 if not self.__writercount: # No more write locks; take our writer position away and # notify waiters of the new circumstances. self.__writer = None self.__condition.notifyAll() elif me in self.__readers: # We are a reader currently, take one nesting depth away. self.__readers[me] -= 1 if not self.__readers[me]: # No more read locks, take our reader position away. del self.__readers[me] if not self.__readers: # No more readers, notify waiters of the new # circumstances. self.__condition.notifyAll() else: raise ValueError("Trying to release unheld lock") finally: self.__condition.release()
def send(s, host, port, window, lock): # Variaveis compartilhadas global max_packages max_packages = 2 global aux_time aux_time = time() global aux_lock aux_lock = Condition() global print_lock print_lock = Condition() # Variavel para controlar o fim do programa global end end = False # Número de sequencia do pacote seqNumber = 0 # Pacotes enviados i = 0 payloads = [ '1', '12', '123', '1234', '12345', '123456', '1234567', '12345678', '123456789' ] # Envia 15 pacotes ao servidor while i != 15: # Demora aleatóriamente entre 0.6 e 1.2 para enviar um novo pacote sleep(random.uniform(0.6, 1.2)) # Envia pacotes com tamanho aleatório entre 1 e 9 de acordo com o indice da lista "payloads" data = payloads[random.randint(0, 8)] # Constroi o pacote para ser enviado packet = p_client(data, seqNumber) # Transforma o pacote em um objeto de bytes packet = pickle.dumps(packet) lock.acquire() # Chega se a quantidades de pacotes enviados é maior do que max_package if i > max_packages: lock.wait() lock.release() s.sendto(packet, (host, port)) print_lock.acquire() print("Pacote com número de sequência {} enviado".format(seqNumber)) print_lock.release() # Atualiza o número de sequencia seqNumber += len(data) # Adiciona o pacote na janela lock.acquire() window[i % 7] = (packet, seqNumber) lock.release() # Incrementa os pacotes enviados i += 1 # Atualiza end end = True
class PooledDB: """Pool for DB-API 2 connections. After you have created the connection pool, you can use connection() to get pooled, steady DB-API 2 connections. """ version = __version__ def __init__(self, creator, mincached=0, maxcached=0, maxshared=0, maxconnections=0, blocking=False, maxusage=None, setsession=None, reset=True, failures=None, ping=1, *args, **kwargs): """Set up the DB-API 2 connection pool. creator: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module mincached: initial number of idle connections in the pool (0 means no connections are made at startup) maxcached: maximum number of idle connections in the pool (0 or None means unlimited pool size) maxshared: maximum number of shared connections (0 or None means all connections are dedicated) When this maximum number is reached, connections are shared if they have been requested as shareable. maxconnections: maximum number of connections generally allowed (0 or None means an arbitrary number of connections) blocking: determines behavior when exceeding the maximum (if this is set to true, block and wait until the number of connections decreases, otherwise an error will be reported) maxusage: maximum number of reuses of a single connection (0 or None means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). setsession: optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to ...", "set time zone ..."] reset: how connections should be reset when returned to the pool (False or None to rollback transcations started with begin(), True to always issue a rollback for safety's sake) failures: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InternalError) is not adequate ping: determines when the connection should be checked with ping() (0 = None = never, 1 = default = whenever fetched from the pool, 2 = when a cursor is created, 4 = when a query is executed, 7 = always, and all other bit combinations of these values) args, kwargs: the parameters that shall be passed to the creator function or the connection constructor of the DB-API 2 module """ try: threadsafety = creator.threadsafety except AttributeError: try: if not callable(creator.connect): raise AttributeError except AttributeError: threadsafety = 2 else: threadsafety = 0 if not threadsafety: raise NotSupportedError("Database module is not thread-safe.") self._creator = creator self._args, self._kwargs = args, kwargs self._maxusage = maxusage self._setsession = setsession self._reset = reset self._failures = failures self._ping = ping if mincached is None: mincached = 0 if maxcached is None: maxcached = 0 if maxconnections is None: maxconnections = 0 if maxcached: if maxcached < mincached: maxcached = mincached self._maxcached = maxcached else: self._maxcached = 0 if threadsafety > 1 and maxshared: self._maxshared = maxshared self._shared_cache = [] # the cache for shared connections else: self._maxshared = 0 if maxconnections: if maxconnections < maxcached: maxconnections = maxcached if maxconnections < maxshared: maxconnections = maxshared self._maxconnections = maxconnections else: self._maxconnections = 0 self._idle_cache = [] # the actual pool of idle connections self._condition = Condition() if not blocking: def wait(): raise TooManyConnections self._condition.wait = wait self._connections = 0 # Establish an initial number of idle database connections: idle = [self.dedicated_connection() for i in range(mincached)] while idle: idle.pop().close() def steady_connection(self): """Get a steady, unpooled DB-API 2 connection.""" return connect(self._creator, self._maxusage, self._setsession, self._failures, self._ping, True, *self._args, **self._kwargs) def connection(self, shareable=True): """Get a steady, cached DB-API 2 connection from the pool. If shareable is set and the underlying DB-API 2 allows it, then the connection may be shared with other threads. """ if shareable and self._maxshared: self._condition.acquire() try: while (not self._shared_cache and self._maxconnections and self._connections >= self._maxconnections): self._condition.wait() if len(self._shared_cache) < self._maxshared: # shared cache is not full, get a dedicated connection try: # first try to get it from the idle cache con = self._idle_cache.pop(0) except IndexError: # else get a fresh connection con = self.steady_connection() else: con._ping_check() # check this connection con = SharedDBConnection(con) self._connections += 1 else: # shared cache full or no more connections allowed self._shared_cache.sort() # least shared connection first con = self._shared_cache.pop(0) # get it while con.con._transaction: # do not share connections which are in a transaction self._shared_cache.insert(0, con) self._condition.wait() self._shared_cache.sort() con = self._shared_cache.pop(0) con.con._ping_check() # check the underlying connection con.share() # increase share of this connection # put the connection (back) into the shared cache self._shared_cache.append(con) self._condition.notify() finally: self._condition.release() con = PooledSharedDBConnection(self, con) else: # try to get a dedicated connection self._condition.acquire() try: while (self._maxconnections and self._connections >= self._maxconnections): self._condition.wait() # connection limit not reached, get a dedicated connection try: # first try to get it from the idle cache con = self._idle_cache.pop(0) except IndexError: # else get a fresh connection con = self.steady_connection() else: con._ping_check() # check connection con = PooledDedicatedDBConnection(self, con) self._connections += 1 finally: self._condition.release() return con def dedicated_connection(self): """Alias for connection(shareable=False).""" return self.connection(False) def unshare(self, con): """Decrease the share of a connection in the shared cache.""" self._condition.acquire() try: con.unshare() shared = con.shared if not shared: # connection is idle, try: # so try to remove it self._shared_cache.remove(con) # from shared cache except ValueError: pass # pool has already been closed finally: self._condition.release() if not shared: # connection has become idle, self.cache(con.con) # so add it to the idle cache def cache(self, con): """Put a dedicated connection back into the idle cache.""" self._condition.acquire() try: if not self._maxcached or len(self._idle_cache) < self._maxcached: con._reset(force=self._reset) # rollback possible transaction # the idle cache is not full, so put it there self._idle_cache.append(con) # append it to the idle cache else: # if the idle cache is already full, con.close() # then close the connection self._connections -= 1 self._condition.notify() finally: self._condition.release() def close(self): """Close all connections in the pool.""" self._condition.acquire() try: while self._idle_cache: # close all idle connections con = self._idle_cache.pop(0) try: con.close() except Exception: pass if self._maxshared: # close all shared connections while self._shared_cache: con = self._shared_cache.pop(0).con try: con.close() except Exception: pass self._connections -= 1 self._condition.notifyAll() finally: self._condition.release() def __del__(self): """Delete the pool.""" try: self.close() except Exception: pass
class VODTransporter(MovieTransport): def __init__(self, dd, dlhash, fileinfo, vodeventfunc): self.wait_sufficient_speed = dd.config.get('wait_sufficient_speed', False) self.player_buffer_time = dd.config.get('player_buffer_time', 5) self.live_buffer_time = dd.config.get('live_buffer_time', 10) self.fileinfo = fileinfo self.dd = dd self.rawserver = dd.rawserver self.downloader = dd.downloader self.storage = dd.storage self.vodeventfunc = vodeventfunc self.log_prefix = 'dd-vod::' + binascii.hexlify(dlhash) + ':' if DEBUG: log(self.log_prefix + '__init__: fileinfo', self.fileinfo, 'wait_sufficient_speed', self.wait_sufficient_speed, 'player_buffer_time', self.player_buffer_time) self.set_mimetype(self.fileinfo['mimetype']) bitrate = self.fileinfo.get('bitrate', None) if bitrate is None: self.bitrate_set = False self.bitrate = 102400 if DEBUG: log(self.log_prefix + '__init__: set fake bitrate: bitrate', self.bitrate) else: self.bitrate_set = True self.bitrate = bitrate if DEBUG: log(self.log_prefix + '__init__: got bitrate: bitrate', self.bitrate) self.data_ready = Condition() self._complete = False self.filestream = None self.prebufprogress = 0.0 self.prebufstart = time.time() self.playable = False self.usernotified = False self.playing = False self.prebuffering = True self.stream_pos = 0 self.outbuf = [] self.stat_outbuf = [] self.outbuflen = 0 self.outbufpos = 0 self.update_prebuffering() self.refill_buffer_task() def set_bitrate(self, bitrate): if DEBUG: log(self.log_prefix + 'set_bitrate: bitrate', bitrate, 'fileinfo', self.fileinfo) self.bitrate = bitrate self.bitrate_set = True def is_playable(self): if not self.playable or self.prebuffering: self.playable = self.prebufprogress == 1.0 and self.enough_buffer() return self.playable def get_prebuffering_progress(self): return self.prebufprogress def complete(self): if DEBUG: log(self.log_prefix + 'complete: ---') if not self._complete: self._complete = True path = self.storage.get_dest_path() self.data_ready.acquire() try: self.filestream = open(path, 'rb') self.filestream.seek(self.stream_pos) if DEBUG: log(self.log_prefix + 'complete: open file and seek: path', path, 'stream_pos', self.stream_pos) self.data_ready.notify() finally: self.data_ready.release() def got_data_observer(self, pos, length): if self.prebuffering: self.rawserver.add_task(self.update_prebuffering, 0.1) return True else: return False def update_prebuffering(self): if not self.prebuffering: return want_len = min(self.bitrate * self.player_buffer_time, self.storage.get_content_length()) avail_len = self.storage.get_available_length(0) if avail_len >= want_len and self.enough_buffer(): self.data_ready.acquire() try: if DEBUG: log(self.log_prefix + 'update_prebuffering: ready: want', want_len, 'avail', avail_len) self.prebuffering = False self.notify_playable() self.data_ready.notify() return True finally: self.data_ready.release() else: if DEBUG: log(self.log_prefix + 'update_prebuffering: not ready: want', want_len, 'avail', avail_len) return False def expected_download_time(self): bytes_left = self.storage.get_amount_left() if bytes_left == 0: return True rate = self.downloader.measure.get_rate() if rate < 0.1: return float(2147483648L) time_left = bytes_left / float(rate) return time_left def expected_playback_time(self): bytes_to_play = self.storage.get_content_length() - self.outbufpos if bytes_to_play <= 0: return 0 if not self.bitrate_set: return float(2147483648L) playback_time = bytes_to_play / float(self.bitrate) return playback_time def enough_buffer(self): try: if not self.bitrate_set: return True if not self.wait_sufficient_speed: return True expected_download_time = self.expected_download_time() expected_playback_time = self.expected_playback_time() if DEBUG: log(self.log_prefix + 'enough_buffer: expected_download_time', expected_download_time, 'expected_playback_time', expected_playback_time) return max(0.0, expected_download_time - expected_playback_time) == 0.0 except: log_exc() return True def expected_buffering_time(self): download_time = self.expected_download_time() playback_time = self.expected_playback_time() if download_time > float(1073741824) and playback_time > float( 1073741824): return float(2147483648L) return abs(download_time - playback_time) def get_playable_after(self): return self.expected_buffering_time() def notify_playable(self): self.prebufprogress = 1.0 self.playable = True if self.usernotified: return mimetype = self.get_mimetype() complete = self.storage.is_finished() if complete: stream = None filename = self.storage.get_dest_path() else: stream = MovieTransportStreamWrapper(self) filename = None try: self.vodeventfunc( self.fileinfo, VODEVENT_START, { 'complete': complete, 'filename': filename, 'mimetype': mimetype, 'stream': stream, 'length': self.storage.get_content_length(), 'bitrate': self.bitrate }) except: log_exc() def get_mimetype(self): return self.mimetype def set_mimetype(self, mimetype): self.mimetype = mimetype def start(self, bytepos=0, force=False): if DEBUG: log(self.log_prefix + 'start: bytepos', bytepos, 'playing', self.playing, 'force', force) if self.playing and not force: return self.downloader.start(bytepos) self.data_ready.acquire() try: self.stream_pos = bytepos self.playing = True if self._complete: if self.filestream is None: path = self.storage.get_dest_path() if DEBUG: log(self.log_prefix + 'start: open file: path', path) self.filestream = open(path, 'rb') if DEBUG: log(self.log_prefix + 'start: seek file: pos', bytepos) self.filestream.seek(bytepos) else: self.outbuf = [] self.stat_outbuf = [] self.outbuflen = 0 self.outbufpos = self.stream_pos finally: self.data_ready.release() self.update_prebuffering() self.refill_buffer() def shutdown(self): if DEBUG: log(self.log_prefix + 'shutdown: ---') self.stop() def stop(self, seek=False): if DEBUG: log(self.log_prefix + 'stop: playing', self.playing, 'seek', seek) if not self.playing: return self.playing = False self.data_ready.acquire() try: self.outbuf = [] self.stat_outbuf = [] self.outbuflen = 0 self.outbufpos = 0 self.prebuffering = False if not seek: if self.filestream is not None: if DEBUG: log(self.log_prefix + 'stop: close filestream') self.filestream.close() self.filestream = None self.data_ready.notify() finally: self.data_ready.release() def seek(self, pos, whence=os.SEEK_SET): length = self.storage.get_content_length() self.data_ready.acquire() try: if whence == os.SEEK_SET: abspos = pos elif whence == os.SEEK_END: if pos > 0: raise ValueError('seeking beyond end of stream') else: abspos = length + pos else: raise ValueError('seeking does not currently support SEEK_CUR') if DEBUG: log(self.log_prefix + 'seek: pos', pos, 'whence', whence, 'length', length, 'abspos', abspos) if self._complete: if self.filestream is None: path = self.storage.get_dest_path() if DEBUG: log(self.log_prefix + 'seek: open file:', path) self.filestream = open(path, 'rb') if DEBUG: log(self.log_prefix + 'seek: seek file: abspos', abspos) self.filestream.seek(abspos) self.stream_pos = abspos else: self.stop(seek=True) self.start(abspos) finally: self.data_ready.release() def read(self, numbytes=None): self.data_ready.acquire() try: data = self.pop(numbytes) if data is None: return self.stream_pos += len(data) if DEBUG: log(self.log_prefix + 'read: update stream pos: stream_pos', self.stream_pos, 'datalen', len(data)) return data except: print_exc() finally: self.data_ready.release() def pop(self, max_size=None): while self.prebuffering and not self.done(): self.data_ready.wait() while not self._complete and not self.outbuf and not self.done(): self.data_ready.wait() if self._complete: if max_size is None: max_size = DEFAULT_READ_SIZE if DEBUG: log(self.log_prefix + 'pop: read from filestream: max_size', max_size) data = self.filestream.read(max_size) return data if not self.outbuf: if DEBUG: log( self.log_prefix + 'pop: empty buffer, return None: _complete', self._complete) return while True: bad_chunk = False start_pos = None total_length = 0 total_data = '' if DEBUG: log(self.log_prefix + 'pop: read data from outbuf') while self.outbuf: pos, data = self.outbuf.pop(0) self.stat_outbuf.pop(0) length = len(data) if max_size is not None and total_length + length > max_size: offset = total_length + length - max_size newpos = pos + length - offset newdata = data[-offset:] data = data[:-offset] oldlength = length length = len(data) if DEBUG: log( self.log_prefix + 'pop: trim chunk to max size: pos', pos, 'oldlen', oldlength, 'max_size', max_size, 'offset', offset, 'total_len', total_length, 'newlen', length, 'newpos', newpos) self.outbuf.insert(0, (newpos, newdata)) self.stat_outbuf.insert(0, (newpos, len(newdata))) if start_pos is None: start_pos = pos total_data += data total_length += length self.outbuflen -= length if DEBUG: log(self.log_prefix + 'pop: outbufpos', self.outbufpos, 'pos', pos, 'start_pos', start_pos, 'len', length, 'total_len', total_length, 'max_size', max_size, 'outbuflen', self.outbuflen) if not start_pos <= self.stream_pos <= start_pos + total_length: if DEBUG: log( self.log_prefix + 'pop: wrong chunk popped, discard: stream_pos', self.stream_pos, 'start_pos', start_pos, 'total_len', total_length) bad_chunk = True break if max_size is None: break if total_length >= max_size: if DEBUG: log( self.log_prefix + 'pop: stop popping, max size reached: total_length', total_length, 'max_size', max_size) break if bad_chunk: continue break return total_data def max_buffer_size(self): return int(self.player_buffer_time * self.bitrate * 2) def done(self): if not self.playing: return True return self.outbufpos == self.storage.get_content_length() and len( self.outbuf) == 0 def refill_buffer(self): self.data_ready.acquire() try: if self.prebuffering or self._complete or not self.playing or self.done( ): return mx = self.max_buffer_size() length = self.storage.get_content_length() while self.outbuflen < mx and self.outbufpos < length: numbytes = mx - self.outbuflen if DEBUG: log( self.log_prefix + 'refill_buffer: read from storage: pos', self.outbufpos, 'numbytes', numbytes, 'outbuflen', self.outbuflen, 'mx', mx) data = self.storage.read(self.outbufpos, numbytes) if not data: if DEBUG: log( self.log_prefix + 'refill_buffer: no data available: pos', self.outbufpos) break datalen = len(data) self.outbuf.append((self.outbufpos, data)) self.stat_outbuf.append((self.outbufpos, datalen)) self.outbuflen += datalen self.outbufpos += datalen self.data_ready.notify() if DEBUG: log( self.log_prefix + 'refill_buffer: got data from storage: datalen', datalen, 'outbufpos', self.outbufpos, 'outbuflen', self.outbuflen) except: log_exc() finally: self.data_ready.release() def refill_buffer_task(self): self.refill_buffer() self.rawserver.add_task(self.refill_buffer_task, REFILL_BUFFER_INTERVAL) def set_wait_sufficient_speed(self, value): if DEBUG: log(self.log_prefix + 'set_wait_sufficient_speed:', value) self.wait_sufficient_speed = value def set_player_buffer_time(self, value): if DEBUG: log(self.log_prefix + 'set_player_buffer_time:', value) self.player_buffer_time = value
class farm: '''init returns an object, we'll call o.run(i), then del o in the end.''' def __init__(self, num_threads, init, it, extendable=False, tn_tmpl=None, reuse=None, handler=None): '''When extendable is True, it means that we need to leave threads ready in case the input list is extended (with extend()). Also the run() function can be invoked several times. The drawback is that underneath the input it is converted to a list, and then manipulated, so it's feasible to start with relatively small input data. Don't forget to call close(), it will clean up all the threads. Or you can use it as a context manager, so this will be called automatically upon __exit__(). If extendable is False, the code is simpler, but it supports iterables however big. tn_tmpl is format() template with {} to be changed to thread's number. reuse tells whether to stash worked sessions for future reuse. If it's a list, it's a global list of warm sessions. A function in the handler parameter is invoked each second, while the processing is postponed, so it shouldn't take long to complete''' self.waiting = 0 self.extendable = extendable if self.extendable: self.arr = list( it) # If it was a generator for example. Make it real. lg.info("Total: {}".format(len(self.arr))) else: # Otherwise leave it as it is, we'll treat it as iterator self.arr = iter(it) self.cond = Condition() self.num_threads = num_threads if num_threads else len(self.arr) # Barrier to wait on for the restart (when extendable) self.barr = Barrier(self.num_threads + 1) self.init = init self.tn_tmpl = tn_tmpl self.reuse = reuse self.handler = handler # Objects living within threads, used to signal them to quit # (by setting quit_flag) self.objects = [] if type(reuse) is list: self.reuse_pool = reuse else: self.reuse_pool = None #if self.reuse: self.reuse_pool=[] self.q = Queue() self.tlist = [] def __del__(self): # Let the user delete them if it was user-supplied if type(self.reuse) is not list: if self.reuse: for i in self.reuse_pool: del i def close(self): # Join our threads if self.extendable: lg.debug('Cancelling threads (break the barrier)') self.barr.abort() # Let those waiting on the barrier to quit lg.info('Joining threads') for i in self.tlist: i.join() lg.info('Finished joining threads') def __enter__(self): return self def __exit__(self, *exc): self.close() def extend(self, arr, end=False): '''If end is True, add to the end''' if not self.extendable: lg.error( 'The farm is not extendable, "extendable" parameter is False') return with self.cond: orig_len = len(self.arr) #if type(arr) is GeneratorType: arr=tuple(arr) if end or self.arr and self.arr[-1] is not None: self.arr += arr else: # When we're quitting self.arr[: 0] = arr # Insert in the beginning, so poison pills won't get lost lg.info("Total after extend: {}".format(len(self.arr))) #lg.info("Notifying: {}".format(len(self.arr)-orig_len)) self.cond.notify(len(self.arr) - orig_len) def print_arr(self): '''Will be printed on farm exit. Beware to call it while the threads are running if you need the actual list.''' with self.cond: print(self.arr) def reusing(self): return type(self.reuse_pool) is list def handle_item(o, i): '''Handles the item with do() function of the class, passing parameters depending on the nature of the argument: can be presented as several arguments to make things easy. do() function must yield one or several results, that will be returned by farm.run()''' lg.info('Item: {}'.format(i)) if isinstance(i, (tuple, list, set)): # Present as arguments yield from o.do(*i) else: yield from o.do(i) #lg.debug('do: {}'.format(res)) #return res def do_extendable(self): '''We need to leave threads ready in case the array is extended. Otherwise we can quit right after do() has completed. Also we can use this farm several times, the objects remain live, so we can extend and invoke another run() to gather the results as many times as we want.''' o = self.init() with suppress(AttributeError): if not o.f: o.f = self # Set to the current farm with self.cond: self.objects.append(o) while True: self.cond.acquire() self.waiting += 1 lg.debug('waiting incremented: {}, len={}'.format( self.waiting, len(self.arr))) if not len(self.arr): if self.waiting == self.num_threads: # No threads left to replenish the array, we should all quit # Adding poison pills if 1: lg.info('Killing all') # Put poison pills for everyone including us self.arr += [None] * (self.num_threads) self.cond.notify( self.num_threads) # Wake up other threads else: lg.info('Killing all') self.arr += [None] * (self.num_threads - 1) self.cond.notify(self.num_threads - 1) self.cond.release() break else: self.cond.wait() # Someone else will kill us lm = len(self.arr) if lm: # Another check for those who have left cond.wait() i = self.arr.pop() self.waiting -= 1 lg.debug('waiting decremented: ' + str(self.waiting)) self.cond.release() if not lm: continue # Someone has stolen our item, snap! if i is None: self.q.put(None) # Mark we're done # Sleep on the condition to let other threads get their pills lg.debug('Sleeping on barrier') try: self.barr.wait() except BrokenBarrierError: # We're to quit break lg.debug('Continuing after barrier') continue # then restart processing the queue for j in farm.handle_item(o, i): self.q.put(j) with self.cond: self.objects.remove(o) del o lg.info("has finished") def do(self): '''if an item from the iterator is a tuple, we explode it to be arguments to do(). Otherwise we pass it verbatim''' #tracker = SummaryTracker() o = None if self.reusing(): # Try to get warm session with self.cond: if len(self.reuse_pool): o = self.reuse_pool.pop() if not o: o = self.init() with suppress(AttributeError): if not o.f: o.f = self # Set to the current farm with self.cond: self.objects.append(o) #lg.warning(len(self.arr)) while True: with self.cond: try: i = next(self.arr) except StopIteration: # empty break if i is None: break # End of queue marker for j in farm.handle_item(o, i): self.q.put(j) #lg.error(asizeof.asizeof(o)) with self.cond: self.objects.remove(o) if self.reusing(): self.reuse_pool.append(o) else: del o # This will release proxy back to the pool self.q.put(None) # Mark the thread has finished lg.info("has finished") #tracker.print_diff() def cancel(self, cnt=0): '''Cancels all the threads in the farm''' # Put poison pills and then signal the threads to stop if not cnt: cnt = self.num_threads # That many threads are running with self.cond: if self.extendable: self.arr += [None] * cnt else: self.arr = chain(repeat(None, cnt), self.arr) self.cond.notify(cnt) for _ in self.objects: _.quit_flag = True def run(self): '''Main function to invoke. When KeyboardInterrupt is received, it sets the quit_flag in all the objects present, retrievers then raise an exception. It's the problem of the do() function to handle it and possibly extend the main list with the item that wasn't handled to show it in the end (for possible restart) self.handler() function is invoked each second if there are no items in the queue to allow for some rudimental auxiliary activity''' # Now start the threads, only once if self.tlist: lg.debug('Restarting threads') self.barr.wait() else: for i in range(self.num_threads): tn = self.tn_tmpl.format(i) if self.tn_tmpl else None t = Thread( target=self.do_extendable if self.extendable else self.do, name=tn) self.tlist.append(t) t.start() cnt = self.num_threads # That many threads are running while True: try: res = self.q.get(timeout=1) except Empty: if self.handler: self.handler() continue # Go back to suck the queue except KeyboardInterrupt: lg.warning( 'Trying to kill nicely, putting {} None'.format(cnt)) self.cancel(cnt) res = None #lg.warning('run: {}'.format(res)) if res != None: yield res continue cnt -= 1 # One thread has completed if not cnt: # All threads are completed (but may still be processing the quit_flag), we'll join them in close() return
class URLHandler(object): def __init__(self, *args, **kwargs): log.debug("create urlhandler {} {}".format(args, kwargs)) self.pending = {} self.name = kwargs.pop('name', None) self.content_handler = kwargs.pop('content_handler', None) self._setup() def _setup(self): self.done = Condition() self.lock = Lock() self.fetcher = make_fetcher(name=self.name, content_handler=self.content_handler) self.fetcher.add_watcher(self) def __getstate__(self): return dict(name=self.name) def __setstate__(self, state): self.__dict__.update(state) self._setup() def is_done(self): return self.count == 0 def thing_to_url(self, t): return t @property def count(self): return len(self.pending) def schedule(self, things): try: self.lock.acquire() self.i_schedule(things) finally: self.lock.release() def i_schedule(self, things): for t in things: self.pending[self.thing_to_url(t)] = t self.fetcher.schedule(self.thing_to_url(t)) def i_handle(self, t, url=None, response=None, exception=None, last_fetched=None): raise NotImplementedError() def __call__(self, watched=None, url=None, response=None, exception=None, last_fetched=None): if url in self.pending: t = self.pending[url] with self.lock: log.debug("RESPONSE url={}, exception={} @ {}".format( url, exception, self.count)) self.i_handle(t, url=url, response=response, exception=exception, last_fetched=last_fetched) del self.pending[url] if self.is_done(): try: self.done.acquire() self.done.notify() finally: self.done.release()
class Application(object): def __init__(self): self.is_authenticating = Condition() self.authorization = None # Bind trakt events Trakt.on('oauth.token_refreshed', self.on_token_refreshed) def authenticate(self): if not self.is_authenticating.acquire(blocking=False): print('Authentication has already been started') return False # Request new device code code = Trakt['oauth/device'].code() print('Enter the code "%s" at %s to authenticate your account' % (code.get('user_code'), code.get('verification_url'))) # Construct device authentication poller poller = Trakt['oauth/device'].poll(**code)\ .on('aborted', self.on_aborted)\ .on('authenticated', self.on_authenticated)\ .on('expired', self.on_expired)\ .on('poll', self.on_poll) # Start polling for authentication token poller.start(daemon=False) # Wait for authentication to complete return self.is_authenticating.wait() def run(self): self.authenticate() if not self.authorization: print('ERROR: Authentication required') exit(1) # Simulate expired token self.authorization['expires_in'] = 0 # Test authenticated calls with Trakt.configuration.oauth.from_response(self.authorization): # Expired token, requests will return `None` print(Trakt['sync/collection'].movies()) with Trakt.configuration.oauth.from_response(self.authorization, refresh=True): # Expired token will be refreshed automatically (as `refresh=True`) print(Trakt['sync/collection'].movies()) with Trakt.configuration.oauth.from_response(self.authorization): # Current token is still valid print(Trakt['sync/collection'].movies()) def on_aborted(self): """Device authentication aborted. Triggered when device authentication was aborted (either with `DeviceOAuthPoller.stop()` or via the "poll" event) """ print('Authentication aborted') # Authentication aborted self.is_authenticating.acquire() self.is_authenticating.notify_all() self.is_authenticating.release() def on_authenticated(self, authorization): """Device authenticated. :param authorization: Authentication token details :type authorization: dict """ # Acquire condition self.is_authenticating.acquire() # Store authorization for future calls self.authorization = authorization print('Authentication successful - authorization: %r' % self.authorization) # Authentication complete self.is_authenticating.notify_all() self.is_authenticating.release() def on_expired(self): """Device authentication expired.""" print('Authentication expired') # Authentication expired self.is_authenticating.acquire() self.is_authenticating.notify_all() self.is_authenticating.release() def on_poll(self, callback): """Device authentication poll. :param callback: Call with `True` to continue polling, or `False` to abort polling :type callback: func """ # Continue polling callback(True) def on_token_refreshed(self, authorization): # OAuth token refreshed, store authorization for future calls self.authorization = authorization print('Token refreshed - authorization: %r' % self.authorization)
class ForkingBase(object): '''Base class for classes which fork children and wait for them to exit. Sub-classes must provide the following data attributes and methods: log - an object of type getmailcore.logging.Logger() ''' def _child_handler(self, sig, stackframe): def notify(): self.__child_exited.acquire() self.__child_exited.notify_all() self.__child_exited.release() self.log.trace('handler called for signal %s' % sig) try: pid, r = os.waitpid(self.child.childpid, 0) except OSError as o: # No children on SIGCHLD. Can't happen? self.log.warning('handler called, but no children (%s)' % o) notify() return signal.signal(signal.SIGCHLD, self.__orig_handler) self.__child_pid = pid self.__child_status = r self.log.trace('handler reaped child %s with status %s' % (pid, r)) notify() def _prepare_child(self): self.log.trace('') self.__child_exited = Condition() self.__child_pid = 0 self.__child_status = None self.__orig_handler = signal.signal(signal.SIGCHLD, self._child_handler) def _wait_for_child(self, childpid): self.__child_exited.acquire() if self.__child_exited.wait( socket.getdefaulttimeout() or 60) == False: # Py2, <Py3.2: always None raise getmailOperationError('waiting child pid %d timed out' % childpid) self.__child_exited.release() if self.__child_pid != childpid: #self.log.error('got child pid %d, not %d' % (pid, childpid)) raise getmailOperationError('got child pid %d, not %d' % (self.__child_pid, childpid)) if os.WIFSTOPPED(self.__child_status): raise getmailOperationError( 'child pid %d stopped by signal %d' % (self.__child_pid, os.WSTOPSIG(self.__child_status))) if os.WIFSIGNALED(self.__child_status): raise getmailOperationError( 'child pid %d killed by signal %d' % (self.__child_pid, os.WTERMSIG(self.__child_status))) if not os.WIFEXITED(self.__child_status): raise getmailOperationError('child pid %d failed to exit' % self.__child_pid) exitcode = os.WEXITSTATUS(self.__child_status) return exitcode def _pipemail(self, msg, delivered_to, received, unixfrom, stdout, stderr): # Write out message msgfile = TemporaryFile23() msgfile.write( msg.flatten(delivered_to, received, include_from=unixfrom)) msgfile.flush() os.fsync(msgfile.fileno()) # Rewind msgfile.seek(0) # Set stdin to read from this file os.dup2(msgfile.fileno(), 0) # Set stdout and stderr to write to files os.dup2(stdout.fileno(), 1) os.dup2(stderr.fileno(), 2) def child_replace_me(self, msg, delivered_to, received, unixfrom, stdout, stderr, args, nolog=False): self._pipemail(msg, delivered_to, received, unixfrom, stdout, stderr) nolog or self.log.debug('about to execl() with args %s\n' % str(args)) os.execl(*args) def forkchild(self, childfun, with_out=True): self.child = child = Namespace() child.stdout = TemporaryFile23() child.stderr = TemporaryFile23() child.childpid = os.fork() if child.childpid != 0: # here (in the parent) self._prepare_child() self.log.debug('spawned child %d\n' % child.childpid) child.exitcode = self._wait_for_child(child.childpid) child.stderr.seek(0) child.err = child.stderr.read().strip().decode() child.stdout.seek(0) if with_out: child.out = child.stdout.read().strip() return child else: #== 0 in the child # calls child_replace_me to execl external command childfun(child.stdout, child.stderr) def get_msginfo(self, msg): msginfo = {} msginfo['sender'] = msg.sender.strip() if msg.recipient != None: rcpnt = msg.recipient.strip() msginfo['recipient'] = rcpnt msginfo['domain'] = rcpnt.lower().split('@')[-1] msginfo['local'] = '@'.join(rcpnt.split('@')[:-1]) self.log.debug('msginfo "%s"\n' % msginfo) return msginfo
class SrRecver: def __init__(self, addr): self.__sock = socket(AF_INET, SOCK_DGRAM) self.__sock.bind(addr) self.__window = [] # 定义窗口中最前端的数据的seq self.__seq_range = [] self.__max_seq = 255 self.__window_size = 10 self.__notify_recv = Condition() self.__recv_cache = [] t = Thread(target=self.__recv_thread) t.start() def recv(self): """ 提供给上层应用调用的接口,用来接收数据。 :return: bytes类型的数据 """ self.__notify_recv.acquire() while len(self.__recv_cache) == 0: self.__notify_recv.wait() self.__notify_recv.release() data = self.__recv_cache[0] del self.__recv_cache[0] return data def __find_index(self, seq): """ 查找seq是不是在窗口中,如果是的话返回其在窗口中的index,如果不在的话返回None :param seq: 要查找的seq :return: """ for i in range(self.__window_size): if self.__seq_range[i] == seq: return i return None def __recv_thread(self): # 初始化seq_range和window为固定大小的数组 for i in range(self.__window_size): self.__seq_range.append(i + 1) self.__window.append(None) while True: data, addr = self.__sock.recvfrom(4096) if lib.checksum(data) == 0xffff: seq = data[1] if seq in self.__seq_range: ack = bytes([255, seq]) ack += (lib.checksum(ack) ^ 0xffff).to_bytes(2, 'big') self.__sock.sendto(ack, addr) index = self.__find_index(seq) self.__window[index] = data # 移动窗口,将接收到的数据移动到缓存中 while self.__window[0] is not None: data = self.__window[0] self.__min_seq = data[1] + 1 if self.__min_seq == self.__max_seq: self.__min_seq = 0 data_len = int.from_bytes(data[2:6], 'big') self.__recv_cache.append(self.__window[0][6:6+data_len]) self.__notify_recv.acquire() self.__notify_recv.notify() self.__notify_recv.release() self.__window.pop(0) self.__seq_range.pop(0) self.__window.append(None) # 维护可以接收的seq范围 seq = self.__seq_range[self.__window_size - 2] + 1 if seq > self.__max_seq: seq = 0 self.__seq_range.append(seq)
class X6500FPGA(BaseWorker): # Constructor, gets passed a reference to the miner core, the X6500Worker, # its FPGA id, and the bitstream version currently running on that FPGA def __init__(self, core, parent, fpga, bitstreamversion): self.parent = parent self.fpga = fpga self.firmware_rev = bitstreamversion # Let our superclass do some basic initialization super(X6500FPGA, self).__init__(core, None) # Initialize wakeup flag for the main thread. # This serves as a lock at the same time. self.wakeup = Condition() # Validate settings, mostly coping them from our parent # Called from the constructor and after every settings change on the parent. def apply_settings(self): self.settings.name = "%s FPGA%d" % (self.parent.settings.name, self.fpga) # Let our superclass handle everything that isn't specific to this worker module super(X6500FPGA, self).apply_settings() # Reset our state. Called both from the constructor and from self.start(). def _reset(self): # Let our superclass handle everything that isn't specific to this worker module super(X6500FPGA, self)._reset() self.stats.temperature = None self.stats.speed = None self.initialramp = True # Start up the worker module. This is protected against multiple calls and concurrency by a wrapper. def _start(self): # Let our superclass handle everything that isn't specific to this worker module super(X6500FPGA, self)._start() # Assume a default job interval to make the core start fetching work for us. # The actual hashrate will be measured (and this adjusted to the correct value) later. self.jobs_per_second = 1. / self.parent.settings.jobinterval # This worker will only ever process one job at once. The work fetcher needs this information # to estimate how many jobs might be required at once in the worst case (after a block was found). self.parallel_jobs = 1 # Reset the shutdown flag for our threads self.shutdown = False # Start up the main thread, which handles pushing work to the device. self.mainthread = Thread(None, self.main, self.settings.name + "_main") self.mainthread.daemon = True self.mainthread.start() # Stut down the worker module. This is protected against multiple calls and concurrency by a wrapper. def _stop(self): # Let our superclass handle everything that isn't specific to this worker module super(X6500FPGA, self)._stop() # Set the shutdown flag for our threads, making them terminate ASAP. self.shutdown = True # Trigger the main thread's wakeup flag, to make it actually look at the shutdown flag. with self.wakeup: self.wakeup.notify() # Wait for the main thread to terminate. self.mainthread.join(2) # This function should interrupt processing of the specified job if possible. # This is necesary to avoid producing stale shares after a new block was found, # or if a job expires for some other reason. If we don't know about the job, just ignore it. # Never attempts to fetch a new job in here, always do that asynchronously! # This needs to be very lightweight and fast. We don't care whether it's a # graceful cancellation for this module because the work upload overhead is low. def notify_canceled(self, job, graceful): # Acquire the wakeup lock to make sure that nobody modifies job/nextjob while we're looking at them. with self.wakeup: # If the currently being processed, or currently being uploaded job are affected, # wake up the main thread so that it can request and upload a new job immediately. if self.job == job: self.wakeup.notify() # Report custom statistics. def _get_statistics(self, stats, childstats): # Let our superclass handle everything that isn't specific to this worker module super(X6500FPGA, self)._get_statistics(stats, childstats) stats.temperature = self.stats.temperature # Main thread entry point # This thread is responsible for fetching work and pushing it to the device. def main(self): # If we're currently shutting down, just die. If not, loop forever, # to recover from possible errors caught by the huge try statement inside this loop. # Count how often the except for that try was hit recently. This will be reset if # there was no exception for at least 5 minutes since the last one. tries = 0 while not self.shutdown: try: # Record our starting timestamp, in order to back off if we repeatedly die starttime = time.time() # Initialize megahashes per second to zero, will be measured later. self.stats.mhps = 0 # Job that the device is currently working on, or that is currently being uploaded. # This variable is used by BaseWorker to figure out the current work source for statistics. self.job = None # Job that was previously being procesed. Has been destroyed, but there might be some late nonces. self.oldjob = None # We keep control of the wakeup lock at all times unless we're sleeping self.wakeup.acquire() # Eat up leftover wakeups self.wakeup.wait(0) # Honor shutdown flag (in case it was a real wakeup) if self.shutdown: break # Set validation success flag to false self.checksuccess = False # Set validation job second iteration flag to false self.seconditeration = False # Initialize hash rate tracking data self.lasttime = None self.lastnonce = None # Initialize malfunction tracking data self.recentshares = 0 self.recentinvalid = 0 # Configure core clock, if the bitstream supports that if self.firmware_rev > 0: self._set_speed(self.parent.settings.initialspeed) # Clear FPGA's nonce queue self.parent.clear_queue(self.fpga) # Send validation job to device job = ValidationJob( self.core, unhexlify( b"00000001c3bf95208a646ee98a58cf97c3a0c4b7bf5de4c89ca04495000005200000000024d1fff8d5d73ae11140e4e48032cd88ee01d48c67147f9a09cd41fdec2e25824f5c038d1a0b350c5eb01f04" )) self._sendjob(job) # Wait for the validation job to complete. The wakeup flag will be set by the listener # thread when the validation job completes. 180 seconds should be sufficient for devices # down to about 50MH/s, for slower devices this timeout will need to be increased. if self.stats.speed: self.wakeup.wait( (2**32 / 1000000. / self.stats.speed) * 1.1) else: self.wakeup.wait(180) # Honor shutdown flag if self.shutdown: break # We woke up, but the validation job hasn't succeeded in the mean time. # This usually means that the wakeup timeout has expired. if not self.checksuccess: raise Exception( "Timeout waiting for validation job to finish") # self.stats.mhps has now been populated by the listener thread self.core.log(self, "Running at %f MH/s\n" % self.stats.mhps, 300, "B") self._update_job_interval() # Main loop, continues until something goes wrong or we're shutting down. while not self.shutdown: # Fetch a job, add 2 seconds safety margin to the requested minimum expiration time. # Blocks until one is available. Because of this we need to release the # wakeup lock temporarily in order to avoid possible deadlocks. self.wakeup.release() job = self.core.get_job(self, self.jobinterval + 2) self.wakeup.acquire() # If a new block was found while we were fetching that job, just discard it and get a new one. if job.canceled: job.destroy() continue # Upload the job to the device self._sendjob(job) # Go through the safety checks and reduce the clock if necessary self.safetycheck() # If the job was already caught by a long poll while we were uploading it, # jump back to the beginning of the main loop in order to immediately fetch new work. # Don't check for the canceled flag before the job was accepted by the device, # otherwise we might get out of sync. if self.job.canceled: continue # Wait while the device is processing the job. If nonces are sent by the device, they # will be processed by the listener thread. If the job gets canceled, we will be woken up. self.wakeup.wait(self.jobinterval) # If something went wrong... except Exception as e: # ...complain about it! self.core.log(self, "%s\n" % traceback.format_exc(), 100, "rB") finally: # We're not doing productive work any more, update stats and destroy current job self._jobend() self.stats.mhps = 0 try: self.wakeup.release() except: pass # If we aren't shutting down, figure out if there have been many errors recently, # and if yes, restart the parent worker as well. if not self.shutdown: tries += 1 if time.time() - starttime >= 300: tries = 0 with self.wakeup: if tries > 5: self.parent.async_restart() return else: self.wakeup.wait(1) # Restart (handled by "while not self.shutdown:" loop above) def notify_nonce_found(self, now, nonce): # Snapshot the current jobs to avoid race conditions oldjob = self.oldjob newjob = self.job # If there is no job, this must be a leftover from somewhere, e.g. previous invocation # or reiterating the keyspace because we couldn't provide new work fast enough. # In both cases we can't make any use of that nonce, so just discard it. if not oldjob and not newjob: return # Pass the nonce that we found to the work source, if there is one. # Do this before calculating the hash rate as it is latency critical. job = None if newjob: if newjob.nonce_found(nonce, oldjob): job = newjob if not job and oldjob: if oldjob.nonce_found(nonce): job = oldjob self.recentshares += 1 if not job: self.recentinvalid += 1 nonceval = struct.unpack("<I", nonce)[0] if isinstance(newjob, ValidationJob): # This is a validation job. Validate that the nonce is correct, and complain if not. if newjob.nonce != nonce: raise Exception( "Mining device is not working correctly (returned %s instead of %s)" % (hexlify(nonce).decode("ascii"), hexlify( newjob.nonce).decode("ascii"))) else: # The nonce was correct if self.firmware_rev > 0: # The FPGA is running overclocker firmware, so we don't need to use this method to calculate the hashrate. # In fact, it will not work because the FPGA will go to sleep after working through all possible nonces. with self.wakeup: self.checksuccess = True self.wakeup.notify() else: if self.seconditeration == True: with self.wakeup: # This is the second iteration. We now know the actual nonce rotation time. delta = (now - newjob.starttime) # Calculate the hash rate based on the nonce rotation time. self.stats.mhps = 2**32 / 1000000. / delta # Update hash rate tracking information self.lasttime = now self.lastnonce = nonceval # Wake up the main thread self.checksuccess = True self.wakeup.notify() else: with self.wakeup: # This was the first iteration. Wait for another one to figure out nonce rotation time. newjob.starttime = now self.seconditeration = True else: if self.firmware_rev == 0: # Adjust hash rate tracking delta = (now - self.lasttime) estimatednonce = int( round(self.lastnonce + self.stats.mhps * 1000000 * delta)) noncediff = nonceval - (estimatednonce & 0xffffffff) if noncediff < -0x80000000: noncediff = noncediff + 0x100000000 elif noncediff > 0x7fffffff: noncediff = noncediff - 0x100000000 estimatednonce = estimatednonce + noncediff # Calculate the hash rate based on adjusted tracking information currentmhps = (estimatednonce - self.lastnonce) / 1000000. / delta weight = min(0.5, delta / 100.) self.stats.mhps = ( 1 - weight) * self.stats.mhps + weight * currentmhps # Update hash rate tracking information self.lasttime = now self.lastnonce = nonceval # This function uploads a job to the device def _sendjob(self, job): # Move previous job to oldjob, and new one to job self.oldjob = self.job self.job = job # Send it to the FPGA start, now = self.parent.send_job(self.fpga, job) # Calculate how long the old job was running if self.oldjob: if self.oldjob.starttime: self.oldjob.hashes_processed( (now - self.oldjob.starttime) * self.stats.mhps * 1000000) self.oldjob.destroy() self.job.starttime = now # This function needs to be called whenever the device terminates working on a job. # It calculates how much work was actually done for the job and destroys it. def _jobend(self, now=None): # Hack to avoid a python bug, don't integrate this into the line above if not now: now = time.time() # Calculate how long the job was actually running and multiply that by the hash # rate to get the number of hashes calculated for that job and update statistics. if self.job != None: if self.job.starttime: self.job.hashes_processed( (now - self.job.starttime) * self.stats.mhps * 1000000) # Destroy the job, which is neccessary to actually account the calculated amount # of work to the worker and work source, and to remove the job from cancelation lists. self.oldjob = self.job self.job.destroy() self.job = None # Check the invalid rate and temperature, and reduce the FPGA clock if these exceed safe values def safetycheck(self): warning = False critical = False if self.recentinvalid >= self.parent.settings.invalidwarning: warning = True if self.recentinvalid >= self.parent.settings.invalidcritical: critical = True if self.stats.temperature: if self.stats.temperature > self.parent.settings.tempwarning: warning = True if self.stats.temperature > self.parent.settings.tempcritical: critical = True threshold = self.parent.settings.warmupstepshares if self.initialramp and not self.recentinvalid else self.parent.settings.speedupthreshold if warning: self.core.log(self, "Detected overload condition!\n", 200, "y") if critical: self.core.log(self, "Detected CRITICAL condition!\n", 100, "rB") if critical: speedstep = -20 self.initialramp = False elif warning: speedstep = -2 self.initialramp = False elif not self.recentinvalid and self.recentshares >= threshold: speedstep = 2 else: speedstep = 0 if self.firmware_rev > 0: if speedstep: self._set_speed(self.stats.speed + speedstep) elif warning or critical: self.core.log( self, "Firmware too old, can not automatically reduce clock!\n", 200, "yB") if critical: self.core.log(self, "Shutting down FPGA to protect it!\n", 100, "rB") self.parent.shutdown_fpga(self.fpga) self.async_stop(2) if speedstep or self.recentshares >= threshold: self.recentinvalid = 0 self.recentshares = 0 def _set_speed(self, speed): speed = min(max(speed, 4), self.parent.settings.maximumspeed) if self.stats.speed == speed: return if speed == self.parent.settings.maximumspeed: self.initialramp = False self.core.log( self, "%s: Setting clock speed to %.2f MHz...\n" % ("Warmup" if self.initialramp else "Tracking", speed), 500, "B") self.parent.set_speed(self.fpga, speed) self.stats.speed = self.parent.get_speed(self.fpga) self.stats.mhps = self.stats.speed self._update_job_interval() if self.stats.speed != speed: self.core.log(self, "Setting clock speed failed!\n", 100, "rB") def _update_job_interval(self): self.core.event(350, self, "speed", self.stats.mhps * 1000, "%f MH/s" % self.stats.mhps, self) # Calculate the time that the device will need to process 2**32 nonces. # This is limited at 60 seconds in order to have some regular communication, # even with very slow devices (and e.g. detect if the device was unplugged). interval = min(60, 2**32 / 1000000. / self.stats.mhps) # Add some safety margin and take user's interval setting (if present) into account. self.jobinterval = min(self.parent.settings.jobinterval, max(0.5, interval * 0.8 - 1)) self.core.log(self, "Job interval: %f seconds\n" % self.jobinterval, 400, "B") # Tell the MPBM core that our hash rate has changed, so that it can adjust its work buffer. self.jobs_per_second = 1. / self.jobinterval self.core.notify_speed_changed(self)
class StreamPump(Thread): def __init__(self, sink, size): Thread.__init__(self) self.rPos = 0 self.wPos = 0 self.buff = bytearray(size) self.size = size self.sink = sink self.cond = Condition() self.isRunning = True def write(self, data): nOfBytes = len(data) overflow = self.wPos + nOfBytes - self.size if overflow >= 0: #print("%d > %d ov" %(overflow,self.rPos)) if self.wPos < self.rPos < self.size or 0 < self.rPos < overflow: print("Error :-(") raise BaseException("Buffer overflow") self.buff[self.wPos:self.size] = data[0:nOfBytes-overflow] self.buff[0:overflow] = data[nOfBytes-overflow:nOfBytes] self.wPos = overflow else: newWpos = self.wPos+nOfBytes #print("%d < %d ov" %(newWpos,self.rPos)) if self.wPos < self.rPos < newWpos: print("Error :-(") raise BaseException("Buffer overflow") self.buff[self.wPos:newWpos] = data self.wPos = newWpos self.cond.acquire() self.cond.notify() self.cond.release() return nOfBytes def run(self): self.isRunning = True while True: while self.rPos == self.wPos: if self.isRunning: self.cond.acquire() self.cond.wait() self.cond.release() else: return # make sure to always use same wPos wPos = self.wPos if self.rPos > wPos: self.sink.write(self.buff[self.rPos:self.size]) self.rPos = 0 self.sink.write(self.buff[self.rPos:wPos]) self.rPos = wPos def finish(self): #print("Finish") self.cond.acquire() self.isRunning = False self.cond.notify() self.cond.release() self.join()
class IbApi(EWrapper): """""" data_filename = "ib_contract_data.db" data_filepath = str(get_file_path(data_filename)) def __init__(self, gateway: BaseGateway): """""" super().__init__() self.gateway = gateway self.gateway_name = gateway.gateway_name self.status = False self.reqid = 0 self.orderid = 0 self.clientid = 0 self.account = "" self.ticks = {} self.orders = {} self.accounts = {} self.contracts = {} self.tick_exchange = {} self.history_req = None self.history_condition = Condition() self.history_buf = [] self.client = IbClient(self) self.thread = Thread(target=self.client.run) def connectAck(self): # pylint: disable=invalid-name """ Callback when connection is established. """ self.status = True self.gateway.write_log("IB TWS连接成功") self.load_contract_data() def connectionClosed(self): # pylint: disable=invalid-name """ Callback when connection is closed. """ self.status = False self.gateway.write_log("IB TWS连接断开") def nextValidId(self, orderId: int): # pylint: disable=invalid-name """ Callback of next valid orderid. """ super().nextValidId(orderId) if not self.orderid: self.orderid = orderId def currentTime(self, time: int): # pylint: disable=invalid-name """ Callback of current server time of IB. """ super().currentTime(time) dt = datetime.fromtimestamp(time) time_string = dt.strftime("%Y-%m-%d %H:%M:%S.%f") msg = f"服务器时间: {time_string}" self.gateway.write_log(msg) def error( self, reqId: TickerId, errorCode: int, errorString: str ): # pylint: disable=invalid-name """ Callback of error caused by specific request. """ super().error(reqId, errorCode, errorString) msg = f"信息通知,代码:{errorCode},内容: {errorString}" self.gateway.write_log(msg) def tickPrice( # pylint: disable=invalid-name self, reqId: TickerId, tickType: TickType, price: float, attrib: TickAttrib ): """ Callback of tick price update. """ super().tickPrice(reqId, tickType, price, attrib) if tickType not in TICKFIELD_IB2VT: return tick = self.ticks[reqId] name = TICKFIELD_IB2VT[tickType] setattr(tick, name, price) # Update name into tick data. contract = self.contracts.get(tick.vt_symbol, None) if contract: tick.name = contract.name # Forex and spot product of IDEALPRO has no tick time and last price. # We need to calculate locally. exchange = self.tick_exchange[reqId] if exchange is Exchange.IDEALPRO: tick.last_price = (tick.bid_price_1 + tick.ask_price_1) / 2 tick.datetime = datetime.now() self.gateway.on_tick(copy(tick)) def tickSize( self, reqId: TickerId, tickType: TickType, size: int ): # pylint: disable=invalid-name """ Callback of tick volume update. """ super().tickSize(reqId, tickType, size) if tickType not in TICKFIELD_IB2VT: return tick = self.ticks[reqId] name = TICKFIELD_IB2VT[tickType] setattr(tick, name, size) self.gateway.on_tick(copy(tick)) def tickString( self, reqId: TickerId, tickType: TickType, value: str ): # pylint: disable=invalid-name """ Callback of tick string update. """ super().tickString(reqId, tickType, value) if tickType != TickTypeEnum.LAST_TIMESTAMP: return tick = self.ticks[reqId] tick.datetime = datetime.fromtimestamp(value) self.gateway.on_tick(copy(tick)) def orderStatus( # pylint: disable=invalid-name self, orderId: OrderId, status: str, filled: float, remaining: float, avgFillPrice: float, permId: int, parentId: int, lastFillPrice: float, clientId: int, whyHeld: str, mktCapPrice: float, ): """ Callback of order status update. """ super().orderStatus( orderId, status, filled, remaining, avgFillPrice, permId, parentId, lastFillPrice, clientId, whyHeld, mktCapPrice, ) orderid = str(orderId) order = self.orders.get(orderid, None) order.traded = filled # To filter PendingCancel status order_status = STATUS_IB2VT.get(status, None) if order_status: order.status = order_status self.gateway.on_order(copy(order)) def openOrder( # pylint: disable=invalid-name self, orderId: OrderId, ib_contract: Contract, ib_order: Order, orderState: OrderState, ): """ Callback when opening new order. """ super().openOrder( orderId, ib_contract, ib_order, orderState ) orderid = str(orderId) order = OrderData( symbol=ib_contract.conId, exchange=EXCHANGE_IB2VT.get( ib_contract.exchange, ib_contract.exchange), type=ORDERTYPE_IB2VT[ib_order.orderType], orderid=orderid, direction=DIRECTION_IB2VT[ib_order.action], volume=ib_order.totalQuantity, gateway_name=self.gateway_name, ) if order.type == OrderType.LIMIT: order.price = ib_order.lmtPrice elif order.type == OrderType.STOP: order.price = ib_order.auxPrice self.orders[orderid] = order self.gateway.on_order(copy(order)) def updateAccountValue( # pylint: disable=invalid-name self, key: str, val: str, currency: str, accountName: str ): """ Callback of account update. """ super().updateAccountValue(key, val, currency, accountName) if not currency or key not in ACCOUNTFIELD_IB2VT: return accountid = f"{accountName}.{currency}" account = self.accounts.get(accountid, None) if not account: account = AccountData(accountid=accountid, gateway_name=self.gateway_name) self.accounts[accountid] = account name = ACCOUNTFIELD_IB2VT[key] setattr(account, name, float(val)) def updatePortfolio( # pylint: disable=invalid-name self, contract: Contract, position: float, marketPrice: float, marketValue: float, averageCost: float, unrealizedPNL: float, realizedPNL: float, accountName: str, ): """ Callback of position update. """ super().updatePortfolio( contract, position, marketPrice, marketValue, averageCost, unrealizedPNL, realizedPNL, accountName, ) if contract.exchange: exchange = EXCHANGE_IB2VT.get(contract.exchange, None) elif contract.primaryExchange: exchange = EXCHANGE_IB2VT.get(contract.primaryExchange, None) else: exchange = Exchange.SMART # Use smart routing for default if not exchange: msg = f"存在不支持的交易所持仓{contract.conId} {contract.exchange} {contract.primaryExchange}" self.gateway.write_log(msg) return ib_size = contract.multiplier if not ib_size: ib_size = 1 price = averageCost / ib_size pos = PositionData( symbol=contract.conId, exchange=exchange, direction=Direction.NET, volume=position, price=price, pnl=unrealizedPNL, gateway_name=self.gateway_name, ) self.gateway.on_position(pos) def updateAccountTime(self, timeStamp: str): # pylint: disable=invalid-name """ Callback of account update time. """ super().updateAccountTime(timeStamp) for account in self.accounts.values(): self.gateway.on_account(copy(account)) def contractDetails(self, reqId: int, contractDetails: ContractDetails): # pylint: disable=invalid-name """ Callback of contract data update. """ super().contractDetails(reqId, contractDetails) # Generate symbol from ib contract details ib_contract = contractDetails.contract if not ib_contract.multiplier: ib_contract.multiplier = 1 symbol = generate_symbol(ib_contract) # Generate contract contract = ContractData( symbol=symbol, exchange=EXCHANGE_IB2VT[ib_contract.exchange], name=contractDetails.longName, product=PRODUCT_IB2VT[ib_contract.secType], size=ib_contract.multiplier, pricetick=contractDetails.minTick, net_position=True, history_data=True, stop_supported=True, gateway_name=self.gateway_name, ) if contract.vt_symbol not in self.contracts: self.gateway.on_contract(contract) self.contracts[contract.vt_symbol] = contract self.save_contract_data() def execDetails( self, reqId: int, contract: Contract, execution: Execution ): # pylint: disable=invalid-name """ Callback of trade data update. """ super().execDetails(reqId, contract, execution) # today_date = datetime.now().strftime("%Y%m%d") trade = TradeData( symbol=contract.conId, exchange=EXCHANGE_IB2VT.get(contract.exchange, contract.exchange), orderid=str(execution.orderId), tradeid=str(execution.execId), direction=DIRECTION_IB2VT[execution.side], price=execution.price, volume=execution.shares, time=datetime.strptime(execution.time, "%Y%m%d %H:%M:%S"), gateway_name=self.gateway_name, ) self.gateway.on_trade(trade) def managedAccounts(self, accountsList: str): """ Callback of all sub accountid. """ super().managedAccounts(accountsList) if not self.account: for account_code in accountsList.split(","): self.account = account_code self.gateway.write_log(f"当前使用的交易账号为{self.account}") self.client.reqAccountUpdates(True, self.account) def historicalData(self, reqId: int, ib_bar: IbBarData): """ Callback of history data update. """ dt = datetime.strptime(ib_bar.date, "%Y%m%d %H:%M:%S") bar = BarData( symbol=self.history_req.symbol, exchange=self.history_req.exchange, datetime=dt, interval=self.history_req.interval, volume=ib_bar.volume, open_price=ib_bar.open, high_price=ib_bar.high, low_price=ib_bar.low, close_price=ib_bar.close, gateway_name=self.gateway_name ) self.history_buf.append(bar) def historicalDataEnd(self, reqId: int, start: str, end: str): """ Callback of history data finished. """ self.history_condition.acquire() self.history_condition.notify() self.history_condition.release() def connect(self, host: str, port: int, clientid: int, account: str): """ Connect to TWS. """ if self.status: return self.clientid = clientid self.account = account self.client.connect(host, port, clientid) self.thread.start() self.client.reqCurrentTime() def close(self): """ Disconnect to TWS. """ if not self.status: return self.status = False self.client.disconnect() def subscribe(self, req: SubscribeRequest): """ Subscribe tick data update. """ if not self.status: return if req.exchange not in EXCHANGE_VT2IB: self.gateway.write_log(f"不支持的交易所{req.exchange}") return # Extract ib contract detail ib_contract = generate_ib_contract(req.symbol, req.exchange) if not ib_contract: self.gateway.write_log("代码解析失败,请检查格式是否正确") return # Get contract data from TWS. self.reqid += 1 self.client.reqContractDetails(self.reqid, ib_contract) # Subscribe tick data and create tick object buffer. self.reqid += 1 self.client.reqMktData(self.reqid, ib_contract, "", False, False, []) tick = TickData( symbol=req.symbol, exchange=req.exchange, datetime=datetime.now(), gateway_name=self.gateway_name, ) self.ticks[self.reqid] = tick self.tick_exchange[self.reqid] = req.exchange def send_order(self, req: OrderRequest): """ Send a new order. """ if not self.status: return "" if req.exchange not in EXCHANGE_VT2IB: self.gateway.write_log(f"不支持的交易所:{req.exchange}") return "" if req.type not in ORDERTYPE_VT2IB: self.gateway.write_log(f"不支持的价格类型:{req.type}") return "" self.orderid += 1 ib_contract = generate_ib_contract(req.symbol, req.exchange) if not ib_contract: return "" ib_order = Order() ib_order.orderId = self.orderid ib_order.clientId = self.clientid ib_order.action = DIRECTION_VT2IB[req.direction] ib_order.orderType = ORDERTYPE_VT2IB[req.type] ib_order.totalQuantity = req.volume ib_order.account = self.account if req.type == OrderType.LIMIT: ib_order.lmtPrice = req.price elif req.type == OrderType.STOP: ib_order.auxPrice = req.price self.client.placeOrder(self.orderid, ib_contract, ib_order) self.client.reqIds(1) order = req.create_order_data(str(self.orderid), self.gateway_name) self.gateway.on_order(order) return order.vt_orderid def cancel_order(self, req: CancelRequest): """ Cancel an existing order. """ if not self.status: return self.client.cancelOrder(int(req.orderid)) def query_history(self, req: HistoryRequest): """""" self.history_req = req self.reqid += 1 ib_contract = generate_ib_contract(req.symbol, req.exchange) if req.end: end = req.end end_str = end.strftime("%Y%m%d %H:%M:%S") else: end = datetime.now() end_str = "" delta = end - req.start days = min(delta.days, 180) # IB only provides 6-month data duration = f"{days} D" bar_size = INTERVAL_VT2IB[req.interval] if req.exchange == Exchange.IDEALPRO: bar_type = "MIDPOINT" else: bar_type = "TRADES" self.client.reqHistoricalData( self.reqid, ib_contract, end_str, duration, bar_size, bar_type, 1, 1, False, [] ) self.history_condition.acquire() # Wait for async data return self.history_condition.wait() self.history_condition.release() history = self.history_buf self.history_buf = [] # Create new buffer list self.history_req = None return history def load_contract_data(self): """""" f = shelve.open(self.data_filepath) self.contracts = f.get("contracts", {}) f.close() for contract in self.contracts.values(): self.gateway.on_contract(contract) self.gateway.write_log("本地缓存合约信息加载成功") def save_contract_data(self): """""" f = shelve.open(self.data_filepath) f["contracts"] = self.contracts f.close()