class NotificationQueue(Queue): """ A queue class that notifies waiting threads when the queue is available for reading.""" def __init__(self): """ Constructor.""" Queue.__init__(self) self.lock = Condition() def get(self): """ Get an item from the queue. This method is thread-safe. Method must be called on a non-empty queue. Return : the first item from the queue Raises: a EmptyQueue exeception """ return Queue.get(self, False) def put(self, item): """ Add a new item to the queue. This method is thread-safe. Threads that are waiting for get an item are notified. item: new item to add to the queue. Return : None """ self.lock.acquire() Queue.put(self, item) self.lock.notify() self.lock.release() def acquire(self): self.lock.acquire() def release(self): self.lock.release() def wait(self): self.lock.wait()
class _EventRef(object): """ Helper class which acts as a threading.Event, but which can be used to pass a reference when signaling the event. """ def __init__(self): self._cond = Condition(Lock()) self._flag = False self._ref = None def isSet(self): return self._flag def set(self, ref): with self._cond: assert self._ref is None self._ref = ref self._flag = True self._cond.notifyAll() def get(self, timeout=None): with self._cond: if not self._flag: self._cond.wait(timeout) if not self._flag: raise TimeoutExceeded('Could not get the reference.') return self._ref def clear(self, ref): with self._cond: self._ref = None self._flag = False
class _CoreScheduleThread(Thread): def __init__(self,threadpool): self.scheduletasks = []; self.tasklock = Lock(); self.condition = Condition(Lock()) self.threadpool = threadpool Thread.__init__(self) def run(self): while True: self.condition.acquire() if len(self.scheduletasks) == 0: self.condition.wait(); else: task = self.scheduletasks.pop(0) if dates.current_timestamps()>=task.nexttime: self.threadpool.execute(task.function,*task.args,**task.kwargs) task.nexttime = dates.current_timestamps()+task.period; else: self.condition.wait(task.nexttime-dates.current_timestamps()) self.addtask(task) self.condition.release() def addtask(self,task): # copy on write self.tasklock.acquire() tasks = [ t for t in self.scheduletasks ] tasks.append(task) tasks.sort(key=lambda task:task.nexttime) self.scheduletasks = tasks self.tasklock.release()
def wait_statement(self, wtime, N=5): ''' ''' start_time = time.time() if self.debug: time.sleep(1) else: if isinstance(wtime, str): wtime = float(self._get_interpolation_value(wtime)) if wtime > N: c = Condition() c.acquire() wd = WaitDialog(wtime=wtime, condition=c, parent=self) do_later(wd.edit_traits, kind='livemodal') c.wait() c.release() do_later(wd.close) else: time.sleep(wtime) msg = 'actual wait time %0.3f s' % (time.time() - start_time) self.log_statement(msg)
class SimpleBlockingQueue(object): def __init__(self): self.items = deque() self.condition = Condition() self.interrupted = False def put(self, item): with self.condition: self.items.pushleft(item) def take(self, block=False): with self.condition: if block: while not self.items: self.condition.wait() if self.interrupted: self.interrupted = False return None elif not self.items: return None return self.items.popright() def interrupt(self): with self.condition: self.interrupted = True self.condition.notifyAll()
class _InterruptableLock(object): def __init__(self, lock): """Creates a new lock.""" self.__lock = lock self.__condition = Condition(Lock()) def __enter__(self): return self.acquire() def __exit__(self, exc_type, exc_value, traceback): self.release() def acquire(self, blocking=True, timeout=-1): """Acquire a lock, blocking or non-blocking. Blocks until timeout, if timeout a positive float and blocking=True. A timeout value of -1 blocks indefinitely, unless blocking=False.""" if not isinstance(timeout, (int, float)): raise TypeError("a float is required") if blocking: # blocking indefinite if timeout == -1: with self.__condition: while not self.__lock.acquire(False): # condition with timeout is interruptable self.__condition.wait(60) return True # same as non-blocking elif timeout == 0: return self.__lock.acquire(False) elif timeout < 0: raise ValueError("timeout value must be strictly positive (or -1)") # blocking finite else: start = time() waited_time = 0 with self.__condition: while waited_time < timeout: if self.__lock.acquire(False): return True else: self.__condition.wait(timeout - waited_time) waited_time = time() - start return False elif timeout != -1: raise ValueError("can't specify a timeout for a non-blocking call") else: # non-blocking return self.__lock.acquire(False) def release(self): """Release a lock.""" self.__lock.release() with self.__condition: self.__condition.notify()
class Event(object): def __init__(self,): self.__cond = Condition(Lock()) self.__flag = False def isSet(self): return self.__flag is_set = isSet def set(self): self.__cond.acquire() try: self.__flag = True self.__cond.notify_all() finally: self.__cond.release() def clear(self): self.__cond.acquire() try: self.__flag = False finally: self.__cond.release() def wait(self, timeout=None): self.__cond.acquire() try: if not self.__flag: self.__cond.wait(timeout) return self.__flag finally: self.__cond.release()
class DirtyBit: def __init__(self): self._mon = RLock() self._rc = Condition(self._mon) self._dirty = 0 def clear(self): self._mon.acquire() self._dirty = 0 #self._rc.notify() only interested in knowing when we are dirty self._mon.release() def set(self): self._mon.acquire() self._dirty = 1 self._rc.notify() self._mon.release() def value(self): return self._dirty def wait(self): self._mon.acquire() self._rc.wait() self._mon.release()
def process_sequence(self, sequence): """ @param sequence: the movement sequence to execute @return: dictionary of sensor readings """ self.log.debug('process sequence') # parse the json message jseq = json.loads(sequence) moveseq = jseq['sequence'] for moves in moveseq: self.log.debug('processing move %s' % moves) condition = Condition() thread_count = 0 for move in moves: thread_count += 1 servo_instruction = self.process_move(move) move_servo_thread = ServoThread(servo_instruction, self.pwm_queue, condition, thread_count, self.update_current_pulse) move_servo_thread.setDaemon(False) move_servo_thread.setName('Servo %d' % servo_instruction['channel']) move_servo_thread.start() condition.acquire() if thread_count > 0: condition.wait() # wait for all threads to finish before doing next loop condition.release()
class BinaryServer(Thread): """ Socket server forwarding request to internal server """ def __init__(self, internal_server, hostname, port): Thread.__init__(self) self.socket_server = None self.hostname = hostname self.port = port self.iserver = internal_server self._cond = Condition() def start(self): with self._cond: Thread.start(self) self._cond.wait() def run(self): logger.warning("Listening on %s:%s", self.hostname, self.port) socketserver.TCPServer.allow_reuse_address = True # get rid of address already in used warning self.socket_server = ThreadingTCPServer((self.hostname, self.port), UAHandler) # self.socket_server.daemon_threads = True # this will force a shutdown of all threads, maybe too hard self.socket_server.internal_server = self.iserver # allow handler to acces server properties with self._cond: self._cond.notify_all() self.socket_server.serve_forever() def stop(self): logger.warning("server shutdown request") self.socket_server.shutdown()
class Node(object): STATE_INIT = 0 STATE_STARTING = 1 STATE_RUNNING = 2 STATE_STOPPED = 3 STATE_PARTITIONED = 4 STATE_FAILED = 5 state_str = {STATE_INIT: "Initial", STATE_STARTING: "Starting", STATE_RUNNING: "Running", STATE_STOPPED: "Stopped", STATE_FAILED: "Failed"} def __init__(self, node_id): self.node_id = node_id self.state = Node.STATE_INIT self.cv_lock = Lock() self.cv = Condition(self.cv_lock) def wait_for_state(self, state): self.cv.acquire() while self.state not in (state, Node.STATE_FAILED): self.cv.wait() self.cv.release() if self.state != state: raise ChistributedException("Node entered failed state while waiting for state %s" % Node.state_str[state]) def set_state(self, state): self.cv.acquire() self.state = state self.cv.notify_all() self.cv.release()
class Consumer(Thread): """Класс потока – обработчика данных""" def __init__(self): """Конструктор класса""" super().__init__() self.condition = Condition() self.received_data = None def run(self): """Код, исполняемый в потоке""" running_sum = 0 count = 0 while True: with self.condition: # Ожидание доступности данных self.condition.wait() # Получение данных data = self.received_data # Обработка данных running_sum += data count += 1 print('Got data: {}\nTotal count: {}\nAverage: {}\n'.format( data, count, running_sum / count)) def send(self, data): """Метод отправки данных в обработчик""" self.received_data = data
class SendBuffer(object): def __init__(self, max_size): self.size = 0 self.frontbuffer = '' self.full = Condition() self.backbuffer = StringIO() self.max_size = max_size def __len__(self): return len(self.frontbuffer) + self.size def write(self, data): dlen = len(data) with self.full: while self.size + dlen > self.max_size and dlen < self.max_size: # if a single write is bigger than the buffer, swallow hard. # No amount of waitng will make it fit. self.full.wait() self.backbuffer.write(data) self.size += dlen def to_sock(self, sock): if not self.frontbuffer: with self.full: buf, self.backbuffer = self.backbuffer, StringIO() self.size = 0 self.full.notify_all() self.frontbuffer = buf.getvalue() written = sock.send(self.frontbuffer) self.frontbuffer = self.frontbuffer[written:]
class DataQueue ( Queue ): def __init__ ( self, count ): Queue.__init__(self) self._count = count self._cond = Condition() def get ( self, block = True, timeout = None ): self._cond.acquire() if self.empty(): if not self._count: raise Empty() elif block: self._cond.wait(timeout) if self.empty() and not self._count: self._cond.release() raise Empty() self._cond.release() return Queue.get(self, block, timeout) def put ( self, data ): self._cond.acquire() self._count = self._count - 1 Queue.put(self, data) if self._count: self._cond.notify() else: self._cond.notifyAll() self._cond.release() def remain ( self ): return self._count + self.unfinished_tasks
class MessageQueue(object): """The message queue used internally by Gossip.""" def __init__(self): self._queue = deque() self._condition = Condition() def pop(self): self._condition.acquire() try: while len(self._queue) < 1: self._condition.wait() return self._queue.pop() finally: self._condition.release() def __len__(self): return len(self._queue) def __deepcopy__(self, memo): newmq = MessageQueue() newmq._queue = copy.deepcopy(self._queue, memo) return newmq def appendleft(self, msg): self._condition.acquire() try: self._queue.appendleft(msg) self._condition.notify() finally: self._condition.release()
class PostBox(list): def __init__(self, *a): list.__init__(self, *a) self.lever = Condition() self.open = True self.done = False def wait(self): self.lever.acquire() if not self.done: self.lever.wait() self.lever.release() return self.result def wakeup(self, data): self.result = data self.lever.acquire() self.done = True self.lever.notifyAll() self.lever.release() def append(self, e): self.lever.acquire() if not self.open: raise BoxClosedErr list.append(self, e) self.lever.release() def close(self): self.lever.acquire() self.open = False self.lever.release()
class Future(object): def __init__(self, func, *args, **kwargs): self.__done = False self.__result = None self.__cond = Condition() self.__func = func self.__args = args self.__kwargs = kwargs self.__except = None def __call__(self): with self.__cond: try: self.__result = self.__func(*self.__args, **self.__kwargs) except: self.__result = None self.__except = sys.exc_info() self.__done = True self.__cond.notify() def result(self): with self.__cond: while not self.__done: self.__cond.wait() if self.__except: exc = self.__except reraise(exc[0], exc[1], exc[2]) result = self.__result return copy.deepcopy(result)
class Heartbeater(object): interval = 5 def __init__(self, api, jobstep_id, interval=None): self.api = api self.jobstep_id = jobstep_id self.cv = Condition() self.finished = Event() if interval is not None: self.interval = interval def wait(self): with self.cv: self.finished.clear() while not self.finished.is_set(): data = self.api.get_jobstep(self.jobstep_id) if data['status']['id'] == 'finished': self.finished.set() break self.cv.wait(self.interval) def close(self): with self.cv: self.finished.set() self.cv.notifyAll()
class ZEventLoopThread(object): def __init__(self, cb=None): self._event_loop = None self._thr = Thread(target=self._thread_func) self._guard = Condition(Lock()) self._thr_callback = cb def start_loop(self): self._thr.start() self._guard.acquire() while self._event_loop is None: self._guard.wait() return self._event_loop def _thread_func(self): # Note: create event loop in the new thread # instead in the constructor event_loop = ZEventLoop() if self._thr_callback: self._thr_callback(event_loop) with self._guard: self._event_loop = event_loop self._guard.notify() event_loop.loop()
class IOManager(Thread): def __init__(self,so): Thread.__init__(self) self.mutexIO = Condition() self.listIO = [] self.handlerIO = so.handlerIO def process(self): for irq in self.listIO: print("Ejecuto IOManager") self.handlerIO.handle(irq) self.removeIOFromListIO() def run(self): while(True): with self.mutexIO: self.mutexIO.wait() self.process() def removeIOFromListIO(self): for reqIO in self.listIO: self.listIO.remove(reqIO) def add(self,iOReq): self.listIO.append(iOReq)
class SharedCell(object): """Shared data for the producer/consumer problem.""" def __init__(self): self._data = -1 self._writeable = True self._condition = Condition() def setData(self, data): """Producer's method to write to shared data.""" self._condition.acquire() while not self._writeable: self._condition.wait() print "%s setting data to %d" % \ (currentThread().getName(), data) self._data = data self._writeable = False self._condition.notify() self._condition.release() def getData(self): """Consumer's method to read from shared data.""" self._condition.acquire() while self._writeable: self._condition.wait() print "%s accessing data %d" % \ (currentThread().getName(), self._data) self._writeable = True self._condition.notify() self._condition.release() return self._data
class CountLatch(object): """Synchronization primitive with the capacity to count and latch. Counting up latches, while reaching zero when counting down un-latches. .. note:: The CountLatch class has been included in Sardana on a provisional basis. Backwards incompatible changes (up to and including removal of the module) may occur if deemed necessary by the core developers. """ def __init__(self): self.count = 0 self.condition = Condition() def count_up(self): """Count up.""" self.condition.acquire() self.count += 1 self.condition.release() def count_down(self, _=None): """Count down.""" self.condition.acquire() self.count -= 1 if self.count <= 0: self.condition.notifyAll() self.condition.release() def wait(self): """Wait until the counter reaches zero.""" self.condition.acquire() while self.count > 0: self.condition.wait() self.condition.release()
def request(self, request_type, request, sequential=False): """ """ cond = Condition() # dict is used as reference type to allow result to be returned from # seperate thread response_cont = {} def on_success(response_type, response): response_cont.update({ "success": True, "type": response_type, "response": response, }) with cond: cond.notify() def on_error(exception): response_cont.update({ "success": False, "exception": exception, }) with cond: cond.notify() with cond: self.request_async(request_type, request, on_success, on_error, sequential) cond.wait() if not response_cont["success"]: raise response_cont["exception"] return response_cont["type"], response_cont["response"]
class Queue(object): def __init__(self): self.lock = Lock() self.cond = Condition(self.lock) self.data = [] self.live = True def put(self, values): self.cond.acquire() self.data.extend(values) self.cond.notify() self.cond.release() def get(self): if not self.cond.acquire(False): return [] self.cond.wait() result = self.data self.data = [] self.cond.release() return result def close(self): self.cond.acquire() self.live = False self.cond.notify() self.cond.release()
class AsyncChannel: # yes, i believe this is just Queue ... i was new to python and couldn't find it def __init__ (self, buffer_size=1): self.buffer = [] self.max_items = buffer_size self.lock = Lock () self.not_empty = Condition (self.lock) self.not_full = Condition (self.lock) def get (self): self.lock.acquire () while len (self.buffer) == 0: self.not_empty.wait () val = self.buffer.pop (0) self.not_full.notifyAll () self.lock.release () return val def put (self, val): self.lock.acquire () while len (self.buffer) == self.max_items: self.not_full.wait () self.buffer.append (val) self.not_empty.notifyAll () self.lock.release ()
class MockSock: def __init__(self): self.sent = six.binary_type() self.send_q = deque() self.recv_q = deque() self.closed = False self.lock = Lock() self.cond = Condition(self.lock) self.timeout = 0.5 def gettimeout(self): return self.timeout def close(self): self.lock.acquire() self.closed = True self.lock.release() def queue_send(self, num_bytes): self.lock.acquire() try: self.send_q.append(num_bytes) self.cond.notify_all() finally: self.lock.release() def send(self, message): self.lock.acquire() try: while not len(self.send_q): self.cond.wait() num_bytes = self.send_q.popleft() if num_bytes < len(message): self.send_q.appendleft(len(message) - num_bytes) sent = min(num_bytes, len(message)) self.sent += message[:sent] return sent finally: self.lock.release() def queue_recv(self, message): self.lock.acquire() try: self.recv_q.append(message) self.cond.notify_all() finally: self.lock.release() def recv(self, max_len): self.lock.acquire() try: while not len(self.recv_q): self.cond.wait() message = self.recv_q.popleft() recd = min(max_len, len(message)) if (recd < len(message)): self.recv_q.appendleft(message[recd:]) return message[:recd] finally: self.lock.release()
class LockedDirectories(object): """ Class that keeps a list of locked directories """ def __init__(self): self.dirs = set() self.cv = Condition() def run_in(self, dir_): """ Start running in the directory and lock it """ self.cv.acquire() while dir_ in self.dirs: self.cv.wait() self.dirs.add(dir_) self.cv.release() def done(self, dir_): """ Finished with the directory, unlock it """ self.cv.acquire() self.dirs.remove(dir_) self.cv.notify_all() self.cv.release()
class RepceJob(object): """class representing message status we can use for waiting on reply""" def __init__(self, cbk): """ - .rid: (process-wise) unique id - .cbk: what we do upon receiving reply """ self.rid = (os.getpid(), thread.get_ident(), time.time()) self.cbk = cbk self.lever = Condition() self.done = False def __repr__(self): return ':'.join([str(x) for x in self.rid]) def wait(self): self.lever.acquire() if not self.done: self.lever.wait() self.lever.release() return self.result def wakeup(self, data): self.result = data self.lever.acquire() self.done = True self.lever.notify() self.lever.release()
class Broadcaster(object): def __init__(self): self._condition = Condition() self._last_id = 0 self._queue = [] @acquire_then_notify def send(self, item): print "Sending message", item self._last_id += 1 self._queue.append((self._last_id, item)) def _find_items(self, since_id=None): if not since_id: found = self._queue else: found = [(id, item) for (id,item) in self._queue if id > since_id] return found @acquire def recv(self, since_id=None, timeout=10): end_time = time() + timeout while time() < end_time: found = self._find_items(since_id) if found: break print "Waiting" self._condition.wait(timeout) print "found %d" % len(found) return found
class ResourcePool(object): def __init__(self, initial_value): super(ResourcePool, self).__init__() self.condition = Condition() self.value = initial_value def acquire(self, amount): with self.condition: while amount > self.value: self.condition.wait() self.value -= amount self.__validate() def release(self, amount): with self.condition: self.value += amount self.__validate() self.condition.notifyAll() def __validate(self): assert 0 <= self.value def __str__(self): return str(self.value) def __repr__(self): return "ResourcePool(%i)" % self.value @contextmanager def acquisitionOf(self, amount): self.acquire(amount) try: yield finally: self.release(amount)
class Runner: def __init__(self): logger.info('Starting performances node') self.robot_name = rospy.get_param('/robot_name') self.running = False self.paused = False self.autopause = False self.pause_time = 0 self.start_time = 0 self.start_timestamp = 0 self.lock = Lock() self.run_condition = Condition() self.running_performance = None self.unload_finished = False # in memory set of properties with priority over params self.variables = {} # References to event subscribing node callbacks self.observers = {} # Performances that already played as alternatives. Used to maximize different performance in single demo self.performances_played = {} self.worker = Thread(target=self.worker) self.worker.setDaemon(True) rospy.init_node('performances') self.services = { 'head_pau_mux': rospy.ServiceProxy('/' + self.robot_name + '/head_pau_mux/select', MuxSelect), 'neck_pau_mux': rospy.ServiceProxy('/' + self.robot_name + '/neck_pau_mux/select', MuxSelect) } self.topics = { 'running_performance': rospy.Publisher('~running_performance', String, queue_size=1), 'look_at': rospy.Publisher('/blender_api/set_face_target', Target, queue_size=1), 'gaze_at': rospy.Publisher('/blender_api/set_gaze_target', Target, queue_size=1), 'head_rotation': rospy.Publisher('/blender_api/set_head_rotation', Float32, queue_size=1), 'emotion': rospy.Publisher('/blender_api/set_emotion_state', EmotionState, queue_size=3), 'gesture': rospy.Publisher('/blender_api/set_gesture', SetGesture, queue_size=3), 'expression': rospy.Publisher('/' + self.robot_name + '/make_face_expr', MakeFaceExpr, queue_size=3), 'kfanimation': rospy.Publisher('/' + self.robot_name + '/play_animation', PlayAnimation, queue_size=3), 'interaction': rospy.Publisher('/behavior_switch', String, queue_size=1), 'bt_control': rospy.Publisher('/behavior_control', Int32, queue_size=1), 'events': rospy.Publisher('~events', Event, queue_size=1), 'chatbot': rospy.Publisher('/' + self.robot_name + '/speech', ChatMessage, queue_size=1), 'speech_events': rospy.Publisher('/' + self.robot_name + '/speech_events', String, queue_size=1), 'soma_state': rospy.Publisher("/blender_api/set_soma_state", SomaState, queue_size=2), 'tts': { 'en': rospy.Publisher('/' + self.robot_name + '/tts_en', String, queue_size=1), 'zh': rospy.Publisher('/' + self.robot_name + '/tts_zh', String, queue_size=1), 'default': rospy.Publisher('/' + self.robot_name + '/tts', String, queue_size=1), }, 'tts_control': rospy.Publisher('/' + self.robot_name + '/tts_control', String, queue_size=1) } self.load_properties() rospy.Service('~reload_properties', Trigger, self.reload_properties_callback) rospy.Service('~set_properties', srv.SetProperties, self.set_properties_callback) rospy.Service('~load', srv.Load, self.load_callback) rospy.Service('~load_performance', srv.LoadPerformance, self.load_performance_callback) rospy.Service('~unload', Trigger, self.unload_callback) rospy.Service('~run', srv.Run, self.run_callback) rospy.Service('~run_by_name', srv.RunByName, self.run_by_name_callback) rospy.Service('~run_full_performance', srv.RunByName, self.run_full_performance_callback) rospy.Service('~resume', srv.Resume, self.resume_callback) rospy.Service('~pause', srv.Pause, self.pause_callback) rospy.Service('~stop', srv.Stop, self.stop) rospy.Service('~current', srv.Current, self.current_callback) # Shared subscribers for nodes rospy.Subscriber('/' + self.robot_name + '/speech_events', String, lambda msg: self.notify('speech_events', msg)) rospy.Subscriber('/' + self.robot_name + '/speech', ChatMessage, self.speech_callback) # Shared subscribers for nodes rospy.Subscriber('/hand_events', String, self.hand_callback) Server(PerformancesConfig, self.reconfig) rospy.Subscriber('/face_training_event', String, self.training_callback) self.worker.start() rospy.spin() def reconfig(self, config, level): with self.lock: self.autopause = config.autopause return config def reload_properties_callback(self, request): self.load_properties() return TriggerResponse(success=True) def unload_callback(self, request): self.unload() return TriggerResponse(success=True) def unload(self): self.stop() with self.lock: if self.running_performance: self.running_performance = None self.topics['running_performance'].publish(String(json.dumps(None))) def set_properties_callback(self, request): self.set_variable(request.id, json.loads(request.properties)) return srv.SetPropertiesResponse(success=True) def load_callback(self, request): return srv.LoadResponse(success=True, performance=json.dumps(self.load(request.id))) def load_performance_callback(self, request): self.load_performance(json.loads(request.performance)) return srv.LoadPerformanceResponse(True) def run_by_name_callback(self, request): self.stop() if not self.load(request.id): return srv.RunByNameResponse(False) return srv.RunByNameResponse(self.run(0.0)) def run_full_performance_callback(self, request): self.stop() performances = self.load_folder(request.id) or self.load(request.id) if not performances: return srv.RunByNameResponse(False) return srv.RunByNameResponse(self.run(0.0, unload_finished=True)) def load_folder(self, id): if id.startswith('shared'): robot_name = 'common' else: robot_name = rospy.get_param('/robot_name') dir_path = os.path.join(rospack.get_path('robots_config'), robot_name, 'performances', id) if os.path.isdir(dir_path): root, dirs, files = next(os.walk(dir_path)) files = fnmatch.filter(files, "*.yaml") if not files: # If no folder is picked one directory # Sub-directories are counted as sub-performances if not dirs: return [] if id in self.performances_played: # All performances played. Pick any but last played if set(self.performances_played[id]) == set(dirs): dirs = self.performances_played[id][:-1] self.performances_played[id] = [] else: # Pick from not played performances dirs = list(set(dirs) - set(self.performances_played[id])) else: self.performances_played[id] = [] # Pick random performance p = random.choice(dirs) self.performances_played[id].append(p) return self.load_folder(os.path.join(id, p)) # make names in folder/file format return self.load(id) return [] def load(self, id): robot_name = 'common' if id.startswith('shared') else rospy.get_param('/robot_name') p = os.path.join(rospack.get_path('robots_config'), robot_name, 'performances', id) if os.path.isdir(p): root, dirs, files = next(os.walk(p)) files = natsorted(fnmatch.filter(files, "*.yaml"), key=lambda f: f.lower()) ids = ["{}/{}".format(id, f[:-5]) for f in files] timelines = [self.get_timeline(i) for i in ids] timelines = [t for t in timelines if t] performance = {'id': id, 'name': os.path.basename(id), 'path': os.path.dirname(id), 'timelines': timelines, 'nodes': self.get_merged_timeline_nodes(timelines)} else: performance = self.get_timeline(id) if performance: self.load_performance(performance) return performance else: return None def get_timeline(self, id): timeline = None robot_name = 'common' if id.startswith('shared') else rospy.get_param('/robot_name') p = os.path.join(rospack.get_path('robots_config'), robot_name, 'performances', id) + '.yaml' if os.path.isfile(p): with open(p, 'r') as f: timeline = yaml.load(f.read()) timeline['id'] = id timeline['name'] = os.path.basename(id) timeline['path'] = os.path.dirname(id) self.validate_timeline(timeline) return timeline def get_timeline_duration(self, timeline): duration = 0 if 'nodes' in timeline and isinstance(timeline['nodes'], list): for node in timeline['nodes']: duration = max(duration, (node['duration'] if 'duration' in node else 0) + node['start_time']) return duration def get_merged_timeline_nodes(self, timelines): merged = [] offset = 0 for timeline in timelines: duration = 0 nodes = timeline.get('nodes', []) nodes = copy.deepcopy(nodes) for node in nodes: duration = max(duration, node['duration'] + node['start_time']) node['start_time'] += offset merged += nodes offset += duration return merged def validate_performance(self, performance): self.validate_timeline(performance) if 'timelines' in performance: for timeline in performance['timelines']: self.validate_timeline(timeline) return performance def validate_timeline(self, timeline): if 'nodes' not in timeline or not isinstance(timeline['nodes'], list): timeline['nodes'] = [] for node in timeline['nodes']: if 'start_time' not in node: node['start_time'] = 0 if node['name'] == 'pause': node['duration'] = 0.1 if 'duration' not in node: node['duration'] = 0 return timeline def load_performance(self, performance): with self.lock: self.validate_performance(performance) self.running_performance = performance self.topics['running_performance'].publish(String(json.dumps(performance))) def run_callback(self, request): return srv.RunResponse(self.run(request.startTime)) def run(self, start_time, unload_finished=False): self.stop() # Wait for worker to stop performance and enter waiting before proceeding self.run_condition.acquire() with self.lock: success = self.running_performance and len(self.running_performance) > 0 if success: self.unload_finished = unload_finished self.running = True self.start_time = start_time self.start_timestamp = time.time() # notify worker thread self.run_condition.notify() self.run_condition.release() return success def resume_callback(self, request): success = self.resume() with self.lock: run_time = self.get_run_time() return srv.ResumeResponse(success, run_time) def resume(self): success = False with self.lock: if self.running and self.paused: run_time = self.get_run_time() self.paused = False self.start_timestamp = time.time() - run_time self.start_time = 0 self.topics['events'].publish(Event('resume', run_time)) success = True return success def stop(self, request=None): stop_time = 0 with self.lock: if self.running: stop_time = self.get_run_time() self.running = False self.paused = False self.topics['tts_control'].publish('shutup') return srv.StopResponse(True, stop_time) def pause_callback(self, request): if self.pause(): with self.lock: return srv.PauseResponse(True, self.get_run_time()) else: return srv.PauseResponse(False, 0) # Pauses current def pause(self): with self.lock: if self.running and not self.paused: self.pause_time = time.time() self.paused = True self.topics['events'].publish(Event('paused', self.get_run_time())) return True else: return False # Returns current performance def current_callback(self, request): with self.lock: current_time = self.get_run_time() running = self.running and not self.paused return srv.CurrentResponse(performance=json.dumps(self.running_performance), current_time=current_time, running=running) def worker(self): self.run_condition.acquire() while True: with self.lock: self.paused = False self.running = False self.topics['events'].publish(Event('idle', 0)) self.run_condition.wait() self.topics['events'].publish(Event('running', self.start_time)) with self.lock: if not self.running_performance: continue behavior = True offset = 0 timelines = self.running_performance['timelines'] if 'timelines' in self.running_performance else [ self.running_performance] for i, timeline in enumerate(timelines): # check if performance is finished without starting running = True nodes = [Node.createNode(node, self, self.start_time - offset, timeline.get('id', '')) for node in timeline['nodes']] pid = timeline.get('id', '') finished = None pause = pid and self.get_property(os.path.dirname(pid), 'pause_behavior') # Pause must be either enabled or not set (by default all performances are # pausing behavior if its not set) if (pause or pause is None) and behavior: # Only pause behavior if its already running. Otherwise Pause behavior have no effect behavior_enabled = False try: behavior_enabled = rospy.get_param("/behavior_enabled") except KeyError: pass if behavior_enabled: self.topics['interaction'].publish('btree_off') behavior = False with self.lock: if not self.running: break while running: with self.lock: run_time = self.get_run_time() if not self.running: self.topics['events'].publish(Event('finished', run_time)) break if self.paused: continue running = False # checks if any nodes still running for k, node in enumerate(nodes): running = node.run(run_time - offset) or running if finished is None: # true if all performance nodes are already finished finished = not running offset += self.get_timeline_duration(timeline) with self.lock: autopause = self.autopause and finished is False and i < len(timelines) - 1 if autopause: self.pause() if not behavior: self.topics['interaction'].publish('btree_on') if self.unload_finished: self.unload_finished = False self.unload() def get_run_time(self): """ Must acquire self.lock in order to safely use this method :return: """ run_time = 0 if self.running: run_time = self.start_time if self.paused: run_time += self.pause_time - self.start_timestamp else: run_time += time.time() - self.start_timestamp return run_time # Notifies register nodes on the events from ROS. def notify(self, event, msg): if event not in self.observers.keys(): return for i in xrange(len(self.observers[event]) - 1, -1, -1): try: self.observers[event][i](msg) except TypeError: # Remove dead methods del self.observers[event][i] # Registers callbacks for specific events. Uses weak reference to allow nodes cleanup after finish. def register(self, event, cb): if not event in self.observers: self.observers[event] = [] m = WeakMethod(cb) self.observers[event].append(m) return m # Allows nodes to unsubscribe from events def unregister(self, event, ref): if event in self.observers: if ref in self.observers[event]: self.observers[event].remove(ref) def hand_callback(self, msg): self.notify('HAND', msg) self.notify(msg.data, msg) def load_properties(self): robot_name = rospy.get_param('/robot_name') robot_path = os.path.join(rospack.get_path('robots_config'), robot_name, 'performances') common_path = os.path.join(rospack.get_path('robots_config'), 'common', 'performances') for path in [common_path, robot_path]: for root, dirnames, filenames in os.walk(path): if '.properties' in filenames: filename = os.path.join(root, '.properties') if os.path.isfile(filename): with open(filename) as f: properties = yaml.load(f.read()) dir = os.path.relpath(root, path) rospy.set_param('/' + os.path.join(self.robot_name, 'webui/performances', dir).strip( "/.") + '/properties', properties) def get_property(self, path, name): param_name = os.path.join('/', self.robot_name, 'webui/performances', path, 'properties', name) return rospy.get_param(param_name, None) def set_variable(self, id, properties): for key, val in properties.iteritems(): rospy.logerr("id {} key {} val {}".format(id, key, val)) if id in self.variables: self.variables[id][key] = val else: self.variables[id] = {key: val} def get_variable(self, id, name): if os.path.dirname(id) in self.variables and name in self.variables[os.path.dirname(id)] \ and self.variables[os.path.dirname(id)][name]: return self.variables[os.path.dirname(id)][name] else: val = None param_name = os.path.join('/', self.robot_name, 'webui/performances', os.path.dirname(id), 'properties/variables', name) if rospy.has_param(param_name): val = rospy.get_param(param_name) if self.is_param(val): if rospy.has_param(val): return str(rospy.get_param(val)) if rospy.has_param("/{}{}".format(self.robot_name, val)): return str(rospy.get_param("/{}{}".format(self.robot_name, val))) return None return val def speech_callback(self, msg): self.notify('SPEECH', msg.utterance) @staticmethod def is_param(param): """ Checks if value is valid param. Has to start with slash """ validator = rospy.names.global_name("param_name") try: validator(param, False) return True except rospy.names.ParameterInvalid: return False def training_callback(self, msg): self.notify('FACE_TRAINING', msg.data)
class Sampler(object): """ Sampler used to play, stop and mix multiple sounds. .. warning:: A single sampler instance should be used at a time. """ def __init__(self, sr=22050, backend='sounddevice', timeout=1): """ :param int sr: samplerate used - all sounds added to the sampler will automatically be resampled if needed (- his can be a CPU consumming task, try to use sound with all identical sampling rate if possible. :param str backend: backend used for playing sound. Can be either 'sounddevice' or 'dummy'. """ self.sr = sr self.sounds = [] self.chunks = Queue(1) self.chunk_available = Condition() self.is_done = Event( ) # new event to prevent play to be called again before the sound is actually played self.timeout = timeout # timeout value for graceful exit of the BackendStream if backend == 'dummy': from .dummy_stream import DummyStream self.BackendStream = DummyStream elif backend == 'sounddevice': from sounddevice import OutputStream self.BackendStream = OutputStream else: raise ValueError("Backend can either be 'sounddevice' or 'dummy'") # TODO: use a process instead? self.play_thread = Thread(target=self.run) self.play_thread.daemon = True self.play_thread.start() def __enter__(self): return self def __exit__(self, *args): self.play_thread.join() def play(self, sound): """ Adds and plays a new Sound to the Sampler. :param sound: sound to play .. note:: If the sound is already playing, it will restart from the beginning. """ # self.is_done.clear() # hold is_done until the sound is played # print("self.is_done", self.is_done.isSet()) if self.sr != sound.sr: raise ValueError( 'You can only play sound with a samplerate of {} (here {}). Use the Sound.resample method for instance.', self.sr, sound.sr) if sound in self.sounds: self.remove(sound) with self.chunk_available: self.sounds.append(sound) sound.playing = True self.chunk_available.notify() # self.is_done.wait() # wait for the sound to be entirely played def remove(self, sound): """ Remove a currently played sound. """ with self.chunk_available: sound.playing = False self.sounds.remove(sound) # Play loop def next_chunks(self): """ Gets a new chunk from all played sound and mix them together. """ with self.chunk_available: while True: playing_sounds = [s for s in self.sounds if s.playing] chunks = [] for s in playing_sounds: try: chunks.append(next(s.chunks)) except StopIteration: s.playing = False self.sounds.remove(s) self.is_done.set( ) # sound was played, release is_done to end the wait in play if chunks: break self.chunk_available.wait() return numpy.mean(chunks, axis=0) def run(self): """ Play loop, i.e. send all sound chunk by chunk to the soundcard. """ self.running = True def chunks_producer(): while self.running: self.chunks.put(self.next_chunks()) t = Thread(target=chunks_producer) t.start() with self.BackendStream(samplerate=self.sr, channels=1) as stream: while self.running: try: stream.write(self.chunks.get(timeout=self.timeout) ) # timeout so stream.write() thread can exit except Empty: self.running = False # let play_thread exit
class ConnectionPool(object): class Connection(object): def __init__(self, mgr, fs_name): self.fs = None self.mgr = mgr self.fs_name = fs_name self.ops_in_progress = 0 self.last_used = time.time() self.fs_id = self.get_fs_id() def get_fs_id(self): fs_map = self.mgr.get('fs_map') for fs in fs_map['filesystems']: if fs['mdsmap']['fs_name'] == self.fs_name: return fs['id'] raise VolumeException( -errno.ENOENT, "Volume '{0}' not found".format(self.fs_name)) def get_fs_handle(self): self.last_used = time.time() self.ops_in_progress += 1 return self.fs def put_fs_handle(self, notify): assert self.ops_in_progress > 0 self.ops_in_progress -= 1 if self.ops_in_progress == 0: notify() def del_fs_handle(self, waiter): if waiter: while self.ops_in_progress != 0: waiter() if self.is_connection_valid(): self.disconnect() else: self.abort() def is_connection_valid(self): fs_id = None try: fs_id = self.get_fs_id() except: # the filesystem does not exist now -- connection is not valid. pass return self.fs_id == fs_id def is_connection_idle(self, timeout): return (self.ops_in_progress == 0 and ((time.time() - self.last_used) >= timeout)) def connect(self): assert self.ops_in_progress == 0 log.debug("Connecting to cephfs '{0}'".format(self.fs_name)) self.fs = cephfs.LibCephFS(rados_inst=self.mgr.rados) log.debug( "Setting user ID and group ID of CephFS mount as root...") self.fs.conf_set("client_mount_uid", "0") self.fs.conf_set("client_mount_gid", "0") log.debug("CephFS initializing...") self.fs.init() log.debug("CephFS mounting...") self.fs.mount(filesystem_name=self.fs_name.encode('utf-8')) log.debug("Connection to cephfs '{0}' complete".format( self.fs_name)) self.mgr._ceph_register_client(self.fs.get_addrs()) def disconnect(self): assert self.ops_in_progress == 0 log.info("disconnecting from cephfs '{0}'".format(self.fs_name)) addrs = self.fs.get_addrs() self.fs.shutdown() self.mgr._ceph_unregister_client(addrs) self.fs = None def abort(self): assert self.ops_in_progress == 0 log.info("aborting connection from cephfs '{0}'".format( self.fs_name)) self.fs.abort_conn() self.fs = None class RTimer(Timer): """ recurring timer variant of Timer """ def run(self): try: while not self.finished.is_set(): self.finished.wait(self.interval) self.function(*self.args, **self.kwargs) self.finished.set() except Exception as e: log.error("ConnectionPool.RTimer: %s", e) raise # TODO: make this configurable TIMER_TASK_RUN_INTERVAL = 30.0 # seconds CONNECTION_IDLE_INTERVAL = 60.0 # seconds def __init__(self, mgr): self.mgr = mgr self.connections = {} self.lock = Lock() self.cond = Condition(self.lock) self.timer_task = ConnectionPool.RTimer( ConnectionPool.TIMER_TASK_RUN_INTERVAL, self.cleanup_connections) self.timer_task.start() def cleanup_connections(self): with self.lock: log.info("scanning for idle connections..") idle_fs = [ fs_name for fs_name, conn in self.connections.items() if conn.is_connection_idle( ConnectionPool.CONNECTION_IDLE_INTERVAL) ] for fs_name in idle_fs: log.info("cleaning up connection for '{}'".format(fs_name)) self._del_fs_handle(fs_name) def get_fs_handle(self, fs_name): with self.lock: conn = None try: conn = self.connections.get(fs_name, None) if conn: if conn.is_connection_valid(): return conn.get_fs_handle() else: # filesystem id changed beneath us (or the filesystem does not exist). # this is possible if the filesystem got removed (and recreated with # same name) via "ceph fs rm/new" mon command. log.warning( "filesystem id changed for volume '{0}', reconnecting..." .format(fs_name)) self._del_fs_handle(fs_name) conn = ConnectionPool.Connection(self.mgr, fs_name) conn.connect() except cephfs.Error as e: # try to provide a better error string if possible if e.args[0] == errno.ENOENT: raise VolumeException( -errno.ENOENT, "Volume '{0}' not found".format(fs_name)) raise VolumeException(-e.args[0], e.args[1]) self.connections[fs_name] = conn return conn.get_fs_handle() def put_fs_handle(self, fs_name): with self.lock: conn = self.connections.get(fs_name, None) if conn: conn.put_fs_handle(notify=lambda: self.cond.notifyAll()) def _del_fs_handle(self, fs_name, wait=False): conn = self.connections.pop(fs_name, None) if conn: conn.del_fs_handle( waiter=None if not wait else lambda: self.cond.wait()) def del_fs_handle(self, fs_name, wait=False): with self.lock: self._del_fs_handle(fs_name, wait) def del_all_handles(self): with self.lock: for fs_name in list(self.connections.keys()): log.info("waiting for pending ops for '{}'".format(fs_name)) self._del_fs_handle(fs_name, wait=True) log.info("pending ops completed for '{}'".format(fs_name)) # no new connections should have been initialized since its # guarded on shutdown. assert len(self.connections) == 0
class SecureDocXMLRPCServer(CustomThreadingMixIn, DocXMLRPCServer): def __init__(self, registerInstance, server_address, keyFile=DEFAULTKEYFILE, certFile=DEFAULTCERTFILE, logRequests=True): """Secure Documenting XML-RPC server. It it very similar to DocXMLRPCServer but it uses HTTPS for transporting XML data. """ DocXMLRPCServer.__init__(self, server_address, SecureDocXMLRpcRequestHandler, logRequests) self.logRequests = logRequests # stuff for doc server try: self.set_server_title(registerInstance.title) except AttributeError: self.set_server_title('default title') try: self.set_server_name(registerInstance.name) except AttributeError: self.set_server_name('default name') if registerInstance.__doc__: self.set_server_documentation(registerInstance.__doc__) else: self.set_server_documentation('default documentation') self.register_introspection_functions() # init stuff, handle different versions: try: SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self) except TypeError: # An exception is raised in Python 2.5 as the prototype of the __init__ # method has changed and now has 3 arguments (self, allow_none, encoding) SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self, False, None) SocketServer.BaseServer.__init__(self, server_address, SecureDocXMLRpcRequestHandler) self.register_instance(registerInstance) # for some reason, have to register instance down here! # SSL socket stuff ctx = SSL.Context(SSL.SSLv23_METHOD) ctx.use_privatekey_file(keyFile) ctx.use_certificate_file(certFile) self.socket = SSL.Connection(ctx, socket.socket(self.address_family, self.socket_type)) self.server_bind() self.server_activate() # requests count and condition, to allow for keyboard quit via CTL-C self.requests = 0 self.rCondition = Condition() def startup(self): 'run until quit signaled from keyboard...' print 'server starting; hit CTRL-C to quit...' while True: try: self.rCondition.acquire() start(self.handle_request, ()) # we do this async, because handle_request blocks! while not self.requests: self.rCondition.wait(timeout=3.0) if self.requests: self.requests -= 1 self.rCondition.release() except KeyboardInterrupt: print "quit signaled, i'm done." return def get_request(self): request, client_address = self.socket.accept() self.rCondition.acquire() self.requests += 1 self.rCondition.notifyAll() self.rCondition.release() return (request, client_address) def listMethods(self): 'return list of method names (strings)' methodNames = self.funcs.keys() methodNames.sort() return methodNames def methodHelp(self, methodName): 'method help' if methodName in self.funcs: return self.funcs[methodName].__doc__ else: raise Exception('method "%s" is not supported' % methodName)
class GbnServer: def __init__(self, addr): # 定义seq最大值为255 self.__BUFSIZE = 255 self.__send_cache = [] self.__seq = 0 self.__pkg_sign = 255 self.__ack_sign = 0 self.__sock = socket(AF_INET, SOCK_DGRAM) self.__sock.bind(addr) self.__client_addr = None self.__window_size = 10 self.__lock = Lock() # 期待的下一个数据包 self.__recv_eseq = 1 self.__recv_cache = [] self.__notify_sender = Condition() self.__notify_recver = Condition() self.__notify_ack = Condition() t = Thread(target=self.__send_thread, daemon=True) t.start() t = Thread(target=self.__ack_thread, daemon=True) t.start() return def __get_seq(self): self.__seq += 1 if self.__seq > self.__BUFSIZE: self.__seq = 0 return self.__seq def __mk_pkt(self, data): temp = bytes([0, self.__get_seq()]) + len(data).to_bytes(4, byteorder='big') + data if len(temp) % 2 != 0: temp += bytes([0]) return temp + (lib.checksum(temp) ^ 0xffff).to_bytes(2, 'big') def send(self, data): """ 使用GBN协议发送数据,这个方法是上层应用的调用,该方法不会发送数据,而是将数据打包放入发送缓存中, 并提醒发送线程 发送数据 。 :param data: 要发送的数据 :return: None """ self.__notify_sender.acquire() self.__send_cache.append(self.__mk_pkt(data)) self.__notify_sender.notify() self.__notify_sender.release() return def recv(self): """ 接收对方发来的数据,当没有数据的时候进入阻塞状态直到对方发来数据。 :return: """ self.__notify_recver.acquire() while len(self.__recv_cache) == 0: self.__notify_recver.wait() data = self.__recv_cache[0] self.__recv_cache.pop(0) return data def __send_thread(self): """ 真正的发送数据的方法,当实例化该类的对象的时候,该方法作为独立的守护线程在后台运行,将发送缓存中的数据(如果 有的话)发送给接收方,如果缓存中没有数据则进入挂起状态。刚开始的时候是进入挂起状态的。 :return: None """ while True: self.__notify_sender.acquire() # 当发送缓存中没有数据的时候,线程进入挂起状态。 while len(self.__send_cache) == 0: self.__notify_sender.wait() self.__notify_sender.release() while len(self.__send_cache) > 0: self.__lock.acquire() for i in range(min(self.__window_size, len(self.__send_cache))): self.__sock.sendto(self.__send_cache[i], self.__client_addr) self.__lock.release() self.__notify_ack.acquire() self.__notify_ack.notify() flag = self.__notify_ack.wait(4) while flag and len(self.__send_cache) > 0: self.__notify_ack.notify() flag = self.__notify_ack.wait(10) self.__notify_ack.release() def __ack_thread(self): """ 当对方关闭连接以至于无法向对方发送还没有发送的数据包的时候,会抛出ConnectionResetError异常 累积确认 :return:无 """ while True: data, addr = self.__sock.recvfrom(1024) if self.__client_addr is None: self.__client_addr = addr if data[0] == 0 and data[1] == self.__recv_eseq and lib.checksum(data) == 0xffff: self.__recv_eseq += 1 if self.__recv_eseq > self.__BUFSIZE: self.__recv_eseq = 0 data_len = int.from_bytes(data[2:6], 'big') self.__recv_cache.append(data[6:6+data_len]) ack = bytes([255, data[1]]) self.__sock.sendto(ack, self.__client_addr) self.__notify_recver.acquire() self.__notify_recver.notify() self.__notify_recver.release() elif len(self.__send_cache) > 0: seq = data[1] while len(self.__send_cache) > 0 and self.__send_cache[0][1] <= seq: self.__lock.acquire() self.__send_cache.pop(0) self.__lock.release() self.__notify_ack.acquire() self.__notify_ack.notify() self.__notify_ack.release()
class CephfsConnectionPool(object): class Connection(object): def __init__(self, mgr: Module_T, fs_name: str): self.fs: Optional["cephfs.LibCephFS"] = None self.mgr = mgr self.fs_name = fs_name self.ops_in_progress = 0 self.last_used = time.time() self.fs_id = self.get_fs_id() def get_fs_id(self) -> int: fs_map = self.mgr.get('fs_map') for fs in fs_map['filesystems']: if fs['mdsmap']['fs_name'] == self.fs_name: return fs['id'] raise CephfsConnectionException( -errno.ENOENT, "FS '{0}' not found".format(self.fs_name)) def get_fs_handle(self) -> "cephfs.LibCephFS": self.last_used = time.time() self.ops_in_progress += 1 return self.fs def put_fs_handle(self, notify: Callable) -> None: assert self.ops_in_progress > 0 self.ops_in_progress -= 1 if self.ops_in_progress == 0: notify() def del_fs_handle(self, waiter: Optional[Callable]) -> None: if waiter: while self.ops_in_progress != 0: waiter() if self.is_connection_valid(): self.disconnect() else: self.abort() def is_connection_valid(self) -> bool: fs_id = None try: fs_id = self.get_fs_id() except: # the filesystem does not exist now -- connection is not valid. pass logger.debug("self.fs_id={0}, fs_id={1}".format(self.fs_id, fs_id)) return self.fs_id == fs_id def is_connection_idle(self, timeout: float) -> bool: return (self.ops_in_progress == 0 and ((time.time() - self.last_used) >= timeout)) def connect(self) -> None: assert self.ops_in_progress == 0 logger.debug("Connecting to cephfs '{0}'".format(self.fs_name)) self.fs = cephfs.LibCephFS(rados_inst=self.mgr.rados) logger.debug( "Setting user ID and group ID of CephFS mount as root...") self.fs.conf_set("client_mount_uid", "0") self.fs.conf_set("client_mount_gid", "0") self.fs.conf_set("client_check_pool_perm", "false") logger.debug("CephFS initializing...") self.fs.init() logger.debug("CephFS mounting...") self.fs.mount(filesystem_name=self.fs_name.encode('utf-8')) logger.debug("Connection to cephfs '{0}' complete".format( self.fs_name)) self.mgr._ceph_register_client(self.fs.get_addrs()) def disconnect(self) -> None: try: assert self.fs assert self.ops_in_progress == 0 logger.info("disconnecting from cephfs '{0}'".format( self.fs_name)) addrs = self.fs.get_addrs() self.fs.shutdown() self.mgr._ceph_unregister_client(addrs) self.fs = None except Exception as e: logger.debug("disconnect: ({0})".format(e)) raise def abort(self) -> None: assert self.fs assert self.ops_in_progress == 0 logger.info("aborting connection from cephfs '{0}'".format( self.fs_name)) self.fs.abort_conn() logger.info("abort done from cephfs '{0}'".format(self.fs_name)) self.fs = None # TODO: make this configurable TIMER_TASK_RUN_INTERVAL = 30.0 # seconds CONNECTION_IDLE_INTERVAL = 60.0 # seconds MAX_CONCURRENT_CONNECTIONS = 5 # max number of concurrent connections per volume def __init__(self, mgr: Module_T): self.mgr = mgr self.connections: Dict[str, List[CephfsConnectionPool.Connection]] = {} self.lock = Lock() self.cond = Condition(self.lock) self.timer_task = RTimer(CephfsConnectionPool.TIMER_TASK_RUN_INTERVAL, self.cleanup_connections) self.timer_task.start() def cleanup_connections(self) -> None: with self.lock: logger.info("scanning for idle connections..") idle_conns = [] for fs_name, connections in self.connections.items(): logger.debug( f'fs_name ({fs_name}) connections ({connections})') for connection in connections: if connection.is_connection_idle( CephfsConnectionPool.CONNECTION_IDLE_INTERVAL): idle_conns.append((fs_name, connection)) logger.info(f'cleaning up connections: {idle_conns}') for idle_conn in idle_conns: self._del_connection(idle_conn[0], idle_conn[1]) def get_fs_handle(self, fs_name: str) -> "cephfs.LibCephFS": with self.lock: try: min_shared = 0 shared_connection = None connections = self.connections.setdefault(fs_name, []) logger.debug( f'[get] volume: ({fs_name}) connection: ({connections})') if connections: min_shared = connections[0].ops_in_progress shared_connection = connections[0] for connection in list(connections): logger.debug( f'[get] connection: {connection} usage: {connection.ops_in_progress}' ) if connection.ops_in_progress == 0: if connection.is_connection_valid(): logger.debug( f'[get] connection ({connection}) can be reused' ) return connection.get_fs_handle() else: # filesystem id changed beneath us (or the filesystem does not exist). # this is possible if the filesystem got removed (and recreated with # same name) via "ceph fs rm/new" mon command. logger.warning( f'[get] filesystem id changed for volume ({fs_name}), disconnecting ({connection})' ) # note -- this will mutate @connections too self._del_connection(fs_name, connection) else: if connection.ops_in_progress < min_shared: min_shared = connection.ops_in_progress shared_connection = connection # when we end up here, there are no "free" connections. so either spin up a new # one or share it. if len(connections ) < CephfsConnectionPool.MAX_CONCURRENT_CONNECTIONS: logger.debug( '[get] spawning new connection since no connection is unused and we still have room for more' ) connection = CephfsConnectionPool.Connection( self.mgr, fs_name) connection.connect() self.connections[fs_name].append(connection) return connection.get_fs_handle() else: assert shared_connection is not None logger.debug( f'[get] using shared connection ({shared_connection})') return shared_connection.get_fs_handle() except cephfs.Error as e: # try to provide a better error string if possible if e.args[0] == errno.ENOENT: raise CephfsConnectionException( -errno.ENOENT, "FS '{0}' not found".format(fs_name)) raise CephfsConnectionException(-e.args[0], e.args[1]) def put_fs_handle(self, fs_name: str, fs_handle: cephfs.LibCephFS) -> None: with self.lock: connections = self.connections.get(fs_name, []) for connection in connections: if connection.fs == fs_handle: logger.debug( f'[put] connection: {connection} usage: {connection.ops_in_progress}' ) connection.put_fs_handle( notify=lambda: self.cond.notifyAll()) def _del_connection(self, fs_name: str, connection: Connection, wait: bool = False) -> None: self.connections[fs_name].remove(connection) connection.del_fs_handle( waiter=None if not wait else lambda: self.cond.wait()) def _del_connections(self, fs_name: str, wait: bool = False) -> None: for connection in list(self.connections.get(fs_name, [])): self._del_connection(fs_name, connection, wait) def del_connections(self, fs_name: str, wait: bool = False) -> None: with self.lock: self._del_connections(fs_name, wait) def del_all_connections(self) -> None: with self.lock: for fs_name in list(self.connections.keys()): logger.info("waiting for pending ops for '{}'".format(fs_name)) self._del_connections(fs_name, wait=True) logger.info("pending ops completed for '{}'".format(fs_name)) # no new connections should have been initialized since its # guarded on shutdown. assert len(self.connections) == 0
class ContinuousPagingSession(object): def __init__(self, stream_id, decoder, row_factory, connection): self.stream_id = stream_id self.decoder = decoder self.row_factory = row_factory self.connection = connection self._condition = Condition() self._stop = False self._page_queue = deque() def on_message(self, result): if isinstance(result, ResultMessage): self.on_page(result) elif isinstance(result, ErrorMessage): self.on_error(result) def on_page(self, result): with self._condition: self._page_queue.appendleft( (result.column_names, result.parsed_rows, None)) self._stop |= result.continuous_paging_last self._condition.notify() if result.continuous_paging_last: self.connection.remove_continuous_paging_session(self.stream_id) def on_error(self, error): with self._condition: self._page_queue.appendleft((None, None, error.to_exception())) self._stop = True self._condition.notify() self.connection.remove_continuous_paging_session(self.stream_id) def results(self): with self._condition: while True: while not self._page_queue and not self._stop: # TODO: need to timeout here somehow self._condition.wait() while self._page_queue: names, rows, err = self._page_queue.pop() if err: raise err self._condition.release() for row in self.row_factory(names, rows): yield row self._condition.acquire() if self._stop: break def cancel(self): log.debug("Canceling paging session %s from %s", self.stream_id, self.connection.host) self.connection.send_msg( CancelMessage(CONTINUOUS_PAGING_OP_TYPE, self.stream_id), self.connection.get_request_id(), self._on_cancel_response) with self._condition: self._stop = True self._condition.notify() def _on_cancel_response(self, response): if isinstance(response, ResultMessage): log.debug("Paging session %s canceled.", self.stream_id) else: log.error( "Failed canceling streaming session %s from %s: %s", self.stream_id, self.connection.host, response.to_exception() if hasattr(response, 'to_exception') else response) self.connection.remove_continuous_paging_session(self.stream_id)
class FileWatcher(Thread): """Looks for new files, and queues them. """ def __init__(self, filename_template, file_queue, frequency): """Looks for new files arriving at the given *frequency*, and queues them. """ Thread.__init__(self) self.queue = file_queue self.template = filename_template self.frequency = datetime.timedelta(minutes=frequency) self.running = True self.cond = Condition() def terminate(self): """Terminate thread. """ self.running = False self.cond.acquire() self.cond.notify() self.cond.release() LOG.debug("Termination request received in FileWatcher") def wait(self, secs): if self.running: self.cond.wait(secs) def run(self): """Run the file watcher. """ filelist = set() sleep_time = 8 while self.running: self.cond.acquire() if isinstance(self.template, (list, tuple)): new_filelist = [] for template in self.template: new_filelist += glob.glob(template) new_filelist = set(new_filelist) else: new_filelist = set(glob.glob(self.template)) files_to_process = list(new_filelist - filelist) filelist = new_filelist files_dict = {} for fil in files_to_process: files_dict[fil] = os.path.getmtime(fil) files_to_process.sort( lambda x, y: cmp(files_dict[x], files_dict[y])) if len(files_to_process) != 0 and self.running: sleep_time = 8 times = [] for i in files_to_process: LOG.debug("queueing %s..." % i) self.queue.put(i) times.append(os.stat(i).st_ctime) times.sort() since_creation = datetime.timedelta(seconds=time.time() - times[-1]) if (self.frequency > since_creation): to_wait = self.frequency - since_creation LOG.info("Waiting at least " + str(to_wait) + " for next file") sleep_time = (to_wait.seconds + to_wait.microseconds / 1000000.0) self.wait(sleep_time) sleep_time = 8 elif self.running: LOG.info("no new file has come, waiting %s secs" % str(sleep_time)) self.wait(sleep_time) if sleep_time < 60: sleep_time *= 2 self.cond.release() LOG.info("FileWatcher terminated.")
class RDMTestThread(Thread): """The RDMResponder tests are closely coupled to the Wrapper (yuck!). So we need to run this all in a separate thread. This is all a bit of a hack and you'll get into trouble if multiple things are running at once... """ RUNNING, COMPLETED, ERROR = range(3) TESTS, COLLECTOR = range(2) def __init__(self, pid_store, logs_directory): super(RDMTestThread, self).__init__() self._pid_store = pid_store self._logs_directory = logs_directory self._terminate = False self._request = None # Guards _terminate and _request self._cv = Condition() self._wrapper = None self._test_state_lock = Lock() # Guards _test_state self._test_state = {} def Stop(self): self._cv.acquire() self._terminate = True self._cv.notify() self._cv.release() def ScheduleTests(self, universe, uid, test_filter, broadcast_write_delay, inter_test_delay, dmx_frame_rate, slot_count): """Schedule the tests to be run. Callable from any thread. Callbable by any thread. Returns: An error message, or None if the tests were scheduled. """ if not self._CheckIfConnected(): return 'Lost connection to OLAD' self._cv.acquire() if self._request is not None: self._cv.release() return 'Existing request pending' self._request = lambda: self._RunTests( universe, uid, test_filter, broadcast_write_delay, inter_test_delay, dmx_frame_rate, slot_count) self._cv.notify() self._cv.release() return None def ScheduleCollector(self, universe, skip_queued_messages): """Schedule the collector to run on a universe. Callable by any thread. Returns: An error message, or None if the collection was scheduled. """ if not self._CheckIfConnected(): return 'Lost connection to OLAD' self._cv.acquire() if self._request is not None: self._cv.release() return 'Existing request pending' self._request = lambda: self._RunCollector(universe, skip_queued_messages) self._cv.notify() self._cv.release() return None def Stat(self): """Check the state of the tests. Callable by any thread. Returns: The status of the tests. """ self._test_state_lock.acquire() state = dict(self._test_state) self._test_state_lock.release() return state def run(self): self._wrapper = ClientWrapper() self._collector = ModelCollector(self._wrapper, self._pid_store) while True: self._cv.acquire() if self._terminate: logging.info('quitting test thread') self._cv.release() return if self._request is not None: request = self._request self._request = None self._cv.release() request() continue # Nothing to do, go into the wait self._cv.wait() self._cv.release() def _UpdateStats(self, tests_completed, total_tests): self._test_state_lock.acquire() self._test_state['tests_completed'] = tests_completed self._test_state['total_tests'] = total_tests self._test_state_lock.release() def _RunTests(self, universe, uid, test_filter, broadcast_write_delay, inter_test_delay, dmx_frame_rate, slot_count): self._test_state_lock.acquire() self._test_state = { 'action': self.TESTS, 'tests_completed': 0, 'total_tests': None, 'state': self.RUNNING, 'duration': 0, } start_time = datetime.now() self._test_state_lock.release() runner = TestRunner.TestRunner(universe, uid, broadcast_write_delay, inter_test_delay, self._pid_store, self._wrapper) for test in TestRunner.GetTestClasses(TestDefinitions): runner.RegisterTest(test) dmx_sender = None if dmx_frame_rate > 0 and slot_count > 0: logging.info( 'Starting DMXSender with slot count %d and FPS of %d' % (slot_count, dmx_frame_rate)) dmx_sender = DMXSender(self._wrapper, universe, dmx_frame_rate, slot_count) try: tests, device = runner.RunTests(test_filter, False, self._UpdateStats) except Exception as e: self._test_state_lock.acquire() self._test_state['state'] = self.ERROR self._test_state['exception'] = str(e) self._test_state['traceback'] = traceback.format_exc() self._test_state_lock.release() return finally: if dmx_sender is not None: dmx_sender.Stop() timestamp = int(time()) end_time = datetime.now() test_parameters = { 'broadcast_write_delay': broadcast_write_delay, 'inter_test_delay': inter_test_delay, 'dmx_frame_rate': dmx_frame_rate, 'dmx_slot_count': slot_count, } log_saver = TestLogger.TestLogger(self._logs_directory) logs_saved = True try: log_saver.SaveLog(uid, timestamp, end_time, tests, device, test_parameters) except TestLogger.TestLoggerException: logs_saved = False self._test_state_lock.acquire() # We can't use total_seconds() since it requires Python 2.7 time_delta = end_time - start_time self._test_state['duration'] = (time_delta.seconds + time_delta.days * 24 * 3600) self._test_state['state'] = self.COMPLETED self._test_state['tests'] = tests self._test_state['logs_saved'] = logs_saved self._test_state['timestamp'] = timestamp self._test_state['uid'] = uid self._test_state_lock.release() def _RunCollector(self, universe, skip_queued_messages): """Run the device model collector for a universe.""" logging.info('Collecting for universe %d' % universe) self._test_state_lock.acquire() self._test_state = { 'action': self.COLLECTOR, 'state': self.RUNNING, } self._test_state_lock.release() try: output = self._collector.Run(universe, skip_queued_messages) except Exception as e: self._test_state_lock.acquire() self._test_state['state'] = self.ERROR self._test_state['exception'] = str(e) self._test_state['traceback'] = traceback.format_exc() self._test_state_lock.release() return self._test_state_lock.acquire() self._test_state['state'] = self.COMPLETED self._test_state['output'] = output self._test_state_lock.release() def _CheckIfConnected(self): """Check if the client is connected to olad. Returns: True if connected, False otherwise. """ # TODO(simon): add this check, remember it needs locking. return True
class OrderedQueueDispatcherPool: """ A thread pool that dispatches messages to a list of receivers. The number of threads is usually smaller than the number of receivers and for each receiver it is guaranteed that messages arrive in the order they were published. No guarantees are given between different receivers. All methods except #start and #stop are reentrant. The pool can be stopped and restarted at any time during the processing but these calls must be single-threaded. Assumptions: - same subscriptions for multiple receivers unlikely, hence filtering done per receiver thread .. codeauthor:: jwienke """ class _Receiver: def __init__(self, receiver): self.receiver = receiver self.queue = Queue() self.processing = False self.processing_mutex = Lock() self.processing_condition = Condition() def _true_filter(self, receiver, message): return True def __init__(self, thread_pool_size, del_func, filter_func=None): """ Construct a new pool. Args: thread_pool_size (int >= 1): number of threads for this pool del_func (callable): the strategy used to deliver messages of type M to receivers of type R. This will most likely be a simple delegate function mapping to a concrete method call. Must be reentrant. callable with two arguments. First is the receiver of a message, second is the message to deliver filter_func (callable): Reentrant function used to filter messages per receiver. Default accepts every message. callable with two arguments. First is the receiver of a message, second is the message to filter. Must return a bool, true means to deliver the message, false rejects it. """ self._logger = get_logger_by_class(self.__class__) if thread_pool_size < 1: raise ValueError("Thread pool size must be at least 1," "{} was given.".format(thread_pool_size)) self._thread_pool_size = int(thread_pool_size) self._del_func = del_func if filter_func is not None: self._filter_func = filter_func else: self._filter_func = self._true_filter self._condition = Condition() self._receivers = [] self._jobsAvailable = False self._started = False self._interrupted = False self._threadPool = [] self._currentPosition = 0 def __del__(self): self.stop() def register_receiver(self, receiver): """ Register a new receiver at the pool. Multiple registrations of the same receiver are possible resulting in being called multiple times for the same message (but effectively this destroys the guarantee about ordering given above because multiple message queues are used for every subscription). Args: receiver: new receiver """ with self._condition: self._receivers.append(self._Receiver(receiver)) self._logger.info("Registered receiver %s", receiver) def unregister_receiver(self, receiver): """ Unregister all registrations of one receiver. Args: receiver: receiver to unregister Returns: True if one or more receivers were unregistered, else False """ removed = None with self._condition: kept = [] for r in self._receivers: if r.receiver == receiver: removed = r else: kept.append(r) self._receivers = kept if removed: with removed.processing_condition: while removed.processing: self._logger.info("Waiting for receiver %s to finish", receiver) removed.processing_condition.wait() return not (removed is None) def push(self, message): """ Push a new message to be dispatched to all receivers in this pool. Args: message: message to dispatch """ with self._condition: for receiver in self._receivers: receiver.queue.put(message) self._jobsAvailable = True self._condition.notify() # XXX: This is disabled because it can trigger this bug for protocol # buffers payloads: # http://code.google.com/p/protobuf/issues/detail?id=454 # See also #1331 # self._logger.debug("Got new message to dispatch: %s", message) def _next_job(self, worker_num): """ Return the next job to process for worker threads. Blocks if there is no job. Args: worker_num: number of the worker requesting a new job Returns: the receiver to work on """ receiver = None with self._condition: got_job = False while not got_job: while (not self._jobsAvailable) and (not self._interrupted): self._logger.debug("Worker %d: no jobs available, waiting", worker_num) self._condition.wait() if (self._interrupted): raise _InterruptedError("Processing was interrupted") # search the next job for _ in range(len(self._receivers)): self._currentPosition = self._currentPosition + 1 real_pos = self._currentPosition % len(self._receivers) if (not self._receivers[real_pos].processing) and \ (not self._receivers[real_pos].queue.empty()): receiver = self._receivers[real_pos] receiver.processing = True got_job = True break if not got_job: self._jobsAvailable = False self._condition.notify() return receiver def _finished_work(self, receiver, worker_num): with self._condition: with receiver.processing_condition: receiver.processing = False receiver.processing_condition.notifyAll() if not receiver.queue.empty(): self._jobsAvailable = True self._logger.debug( "Worker %d: new jobs available, " "notifying one", worker_num) self._condition.notify() def _worker(self, worker_num): """ Threaded worker method. Args: worker_num: number of this worker thread """ try: while True: receiver = self._next_job(worker_num) message = receiver.queue.get(True, None) self._logger.debug("Worker %d: got message %s for receiver %s", worker_num, message, receiver.receiver) if self._filter_func(receiver.receiver, message): self._logger.debug( "Worker %d: delivering message %s for receiver %s", worker_num, message, receiver.receiver) self._del_func(receiver.receiver, message) self._logger.debug( "Worker %d: delivery for receiver %s finished", worker_num, receiver.receiver) self._finished_work(receiver, worker_num) except _InterruptedError: pass def start(self): """ Start processing and return immediately. Raises: RuntimeError: if the pool was already started and is running """ with self._condition: if self._started: raise RuntimeError("Pool already running") self._interrupted = False for i in range(self._thread_pool_size): worker = Thread(target=self._worker, args=[i]) worker.setDaemon(True) worker.start() self._threadPool.append(worker) self._started = True self._logger.info("Started pool with %d threads", self._thread_pool_size) def stop(self): """Block until every thread has stopped working.""" self._logger.info("Starting to stop thread pool by wating for workers") with self._condition: self._interrupted = True self._condition.notifyAll() for worker in self._threadPool: self._logger.debug("Joining worker %s", worker) worker.join() self._threadPool = [] self._started = False self._logger.info("Stopped thread pool")
class IPAllocatorDHCP(IPAllocator): def __init__(self, assigned_ip_blocks: Set[ip_network], ip_state_map: IpDescriptorMap, dhcp_store: MutableMapping[str, DHCPDescriptor], gw_info: UplinkGatewayInfo, retry_limit: int = 300, iface: str = "dhcp0"): """ Allocate IP address for SID using DHCP server. SID is mapped to MAC address using function defined in mac.py then this mac address used in DHCP request to allocate new IP from DHCP server. This IP is also cached to improve performance in case of reallocation for same SID in short period of time. Args: assigned_ip_blocks: set of IP blocks, populated from DHCP. ip_state_map: maintains state of IP allocation to UE. dhcp_store: maintains DHCP transaction for each active MAC address gw_info_map: maintains uplink GW info retry_limit: try DHCP request iface: DHCP interface. """ self._ip_state_map = ip_state_map # {state=>{ip=>ip_desc}} self._assigned_ip_blocks = assigned_ip_blocks self.dhcp_wait = Condition() self._dhcp_client = DHCPClient(dhcp_wait=self.dhcp_wait, dhcp_store=dhcp_store, gw_info=gw_info, iface=iface) self._retry_limit = retry_limit # default wait for two minutes self._dhcp_client.run() def add_ip_block(self, ipblock: ip_network): logging.warning("No need to allocate block for DHCP allocator: %s", ipblock) def remove_ip_blocks(self, *ipblocks: List[ip_network], _force: bool = False) -> List[ip_network]: logging.warning("trying to delete ipblock from DHCP allocator: %s", ipblocks) return [] def list_added_ip_blocks(self) -> List[ip_network]: return list(deepcopy(self._assigned_ip_blocks)) def list_allocated_ips(self, ipblock: ip_network) -> List[ip_address]: """ List IP addresses allocated from a given IP block Args: ipblock (ipaddress.ip_network): ip network to add e.g. ipaddress.ip_network("10.0.0.0/24") Return: list of IP addresses (ipaddress.ip_address) """ return [ ip for ip in self._ip_state_map.list_ips(IPState.ALLOCATED) if ip in ipblock ] def alloc_ip_address(self, sid: str) -> IPDesc: """ Assumption: one-to-one mappings between SID and IP. Args: sid (string): universal subscriber id Returns: ipaddress.ip_address: IP address allocated Raises: NoAvailableIPError: if run out of available IP addresses """ mac = create_mac_from_sid(sid) LOG.debug("allocate IP for %s mac %s", sid, mac) dhcp_desc = self._dhcp_client.get_dhcp_desc(mac) LOG.debug("got IP from redis: %s", dhcp_desc) if dhcp_allocated_ip(dhcp_desc) is not True: dhcp_desc = self._alloc_ip_address_from_dhcp(mac) if dhcp_allocated_ip(dhcp_desc): ip_block = ip_network(dhcp_desc.subnet) ip_desc = IPDesc(ip=ip_address(dhcp_desc.ip), state=IPState.ALLOCATED, sid=sid, ip_block=ip_block, ip_type=IPType.DHCP) LOG.debug("Got IP after sending DHCP requests: %s", ip_desc) self._assigned_ip_blocks.add(ip_block) return ip_desc else: raise NoAvailableIPError("No available IP addresses From DHCP") def release_ip(self, ip_desc: IPDesc): """ Release IP address, this involves following steps. 1. send DHCP protocol packet to release the IP. 2. update IP block list. 3. update IP from ip-state. Args: ip_desc, release needs following info from IPDesc. sid: SID, used to get mac address. ip: IP assigned to this SID ip_block: IP block of the IP address. Returns: None """ self._dhcp_client.release_ip_address(create_mac_from_sid(ip_desc.sid)) # Remove the IP from free IP list, since DHCP is the # owner of this IP self._ip_state_map.remove_ip_from_state(ip_desc.ip, IPState.FREE) list_allocated_ips = self._ip_state_map.list_ips(IPState.ALLOCATED) for ipaddr in list_allocated_ips: if ipaddr in ip_desc.ip_block: # found the IP, do not remove this ip_block return ip_block_network = ip_network(ip_desc.ip_block) if ip_block_network in self._assigned_ip_blocks: self._assigned_ip_blocks.remove(ip_block_network) logging.debug("del: _assigned_ip_blocks %s ipblock %s", self._assigned_ip_blocks, ip_desc.ip_block) def stop_dhcp_sniffer(self): self._dhcp_client.stop() def _alloc_ip_address_from_dhcp(self, mac: MacAddress) -> DHCPDescriptor: retry_count = 0 with self.dhcp_wait: dhcp_desc = None while (retry_count < self._retry_limit and dhcp_allocated_ip(dhcp_desc) is not True): if retry_count % DEFAULT_DHCP_REQUEST_RETRY_FREQUENCY == 0: self._dhcp_client.send_dhcp_packet(mac, DHCPState.DISCOVER) self.dhcp_wait.wait(timeout=DEFAULT_DHCP_REQUEST_RETRY_DELAY) dhcp_desc = self._dhcp_client.get_dhcp_desc(mac) retry_count = retry_count + 1 return dhcp_desc
class TileAgent: ST_IDLE = 0 ST_RUN = 1 ST_PAUSE = 2 ST_CLOSING = 3 TILE_VALID = 0x00 TILE_NOT_IN_MEM = 0x01 TILE_NOT_IN_DISK = 0x02 TILE_EXPIRE = 0x03 TILE_REQ = 0x10 TILE_REQ_FAILED = 0x20 #properties from map_desc @property def map_id(self): return self.__map_desc.map_id @property def map_title(self): return self.__map_desc.map_title @property def level_min(self): return self.__map_desc.level_min @property def level_max(self): return self.__map_desc.level_max @property def url_template(self): return self.__map_desc.url_template @property def server_parts(self): return self.__map_desc.server_parts @property def invert_y(self): return self.__map_desc.invert_y @property def coord_sys(self): return self.__map_desc.coord_sys @property def lower_corner(self): return self.__map_desc.lower_corner @property def upper_corner(self): return self.__map_desc.upper_corner @property def expire_sec(self): return self.__map_desc.expire_sec @property def tile_format(self): return self.__map_desc.tile_format @property def state(self): return self.__state def __init__(self, map_desc, cache_dir, auto_start=False): self.__map_desc = map_desc.clone() self.__state = self.ST_IDLE #local cache self.__cache_dir = cache_dir self.__disk_cache = None #memory cache self.__mem_cache = MemoryCache(self.TILE_NOT_IN_MEM, is_concurrency=True) #download helpers self.__MAX_WORKS = 3 self.__download_lock = Lock() self.__download_cv = Condition(self.__download_lock) self.__workers = {} #self.__workers_lock = Lock() #self.__workers_cv = Condition(self.__workers_lock) self.__req_queue = OrderedDict() #self.__req_lock = Lock() #self.__req_cv = Condition(self.__req_lock) self.__download_monitor = Thread(target=self.__runDownloadMonitor) if auto_start: self.start() def start(self): #create cache dir for the map self.__disk_cache = DBDiskCache(self.__cache_dir, self.__map_desc, conf.DB_SCHEMA) self.__disk_cache.start() #start download thread self.__state = self.ST_RUN self.__download_monitor.start() def close(self): #notify download monitor to exit with self.__download_cv: self.__state = self.ST_CLOSING self.__download_cv.notify() self.__download_monitor.join() #close resources if self.__disk_cache is not None: self.__disk_cache.close() def pause(self): with self.__download_cv: if self.__state == self.ST_RUN: self.__state = self.ST_PAUSE logging.debug("[%s] Change status from run to pause" % (self.map_id, )) self.__download_cv.notify() def resume(self): with self.__download_cv: if self.__state == self.ST_PAUSE: self.__state = self.ST_RUN logging.debug("[%s] Change status from pasue to run" % (self.map_id, )) self.__download_cv.notify() def isSupportedLevel(self, level): return self.level_min <= level and level <= self.level_max def getCachePath(self): return os.path.join(self.__cache_dir, self.map_id) def genTileId(self, level, x, y): return "%s-%d-%d-%d" % (self.map_id, level, x, y) @classmethod def flipY(cls, y, level): return (1 << level) - 1 - y def genTileUrl(self, level, x, y): if self.invert_y: y = self.flipY(y, level) url = self.url_template if self.server_parts: url = url.replace("{$serverpart}", random.choice(self.server_parts)) url = url.replace("{$x}", str(x)) url = url.replace("{$y}", str(y)) url = url.replace("{$z}", str(level)) #logging.critical('url: ' + url) return url def __downloadTile(self, id, req): level, x, y, status, cb = req #unpack the req tile_data = None try: url = self.genTileUrl(level, x, y) logging.info("[%s] DL %s" % (self.map_id, url)) with urllib.request.urlopen(url, timeout=30) as response: tile_data = response.read() logging.info('[%s] DL %s [OK]' % (self.map_id, url)) except Exception as ex: logging.warning('[%s] DL %s [FAILED][%s]' % (self.map_id, url, str(ex))) #as failed, and not save to memory/disk if self.__state == self.ST_CLOSING: return None if tile_data is None: #save to memory status = self.TILE_REQ_FAILED | (status & 0x0F) self.__mem_cache.set(id, status) return None #get tile_img, and save to memory tile_img = None try: tile_img = Image.open(BytesIO(tile_data)) self.__mem_cache.set(id, self.TILE_VALID, tile_img) except Exception as ex: logging.error("[%s] Error to open tile data: %s" % (self.map_id, str(ex))) return None #save tile_data to disk try: self.__disk_cache.put(level, x, y, tile_data) except Exception as ex: logging.error("[%s] Error to save tile data: %s" % (self.map_id, str(ex))) return tile_img #The therad to download def __runDownloadJob(self, id, req): #do download tile_img = self.__downloadTile(id, req) #the download is done #(do this before thread exit, to ensure monitor is notified) with self.__download_cv: self.__workers.pop(id, None) self.__download_cv.notify() #premature done if self.__state == self.ST_CLOSING: return #invoke cb. cb may be blocking, so do this AFTER removing the thread from __workers if tile_img is not None: level, x, y, status, cb = req #unpack the req if cb is not None: tile_info = (self.map_id, level, x, y) try: cb(tile_info) except Exception as ex: logging.warning( "[%s] Invoke cb of download tile error: %s" % (self.map_id, str(ex))) #The thread to handle all download requests def __runDownloadMonitor(self): def no_worker(): return len(self.__workers) == 0 #http_handler = urllib.request.HTTPHandler() #opener = urllib.request.build_opener(http_handler) #urllib.request.install_opener(opener) while True: #wait for requests with self.__download_cv: self.__download_cv.wait() if self.__state == self.ST_CLOSING: logging.debug( "[%s] status(closing), download monitor closing" % (self.map_id, )) break elif self.__state == self.ST_PAUSE: logging.debug("[%s] status(pause), continue to wait" % (self.map_id, )) continue if len(self.__req_queue) > 0 and len( self.__workers) < self.__MAX_WORKS: #the req id, req = self.__req_queue.popitem() #LIFO if id in self.__workers: logging.warning( "[%s] Opps! the req is DUP and in progress." % (self.map_id, )) #should not happen else: #create the job and run the worker job = lambda: self.__runDownloadJob(id, req) worker = Thread(name=id, target=job) self.__workers[id] = worker worker.start() #todo: interrupt urllib.request.openurl to stop download workers. #http_handler.close() with self.__download_cv: #self.__download_cv.wait_for(no_worker) self.__state = self.ST_IDLE logging.debug("[%s] status(idle), download monitor closed" % (self.map_id, )) def __requestTile(self, id, req): #check and add to req queue with self.__download_cv: if id in self.__req_queue: return if id in self.__workers: return #add the req self.__req_queue[id] = req self.__download_cv.notify() def __getTileFromDisk(self, level, x, y): try: data, ts = self.__disk_cache.get(level, x, y) if data is not None: img = Image.open(BytesIO(data)) return img, ts except Exception as ex: logging.warning("[%s] Error to read tile data: %s" % (self.map_id, str(ex))) return None, None def __getTile(self, level, x, y, req_type=None, cb=None): #check level if level > self.level_max or level < self.level_min: raise ValueError("level is out of range") id = self.genTileId(level, x, y) img, status, ts = self.__mem_cache.get(id) #READ FROM memory status_bak = status if status == self.TILE_VALID: return img if (status & 0xF0) == self.TILE_REQ: return img # None or Expire if (status & 0xF0) == self.TILE_REQ_FAILED: if (time.time() - ts) < 60: #todo: user to specify retry period return None status &= 0x0F #remove req_failed status if status == self.TILE_NOT_IN_MEM: img, ts = self.__getTileFromDisk(level, x, y) #READ FROM disk if img is None: status = self.TILE_NOT_IN_DISK elif ts and self.expire_sec and (time.time() - ts) > self.expire_sec: status = self.TILE_EXPIRE else: self.__mem_cache.set(id, self.TILE_VALID, img) return img #check status, should be 'not in disk' or 'expire' if status == self.TILE_NOT_IN_DISK: pass elif status == self.TILE_EXPIRE: logging.warning("[%s] Tile(%d,%d,%d) is expired" % (self.map_id, level, x, y)) else: logging.critical("[%s] Error: unexpected tile status: %d" % (self.map_id, status)) return None #req or not if not req_type: if status != status_bak: self.__mem_cache.set(id, status) return img else: status |= self.TILE_REQ self.__mem_cache.set(id, status) if req_type == "async": self.__requestTile(id, (level, x, y, status, cb)) return img else: # sync return self.__downloadTile(id, (level, x, y, status, None)) def __genMagnifyFakeTile(self, level, x, y, diff=1): side = to_pixel(1, 1)[0] for i in range(1, diff + 1): scale = 2**i img = self.__getTile(level - i, int(x / scale), int(y / scale)) if img: step = int(side / scale) px = step * (x % scale) py = step * (y % scale) img = img.crop((px, py, px + step, py + step)) img = img.resize((side, side)) #magnify return img return None def __genMinifyFakeTile(self, level, x, y, diff=1): bg = 'lightgray' side = to_pixel(1, 1)[0] for i in range(1, diff + 1): scale = 2**i img = Image.new("RGBA", (side * scale, side * scale), bg) has_tile = False #paste tiles for p in range(scale): for q in range(scale): t = self.__getTile(level + i, x * scale + p, y * scale + q) if t: img.paste(t, (p * side, q * side)) has_tile = True #minify if has_tile: img = img.resize((side, side)) return img return None #gen fake from lower/higher level #return None if not avaliable def __genFakeTile(self, level, x, y): #gen from lower level level_diff = min(level - self.level_min, 3) img = self.__genMagnifyFakeTile(level, x, y, level_diff) if img: return img #gen from upper level level_diff = min(self.level_max - level, 1) img = self.__genMinifyFakeTile(level, x, y, level_diff) if img: return img return None # @cb is only for req_type == "async" to nitify the tile is done, # which call cb(tile_info), tile_info = (map_id, level, x, y) def getTile(self, level, x, y, req_type, cb=None, allow_fake=True): img = self.__getTile(level, x, y, req_type, cb) if img is not None: img.is_fake = False return img if allow_fake: img = self.__genFakeTile(level, x, y) if img is not None: img.is_fake = True return img return None
class ServiceRegistrationInfo(_BaseObject): """ Service Registration instances are used to register and expose services onto a DXL fabric. DXL Services are exposed to the DXL fabric and are invoked in a fashion similar to RESTful web services. Communication between an invoking client and the DXL service is one-to-one (request/response). Each service is identified by the "topics" it responds to. Each of these "topics" can be thought of as a method that is being "invoked" on the service by the remote client. Multiple service "instances" can be registered with the DXL fabric that respond to the same "topics". When this occurs (unless explicitly overridden by the client) the fabric will select the particular instance to route the request to (by default round-robin). Multiple service instances can be used to increase scalability and fault-tolerance. The following demonstrates registering a service that responds to a single topic with the DXL fabric: .. code-block:: python from dxlclient.callbacks import RequestCallback from dxlclient.message import Response from dxlclient.service import ServiceRegistrationInfo class MyRequestCallback(RequestCallback): def on_request(self, request): # Extract information from request print request.payload.decode() # Create the response message res = Response(request) # Populate the response payload res.payload = "pong".encode() # Send the response dxl_client.send_response(res) # Create service registration object info = ServiceRegistrationInfo(dxl_client, "/mycompany/myservice") # Add a topic for the service to respond to info.add_topic("/testservice/testrequesttopic", MyRequestCallback()) # Register the service with the fabric (wait up to 10 seconds for registration to complete) dxl_client.register_service_sync(info, 10) **NOTE:** A service is only considered "active" if there are references to the :class:`ServiceRegistrationInfo` instance. If no references to the info object exist, it will be destructed, and the service will be automatically *unregistered* from the fabric. The following demonstrates a client that is invoking the service in the example above: .. code-block:: python from dxlclient.message import Request, Message # Create the request message req = Request("/testservice/testrequesttopic") # Populate the request payload req.payload = "ping".encode() # Send the request and wait for a response (synchronous) res = dxl_client.sync_request(req) # Extract information from the response (if an error did not occur) if res.message_type != Message.MESSAGE_TYPE_ERROR: print res.payload.decode() """ def __init__(self, client, service_type): """ Constructor parameters: :param client: The :class:`dxlclient.client.DxlClient` instance that will expose this service :param service_type: A textual name for the service. For example, "/mycompany/myservice" """ super(ServiceRegistrationInfo, self).__init__() # if not isinstance(channels, list): # raise ValueError('Channels should be a list') # if not channels: # raise InvalidServiceException('Channel list is empty') if not service_type: raise ValueError("Undefined service name") #The service type or name prefix self._service_type = service_type # The unique service ID self._service_id = UuidGenerator.generate_id_as_string() # The map of registered channels and their associated callbacks self._callbacks_by_topic = {} #The map of meta data associated with this service (name-value pairs) self._metadata = {} # List of destination tenants self._destination_tenant_guids = [] # The Time-To-Live (TTL) of the service registration (default: 60 minutes) self._ttl = 60 # minutes # The minimum Time-To-Live (TTL) of the service registration (default: 10 minutes) self._ttl_lower_limit = 10 # Internal client reference self._dxl_client = client # Registration sync object self._registration_sync = Condition() # Whether at least one registration has occurred self._registration_occurred = False # Whether at least one unregistration registration has occurred self._unregistration_occurred = False self._destroy_lock = RLock() self._destroyed = False def __del__(self): """destructor""" super(ServiceRegistrationInfo, self).__del__() self._destroy() def _destroy(self, unregister=True): """ Destroys the service registration :param unregister: Whether to unregister the service from the fabric """ with self._destroy_lock: if not self._destroyed: if unregister and self._dxl_client: try: self._dxl_client.unregister_service_async(self) except Exception: # Currently ignoring this as it can occur due to the fact that we are # attempting to unregister a service that was never registered pass self._dxl_client = None self._destroyed = True @property def service_type(self): """ A textual name for the service. For example, "/mycompany/myservice" """ return self._service_type @property def service_id(self): """ A unique identifier for the service instance (automatically generated when the :class:`ServiceRegistrationInfo` object is constructed) """ return self._service_id @property def metadata(self): """ A dictionary of name-value pairs that are sent as part of the service registration. Brokers provide a registry service that allows for registered services and their associated meta-information to be inspected. The metadata is typically used to include information such as the versions for products that are exposing DXL services, etc. """ return self._metadata @metadata.setter def metadata(self, metadata): self._metadata = metadata @property def ttl(self): """ The interval (in minutes) at which the client will automatically re-register the service with the DXL fabric (defaults to 60 minutes) """ return self._ttl @ttl.setter def ttl(self, ttl): self._ttl = ttl @property def topics(self): """ Returns a tuple containing the topics that the service responds to """ return tuple(self._callbacks_by_topic.keys()) def add_topic(self, topic, callback): """ Registers a topic for the service to respond to along with the :class:`dxlclient.callbacks.RequestCallback` that will be invoked. :param topic: The topic for the service to respond to :param callback: The :class:`dxlclient.callbacks.RequestCallback` that will be invoked when a :class:`dxlclient.message.Request` message is received """ # TODO: use dictionary get method try: callbacks = self._callbacks_by_topic[topic] except KeyError: callbacks = set() self._callbacks_by_topic[topic] = callbacks finally: callbacks.add(callback) def add_topics(self, callbacks_by_topic): """ Registers a set of topics for the service to respond to along with their associated :class:`dxlclient.callbacks.RequestCallback` instances as a dictionary :param callbacks_by_topic: Dictionary containing a set of topics for the service to respond to along with their associated :class:`dxlclient.callbacks.RequestCallback` instances """ if not isinstance(callbacks_by_topic, dict): raise ValueError("Channel and callback should be a dictionary") if not callbacks_by_topic: raise ValueError("Undefined channel") for channel, callback in callbacks_by_topic.iteritems(): self.add_topic(channel, callback) @property def destination_tenant_guids(self): """ The set of tenant identifiers that the service will be available to. Setting this value will limit which tenants can invoke the service. """ return self._destination_tenant_guids @destination_tenant_guids.setter def destination_tenant_guids(self, tenant_guids=None): if tenant_guids is None: tenant_guids = [] self._destination_tenant_guids = tenant_guids def _wait_for_registration_notification(self, wait_time, is_register): """ Waits for a registration notification (register or unregister). :param waitTime: The amount of time to wait. :param isRegister Whether we are waiting for a register or unregister notification. :return: None. """ with self._registration_sync: if wait_time > 0: self._registration_sync.wait(wait_time) else: raise DxlException( "Timeout waiting for service related notification") def _wait_for_registration(self, timeout): """ Waits for the service to be registered with the broker for the first time. :param timeout: The amount of time to wait for the registration to occur. :return: None. """ with self._registration_sync: end_time = int(time.time()) + timeout while not self._registration_occurred: self._wait_for_registration_notification( end_time - int(time.time()), True) def _notify_registration_succeeded(self): """ Invoked when the service has been successfully registered with a broker. :return: None. """ with self._registration_sync: self._registration_occurred = True self._unregistration_occurred = False self._registration_sync.notify_all() # @synchronized def _wait_for_unregistration(self, timeout): """ Waits for the service to be unregistered with the broker for the first time. :param timeout: The amount of time to wait for the unregistration to occur. :return: None. """ end_time = int(time.time()) + timeout with self._registration_sync: while not self._unregistration_occurred: self._wait_for_registration_notification( end_time - int(time.time()), False) #@synchronized def _notify_unregistration_succeeded(self): """ Invoked when the service has been successfully unregistered with a broker. :return: None. """ with self._registration_sync: self._registration_occurred = False self._unregistration_occurred = True self._registration_sync.notify_all()
class SecondaryMonitor(Thread): """ Monitor data from secondary port and send programs to robot """ def __init__(self, host): Thread.__init__(self) self.logger = logging.getLogger("ursecmon") self._parser = ParserUtils() self._dict = {} self._dictLock = Lock() self.host = host self.connect() self._get_version() self._prog_queue = [] self._prog_queue_lock = Lock() self._dataqueue = bytes() self._trystop = False # to stop thread self.running = False # True when robot is on and listening self._dataEvent = Condition() self.lastpacket_timestamp = 0 self.start() try: self.wait() # make sure we got some data before someone calls us except Exception as ex: self.close() raise ex def _get_version(self): tmp = self._s_secondary.recv(1024) tmpdict = self._parser.parse(tmp) self._dict = tmpdict if "VersionMessage" in self._dict: self._parser.version = ( self._dict["VersionMessage"]['majorVersion'], self._dict["VersionMessage"]['minorVersion']) else: self._parser.version = (0, 0) def connect(self): secondary_port = 30002 # Secondary client interface on Universal Robots self._s_secondary = socket.create_connection( (self.host, secondary_port), timeout=2.0) def send_program(self, prog): """ send program to robot in URRobot format If another program is send while a program is running the first program is aborded. """ prog.strip() self.logger.debug("Enqueueing program: %s", prog) if not isinstance(prog, bytes): prog = prog.encode() data = Program(prog + b"\n") with data.condition: with self._prog_queue_lock: self._prog_queue.append(data) data.condition.wait(timeout=2.0) self.logger.debug("program sent: %s", data) def run(self): """ check program execution status in the secondary client data packet we get from the robot This interface uses only data from the secondary client interface (see UR doc) Only the last connected client is the primary client, so this is not guaranted and we cannot rely on information to the primary client. """ while not self._trystop: with self._prog_queue_lock: if len(self._prog_queue) > 0: data = self._prog_queue.pop(0) self._s_secondary.send(data.program) with data.condition: data.condition.notify_all() data = self._get_data() try: tmpdict = self._parser.parse(data) with self._dictLock: self._dict = tmpdict except ParsingException as ex: self.logger.warning( "Error parsing one packet from urrobot: %s", ex) continue if "RobotModeData" not in self._dict: self.logger.warning( "Got a packet from robot without RobotModeData, strange ..." ) continue self.lastpacket_timestamp = time.time() rmode = 0 if self._parser.version >= (3, 0): rmode = 7 if self._parser.version >= (5, 8): if self._dict["RobotModeData"]["robotMode"] == rmode \ and self._dict["RobotModeData"]["isRealRobotEnabled"] is True \ and self._dict["RobotModeData"]["isEmergencyStopped"] is False \ and self._dict["RobotModeData"]["isProtectiveStopped"] is False \ and self._dict["RobotModeData"]["isRealRobotConnected"] is True \ and self._dict["RobotModeData"]["isRobotPowerOn"] is True: self.running = True else: if self._dict["RobotModeData"]["robotMode"] == rmode \ and self._dict["RobotModeData"]["isRealRobotEnabled"] is True \ and self._dict["RobotModeData"]["isEmergencyStopped"] is False \ and self._dict["RobotModeData"]["isSecurityStopped"] is False \ and self._dict["RobotModeData"]["isRobotConnected"] is True \ and self._dict["RobotModeData"]["isPowerOnRobot"] is True: self.running = True else: if self.running: self.logger.error("Robot not running: " + str(self._dict["RobotModeData"])) self.running = False with self._dataEvent: # print("X: new data") self._dataEvent.notifyAll() def _get_data(self): """ returns something that looks like a packet, nothing is guaranted """ while True: # self.logger.debug("data queue size is: {}".format(len(self._dataqueue))) ans = self._parser.find_first_packet(self._dataqueue[:]) if ans: self._dataqueue = ans[1] # self.logger.debug("found packet of size {}".format(len(ans[0]))) return ans[0] else: # self.logger.debug("Could not find packet in received data") try: tmp = self._s_secondary.recv(1024) except socket.timeout as e: self._s_secondary.close() time.sleep(1.0) self.connect() raise TimeoutException( "Did not receive a valid data packet from robot in {}". format(e)) self._dataqueue += tmp def wait(self, timeout=2.0): """ wait for next data packet from robot """ tstamp = self.lastpacket_timestamp with self._dataEvent: try: self._dataEvent.wait(timeout) except socket.timeout as e: self._s_secondary.close() time.sleep(1.0) self.connect() raise TimeoutException( "Did not receive a valid data packet from robot in {}". format(e)) if tstamp == self.lastpacket_timestamp: raise TimeoutException( "Did not receive a valid data packet from robot in {}". format(timeout)) if self._parser.version >= (5, 8): isProtectiveStopped = self._dict["RobotModeData"][ "isProtectiveStopped"] else: isProtectiveStopped = self._dict["RobotModeData"][ "isSecurityStopped"] if isProtectiveStopped is True: raise ProtectiveStopException("Protective stopped") def get_cartesian_info(self, wait=False): if wait: self.wait() with self._dictLock: if "CartesianInfo" in self._dict: return self._dict["CartesianInfo"] else: return None def get_all_data(self, wait=False): """ return last data obtained from robot in dictionnary format """ if wait: self.wait() with self._dictLock: return self._dict.copy() def get_joint_data(self, wait=False): if wait: self.wait() with self._dictLock: if "JointData" in self._dict: return self._dict["JointData"] else: return None def get_digital_out(self, nb, wait=False): if wait: self.wait() with self._dictLock: output = self._dict["MasterBoardData"]["digitalOutputBits"] mask = 1 << nb if output & mask: return 1 else: return 0 def get_digital_out_bits(self, wait=False): if wait: self.wait() with self._dictLock: return self._dict["MasterBoardData"]["digitalOutputBits"] def get_digital_in(self, nb, wait=False): if wait: self.wait() with self._dictLock: output = self._dict["MasterBoardData"]["digitalInputBits"] mask = 1 << nb if output & mask: return 1 else: return 0 def get_digital_in_bits(self, wait=False): if wait: self.wait() with self._dictLock: return self._dict["MasterBoardData"]["digitalInputBits"] def get_analog_in(self, nb, wait=False): if wait: self.wait() with self._dictLock: return self._dict["MasterBoardData"]["analogInput" + str(nb)] def get_analog_inputs(self, wait=False): if wait: self.wait() with self._dictLock: return self._dict["MasterBoardData"]["analogInput0"], self._dict[ "MasterBoardData"]["analogInput1"] def is_program_running(self, wait=False): """ return True if robot is executing a program Rmq: The refresh rate is only 10Hz so the information may be outdated """ if wait: self.wait() with self._dictLock: return self._dict["RobotModeData"]["isProgramRunning"] def is_protective_stopped(self, wait=False): if wait: self.wait() with self._dictLock: if self._parser.version >= (5, 8): val = self.secmon._dict["RobotModeData"]["isProtectiveStopped"] else: val = self.secmon._dict["RobotModeData"]["isSecurityStopped"] return val def close(self): self._trystop = True self.join() # with self._dataEvent: #wake up any thread that may be waiting for data before we close. Should we do that? # self._dataEvent.notifyAll() if self._s_secondary: with self._prog_queue_lock: self._s_secondary.close()
class SSDP(Thread): def __init__(self, host, server_name, url, description, manufacturer, manufacturer_url, model, **kwargs): self.__host = host # Note: this is the LAN IP address! self.__server_name = server_name self.__url = url self.__description = description self.__manufacturer = manufacturer self.__manufacturer_url = manufacturer_url self.__model = model self.__ssdp_broadcast_interval = kwargs.get( "ssdp_broadcast_interval", 15) # optional, default 15 seconds self.__myhostname = socket.gethostname() # a steady uuid: stays the same as long as hostname and ip address stay the same: self.__uuid = uuid.uuid3(uuid.NAMESPACE_DNS, self.__myhostname + self.__host) # Create the SSDP broadcast message self.__mySSDPbroadcast = f"""NOTIFY * HTTP/1.1 HOST: 239.255.255.250:1900 CACHE-CONTROL: max-age=60 LOCATION: {self.__url}/description.xml SERVER: {self.__server_name} NT: upnp:rootdevice USN: uuid:{self.__uuid}::upnp:rootdevice NTS: ssdp:alive OPT: "http://schemas.upnp.org/upnp/1/0/"; ns=01 """ self.__mySSDPbroadcast = self.__mySSDPbroadcast.replace( "\n", "\r\n").encode("utf-8") # Create the XML info (description.xml) self.__myxml = f"""<?xml version="1.0" encoding="UTF-8" ?> <root xmlns="urn:schemas-upnp-org:device-1-0"> <specVersion> <major>1</major> <minor>0</minor> </specVersion> <URLBase>{self.__url}</URLBase> <device> <deviceType>urn:schemas-upnp-org:device:Basic:1</deviceType> <friendlyName>{self.__server_name} ({self.__myhostname})</friendlyName> <manufacturer>{self.__manufacturer}</manufacturer> <manufacturerURL>{self.__manufacturer_url}</manufacturerURL> <modelDescription>{self.__model} </modelDescription> <modelName>{self.__model}</modelName> <modelNumber> </modelNumber> <modelDescription>{self.__description}</modelDescription> <modelURL>{self.__manufacturer_url}</modelURL> <UDN>uuid:{self.__uuid}</UDN> <presentationURL>{self.__url}</presentationURL> </device> </root>""" self.__stop = False self.__condition = Condition(Lock()) super().__init__() def stop(self): logging.info("Stopping SSDP") self.__stop = True with self.__condition: self.__condition.notify() def run(self): logging.info("Serving SSDP on %s as %s", self.__host, self.__server_name) # the standard multicast settings for SSDP: MCAST_GRP = "239.255.255.250" MCAST_PORT = 1900 MULTICAST_TTL = 2 while not self.__stop: # Do network stuff # Create socket, send the broadcast, and close the socket again try: with socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) as sock: sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL) sock.sendto(self.__mySSDPbroadcast, (MCAST_GRP, MCAST_PORT)) except: # probably no network pass # Wait until awoken or timeout is up with self.__condition: self.__condition.wait(self.__ssdp_broadcast_interval) def serve_xml(self): """Returns an XML-structure based on the information being served by this service, returns nothing if not running""" if self.__stop: return return self.__myxml
def Acquire(self, timeout=2): #print("Locks : " + str(self.locks) + ". Trying to acquire lock")# at\n" + ''.join(traceback.format_stack())) with self._lock: if self.cond is None: if self.lock.acquire(False): return self._GotLock() elif self.locks == 0: print( "LOCKS SHOULDVE BEEN UNLOCKED BUT CANT BE ACQUIRED. RECREATING LOCK" ) self.lock = self.TLock() if not self.lock.acquire(False): raise Exception("What the f**k") return self._GotLock() if timeout <= 0 and self.cond is None and self.lock.acquire(): return self._GotLock() cond = Condition(RLock()) start = now = time() end = now + timeout tup = (cond, currentThread().name, start) with cond: with self._lock: name = currentThread().name #print("WAITING FOR CONDITION %s" % name) self._waitingList.append(tup) while now < end: if (self.cond == cond or self.cond is None) and self.lock.acquire(False): with self._lock: self.cond = None #print("GOT LOCK %s %g" % (name, (now-start))) return self._GotLock(tup) else: if tup not in self._waitingList: if now < end: with self._lock: name = currentThread().name #print("WAITING AGAIN FOR CONDITION %s %d" % (name, self.locks)) self._waitingList.append(tup) #cond.acquire() self._lock.release() waitTime = min(end - now, 1) try: cond.wait(waitTime) except Exception: cond.acquire() cond.wait(waitTime) self._lock.acquire() now = time() with self._lock: if self.cond == cond or cond is None: with self._lock: self.cond = None return self._GotLock(tup) elif tup in self._waitingList: self._waitingList.remove(tup) if self.cond: selfcond = str(id(self.cond)) else: selfcond = "None" if self.e: raise Exception() s0 = "%s %s %d Lock acquiring timeout (%gs) Locks remain : %d %s " % ( currentThread().name, str( id(cond)), self.id, now - start, self.locks, selfcond) s = s0 + "at\n" + FormatStack() + '\n' i = 0 for trace in self.stackTraces: s = s + ("[STACKTRACE:%d]\n%s" % (i, trace)) i += 1 s = s + '\nCurrent execution:\n' + FormatThreadStack(self.owner) print(s) self.e = s0 raise Exception(s0)
class Q(ABC): def __init__(self, maxsize_queue=1000, numbers_threads=1, thread_status=False): super().__init__() self.__numbers_threads = numbers_threads self.__thread_status = thread_status self.__q = queue.Queue(maxsize=maxsize_queue) self.condition = Condition(Lock()) self.waiting = False self.started = False @abstractmethod def process_item(self, item) -> None: pass @abstractmethod def empty_queue_for_lock(self) -> None: pass @abstractmethod def process_status(self) -> None: pass def apply_lock(self): if self.waiting == False: with self.condition: self.waiting = True self.condition.wait() self.waiting = False def apply_unlock(self): if self.waiting == True: with self.condition: self.condition.notify_all() def __worker(self): while self.started == True: try: if self.__q.empty() == True: self.empty_queue_for_lock() else: item = self.__q.get() self.process_item(item) self.__q.task_done() except: pass def __worker__status(self): while self.started == True: try: self.process_status() except: pass def q_values(self): return self.__q def put_nowait_item(self, item: FrameDTO = None) -> None: self.__q.put_nowait(item) self.apply_unlock() def put_item(self, item: FrameDTO = None, block=True, timeout=None) -> None: self.__q.put(item, block, timeout) self.apply_unlock() def run_queue(self) -> None: if self.started == True: return self.started = True self.threads = [] for i in range(self.__numbers_threads): thr = Thread(target=self.__worker, daemon=True) thr.start() self.threads.append(thr) print("self.__thread_status", self.__thread_status) if self.__thread_status == True: thr = Thread(target=self.__worker__status, daemon=True) thr.start() self.threads.append(thr) def join_queue(self) -> None: self.__q.join() def stop_queue(self) -> None: self.started = False
class Application: def __init__(self): root = tk.Tk() root.title(CONFIG['win']['title']) root.geometry(CONFIG['win']['geometry']) root.resizable(*CONFIG['win']['resizable']) root.protocol('WM_DELETE_WINDOW', self.destructor) self.widgets = {} self.widgets['root'] = root # gui self.fonts = None self.themes = None self.init_fonts() self.init_themes() self.curr_theme = CONFIG['win']['default_theme'] # flags self.stopped = True self.stoppable = False self.exit = False self.feedback_required = False self.feedback_provided = False self.feedback_frame = False # status self.rl_session = None self.env = None self.agent = None self.feedback_id = None self.time_secs = 0 self.last_time_secs = None self.evaluation = 0 self.attempts = 0 self.secret = np.full(CONFIG['rl']['code_len'], None) # camera self.feedback_highlighter = FeedbackHighlighter( CONFIG['highlighter']['fps'], CONFIG['highlighter']['res'], CONFIG['highlighter']['format'], CONFIG['highlighter']['duration'], CONFIG['highlighter']['video_path']) self.emotion_analyzer = EmotionAnalyzer( CONFIG['analyzer']['docker_image_repository'], CONFIG['analyzer']['docker_image_tag'], CONFIG['analyzer']['video_path'], CONFIG['analyzer']['csv_path']) self.vcap = self.init_camera() # mainloop self.mainloop_cv = Condition() self.stoppable_mutex = Lock() self.mainloop_thread = self.mainloop() self.mainloop_thread.start() # customtk-------------------------------------------------- def custom_label(self, master, x, y, height, width, *args, **kwargs): frame = tk.Frame(master, height=height, width=width) frame.pack_propagate(0) frame.place(x=x, y=y) label = tk.Label(frame, *args, **kwargs) label.pack(fill=tk.BOTH, expand=1) return label def custom_button(self, master, x, y, height, width, *args, **kwargs): frame = tk.Frame(master, height=height, width=width) frame.pack_propagate(0) frame.place(x=x, y=y) button = tk.Button(frame, *args, **kwargs) button.pack(fill=tk.BOTH, expand=1) return button def custom_option_menu(self, master, x, y, height, width, value, values, *args, **kwargs): frame = tk.Frame(master, height=height, width=width) frame.pack_propagate(0) frame.place(x=x, y=y) option_menu = tk.OptionMenu(frame, value, values, *args, **kwargs) option_menu.pack(fill=tk.BOTH, expand=1) return option_menu # init------------------------------------------------------ def init_fonts(self): self.fonts = [] for font in STYLES['fonts']: self.fonts.append( tkFont.Font( name=font['font_name'], family=font['font'], size=font['font_size'], weight=font['font_weight'], )) def init_themes(self): self.themes = {} for theme in STYLES['themes']: self.themes[theme['name']] = theme['widgets'] def init_gui(self): # video preview video_preview_frame = tk.Frame(master=self.widgets['root'], width=640, height=510) video_preview_frame.place(x=40, y=40) video_preview_title = self.custom_label(video_preview_frame, 0, 0, 30, 640) video_preview_content = self.custom_label(video_preview_frame, 0, 30, 480, 640) self.widgets['video_preview_frame'] = video_preview_frame self.widgets['video_preview_title'] = video_preview_title self.widgets['video_preview_content'] = video_preview_content # attempts attempts_frame = tk.Frame(master=self.widgets['root'], width=200, height=110) attempts_frame.place(x=720, y=40) attempts_title = self.custom_label(attempts_frame, 0, 0, 30, 200) attempts_content = self.custom_label(attempts_frame, 0, 30, 80, 200) self.widgets['attempts_frame'] = attempts_frame self.widgets['attempts_title'] = attempts_title self.widgets['attempts_content'] = attempts_content # timer timer_frame = tk.Frame(master=self.widgets['root'], width=200, height=110) timer_frame.place(x=960, y=40) timer_title = self.custom_label(timer_frame, 0, 0, 30, 200) timer_content = self.custom_label(timer_frame, 0, 30, 80, 200) self.widgets['timer_frame'] = timer_frame self.widgets['timer_title'] = timer_title self.widgets['timer_content'] = timer_content # code selector code_selector_frame = tk.Frame(master=self.widgets['root'], width=440, height=360) code_selector_frame.place(x=720, y=190) code_selector_title = self.custom_label(code_selector_frame, 0, 0, 30, 440) code_selector_content = self.custom_label(code_selector_frame, 34, 54, 283, 373) code_selector_buttons = np.empty( (CONFIG['rl']['code_len'], CONFIG['rl']['no_actions']), dtype=object) for step in range(CONFIG['rl']['code_len']): for action in range(CONFIG['rl']['no_actions']): padx = ( 0, 20) if action != CONFIG['rl']['no_actions'] - 1 else ( 0, 0) pady = (0, 24) tmp_frame = tk.Frame(code_selector_content, height=78, width=78) tmp_frame.pack_propagate(0) tmp_frame.grid(row=step, column=action, padx=padx, pady=pady) code_selector_buttons[step][action] = tk.Button( tmp_frame, text=str(action), command=None) code_selector_buttons[step][action].pack(fill=tk.BOTH, expand=1) self.widgets['code_selector_frame'] = code_selector_frame self.widgets['code_selector_title'] = code_selector_title self.widgets['code_selector_content'] = code_selector_content self.widgets['code_selector_buttons'] = code_selector_buttons # feedback evaluation feedback_evaluation_frame = tk.Frame(master=self.widgets['root'], width=640, height=110) feedback_evaluation_frame.place(x=40, y=590) feedback_evaluation_title = self.custom_label( feedback_evaluation_frame, 0, 0, 30, 640) feedback_evaluation_scale = tk.Scale( feedback_evaluation_frame, from_=CONFIG['rl']['min_evaluation'], to=CONFIG['rl']['max_evaluation'], length=480, resolution=0.1) feedback_evaluation_scale.place(x=40, y=40) feedback_evaluation_button = self.custom_button( feedback_evaluation_frame, 550, 42, 50, 50) self.widgets[ 'feedback_evaluation_frame'] = feedback_evaluation_frame self.widgets[ 'feedback_evaluation_title'] = feedback_evaluation_title self.widgets[ 'feedback_evaluation_scale'] = feedback_evaluation_scale self.widgets[ 'feedback_evaluation_button'] = feedback_evaluation_button # feedback indicator feedback_indicator_frame = tk.Frame(master=self.widgets['root'], width=200, height=110) feedback_indicator_frame.place(x=720, y=590) feedback_indicator_title = self.custom_label( feedback_indicator_frame, 0, 0, 30, 200) feedback_indicator_content = self.custom_label( feedback_indicator_frame, 0, 30, 80, 200) self.widgets['feedback_indicator_frame'] = feedback_indicator_frame self.widgets['feedback_indicator_title'] = feedback_indicator_title self.widgets[ 'feedback_indicator_content'] = feedback_indicator_content # agent code code_frame = tk.Frame(master=self.widgets['root'], width=200, height=110) code_frame.place(x=960, y=590) code_title = self.custom_label(code_frame, 0, 0, 30, 200) code_content = self.custom_label(code_frame, 0, 30, 80, 200) self.widgets['code_frame'] = code_frame self.widgets['code_title'] = code_title self.widgets['code_content'] = code_content # control buttons flow_button = self.custom_button(self.widgets['root'], 40, 740, 40, 200) reset_button = self.custom_button(self.widgets['root'], 280, 740, 40, 200) self.widgets['flow_button'] = flow_button self.widgets['reset_button'] = reset_button # theme selector theme = tk.StringVar(self.widgets['root']) theme.set(self.curr_theme) theme_selector = self.custom_option_menu( self.widgets['root'], 960, 740, 40, 200, theme, *self.themes.keys(), command=self.on_theme_changed) self.widgets['theme'] = theme self.widgets['theme_selector'] = theme_selector def init_camera(self): for camera in range(3): vcap = cv2.VideoCapture(camera) if vcap is not None and vcap.isOpened(): return vcap return cv2.VideoCapture() def init_listeners(self): for step in range(CONFIG['rl']['code_len']): for action in range(CONFIG['rl']['no_actions']): self.widgets['code_selector_buttons'][step][ action].configure( command=self.on_code_selector_button_clicked( step, action)) self.widgets['feedback_evaluation_button'].configure( command=self.on_feedback_evaluation_button_clicked) self.widgets['reset_button'].configure( command=self.on_reset_button_clicked) self.widgets['flow_button'].configure( command=self.on_flow_button_clicked) def init_rl_session(self): return { 'session_id': CONFIG['rl']['session_prefix'] + str(int(time.time() * 1000)), 'config': { 'secret': sorted(list(self.secret)), 'no_pegs': self.env.action_space.n, 'code_len': len(self.env.secret), 'alpha': self.agent.alpha, 'gamma': self.agent.gamma, 'epsilon': self.agent.epsilon, 'beta': self.agent.beta, 'exploration_mode': self.agent.exploration_mode, 'epsilon_decay': self.agent.epsilon_decay, 'epsilon_low': self.agent.epsilon_low }, 'result': { 'guessed': None, 'optimal': None, 'qmatrix': None, 'attempts': None, 'time': None }, 'feedback': {} } # gui------------------------------------------------------- def apply_theme(self): theme = self.themes[self.curr_theme] # root self.widgets['root'].configure( background=theme['root']['background']) # video preview self.widgets['video_preview_frame'].configure( bg=theme['video_preview_frame']['background']) self.widgets['video_preview_title'].configure( text=theme['video_preview_title']['text'], bg=theme['video_preview_title']['background'], fg=theme['video_preview_title']['foreground'], font=tkFont.Font(name=theme['video_preview_title']['font'], exists=True)) self.widgets['video_preview_content'].configure( bg=theme['video_preview_content']['background'], fg=theme['video_preview_content']['foreground'], font=tkFont.Font(name=theme['video_preview_content']['font'], exists=True)) # attempts self.widgets['attempts_frame'].configure( bg=theme['attempts_frame']['background']) self.widgets['attempts_title'].configure( text=theme['attempts_title']['text'], bg=theme['attempts_title']['background'], fg=theme['attempts_title']['foreground'], font=tkFont.Font(name=theme['attempts_title']['font'], exists=True)) self.widgets['attempts_content'].configure( bg=theme['attempts_content']['background'], fg=theme['attempts_content']['foreground'], font=tkFont.Font(name=theme['attempts_content']['font'], exists=True)) # timer self.widgets['timer_frame'].configure( bg=theme['timer_frame']['background']) self.widgets['timer_title'].configure( text=theme['timer_title']['text'], bg=theme['timer_title']['background'], fg=theme['timer_title']['foreground'], font=tkFont.Font(name=theme['timer_title']['font'], exists=True)) self.widgets['timer_content'].configure( bg=theme['timer_content']['background'], fg=theme['timer_content']['foreground'], font=tkFont.Font(name=theme['timer_content']['font'], exists=True)) # code selector self.widgets['code_selector_frame'].configure( bg=theme['code_selector_frame']['background']) self.widgets['code_selector_title'].configure( text=theme['code_selector_title']['text'], bg=theme['code_selector_title']['background'], fg=theme['code_selector_title']['foreground'], font=tkFont.Font(name=theme['code_selector_title']['font'], exists=True)) self.widgets['code_selector_content'].configure( bg=theme['code_selector_content']['background']) for step in range(CONFIG['rl']['code_len']): for action in range(CONFIG['rl']['no_actions']): self.widgets['code_selector_buttons'][step][ action].configure( bg=theme['code_selector_button']['background'], fg=theme['code_selector_button']['foreground'], activebackground=theme['code_selector_button'] ['background_active'], activeforeground=theme['code_selector_button'] ['foreground_active'], disabledforeground=theme['code_selector_button'] ['foreground_disabled'], font=tkFont.Font( name=theme['code_selector_button']['font'], exists=True), highlightthickness=0, bd=0) # feedback evaluation self.widgets['feedback_evaluation_frame'].configure( bg=theme['feedback_evaluation_frame']['background']) self.widgets['feedback_evaluation_title'].configure( text=theme['feedback_evaluation_title']['text'], bg=theme['feedback_evaluation_title']['background'], fg=theme['feedback_evaluation_title']['foreground'], font=tkFont.Font( name=theme['feedback_evaluation_title']['font'], exists=True)) self.widgets['feedback_evaluation_scale'].configure( tickinterval=1, orient=tk.HORIZONTAL, bg=theme['feedback_evaluation_scale']['background'], fg=theme['feedback_evaluation_scale']['foreground'], troughcolor=theme['feedback_evaluation_scale']['trough'], font=tkFont.Font( name=theme['feedback_evaluation_scale']['font'], exists=True), highlightthickness=0, bd=0) self.widgets['feedback_evaluation_button'].configure( text=theme['feedback_evaluation_button']['text'], bg=theme['feedback_evaluation_button']['background'], fg=theme['feedback_evaluation_button']['foreground'], activebackground=theme['feedback_evaluation_button'] ['background_active'], activeforeground=theme['feedback_evaluation_button'] ['foreground_active'], disabledforeground=theme['feedback_evaluation_button'] ['foreground_disabled'], font=tkFont.Font( name=theme['feedback_evaluation_button']['font'], exists=True), highlightthickness=0, bd=0, ) # feedback indicator self.widgets['feedback_indicator_frame'].configure( bg=theme['feedback_indicator_frame']['background']) self.widgets['feedback_indicator_title'].configure( text=theme['feedback_indicator_title']['text'], bg=theme['feedback_indicator_title']['background'], fg=theme['feedback_indicator_title']['foreground'], font=tkFont.Font( name=theme['feedback_indicator_title']['font'], exists=True)) self.widgets['feedback_indicator_content'].configure( text=theme['feedback_indicator_content']['text'], bg=theme['feedback_indicator_content']['background'], fg=theme['feedback_indicator_content']['foreground'], font=tkFont.Font( name=theme['feedback_indicator_content']['font'], exists=True)) # agent code self.widgets['code_frame'].configure( bg=theme['code_frame']['background']) self.widgets['code_title'].configure( text=theme['code_title']['text'], bg=theme['code_title']['background'], fg=theme['code_title']['foreground'], font=tkFont.Font(name=theme['code_title']['font'], exists=True)) self.widgets['code_content'].configure( bg=theme['code_content']['background'], fg=theme['code_content']['foreground'], font=tkFont.Font(name=theme['code_content']['font'], exists=True)) # control buttons self.widgets['flow_button'].configure( bg=theme['flow_button']['background'], fg=theme['flow_button']['foreground'], activebackground=theme['flow_button']['background_active'], activeforeground=theme['flow_button']['foreground_active'], disabledforeground=theme['flow_button']['foreground_disabled'], font=tkFont.Font(name=theme['flow_button']['font'], exists=True), highlightthickness=0, bd=0) self.widgets['reset_button'].configure( text=theme['reset_button']['text'], bg=theme['reset_button']['background'], fg=theme['reset_button']['foreground'], activebackground=theme['reset_button']['background_active'], activeforeground=theme['reset_button']['foreground_active'], disabledforeground=theme['reset_button'] ['foreground_disabled'], font=tkFont.Font(name=theme['reset_button']['font'], exists=True), highlightthickness=0, bd=0) # theme selector self.widgets['theme_selector'].config( bg=theme['theme_selector']['background'], fg=theme['theme_selector']['foreground'], font=tkFont.Font(name=theme['theme_selector']['font'], exists=True), activebackground=theme['theme_selector']['background_active'], activeforeground=theme['theme_selector']['foreground_active'], highlightthickness=0, bd=0, relief=tk.FLAT, indicatoron=0, direction='above') self.refresh() def refresh(self, refresh_type='all'): if refresh_type == 'all': self.update_timer() self.update_attempts() self.update_feedback_indicator() self.update_code() self.update_feedback_evaluation_scale() self.update_feedback_evaluation_button() self.update_flow_button() self.update_reset_button() self.update_code_selector() elif refresh_type == 'rl': self.update_attempts() self.update_code() self.update_feedback_indicator() self.update_feedback_evaluation_scale() self.update_feedback_evaluation_button() self.update_flow_button() self.update_reset_button() def update_attempts(self): attempts_str = str(self.attempts).replace('', ' ')[1:-1] self.widgets['attempts_content'].configure(text=attempts_str) def update_timer(self): mins, secs = divmod(int(round(self.time_secs)), 60) time_secs_str = str(mins).zfill(2) + ':' + str(secs).zfill(2) time_secs_str = time_secs_str.replace('', ' ')[1:-1] self.widgets['timer_content'].configure(text=time_secs_str) def update_feedback_indicator(self): theme = self.themes[self.curr_theme] if self.feedback_required: fg = theme['feedback_indicator_content']['foreground_required'] else: fg = theme['feedback_indicator_content'][ 'foreground_not_required'] self.widgets['feedback_indicator_content'].configure(fg=fg) def update_code(self): code = None if self.agent is not None: code = list(self.agent.curr_state) if code is None or len(code) == 0: theme = self.themes[self.curr_theme] code_str = theme['code_content']['text_empty'] else: code_str = '{' + str(code)[1:-1] + '}' self.widgets['code_content'].configure(text=code_str) def update_feedback_evaluation_scale(self): theme = self.themes[self.curr_theme] if self.feedback_required: state = tk.NORMAL troughcolor = theme['feedback_evaluation_scale']['trough'] else: state = tk.DISABLED troughcolor = theme['feedback_evaluation_scale'][ 'trough_disabled'] self.widgets['feedback_evaluation_scale'].configure( state=state, troughcolor=troughcolor) def update_feedback_evaluation_button(self): if self.feedback_required: state = tk.NORMAL else: state = tk.DISABLED self.widgets['feedback_evaluation_button'].configure(state=state) def update_flow_button(self): theme = self.themes[self.curr_theme] if (self.env is not None and self.env.is_guessed()) or not (self.stoppable or self.stopped): bg = theme['flow_button']['background_disabled'] state = tk.DISABLED text = self.widgets['flow_button']['text'] else: bg = theme['flow_button']['background'] state = tk.NORMAL if self.stopped: text = theme['flow_button']['text_start'] else: text = theme['flow_button']['text_stop'] self.widgets['flow_button'].configure(state=state, bg=bg, text=text) def update_reset_button(self): theme = self.themes[self.curr_theme] if self.stopped: bg = theme['reset_button']['background'] state = tk.NORMAL else: bg = theme['reset_button']['background_disabled'] state = tk.DISABLED self.widgets['reset_button'].configure(state=state, bg=bg) def update_code_selector_button(self, step, action): theme = self.themes[self.curr_theme] if self.secret[step] is None: bg = theme['code_selector_button']['background'] fg = theme['code_selector_button']['foreground'] state = tk.NORMAL else: if action == self.secret[step]: bg = theme['code_selector_button']['background_selected'] fg = theme['code_selector_button']['foreground_selected'] if self.stopped: state = tk.NORMAL else: state = tk.DISABLED else: bg = theme['code_selector_button']['background_disabled'] fg = theme['code_selector_button']['foreground'] state = tk.DISABLED self.widgets['code_selector_buttons'][step][action].configure( bg=bg, fg=fg, state=state) def update_code_selector(self): for step in range(CONFIG['rl']['code_len']): for action in range(CONFIG['rl']['no_actions']): self.update_code_selector_button(step, action) def flash_code_selector_button(self, step, action, flash_bg_color, flash_count=3, delay=250): if flash_count > 0: self.widgets['code_selector_buttons'][step][action].configure( background=flash_bg_color) self.widgets['code_selector_buttons'][step][action].after( delay / 2, lambda: self.update_code_selector_button(step, action)) self.widgets['code_selector_buttons'][step][action].after( delay, lambda: self.flash_code_selector_button( step, action, flash_bg_color, flash_count - 1, delay)) def flash_error_code_selector(self): theme = self.themes[self.curr_theme] for step in range(CONFIG['rl']['code_len']): if self.secret[step] is None: for action in range(CONFIG['rl']['no_actions']): self.flash_code_selector_button( step, action, theme['code_selector_button']['flash_error']) def flash_guessed_code_selector(self): theme = self.themes[self.curr_theme] for step in range(CONFIG['rl']['code_len']): for action in range(CONFIG['rl']['no_actions']): self.flash_code_selector_button( step, action, theme['code_selector_button']['flash_guessed'], flash_count=3, delay=500) def flash_action_code_selector(self, action): theme = self.themes[self.curr_theme] for step in range(CONFIG['rl']['code_len']): self.flash_code_selector_button( step, action, theme['code_selector_button']['flash_action'], flash_count=1, delay=1500) # listeners------------------------------------------------- def on_code_selector_button_clicked(self, step, action): def on_code_selector_button_clicked_listener(): if self.secret[step] is None: self.secret[step] = action else: self.secret[step] = None self.update_code_selector() return on_code_selector_button_clicked_listener def on_flow_button_clicked(self): self.stoppable_mutex.acquire() if self.stopped and (None in self.secret): self.flash_error_code_selector() else: with self.mainloop_cv: self.stopped = not self.stopped self.mainloop_cv.notifyAll() if not self.stopped: self.update_code_selector() self.timer() else: self.last_time_secs = None self.update_reset_button() self.update_flow_button() self.stoppable_mutex.release() def on_reset_button_clicked(self): if self.rl_session is not None: self.fill_rl_session_result() DB.insert(self.rl_session) self.rl_session = None self.reset() self.refresh() def on_theme_changed(self, theme): self.curr_theme = theme self.apply_theme() def on_feedback_evaluation_button_clicked(self): self.evaluation = self.widgets['feedback_evaluation_scale'].get() self.feedback_id = CONFIG['highlighter'][ 'video_name_prefix'] + str(int(time.time() * 1000)) self.rl_session['feedback'][self.feedback_id] = { 'evaluation': self.evaluation, 'attempt': list(self.agent.curr_state), 'time': str(int(self.time_secs / 60)).zfill(2) + ':' + str(int(self.time_secs % 60)).zfill(2) } with self.mainloop_cv: self.feedback_required = False self.mainloop_cv.notifyAll() self.feedback_provided = True self.feedback_frame = True self.update_feedback_indicator() self.update_feedback_evaluation_scale() self.update_feedback_evaluation_button() # status---------------------------------------------------- def destructor(self): self.exit = True with self.mainloop_cv: self.mainloop_cv.notifyAll() self.widgets['root'].destroy() self.vcap.release() cv2.destroyAllWindows() def reset(self): self.feedback_provided = False self.feedback_frame = False self.rl_session = None with self.mainloop_cv: self.feedback_required = False self.stopped = True self.mainloop_cv.notifyAll() self.time_secs = 0 self.last_time_secs = None self.evaluation = 0 self.attempts = 0 self.secret = np.full(CONFIG['rl']['code_len'], None) self.env = None self.agent = None def fill_rl_session_result(self): qmatrix = {} for state in self.agent.qmatrix.keys(): state_str = '{' + str(list(state))[1:-1] + '}' qmatrix[state_str] = { 'qvalues': str(list(self.agent.qmatrix[state]['qvalues'])), 'td_errors': str(list(self.agent.qmatrix[state]['td_errors'])), 'td_errors_variations': str(list(self.agent.qmatrix[state]['td_errors_delta'])), 'visits': self.agent.qmatrix[state]['visits'] } time_str = str(int(self.time_secs / 60)).zfill(2) + ':' + str( int(self.time_secs % 60)).zfill(2) self.rl_session['result']['guessed'] = self.env.is_guessed() self.rl_session['result']['optimal'] = list( self.agent.get_optimal()) self.rl_session['result']['qmatrix'] = qmatrix self.rl_session['result']['attempts'] = self.attempts self.rl_session['result']['time'] = time_str for feedback_id in self.rl_session['feedback'].keys(): csv_path = CONFIG['analyzer'][ 'csv_path'] + '/' + feedback_id + '.csv' if os.path.isfile(csv_path): self.rl_session['feedback'][feedback_id][ 'csv_path'] = csv_path else: self.rl_session['feedback'][feedback_id]['csv_path'] = None # services-------------------------------------------------- def webcam(self): success, frame = self.vcap.read() if success: video_path = self.feedback_highlighter.scroll( cv2.flip(frame, 1), self.feedback_frame, self.feedback_id) self.feedback_frame = False if video_path is not None: Thread(target=lambda: self.emotion_analyzer.analyze( os.path.basename(video_path))).start() rgba_frame = cv2.cvtColor(cv2.flip(frame, 1), cv2.COLOR_BGR2RGBA) img = Image.fromarray(rgba_frame) imgtk = ImageTk.PhotoImage(image=img) self.widgets['video_preview_content'].imgtk = imgtk self.widgets['video_preview_content'].configure(image=imgtk, text='') else: self.vcap.release() self.vcap = self.init_camera() theme = self.themes[self.curr_theme] self.widgets['video_preview_content'].configure( text=theme['video_preview_content']['text_error'], image='') self.widgets['video_preview_content'].after( 1000 / CONFIG['vcap']['fps'], self.webcam) def timer(self): if not self.stopped: self.last_time_secs = self.last_time_secs or time.time() now_time_secs = time.time() self.time_secs = (self.time_secs + now_time_secs - self.last_time_secs) % 3600 self.last_time_secs = now_time_secs self.update_timer() self.widgets['timer_content'].after(1000, self.timer) def mainloop(self): def mainloop_thread(): while not self.exit: # Verifica se l'applicazione è stata fermata o se è richiesto un feedback # in caso affermativo rimane in attesa passiva with self.mainloop_cv: while self.stopped or self.feedback_required: # In caso di uscita salva i dati se presenti if self.exit: if self.rl_session is not None: self.fill_rl_session_result() DB.insert(self.rl_session) self.rl_session = None return self.mainloop_cv.wait() # Inializzazione sessione RL if self.rl_session is None: self.env = gym.make(CONFIG['rl']['gym'], no_pegs=CONFIG['rl']['no_actions'], secret=self.secret, random_seed=np.random.randint( np.iinfo(np.int32).max)) self.agent = Agent(self.env) self.rl_session = self.init_rl_session() else: # Ignora le eccezioni sul mainloop (grafica) quando si esce # dall'applicazione senza che lo step RL sia terminato try: # Disabilita il pulsante di STOP self.stoppable_mutex.acquire() if self.stopped: continue self.stoppable = False self.update_flow_button() self.stoppable_mutex.release() # Se è stato fornito un feedback aggiorna la matrice Q if self.feedback_provided: self.agent.update_qmatrix(self.evaluation) self.feedback_provided = False self.agent.curr_state = self.env.reset() print(self.agent.qmatrix_to_str()) # Altrimenti scegli un'azione da eseguire elif not self.feedback_required: action = self.agent.get_action() self.feedback_required = self.agent.take_action( action) # Se l'azione è terminale incrementa gli attempts if self.feedback_required: self.attempts += 1 # Se il multiset finale è corretto interrompi e salva i dati if self.env.is_guessed(): self.agent.update_qmatrix( CONFIG['rl']['max_evaluation']) self.fill_rl_session_result() DB.insert(self.rl_session) self.rl_session = None self.stopped = True self.feedback_required = False self.flash_guessed_code_selector() # Altrimenti flash azione else: self.flash_action_code_selector(action) # Altrimenti flash azione else: self.flash_action_code_selector(action) # Riabilita il pulsante di STOP time.sleep(0.5) self.stoppable = True self.refresh('rl') #time.sleep(CONFIG['rl']['epoch_delay']) except: raise self.init_gui() self.init_listeners() self.apply_theme() self.webcam() return Thread(target=mainloop_thread)
# Notify that the count was increment event_count_condition.notify_all() # Register the callback with the client client.add_event_callback(EVENT_TOPIC, MyEventCallback()) # # Send events # # Record the start time start = time.time() # Loop and send the events for event_id in range(TOTAL_EVENTS): # Create the event event = Event(EVENT_TOPIC) # Set the payload event.payload = str(event_id).encode() # Send the event client.send_event(event) # Wait until all events have been received print("Waiting for events to be received...") with event_count_condition: while event_count[0] < TOTAL_EVENTS: event_count_condition.wait() # Print the elapsed time print("Elapsed time (ms): " + str((time.time() - start) * 1000))
class RPC: def __init__(self, infile, outfile, onRequest, onNotification, onError): self.infile = infile self.outfile = outfile self.onRequest = onRequest self.onNotification = onNotification self.onError = onError self.mid = 0 self.queue = {} self.cv = Condition() self.result = None def incMid(self) -> int: mid = self.mid self.mid += 1 return mid def message(self, contentDict: Dict[str, Any]) -> None: content = json.dumps(contentDict) message = ("Content-Length: {}\r\n\r\n" "{}".format(len(content), content)) logger.debug(' => ' + content) self.outfile.write(message) self.outfile.flush() def call(self, method: str, params: Dict[str, Any], cb=None): """ @param cb: func. Callback to handle result. If None, turn to sync call. """ mid = self.incMid() if cb is not None: self.queue[mid] = cb contentDict = { "jsonrpc": "2.0", "method": method, "params": params, "id": mid, } # type: Dict[str, Any] self.message(contentDict) if cb is not None: return with self.cv: while self.result is None: self.cv.wait() result = self.result self.result = None return result def notify(self, method: str, params: Dict[str, Any]) -> None: contentDict = { "jsonrpc": "2.0", "method": method, "params": params, } # type: Dict[str, Any] self.message(contentDict) def serve(self): contentLength = 0 while not self.infile.closed: line = self.infile.readline().strip() if line: header, value = line.split(":") if header == "Content-Length": contentLength = int(value) else: content = self.infile.read(contentLength) logger.debug(' <= ' + content) self.handle(json.loads(content)) def handle(self, message: Dict[str, Any]): if "error" in message: # error if "id" in message: mid = message["id"] del self.queue[mid] try: self.onError(message["error"]) except: logger.exception("Exception in RPC.onError.") elif "result" in message: # result mid = message['id'] result = message["result"] if mid in self.queue: # async call try: self.queue[mid](result) except: logger.exception("Exception in RPC request callback.") del self.queue[mid] else: # sync call with self.cv: self.result = result self.cv.notify() elif "method" in message: # request/notification if "id" in message: # request try: self.onRequest(message) except: logger.exception("Exception in RPC.onRequest") else: try: self.onNotification(message) except: logger.exception("Exception in RPC.onNotification") else: logger.error('Unexpected')
class PLCObject(object): def __init__(self, WorkingDir, argv, statuschange, evaluator, pyruntimevars): self.workingdir = WorkingDir # must exits already self.tmpdir = os.path.join(WorkingDir, 'tmp') if os.path.exists(self.tmpdir): shutil.rmtree(self.tmpdir) os.mkdir(self.tmpdir) # FIXME : is argv of any use nowadays ? self.argv = [WorkingDir] + argv # force argv[0] to be "path" to exec... self.statuschange = statuschange self.evaluator = evaluator self.pyruntimevars = pyruntimevars self.PLCStatus = PlcStatus.Empty self.PLClibraryHandle = None self.PLClibraryLock = Lock() # Creates fake C funcs proxies self._InitPLCStubCalls() self._loading_error = None self.python_runtime_vars = None self.TraceThread = None self.TraceLock = Lock() self.Traces = [] self.DebugToken = 0 self._init_blobs() # First task of worker -> no @RunInMain def AutoLoad(self, autostart): # Get the last transfered PLC try: self.CurrentPLCFilename = open( self._GetMD5FileName(), "r").read().strip() + lib_ext self.PLCStatus = PlcStatus.Stopped if autostart: if self.LoadPLC(): self.StartPLC() return except Exception: self.PLCStatus = PlcStatus.Empty self.CurrentPLCFilename = None self.StatusChange() def StatusChange(self): if self.statuschange is not None: for callee in self.statuschange: callee(self.PLCStatus) def LogMessage(self, *args): if len(args) == 2: level, msg = args else: level = LogLevelsDefault msg, = args PLCprint(msg) if self._LogMessage is not None: return self._LogMessage(level, msg, len(msg)) return None @RunInMain def ResetLogCount(self): if self._ResetLogCount is not None: self._ResetLogCount() # used internaly def GetLogCount(self, level): if self._GetLogCount is not None: return int(self._GetLogCount(level)) elif self._loading_error is not None and level == 0: return 1 @RunInMain def GetLogMessage(self, level, msgid): tick = ctypes.c_uint32() tv_sec = ctypes.c_uint32() tv_nsec = ctypes.c_uint32() if self._GetLogMessage is not None: maxsz = len(self._log_read_buffer)-1 sz = self._GetLogMessage(level, msgid, self._log_read_buffer, maxsz, ctypes.byref(tick), ctypes.byref(tv_sec), ctypes.byref(tv_nsec)) if sz and sz <= maxsz: self._log_read_buffer[sz] = '\x00' return self._log_read_buffer.value, tick.value, tv_sec.value, tv_nsec.value elif self._loading_error is not None and level == 0: return self._loading_error, 0, 0, 0 return None def _GetMD5FileName(self): return os.path.join(self.workingdir, "lasttransferedPLC.md5") def _GetLibFileName(self): return os.path.join(self.workingdir, self.CurrentPLCFilename) def _LoadPLC(self): """ Load PLC library Declare all functions, arguments and return values """ md5 = open(self._GetMD5FileName(), "r").read() self.PLClibraryLock.acquire() try: self._PLClibraryHandle = dlopen(self._GetLibFileName()) self.PLClibraryHandle = ctypes.CDLL(self.CurrentPLCFilename, handle=self._PLClibraryHandle) self.PLC_ID = ctypes.c_char_p.in_dll(self.PLClibraryHandle, "PLC_ID") if len(md5) == 32: self.PLC_ID.value = md5 self._startPLC = self.PLClibraryHandle.startPLC self._startPLC.restype = ctypes.c_int self._startPLC.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p)] self._stopPLC_real = self.PLClibraryHandle.stopPLC self._stopPLC_real.restype = None self._PythonIterator = getattr(self.PLClibraryHandle, "PythonIterator", None) if self._PythonIterator is not None: self._PythonIterator.restype = ctypes.c_char_p self._PythonIterator.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_void_p)] self._stopPLC = self._stopPLC_real else: # If python confnode is not enabled, we reuse _PythonIterator # as a call that block pythonthread until StopPLC self.PlcStopping = Event() def PythonIterator(res, blkid): self.PlcStopping.clear() self.PlcStopping.wait() return None self._PythonIterator = PythonIterator def __StopPLC(): self._stopPLC_real() self.PlcStopping.set() self._stopPLC = __StopPLC self._ResetDebugVariables = self.PLClibraryHandle.ResetDebugVariables self._ResetDebugVariables.restype = None self._RegisterDebugVariable = self.PLClibraryHandle.RegisterDebugVariable self._RegisterDebugVariable.restype = None self._RegisterDebugVariable.argtypes = [ctypes.c_int, ctypes.c_void_p] self._FreeDebugData = self.PLClibraryHandle.FreeDebugData self._FreeDebugData.restype = None self._GetDebugData = self.PLClibraryHandle.GetDebugData self._GetDebugData.restype = ctypes.c_int self._GetDebugData.argtypes = [ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_void_p)] self._suspendDebug = self.PLClibraryHandle.suspendDebug self._suspendDebug.restype = ctypes.c_int self._suspendDebug.argtypes = [ctypes.c_int] self._resumeDebug = self.PLClibraryHandle.resumeDebug self._resumeDebug.restype = None self._ResetLogCount = self.PLClibraryHandle.ResetLogCount self._ResetLogCount.restype = None self._GetLogCount = self.PLClibraryHandle.GetLogCount self._GetLogCount.restype = ctypes.c_uint32 self._GetLogCount.argtypes = [ctypes.c_uint8] self._LogMessage = self.PLClibraryHandle.LogMessage self._LogMessage.restype = ctypes.c_int self._LogMessage.argtypes = [ctypes.c_uint8, ctypes.c_char_p, ctypes.c_uint32] self._log_read_buffer = ctypes.create_string_buffer(1 << 14) # 16K self._GetLogMessage = self.PLClibraryHandle.GetLogMessage self._GetLogMessage.restype = ctypes.c_uint32 self._GetLogMessage.argtypes = [ctypes.c_uint8, ctypes.c_uint32, ctypes.c_char_p, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32)] self._loading_error = None except Exception: self._loading_error = traceback.format_exc() PLCprint(self._loading_error) return False finally: self.PLClibraryLock.release() return True @RunInMain def LoadPLC(self): res = self._LoadPLC() if res: self.PythonRuntimeInit() else: self._FreePLC() return res @RunInMain def UnLoadPLC(self): self.PythonRuntimeCleanup() self._FreePLC() def _InitPLCStubCalls(self): """ create dummy C func proxies """ self._startPLC = lambda x, y: None self._stopPLC = lambda: None self._ResetDebugVariables = lambda: None self._RegisterDebugVariable = lambda x, y: None self._IterDebugData = lambda x, y: None self._FreeDebugData = lambda: None self._GetDebugData = lambda: -1 self._suspendDebug = lambda x: -1 self._resumeDebug = lambda: None self._PythonIterator = lambda: "" self._GetLogCount = None self._LogMessage = None self._GetLogMessage = None self._PLClibraryHandle = None self.PLClibraryHandle = None def _FreePLC(self): """ Unload PLC library. This is also called by __init__ to create dummy C func proxies """ self.PLClibraryLock.acquire() try: # Unload library explicitely if getattr(self, "_PLClibraryHandle", None) is not None: dlclose(self._PLClibraryHandle) # Forget all refs to library self._InitPLCStubCalls() finally: self.PLClibraryLock.release() return False def PythonRuntimeCall(self, methodname, use_evaluator=True, reverse_order=False): """ Calls init, start, stop or cleanup method provided by runtime python files, loaded when new PLC uploaded """ methods = self.python_runtime_vars.get("_runtime_%s" % methodname, []) if reverse_order: methods = reversed(methods) for method in methods: if use_evaluator: _res, exp = self.evaluator(method) else: _res, exp = default_evaluator(method) if exp is not None: self.LogMessage(0, '\n'.join(traceback.format_exception(*exp))) # used internaly def PythonRuntimeInit(self): MethodNames = ["init", "start", "stop", "cleanup"] self.python_runtime_vars = globals().copy() self.python_runtime_vars.update(self.pyruntimevars) parent = self class PLCSafeGlobals(object): def __getattr__(self, name): try: t = parent.python_runtime_vars["_"+name+"_ctype"] except KeyError: raise KeyError("Try to get unknown shared global variable : %s" % name) v = t() parent.python_runtime_vars["_PySafeGetPLCGlob_"+name](ctypes.byref(v)) return parent.python_runtime_vars["_"+name+"_unpack"](v) def __setattr__(self, name, value): try: t = parent.python_runtime_vars["_"+name+"_ctype"] except KeyError: raise KeyError("Try to set unknown shared global variable : %s" % name) v = parent.python_runtime_vars["_"+name+"_pack"](t, value) parent.python_runtime_vars["_PySafeSetPLCGlob_"+name](ctypes.byref(v)) self.python_runtime_vars.update({ "PLCGlobals": PLCSafeGlobals(), "WorkingDir": self.workingdir, "PLCObject": self, "PLCBinary": self.PLClibraryHandle, "PLCGlobalsDesc": []}) for methodname in MethodNames: self.python_runtime_vars["_runtime_%s" % methodname] = [] try: filenames = os.listdir(self.workingdir) filenames.sort() for filename in filenames: name, ext = os.path.splitext(filename) if name.upper().startswith("RUNTIME") and ext.upper() == ".PY": execfile(os.path.join(self.workingdir, filename), self.python_runtime_vars) for methodname in MethodNames: method = self.python_runtime_vars.get("_%s_%s" % (name, methodname), None) if method is not None: self.python_runtime_vars["_runtime_%s" % methodname].append(method) except Exception: self.LogMessage(0, traceback.format_exc()) raise self.PythonRuntimeCall("init", use_evaluator=False) self.PythonThreadCondLock = Lock() self.PythonThreadCond = Condition(self.PythonThreadCondLock) self.PythonThreadCmd = "Wait" self.PythonThread = Thread(target=self.PythonThreadProc, name="PLCPythonThread") self.PythonThread.start() # used internaly def PythonRuntimeCleanup(self): if self.python_runtime_vars is not None: self.PythonThreadCommand("Finish") self.PythonThread.join() self.PythonRuntimeCall("cleanup", use_evaluator=False, reverse_order=True) self.python_runtime_vars = None def PythonThreadLoop(self): res, cmd, blkid = "None", "None", ctypes.c_void_p() compile_cache = {} while True: cmd = self._PythonIterator(res, blkid) FBID = blkid.value if cmd is None: break try: self.python_runtime_vars["FBID"] = FBID ccmd, AST = compile_cache.get(FBID, (None, None)) if ccmd is None or ccmd != cmd: AST = compile(cmd, '<plc>', 'eval') compile_cache[FBID] = (cmd, AST) result, exp = self.evaluator(eval, AST, self.python_runtime_vars) if exp is not None: res = "#EXCEPTION : "+str(exp[1]) self.LogMessage(1, ('PyEval@0x%x(Code="%s") Exception "%s"') % ( FBID, cmd, '\n'.join(traceback.format_exception(*exp)))) else: res = str(result) self.python_runtime_vars["FBID"] = None except Exception as e: res = "#EXCEPTION : "+str(e) self.LogMessage(1, ('PyEval@0x%x(Code="%s") Exception "%s"') % (FBID, cmd, str(e))) def PythonThreadProc(self): while True: self.PythonThreadCondLock.acquire() cmd = self.PythonThreadCmd while cmd == "Wait": self.PythonThreadCond.wait() cmd = self.PythonThreadCmd self.PythonThreadCmd = "Wait" self.PythonThreadCondLock.release() if cmd == "Activate": self.PythonRuntimeCall("start") self.PythonThreadLoop() self.PythonRuntimeCall("stop", reverse_order=True) else: # "Finish" break def PythonThreadCommand(self, cmd): self.PythonThreadCondLock.acquire() self.PythonThreadCmd = cmd self.PythonThreadCond.notify() self.PythonThreadCondLock.release() def _fail(msg): self.LogMessage(0, msg) self.PLCStatus = PlcStatus.Broken self.StatusChange() def PreStartPLC(self): """ Here goes actions to be taken just before PLC starts, with all libraries and python object already created. For example : restore saved proprietary parameters """ pass @RunInMain def StartPLC(self): if self.PLClibraryHandle is None: if not self.LoadPLC(): self._fail(_("Problem starting PLC : can't load PLC")) self.PreStartPLC() if self.CurrentPLCFilename is not None and self.PLCStatus == PlcStatus.Stopped: c_argv = ctypes.c_char_p * len(self.argv) res = self._startPLC(len(self.argv), c_argv(*self.argv)) if res == 0: self.PLCStatus = PlcStatus.Started self.StatusChange() self.PythonThreadCommand("Activate") self.LogMessage("PLC started") else: self._fail(_("Problem starting PLC : error %d" % res)) @RunInMain def StopPLC(self): if self.PLCStatus == PlcStatus.Started: self.LogMessage("PLC stopped") self._stopPLC() self.PLCStatus = PlcStatus.Stopped self.StatusChange() if self.TraceThread is not None: self.TraceThread.join() self.TraceThread = None return True return False def GetPLCstatus(self): try: return self._GetPLCstatus() except EOFError: return (PlcStatus.Disconnected, None) @RunInMain def _GetPLCstatus(self): return self.PLCStatus, map(self.GetLogCount, xrange(LogLevelsCount)) @RunInMain def GetPLCID(self): return getPSKID(partial(self.LogMessage, 0)) def _init_blobs(self): self.blobs = {} if os.path.exists(self.tmpdir): shutil.rmtree(self.tmpdir) os.mkdir(self.tmpdir) @RunInMain def SeedBlob(self, seed): blob = (mkstemp(dir=self.tmpdir) + (hashlib.new('md5'),)) _fd, _path, md5sum = blob md5sum.update(seed) newBlobID = md5sum.digest() self.blobs[newBlobID] = blob return newBlobID @RunInMain def AppendChunkToBlob(self, data, blobID): blob = self.blobs.pop(blobID, None) if blob is None: return None fd, _path, md5sum = blob md5sum.update(data) newBlobID = md5sum.digest() os.write(fd, data) self.blobs[newBlobID] = blob return newBlobID @RunInMain def PurgeBlobs(self): for fd, _path, _md5sum in self.blobs.values(): os.close(fd) self._init_blobs() def _BlobAsFile(self, blobID, newpath): blob = self.blobs.pop(blobID, None) if blob is None: raise Exception(_("Missing data to create file: {}").format(newpath)) fd, path, _md5sum = blob fobj = os.fdopen(fd) fobj.flush() os.fsync(fd) fobj.close() shutil.move(path, newpath) def _extra_files_log_path(self): return os.path.join(self.workingdir, "extra_files.txt") def RepairPLC(self): self.PurgePLC() MainWorker.quit() @RunInMain def PurgePLC(self): extra_files_log = self._extra_files_log_path() old_PLC_filename = os.path.join(self.workingdir, self.CurrentPLCFilename) \ if self.CurrentPLCFilename is not None \ else None try: allfiles = open(extra_files_log, "rt").readlines() allfiles.extend([extra_files_log, old_PLC_filename, self._GetMD5FileName()]) except Exception: self.LogMessage("No files to purge") allfiles = [] for filename in allfiles: if filename: filename = filename.strip() try: os.remove(os.path.join(self.workingdir, filename)) except Exception: self.LogMessage("Couldn't purge " + filename) self.PLCStatus = PlcStatus.Empty # TODO: PLCObject restart @RunInMain def NewPLC(self, md5sum, plc_object, extrafiles): if self.PLCStatus in [PlcStatus.Stopped, PlcStatus.Empty, PlcStatus.Broken]: NewFileName = md5sum + lib_ext extra_files_log = self._extra_files_log_path() new_PLC_filename = os.path.join(self.workingdir, NewFileName) self.UnLoadPLC() self.PurgePLC() self.LogMessage("NewPLC (%s)" % md5sum) try: # Create new PLC file self._BlobAsFile(plc_object, new_PLC_filename) # Then write the files log = open(extra_files_log, "w") for fname, blobID in extrafiles: fpath = os.path.join(self.workingdir, fname) self._BlobAsFile(blobID, fpath) log.write(fname+'\n') # Store new PLC filename based on md5 key with open(self._GetMD5FileName(), "w") as f: f.write(md5sum) f.flush() os.fsync(f.fileno()) # Store new PLC filename self.CurrentPLCFilename = NewFileName except Exception: self.PLCStatus = PlcStatus.Broken self.StatusChange() PLCprint(traceback.format_exc()) return False if self.LoadPLC(): self.PLCStatus = PlcStatus.Stopped else: self.PLCStatus = PlcStatus.Broken self.StatusChange() return self.PLCStatus == PlcStatus.Stopped return False def MatchMD5(self, MD5): try: last_md5 = open(self._GetMD5FileName(), "r").read() return last_md5 == MD5 except Exception: pass return False @RunInMain def SetTraceVariablesList(self, idxs): """ Call ctype imported function to append these indexes to registred variables in PLC debugger """ self.DebugToken += 1 if idxs: # suspend but dont disable if self._suspendDebug(False) == 0: # keep a copy of requested idx self._ResetDebugVariables() for idx, iectype, force in idxs: if force is not None: c_type, _unpack_func, pack_func = \ TypeTranslator.get(iectype, (None, None, None)) force = ctypes.byref(pack_func(c_type, force)) self._RegisterDebugVariable(idx, force) self._TracesSwap() self._resumeDebug() return self.DebugToken else: self._suspendDebug(True) return None def _TracesSwap(self): self.LastSwapTrace = time() if self.TraceThread is None and self.PLCStatus == PlcStatus.Started: self.TraceThread = Thread(target=self.TraceThreadProc, name="PLCTrace") self.TraceThread.start() self.TraceLock.acquire() Traces = self.Traces self.Traces = [] self.TraceLock.release() return Traces @RunInMain def GetTraceVariables(self, DebugToken): if DebugToken is not None and DebugToken == self.DebugToken: return self.PLCStatus, self._TracesSwap() return PlcStatus.Broken, [] def TraceThreadProc(self): """ Return a list of traces, corresponding to the list of required idx """ self._resumeDebug() # Re-enable debugger while self.PLCStatus == PlcStatus.Started: tick = ctypes.c_uint32() size = ctypes.c_uint32() buff = ctypes.c_void_p() TraceBuffer = None self.PLClibraryLock.acquire() res = self._GetDebugData(ctypes.byref(tick), ctypes.byref(size), ctypes.byref(buff)) if res == 0: if size.value: TraceBuffer = ctypes.string_at(buff.value, size.value) self._FreeDebugData() self.PLClibraryLock.release() # leave thread if GetDebugData isn't happy. if res != 0: break if TraceBuffer is not None: self.TraceLock.acquire() lT = len(self.Traces) if lT != 0 and lT * len(self.Traces[0]) > 1024 * 1024: self.Traces.pop(0) self.Traces.append((tick.value, TraceBuffer)) self.TraceLock.release() # TraceProc stops here if Traces not polled for 3 seconds traces_age = time() - self.LastSwapTrace if traces_age > 3: self.TraceLock.acquire() self.Traces = [] self.TraceLock.release() self._suspendDebug(True) # Disable debugger break self.TraceThread = None def RemoteExec(self, script, *kwargs): try: exec(script, kwargs) except Exception: _e_type, e_value, e_traceback = sys.exc_info() line_no = traceback.tb_lineno(get_last_traceback(e_traceback)) return (-1, "RemoteExec script failed!\n\nLine %d: %s\n\t%s" % (line_no, e_value, script.splitlines()[line_no - 1])) return (0, kwargs.get("returnVal", None))
class Api(vnokcoin_spot_usd.OkcoinApi): """OKCOIN的API实现""" #---------------------------------------------------------------------- def __init__(self, gateway): """Constructor""" super(Api, self).__init__() self.gateway = gateway # gateway对象 self.gatewayName = gateway.gatewayName # gateway对象名称 self.cbDict = {} self.tickDict = {} self.orderDict = {} self.lastOrderID = '' self.orderCondition = Condition() self.trade_password = False #self.initCallback() self.tradeFlag = False self.tradeFlag_2 = True self.logger = rwLoggerFunction() #self.strategyName='' #---------------------------------------------------------------------- def qryInstruments(self): """查询合约""" params = {'accountId': self.accountId} self.getInstruments(params) # ---------------------------------------------------------------------- def qryGenerateCnyContract(self): l = self.generateCnyContract() for contract in l: contract.gatewayName = self.gatewayName self.gateway.onContract(contract) # ---------------------------------------------------------------------- def onMessage(self, ws, evt): """信息推送""" data = json.loads(evt) channel = data[RESPONSE_CHANNEL] callback = self.cbDict[channel] callback(data) #---------------------------------------------------------------------- def onError(self, evt): """错误推送""" error = VtErrorData() error.gatewayName = self.gatewayName error.errorMsg = str(evt) self.gateway.onError(error) #---------------------------------------------------------------------- def onClose(self, ws): """接口断开""" self.gateway.connected = True self.writeLog(u'服务器连接断开') #---------------------------------------------------------------------- def onOpen(self, ws): pass #---------------------------------------------------------------------- def writeLog(self, content): """快速记录日志""" log = VtLogData() log.gatewayName = self.gatewayName log.logContent = content self.gateway.onLog(log) #---------------------------------------------------------------------- def initCallback(self): """初始化回调函数""" pass #---------------------------------------------------------------------- def generateSpecificContract(self, contract, symbol): """生成合约""" new = copy(contract) new.symbol = symbol new.vtSymbol = symbol #new.vtSymbol = EXCHANGE_NAME + CONNECTION_MARK + symbol new.name = symbol return new #---------------------------------------------------------------------- def generateCnyContract(self): """生成CNY合约信息""" contractList = [] contract = VtContractData() contract.exchange = EXCHANGE_NAME contract.productClass = PRODUCT_SPOT contract.size = 1 contract.priceTick = 0.01 #contract.strategyName = self.strategyName contractList.append( self.generateSpecificContract(contract, BTC_USD_SPOT)) return contractList # ---------------------------------------------------------------------- # def getTickerInfo(self,symbol): # """查询行情数据""" # paramsDict = {"ticker_depth": 'ticker', "symbol": symbol} # self.sendRequest(paramsDict, self.onTicker, API_FLAG_TICKER) # ---------------------------------------------------------------------- def onTicker(self, data): """""" if 'ticker' not in data: return ticker = data['ticker'] symbol = BTC_USD_SPOT #vtSymbol= EXCHANGE_NAME+'_'+symbol vtSymbol = symbol if vtSymbol not in self.tickDict: tick = VtTickData() tick.symbol = symbol tick.vtSymbol = vtSymbol tick.gatewayName = self.gatewayName tick.exchange = EXCHANGE_NAME self.tickDict[vtSymbol] = tick else: tick = self.tickDict[vtSymbol] tick.highPrice = float(ticker['high']) tick.lowPrice = float(ticker['low']) tick.lastPrice = float(ticker['last']) tick.volume = float(ticker['vol']) tick.date, tick.time = generateDateTime(data['date']) newtick = copy(tick) self.gateway.onTick(newtick) # ---------------------------------------------------------------------- # def getDepthInfo(self,symbol): # """查询行情数据""" # paramsDict = {"ticker_depth": 'depth', "symbol": symbol} # self.sendRequest(paramsDict, self.onDepth, API_FLAG_TICKER) # ---------------------------------------------------------------------- def onDepth(self, data): """""" # if 'asks' not in data: # return # # symbol = channelSymbolMap[data['symbol']] # # if symbol not in self.tickDict: # tick = VtTickData() # tick.symbol = symbol # tick.vtSymbol = symbol # tick.gatewayName = self.gatewayName # self.tickDict[symbol] = tick # else: # tick = self.tickDict[symbol] # # tick.bidPrice1, tick.bidVolume1 = data['bids'][0] # tick.bidPrice2, tick.bidVolume2 = data['bids'][1] # tick.bidPrice3, tick.bidVolume3 = data['bids'][2] # tick.bidPrice4, tick.bidVolume4 = data['bids'][3] # tick.bidPrice5, tick.bidVolume5 = data['bids'][4] # # tick.askPrice1, tick.askVolume1 = data['asks'][0] # tick.askPrice2, tick.askVolume2 = data['asks'][1] # tick.askPrice3, tick.askVolume3 = data['asks'][2] # tick.askPrice4, tick.askVolume4 = data['asks'][3] # tick.askPrice5, tick.askVolume5 = data['asks'][4] pass #newtick = copy(tick) #self.gateway.onTick(newtick) # ---------------------------------------------------------------------- def getOrders(self): """查询正在进行的委托订单""" # timestamp = long(time.time()) # paramsDict = {"access_key": self.apiKey, "secret_key": self.secretKey, # "created": timestamp, "coin_type": 1, "method": 'get_orders'} # sign = signature(paramsDict) # del paramsDict["secret_key"] # paramsDict['sign'] = sign # self.sendRequest(paramsDict, self.onGetOrders) pass # ---------------------------------------------------------------------- def onGetOrders(self, data): """回调函数""" # if len(data) == 0: # return # for d in data: # order = VtOrderData() # order.gatewayName = self.gatewayName # # order.symbol = BTC_USD_SPOT # order.exchange = EXCHANGE_OKCOIN # order.vtSymbol = '.'.join([order.symbol, order.exchange]) # order.orderID = str(d['id']) # order.direction, priceType = priceTypeMap[str(d['type'])] # order.offset = orderTypeMap[str(d['type'])] # #order.status = orderStatusMap[str(d['status'])] # # order.price = d['order_price'] # order.totalVolume = d['order_amount'] # order.tradeVolume = d['processed_amount'] # order.orderTime = generateDateTimeStamp(d['order_time']) # # order.vtOrderID = '.'.join([self.gatewayName, order.orderID]) # # self.gateway.onOrder(order) # # self.orderDict[order.orderID] = order pass #self.writeLog(u'委托信息查询完成') # ---------------------------------------------------------------------- def getTrades(self): #print "okcoin getTrades start" self.tradeFlag = False ORDER_INFO_RESOURCE = "/api/v1/order_info.do" params = { 'api_key': self.apiKey, 'symbol': 'btc_usd', 'order_id': self.lastOrderID } params['sign'] = buildMySign(params, self.secretKey) self.sendRequest(params, self.onGetTrade, ORDER_INFO_RESOURCE) #print "okcoin getTrades end" #---------------------------------------------------------------------- def onGetTrade(self, result): #print "okcoin onGetTrade start" """回调函数""" if 'orders' not in result: print 'Trade Data Error' return ordersData = result['orders'] if len(ordersData) == 0: print 'no Trade Data ' return for data in ordersData: order = VtOrderData() order.gatewayName = self.gatewayName order.symbol = BTC_USD_SPOT order.exchange = EXCHANGE_OKCOIN order.vtSymbol = '.'.join([order.symbol, order.exchange]) order.orderID = str(data['order_id']) order.direction, priceType = priceTypeMap[str(data['type'])] order.offset = tradeTypeMap[str(data['type'])] # order.status = orderStatusMap[str(d['status'])] order.price = data['price'] order.totalVolume = data['amount'] order.tradeVolume = data['deal_amount'] order.status = tradeStatusMap[str(data['status'])] #order.orderTime = generateDateTimeStamp(d['order_time']) order.vtOrderID = '.'.join([self.gatewayName, order.orderID]) self.gateway.onOrder(order) #self.orderDict[order.orderID] = order if 'status' in data and tradeStatusMap[str( data['status'])] == TRADER_STATUS_DEAL: trade = VtTradeData() trade.gatewayName = self.gatewayName trade.symbol = BTC_USD_SPOT trade.exchange = EXCHANGE_OKCOIN trade.vtSymbol = '.'.join([trade.symbol, trade.exchange]) trade.orderID = str(data['order_id']) trade.tradeID = str(data['order_id']) trade.direction, priceType = priceTypeMap[str(data['type'])] trade.offset = tradeTypeMap[str(data['type'])] trade.price = data['avg_price'] trade.volume = float(data['deal_amount']) trade.status = tradeStatusMap[str(data['status'])] trade.vtOrderID = '.'.join([self.gatewayName, trade.orderID]) trade.vtTradeID = '.'.join([self.gatewayName, trade.tradeID]) self.gateway.onTrade(trade) #self.orderDict[trade.orderID] = trade #print "okcoin onGetTrade end" self.writeLog(u'成交信息查询完成') # ---------------------------------------------------------------------- def spotUserInfo(self): """查询现货账户""" USERINFO_RESOURCE = "/api/v1/userinfo.do" params = {} params['api_key'] = self.apiKey params['sign'] = buildMySign(params, self.secretKey) self.sendRequest(params, self.onSpotUserInfo, USERINFO_RESOURCE) # ---------------------------------------------------------------------- def onSpotUserInfo(self, data): """回调函数""" # 持仓信息 for symbol in ['btc', 'usd']: pos = VtPositionData() pos.gatewayName = self.gatewayName pos.symbol = symbol pos.vtSymbol = symbol pos.vtPositionName = symbol + "_" + self.gatewayName pos.direction = DIRECTION_NET funds = data['info']['funds'] pos.frozen = float(funds['freezed']['%s' % symbol]) pos.position = float(funds['free']['%s' % symbol]) self.gateway.onPosition(pos) account = VtAccountData() account.gatewayName = self.gatewayName account.accountID = self.gatewayName account.vtAccountID = account.accountID account.balance = float(funds['asset']['total']) self.gateway.onAccount(account) # ---------------------------------------------------------------------- def sendOrder(self, params): #print "okcoin sendOrder start" self.lastOrderID = '' """发送委托""" if params.direction == DIRECTION_LONG: direction = 'buy' else: direction = 'sell' TRADE_RESOURCE = "/api/v1/trade.do" paramsDict = { 'api_key': self.apiKey, 'symbol': 'btc_usd', 'type': direction } if params.orderStyle == 1: self.tradeFlag_2 = False if params.price: paramsDict['price'] = params.price if params.volume: paramsDict['amount'] = params.volume paramsDict['sign'] = buildMySign(paramsDict, self.secretKey) #print "okcoin sendOrder start_2" self.sendRequest(paramsDict, self.onSendOrder, TRADE_RESOURCE) #print "okcoin sendOrder start_3" # 等待发单回调推送委托号信息 self.orderCondition.acquire() self.orderCondition.wait() self.orderCondition.release() vtOrderID = '.'.join([self.gatewayName, self.lastOrderID]) #print "okcoin sendOrder end" return vtOrderID # ---------------------------------------------------------------------- def onSendOrder(self, data): #print "okcoin onSendOrder start" if 'result' in data and data['result']: self.lastOrderID = str(data['order_id']) self.tradeFlag = True #self.logger.setInfoLog('onSend_okcoin:' + 'ID:' + self.lastOrderID) print(u'okcoin onSendOrder Sucess:', self.lastOrderID) else: print(u'OKCOIN下单失败,请查询账户资金额度') print "okcoin onSendOrder start_2" # 收到委托号后,通知发送委托的线程返回委托号 self.orderCondition.acquire() self.orderCondition.notify() self.orderCondition.release() #print "okcoin onSendOrder end" # ---------------------------------------------------------------------- def cancelOrder(self, params): """发送撤單""" CANCEL_ORDER_RESOURCE = "/api/v1/cancel_order.do" params = { 'api_key': self.apiKey, 'symbol': 'btc_cny', 'order_id': params.orderID } params['sign'] = buildMySign(params, self.__secretkey) return self.sendRequest(params, self.onCancelOrder, CANCEL_ORDER_RESOURCE) # ---------------------------------------------------------------------- # 从策略例调用的接口 def getTrades_huotou(self, orderID): """查询最近的成交订单""" ORDER_INFO_RESOURCE = "/api/v1/order_info.do" params = { 'api_key': self.apiKey, 'symbol': 'btc_usd', 'order_id': orderID } params['sign'] = buildMySign(params, self.secretKey) result = httpPost(OKCOIN_HOST, ORDER_INFO_RESOURCE, params) if result['result'] and len(result['orders']) > 0: orderStatus = self.onGetTrade(result) return orderStatus else: return False # ---------------------------------------------------------------------- def onCancelOrder(self, data): # if data['result'] == 'success': # print "撤单完成" pass # ---------------------------------------------------------------------- def generateUsdContract(self): """生成USD合约信息""" pass # ---------------------------------------------------------------------- def onSpotTrade(self, data): """委托回报""" pass # ---------------------------------------------------------------------- def onSpotCancelOrder(self, data): """撤单回报""" pass # ---------------------------------------------------------------------- def spotSendOrder(self, req): """发单""" pass # ---------------------------------------------------------------------- def spotCancel(self, req): """撤单""" pass # ---------------------------------------------------------------------- def onSpotSubTrades(self, data): pass # ---------------------------------------------------------------------- def onSpotOrderInfo(self, data): pass
class MypyEventHandler(BaseThread): def __init__(self, dep_graph, queueing_handler, file_cache, compact, num_workers): # type: (ModuleGraph, MypyQueueingHandler, MypyFileCache, bool, int) -> None self.dep_graph = dep_graph self.worker_pool = [] # type: List[MypyWorker] self.task_pool = [] # type: List[MypyTask] self.task_cond = Condition() self.queueing_handler = queueing_handler self.file_cache = file_cache self.compact = compact self.num_workers = num_workers super(MypyEventHandler, self).__init__() def on_deleted(self, event): # type: (FileSystemEvent) -> None pass def on_created(self, event): # type: (FileSystemEvent) -> None pass def _add_task(self, task, index=None): # type: (MypyTask, Optional[int]) -> None if task in self.task_pool: return if index is None: self.task_pool.append(task) else: self.task_pool.insert(index, task) def _find_modified_module(self, src_path): # type: (str) -> Optional[Module] modified_module = None for module_ in self.dep_graph.listModules(): if module_.filename == src_path: modified_module = module_ break return modified_module def _find_dependencies(self, root_module): # type: (Module) -> Set[str] modified_modules = {root_module} dependencies_to_check = set() def check_module(mod): # type: (Module) -> None # If it's an __init__ file then we might have changed the interface to # this module, so add it to the set of modified modules. This will cause # us to iterate all modules again. if mod.modname.endswith('__init__'): modified_modules.add(mod) else: dependencies_to_check.add(os.path.abspath(mod.filename)) while True: modified_modules_size = len(modified_modules) for module_ in self.dep_graph.listModules(): for import_name in module_.imports: modified_modnames = { mod.modname for mod in modified_modules } if import_name in modified_modnames: check_module(module_) break elif '{}.__init__'.format( import_name) in modified_modnames: check_module(module_) break # Run until we haven't added any new modules to the set of modified modules. # We do this to catch the following cases: # (1) an init exports another init's methods (uncommon) # (2) we have a.py (modified) <- __init__.py <- b.py and we happen to scan b.py first (more common) if modified_modules_size == len(modified_modules): break dependencies_to_check.update({ os.path.abspath(module_.filename) for module_ in modified_modules }) return dependencies_to_check def _ensure_workers(self): # type: () -> None while len(self.worker_pool) < self.num_workers: worker = MypyWorker(self.task_pool, self.task_cond, self.file_cache, self.compact) self.worker_pool.append(worker) worker.start() def _disable_workers(self): # type: () -> None # Prevent workers from consuming any tasks in the queue until we're ready. for worker in self.worker_pool: worker.run_tasks = False def _enable_workers(self): # type: () -> None # Mark all workers to start running tasks and interrupt any # workers with tasks that will need to be re-run. for worker in self.worker_pool: worker.run_tasks = True if worker.current_task is None: continue if worker.current_task in self.task_pool: worker.current_task.interrupt() self.task_cond.notify_all() def _wait_until_tasks_completed(self): # type: () -> None start_size = len(self.task_pool) while len(self.task_pool ) > 0 and not self.queueing_handler.has_new_events: curr_size = len(self.task_pool) total_completed_tasks = start_size - curr_size sys.stdout.write(' {}/{}\r'.format(total_completed_tasks, start_size)) sys.stdout.flush() self.task_cond.wait() if self.queueing_handler.has_new_events: print('Detected new changes, interrupting...') else: # Even though all tasks have been pulled from the task_pool, # they might not have been completed, so we have to wait until all # workers have cleared their current_task field. all_clear = False while not all_clear: all_clear = True for worker in self.worker_pool: if worker.current_task is not None: all_clear = False break if not all_clear: self.task_cond.wait() def on_modified(self, event): # type: (FileSystemEvent) -> None print_divider('TYPECHECKING', newline_before=True) self.task_cond.acquire() modified_module = self._find_modified_module(event.src_path) if modified_module is None: print('Unable to find module for modified file {}'.format( event.src_path)) dependencies_to_check = {event.src_path} else: dependencies_to_check = self._find_dependencies(modified_module) # Add the modified file first so it's the first one to be checked. self._add_task(MypyTask(os.path.abspath(modified_module.filename)), index=0) for filename in dependencies_to_check: self._add_task(MypyTask(filename)) self._ensure_workers() self._enable_workers() self._wait_until_tasks_completed() self._disable_workers() print_divider('DONE') self.task_cond.release() def run(self): # type: () -> None while True: self.queueing_handler.next_event(self)
class TradeAgent(object): def __init__(self, td_front, broker_id, user_id, password): self.request_id = 1 self.broker_id = broker_id self.user_id = user_id self.password = password self.callback = None self.qry_cond = Condition() self.rsp_qry_queue = Queue() self.cond = Condition() self.data_queue = Queue() self.qry_results = {} self.instrument_dict = {} self.position_dict = None self.trading_account = None self.trade_dict = None self.order_dict = None self.session_id = None self.front_id = None self.max_order_ref = None self.api = MyTraderApi(broker_id, user_id, password, self.rsp_qry_queue, self.data_queue) thread = Thread(target=self.init, args=(td_front, )) thread.start() thread_rsp_qry = Thread(target=self.process_rsp_qry) thread_rsp_qry.start() thread_data = Thread(target=self.process_data) thread_data.start() def init(self, td_front): self.api.RegisterFront(td_front) self.api.SubscribePrivateTopic(2) self.api.SubscribePublicTopic(2) self.api.Init() self.api.Join() def process_rsp_qry(self): while 1: msg = self.rsp_qry_queue.get() if not isinstance(msg, tuple): continue request_id, result, is_last = msg self.qry_cond.acquire() if request_id not in self.qry_results: self.qry_results[request_id] = { 'results': [], 'status': False, } if result: self.qry_results[request_id]['results'].append(result) if is_last: self.qry_results[request_id]['status'] = True self.qry_cond.notifyAll() self.qry_cond.release() def process_data(self): while 1: data = self.data_queue.get() if isinstance( data, ApiStruct.Order ) and data.ExchangeID and data.TraderID and data.OrderLocalID: self.order_dict[(data.ExchangeID, data.TraderID, data.OrderLocalID)] = data elif isinstance( data, ApiStruct.Trade) and data.ExchangeID and data.TradeID: self.trade_dict[(data.ExchangeID, data.TradeID)] = data if self.callback: self.callback(data) def ready(self, timeout=3): request_time = time.time() while not self.api.ready and time.time() < request_time + timeout: time.sleep(1) if self.api.ready and self.api.user_login: self.front_id = self.api.user_login.FrontID self.session_id = self.api.user_login.SessionID self.max_order_ref = int(self.api.user_login.MaxOrderRef) print 'trade agent initial succeed' return True else: print 'trade agent inital failed' return False def load_instruments(self, instrument_ids): for instrument_id in instrument_ids: self.query_instrument(instrument_id) time.sleep(1.1) def _get_results(self, timeout, request_id): self.qry_cond.acquire() self.qry_results[request_id] = { 'results': [], 'status': False, } request_time = time.time() while not self.qry_results[request_id][ 'status'] and request_time + timeout > time.time(): self.qry_cond.wait(0.1) self.qry_cond.release() ret = self.qry_results.pop(request_id, {}) if not ret.get('status'): return None return ret.get('results') def query_settlement_info(self, timeout=3): req = ApiStruct.QrySettlementInfo(BrokerID=self.broker_id, InvestorID=self.user_id) self.request_id += 1 request_id = self.request_id ret = self.api.ReqQrySettlementInfo(req, request_id) if ret != 0: print 'query settlement info failed', ret return None results = self._get_results(timeout, request_id) return results def query_settlement_info_confirm(self, timeout=3): req = ApiStruct.QrySettlementInfoConfirm(BrokerID=self.broker_id, InvestorID=self.user_id) self.request_id += 1 request_id = self.request_id ret = self.api.ReqQrySettlementInfoConfirm(req, request_id) if ret != 0: print 'query settlement info confirm failed', ret return None results = self._get_results(timeout, request_id) if results: print results[0] print 'settlement info already confirmed.' return results def query_order(self, timeout=3): req = ApiStruct.QryOrder(BrokerID=self.broker_id, InvestorID=self.user_id) self.request_id += 1 request_id = self.request_id ret = self.api.ReqQryOrder(req, request_id) if ret != 0: print 'query order failed', ret return None results = self._get_results(timeout, request_id) if results is None: return None self.order_dict = {(result.ExchangeID, result.TraderID, result.OrderLocalID): result for result in results} return self.order_dict def query_trade(self, timeout=3): req = ApiStruct.QryTrade(BrokerID=self.broker_id, InvestorID=self.user_id) self.request_id += 1 request_id = self.request_id ret = self.api.ReqQryTrade(req, request_id) if ret != 0: print 'query trade failed', ret return None results = self._get_results(timeout, request_id) if results is None: return None self.trade_dict = {(result.ExchangeID, result.TradeID): result for result in results} return self.trade_dict def query_position(self, instrument_id=None, timeout=3): req = ApiStruct.QryInvestorPosition(BrokerID=self.broker_id, InvestorID=self.user_id) if instrument_id: req.InstrumentID = instrument_id self.request_id += 1 request_id = self.request_id ret = self.api.ReqQryInvestorPosition(req, request_id) if ret != 0: print 'query position failed', ret return None results = self._get_results(timeout, request_id) if results is None: return None self.position_dict = {} for result in results: if result.InstrumentID is None or result.PosiDirection is None or result.Position is None: continue if (result.InstrumentID, result.PosiDirection) not in self.position_dict: self.position_dict[(result.InstrumentID, result.PosiDirection)] = [] self.position_dict[(result.InstrumentID, result.PosiDirection)].append(result) return self.position_dict def query_position_detail(self, timeout=3): req = ApiStruct.QryInvestorPositionDetail(BrokerID=self.broker_id, InvestorID=self.user_id) self.request_id += 1 request_id = self.request_id ret = self.api.ReqQryInvestorPositionDetail(req, request_id) if ret != 0: print 'query position detail failed', ret return None results = self._get_results(timeout, request_id) return results def query_trading_account(self, timeout=3): req = ApiStruct.QryTradingAccount(BrokerID=self.broker_id, InvestorID=self.user_id) self.request_id += 1 request_id = self.request_id ret = self.api.ReqQryTradingAccount(req, request_id) if ret != 0: print 'query trading account failed', ret return None results = self._get_results(timeout, request_id) if not results: return None self.trading_account = results[0] return self.trading_account def query_instrument(self, instrument_id, timeout=3): req = ApiStruct.QryInstrument(InstrumentID=instrument_id) self.request_id += 1 request_id = self.request_id ret = self.api.ReqQryInstrument(req, request_id) if ret != 0: print 'query instrument {} failed'.format(instrument_id), ret return None results = self._get_results(timeout, request_id) if not results: return None self.instrument_dict[instrument_id] = results[0] print results[0] return results[0] def query_depth_market_data(self, instrument_id, timeout=3): req = ApiStruct.QryDepthMarketData(InstrumentID=instrument_id) self.request_id += 1 request_id = self.request_id ret = self.api.ReqQryDepthMarketData(req, request_id) if ret != 0: print 'query {} depth market data failed'.format( instrument_id), ret return None results = self._get_results(timeout, request_id) if not results: return None return results[0] def settlement_info_confirm(self, timeout=3): req = ApiStruct.SettlementInfoConfirm(BrokerID=self.broker_id, InvestorID=self.user_id) self.request_id += 1 request_id = self.request_id ret = self.api.ReqSettlementInfoConfirm(req, request_id) if ret != 0: print 'settlement info confirm failed', ret return False results = self._get_results(timeout, request_id) if results is None: return False print results[0] print 'settlement info confirm succeed.' return True def _make_order(self, instrument_id, direction, volume): self.max_order_ref += 1 return ApiStruct.InputOrder( BrokerID=self.broker_id, InvestorID=self.user_id, InstrumentID=instrument_id, OrderRef=str(self.max_order_ref), Direction=direction, CombOffsetFlag=ApiStruct.OF_Open, CombHedgeFlag=ApiStruct.HF_Speculation, VolumeTotalOriginal=volume, ContingentCondition=ApiStruct.CC_Immediately, VolumeCondition=ApiStruct.VC_AV, MinVolume=1, ForceCloseReason=ApiStruct.FCC_NotForceClose, IsAutoSuspend=0, UserForceClose=0) def _set_price(self, order, direction, limit_price, time_condition, market_data): price = None if limit_price == 0: if market_data: if direction == ApiStruct.D_Buy: # 取对手价 price = market_data.AskPrice1 else: # 取对手价 price = market_data.BidPrice1 time_condition = ApiStruct.TC_IOC else: price = limit_price if price is None: return False order.LimitPrice = price order.TimeCondition = time_condition return True def buy(self, instrument_id, volume, limit_price=0, time_condition=ApiStruct.TC_GFD, market_data=None): direction = ApiStruct.D_Buy order = self._make_order(instrument_id, direction, volume) order.OrderPriceType = ApiStruct.OPT_LimitPrice if not self._set_price(order, direction, limit_price, time_condition, market_data): return None self.request_id += 1 self.api.ReqOrderInsert(order, self.request_id) return order def sell(self, instrument_id, volume, limit_price=0, time_condition=ApiStruct.TC_GFD, market_data=None): direction = ApiStruct.D_Sell order = self._make_order(instrument_id, direction, volume) order.CombOffsetFlag = ApiStruct.OF_Close order.OrderPriceType = ApiStruct.OPT_LimitPrice if not self._set_price(order, direction, limit_price, time_condition, market_data): return None self.request_id += 1 self.api.ReqOrderInsert(order, self.request_id) return order def sell_today(self, instrument_id, volume, limit_price=0, time_condition=ApiStruct.TC_GFD, market_data=None): direction = ApiStruct.D_Sell order = self._make_order(instrument_id, direction, volume) order.CombOffsetFlag = ApiStruct.OF_CloseToday order.OrderPriceType = ApiStruct.OPT_LimitPrice if not self._set_price(order, direction, limit_price, time_condition, market_data): return None self.request_id += 1 self.api.ReqOrderInsert(order, self.request_id) return order def short(self, instrument_id, volume, limit_price=0, time_condition=ApiStruct.TC_GFD, market_data=None): direction = ApiStruct.D_Sell order = self._make_order(instrument_id, direction, volume) order.OrderPriceType = ApiStruct.OPT_LimitPrice if not self._set_price(order, direction, limit_price, time_condition, market_data): return None self.request_id += 1 self.api.ReqOrderInsert(order, self.request_id) return order def cover(self, instrument_id, volume, limit_price=0, time_condition=ApiStruct.TC_GFD, market_data=None): direction = ApiStruct.D_Buy order = self._make_order(instrument_id, direction, volume) order.CombOffsetFlag = ApiStruct.OF_Close order.OrderPriceType = ApiStruct.OPT_LimitPrice if not self._set_price(order, direction, limit_price, time_condition, market_data): return None self.request_id += 1 self.api.ReqOrderInsert(order, self.request_id) return order def cover_today(self, instrument_id, volume, limit_price=0, time_condition=ApiStruct.TC_GFD, market_data=None): direction = ApiStruct.D_Buy order = self._make_order(instrument_id, direction, volume) order.CombOffsetFlag = ApiStruct.OF_CloseToday order.OrderPriceType = ApiStruct.OPT_LimitPrice if not self._set_price(order, direction, limit_price, time_condition, market_data): return None self.request_id += 1 self.api.ReqOrderInsert(order, self.request_id) return order def open(self, instrument_id, posi_direction, volume, limit_price=0, time_condition=ApiStruct.TC_GFD, market_data=None): if posi_direction == ApiStruct.PD_Long: return self.buy(instrument_id, volume, limit_price, time_condition, market_data) elif posi_direction == ApiStruct.PD_Short: return self.short(instrument_id, volume, limit_price, time_condition, market_data) def close(self, instrument_id, posi_direction, volume, limit_price=0, time_condition=ApiStruct.TC_GFD, market_data=None): if posi_direction == ApiStruct.PD_Long: return self.sell(instrument_id, volume, limit_price, time_condition, market_data) elif posi_direction == ApiStruct.PD_Short: return self.cover(instrument_id, volume, limit_price, time_condition, market_data) def close_today(self, instrument_id, posi_direction, volume, limit_price=0, time_condition=ApiStruct.TC_GFD, market_data=None): if posi_direction == ApiStruct.PD_Long: return self.sell_today(instrument_id, volume, limit_price, time_condition, market_data) elif posi_direction == ApiStruct.PD_Short: return self.cover_today(instrument_id, volume, limit_price, time_condition, market_data) def refer(self, instrument_id, exchange_id, order_sys_id): order_action = ApiStruct.InputOrderAction( BrokerID=self.broker_id, InvestorID=self.user_id, InstrumentID=instrument_id, ActionFlag=ApiStruct.AF_Delete, ExchangeID=exchange_id, OrderSysID=order_sys_id) self.request_id += 1 return self.api.ReqOrderAction(order_action, self.request_id) == 0 def refer_local(self, instrument_id, order_ref): order_action = ApiStruct.InputOrderAction( BrokerID=self.broker_id, InvestorID=self.user_id, InstrumentID=instrument_id, ActionFlag=ApiStruct.AF_Delete, FrontID=self.front_id, SessionID=self.session_id, OrderRef=order_ref) self.request_id += 1 return self.api.ReqOrderAction(order_action, self.request_id) == 0
class PooledDB: """Pool for DB-API 2 connections. After you have created the connection pool, you can use connection() to get pooled, steady DB-API 2 connections. """ version = __version__ def __init__(self, creator, mincached=0, maxcached=0, maxshared=0, maxconnections=0, blocking=False, maxusage=None, setsession=None, failures=None, *args, **kwargs): """Set up the DB-API 2 connection pool. creator: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module mincached: initial number of idle connections in the pool (0 means no connections are made at startup) maxcached: maximum number of idle connections in the pool (0 or None means unlimited pool size) maxshared: maximum number of shared connections (0 or None means all connections are dedicated) When this maximum number is reached, connections are shared if they have been requested as shareable. maxconnections: maximum number of connections generally allowed (0 or None means an arbitrary number of connections) blocking: determines behavior when exceeding the maximum (if this is set to true, block and wait until the number of connections decreases, otherwise an error will be reported) maxusage: maximum number of reuses of a single connection (0 or None means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). setsession: optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to ...", "set time zone ..."] failures: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InternalError) is not adequate args, kwargs: the parameters that shall be passed to the creator function or the connection constructor of the DB-API 2 module """ try: threadsafety = creator.threadsafety except AttributeError: try: if not callable(creator.connect): raise AttributeError except AttributeError: threadsafety = 2 else: threadsafety = 0 if not threadsafety: raise NotSupportedError("Database module is not thread-safe.") self._creator = creator self._args, self._kwargs = args, kwargs self._maxusage = maxusage self._setsession = setsession self._failures = failures if mincached is None: mincached = 0 if maxcached is None: maxcached = 0 if maxconnections is None: maxconnections = 0 if maxcached: if maxcached < mincached: maxcached = mincached self._maxcached = maxcached else: self._maxcached = 0 if threadsafety > 1 and maxshared: self._maxshared = maxshared self._shared_cache = [] # the cache for shared connections else: self._maxshared = 0 if maxconnections: if maxconnections < maxcached: maxconnections = maxcached if maxconnections < maxshared: maxconnections = maxshared self._maxconnections = maxconnections else: self._maxconnections = 0 self._idle_cache = [] # the actual pool of idle connections self._condition = Condition() if not blocking: def wait(): raise TooManyConnections self._condition.wait = wait self._connections = 0 # Establish an initial number of idle database connections: idle = [self.dedicated_connection() for i in range(mincached)] while idle: idle.pop().close() def steady_connection(self): """Get a steady, unpooled DB-API 2 connection.""" return connect(self._creator, self._maxusage, self._setsession, self._failures, True, *self._args, **self._kwargs) def connection(self, shareable=True): """"Get a steady, cached DB-API 2 connection from the pool. If shareable is set and the underlying DB-API 2 allows it, then the connection may be shared with other threads. """ if shareable and self._maxshared: self._condition.acquire() try: while (not self._shared_cache and self._maxconnections and self._connections >= self._maxconnections): self._condition.wait() if len(self._shared_cache) < self._maxshared: # shared cache is not full, get a dedicated connection try: # first try to get it from the idle cache con = self._idle_cache.pop(0) except IndexError: # else get a fresh connection con = self.steady_connection() con = SharedDBConnection(con) self._connections += 1 else: # shared cache full or no more connections allowed self._shared_cache.sort() # least shared connection first con = self._shared_cache.pop(0) # get it con.share() # increase share of this connection # put the connection (back) into the shared cache self._shared_cache.append(con) self._condition.notify() finally: self._condition.release() con = PooledSharedDBConnection(self, con) else: # try to get a dedicated connection self._condition.acquire() try: while (self._maxconnections and self._connections >= self._maxconnections): self._condition.wait() # connection limit not reached, get a dedicated connection try: # first try to get it from the idle cache con = self._idle_cache.pop(0) except IndexError: # else get a fresh connection con = self.steady_connection() con = PooledDedicatedDBConnection(self, con) self._connections += 1 finally: self._condition.release() return con def dedicated_connection(self): """Alias for connection(shareable=False).""" return self.connection(False) def unshare(self, con): """Decrease the share of a connection in the shared cache.""" self._condition.acquire() try: con.unshare() shared = con.shared if not shared: # connection is idle, try: # so try to remove it self._shared_cache.remove(con) # from shared cache except ValueError: pass # pool has already been closed finally: self._condition.release() if not shared: # connection has become idle, self.cache(con.con) # so add it to the idle cache def cache(self, con): """Put a dedicated connection back into the idle cache.""" self._condition.acquire() try: if not self._maxcached or len(self._idle_cache) < self._maxcached: # the idle cache is not full, so put it there, but try: # before returning the connection back to the pool, con.rollback() # perform a rollback # in order to prevent uncommited actions from being # unintentionally commited by some other thread except Exception: # if an error occurs (no transaction, not supported) pass # then it will be silently ignored self._idle_cache.append(con) # append it to the idle cache else: # if the idle cache is already full, con.close() # then close the connection self._connections -= 1 self._condition.notify() finally: self._condition.release() def close(self): """Close all connections in the pool.""" self._condition.acquire() try: while self._idle_cache: # close all idle connections con = self._idle_cache.pop(0) try: con.close() except Exception: pass if self._maxshared: # close all shared connections while self._shared_cache: con = self._shared_cache.pop(0).con try: con.close() except Exception: pass self._connections -= 1 self._condition.notifyAll() finally: self._condition.release() def __del__(self): """Delete the pool.""" try: self.close() except Exception: pass
test = 1 # 2. queue import Queue # deque 是线程安全的 detail_url_queue = Queue(maxsize=1000) # 线程同步: 保证调用同一变量的先后执行 from threading import Lock, RLock, Condition lock = Lock() # lock.acquire() lock.release() lock = RLock() # 可重入的锁, 可连续调用acquire lock.acquire() lock.release() cond = Condition() cond.acquire() cond.wait() cond.notify() cond.release() # 进程间通信 from multiprocessing import Process, Queue, Pool, Manager, Pipe # Queue # 要使用multiprocessing, 或是manager的 # multiprocessing中的queue不能用于pool进程池 # pool中的进程间通信需要使用manager中的queue queue = Queue() manager = Manager() m_queue = manager.Queue(10) # pipe pipe的性能高于queue pipe只能适用于两个进程 receive_pipe, send_pipe = Pipe()
class Dispatcher(InstrumentedThread): def __init__(self, timeout=10): super().__init__(name='Dispatcher') self._timeout = timeout self._msg_type_handlers = {} self._in_queue = queue.PriorityQueue() self._send_message = {} self._send_last_message = {} self._message_information = {} self._condition = Condition() self._dispatch_timers = {} self._priority = {} def _get_dispatch_timer(self, tag): if tag not in self._dispatch_timers: self._dispatch_timers[tag] = COLLECTOR.timer( 'dispatch_execution_time', tags={"handler": tag}, instance=self) return self._dispatch_timers[tag] def add_send_message(self, connection, send_message): """Adds a send_message function to the Dispatcher's dictionary of functions indexed by connection. Args: connection (str): A locally unique identifier provided by the receiver of messages. send_message (fn): The method that should be called by the dispatcher to respond to messages which arrive via connection. """ self._send_message[connection] = send_message LOGGER.debug("Added send_message function " "for connection %s", connection) def add_send_last_message(self, connection, send_last_message): """Adds a send_last_message function to the Dispatcher's dictionary of functions indexed by connection. Args: connection (str): A locally unique identifier provided by the receiver of messages. send_last_message (fn): The method that should be called by the dispatcher to respond to messages which arrive via connection, when the connection should be closed after the message has been sent. """ self._send_last_message[connection] = send_last_message LOGGER.debug("Added send_last_message function " "for connection %s", connection) def remove_send_message(self, connection): """Removes a send_message function previously registered with the Dispatcher. Args: connection (str): A locally unique identifier provided by the receiver of messages. """ if connection in self._send_message: del self._send_message[connection] LOGGER.debug("Removed send_message function " "for connection %s", connection) else: LOGGER.debug("Attempted to remove send_message " "function for connection %s, but no " "send_message function was registered", connection) def remove_send_last_message(self, connection): """Removes a send_last_message function previously registered with the Dispatcher. Args: connection (str): A locally unique identifier provided by the receiver of messages. """ if connection in self._send_last_message: del self._send_last_message[connection] LOGGER.debug("Removed send_last_message function " "for connection %s", connection) else: LOGGER.debug("Attempted to remove send_last_message " "function for connection %s, but no " "send_last_message function was registered", connection) def dispatch(self, connection, message, connection_id): if message.message_type in self._msg_type_handlers: priority = self._priority.get(message.message_type, Priority.LOW) message_id = _gen_message_id() self._message_information[message_id] = ( connection, connection_id, message, _ManagerCollection( self._msg_type_handlers[message.message_type]) ) self._in_queue.put_nowait((priority, message_id)) queue_size = self._in_queue.qsize() if queue_size > 10: LOGGER.debug("Dispatch incoming queue size: %s", queue_size) else: LOGGER.info("received a message of type %s " "from %s but have no handler for that type", get_enum_name(message.message_type), connection_id) def add_handler(self, message_type, handler, executor, priority=None): if not isinstance(handler, Handler): raise TypeError("%s is not a Handler subclass" % handler) if message_type not in self._msg_type_handlers: self._msg_type_handlers[message_type] = [ _HandlerManager(executor, handler)] else: self._msg_type_handlers[message_type].append( _HandlerManager(executor, handler)) if priority is not None: self._priority[message_type] = priority def set_message_priority(self, message_type, priority): self._priority[message_type] = priority def _process(self, message_id): _, connection_id, \ message, collection = self._message_information[message_id] try: handler_manager = next(collection) except IndexError: # IndexError is raised if done with handlers del self._message_information[message_id] return timer_tag = type(handler_manager.handler).__name__ timer_ctx = self._get_dispatch_timer(timer_tag).time() def do_next(result): timer_ctx.stop() try: self._determine_next(message_id, result) except Exception: # pylint: disable=broad-except LOGGER.exception( "Unhandled exception while determining next") handler_manager.execute(connection_id, message.content, do_next) def _determine_next(self, message_id, result): if result is None: LOGGER.debug('Ignoring None handler result, likely due to an ' 'unhandled error while executing the handler') return if result.status == HandlerStatus.DROP: del self._message_information[message_id] elif result.status == HandlerStatus.PASS: self._process(message_id) elif result.status == HandlerStatus.RETURN_AND_PASS: connection, connection_id, \ original_message, _ = self._message_information[message_id] if result.message_out and result.message_type: message = validator_pb2.Message( content=result.message_out.SerializeToString(), correlation_id=original_message.correlation_id, message_type=result.message_type) try: self._send_message[connection](msg=message, connection_id=connection_id) except KeyError: LOGGER.info("Can't send message %s back to " "%s because connection %s not in dispatcher", get_enum_name(message.message_type), connection_id, connection) self._process(message_id) else: LOGGER.error("HandlerResult with status of RETURN_AND_PASS " "is missing message_out or message_type") elif result.status == HandlerStatus.RETURN: connection, connection_id, \ original_message, _ = self._message_information[message_id] del self._message_information[message_id] if result.message_out and result.message_type: message = validator_pb2.Message( content=result.message_out.SerializeToString(), correlation_id=original_message.correlation_id, message_type=result.message_type) try: self._send_message[connection](msg=message, connection_id=connection_id) except KeyError: LOGGER.info("Can't send message %s back to " "%s because connection %s not in dispatcher", get_enum_name(message.message_type), connection_id, connection) else: LOGGER.error("HandlerResult with status of RETURN " "is missing message_out or message_type") elif result.status == HandlerStatus.RETURN_AND_CLOSE: connection, connection_id, \ original_message, _ = self._message_information[message_id] del self._message_information[message_id] if result.message_out and result.message_type: message = validator_pb2.Message( content=result.message_out.SerializeToString(), correlation_id=original_message.correlation_id, message_type=result.message_type) try: LOGGER.warning( "Sending hang-up in reply to %s to connection %s", get_enum_name(original_message.message_type), connection_id) self._send_last_message[connection]( msg=message, connection_id=connection_id) except KeyError: LOGGER.info("Can't send last message %s back to " "%s because connection %s not in dispatcher", get_enum_name(message.message_type), connection_id, connection) else: LOGGER.error("HandlerResult with status of RETURN_AND_CLOSE " "is missing message_out or message_type") with self._condition: if not self._message_information: self._condition.notify() def run(self): while True: try: _, msg_id = self._in_queue.get() if msg_id == -1: break self._process(msg_id) except Exception: # pylint: disable=broad-except LOGGER.exception("Unhandled exception while dispatching") def stop(self): self._in_queue.put_nowait((Priority.HIGH, -1)) def block_until_complete(self): """Blocks until no more messages are in flight, useful for unit tests. """ with self._condition: if self._message_information: self._condition.wait()
class QueueMessageHandler(MessageHandler, Thread): def __init__(self, previous_handler): Thread.__init__(self) MessageHandler.__init__(self, previous_handler) self.daemon = True self.queue = [] self.c = Condition() self.alive = True self.start() def handle_message(self, msg): with self.c: should_notify = len(self.queue) == 0 self.queue.append(msg) if len(self.queue) > self.queue_length: del self.queue[0:len(self.queue) - self.queue_length] if should_notify: self.c.notify() def transition(self): if self.throttle_rate == 0 and self.queue_length == 0: self.finish() return MessageHandler(self) elif self.queue_length == 0: self.finish() return ThrottleMessageHandler(self) else: with self.c: if len(self.queue) > self.queue_length: del self.queue[0:len(self.queue) - self.queue_length] self.c.notify() return self def finish(self): """ If throttle was set to 0, this pushes all buffered messages """ # Notify the thread to finish with self.c: self.alive = False self.c.notify() self.join() def run(self): while self.alive: with self.c: while self.alive and (self.time_remaining() > 0 or len(self.queue) == 0): if len(self.queue) == 0: self.c.wait() else: self.c.wait(self.time_remaining()) if self.alive and self.time_remaining() == 0 and len( self.queue) > 0: try: MessageHandler.handle_message(self, self.queue[0]) except: traceback.print_exc(file=sys.stderr) del self.queue[0] while self.time_remaining() == 0 and len(self.queue) > 0: try: MessageHandler.handle_message(self, self.queue[0]) except: traceback.print_exc(file=sys.stderr) del self.queue[0]
class System: def __init__(self, config): self.tasks = deque() self.config = config self.scheduler = Scheduler(self) self._messages = deque() self.message_manager = MessageManager(self) self.events_manager = SystemEventsManager(self) self._a_context = None self._a_ref = None self._result = None self._trigger = Condition() self._is_running = False self._active_schedulers = 0 @property def is_running(self): return self._is_running def start(self): self._is_running = True self.events_manager.start() self.scheduler.start() with self._trigger: self._trigger.wait() def stop(self): self._is_running = False with self.events_manager.trigger: self.events_manager.trigger.notify_all() self.events_manager.join() with self.scheduler.trigger: self.scheduler.trigger.notify_all() self.scheduler.join() def spawn(self, actor_class, *args, **kwargs): return self.scheduler.spawn_actor( SpawnRequest( parent_ref=ActorRef(0, 0), actor_class=actor_class, args=args, kwargs=kwargs ) ) def send(self, target_ref, message): self.message_manager.send( Message( sender_ref=ActorRef(0, 0), target_ref=target_ref, message=message ) ) def ask(self, target_ref, message, timeout=None): context = uuid.uuid4().int self._a_ref = target_ref self._a_context = context self._result = None self.message_manager.send( Message( sender_ref=ActorRef(0, 0), target_ref=target_ref, message=message, context=context ) ) with self._trigger: self._trigger.wait(timeout) result = self._result self._result = None return result
class MainThreadController(object): def __init__(self): if not is_main_thread(): raise Exception( "A controller can only be created from the main thread") self._main_wake_up = Condition() self._main_wake_up.acquire() self._obj = None self._caller_wake_up = None self._args = None self._kwargs = None self._return = None self._quit = False def invoke(self, obj, *args, **kwargs): if is_main_thread(): return obj.__call__(*args, **kwargs) released = False self._main_wake_up.acquire() try: self._caller_wake_up = Condition() with self._caller_wake_up: self._obj = obj self._args = args self._kwargs = kwargs # tell the main thread to wake up self._main_wake_up.notify_all() self._main_wake_up.release() released = True self._caller_wake_up.wait() # tell the main thread that we received the result: self._caller_wake_up.notify_all() ret = self._return return ret finally: self._obj = None self._args = None self._kwargs = None self._caller_wake_up = None self._return = None if not released: self._main_wake_up.release() def quit(self): self._main_wake_up.acquire() try: self._quit = True self._main_wake_up.notify_all() finally: self._main_wake_up.release() def run(self): if not is_main_thread(): raise Exception("run can only be called from the main thread!") from . import signals def signal_handler(signal, frame): self._quit = True signals.add_sigint_handler(signal_handler) while True: try: self._main_wake_up.wait(1.0) except KeyboardInterrupt: self._quit = True if self._quit: return elif self._caller_wake_up is None: # we timed out continue with self._caller_wake_up: self._return = self._obj.__call__(*self._args, **self._kwargs) self._caller_wake_up.notify_all() # wait for the calling thread to confirm it received the result while not self._caller_wake_up.wait(1.0): if self._quit: return