def run(conn): """Function to handle running implosion generation in separate :py:class:`multithreading.Process` :param conn: A connection, i.e. one end of a `Pipe()` """ # Need duck-checking instead of real type-checking... assert hasattr(conn, 'send') and hasattr(conn, 'recv') # Get the implosion object from the pipe: imp = conn.recv() assert isinstance(imp, Implosion) connLock = Lock() # Run in a separate thread in this process: def impRun(): nonlocal imp, conn try: imp.generate() except Exception as e: connLock.acquire() conn.send(e) connLock.release() t = Thread(target=impRun) t.start() while t.is_alive(): connLock.acquire() conn.send(imp.progress()) connLock.release() time.sleep(0.01) # When the thread is done, send the Implosion object back: conn.send(imp)
class PriorityQueue: """ a non blocking priority queue """ def __init__(self): self.queue = [] self.lock = Lock() def __iter__(self): return iter(self.queue) def __delitem__(self, key): del self.queue[key] def put(self, element): self.lock.acquire() heappush(self.queue, element) self.lock.release() def get(self): """ return element or None """ self.lock.acquire() try: el = heappop(self.queue) return el except IndexError: return None, None finally: self.lock.release()
class Reader(Thread): def __init__(self): super(Reader, self).__init__() self.buffer = [] self.lock = Lock() def run(self): while True: self.lock.acquire() self.buffer.append(getch()) self.lock.release() sleep(0) def read(self): self.start() while True: if self.buffer: # The reader may be in the middle of receiving multi-byte # input; wait long enough for this to finish. sleep(0.1) self.lock.acquire() chars, self.buffer = self.buffer, [] self.lock.release() yield chars
class DataWindow(Thread): def __init__(self,data_adapter): Thread.__init__(self) self.win = N.zeros((100,3)) self.winlock = Lock() self.data_adapter = data_adapter def run(self): self.data_adapter.start() self.running = True while self.running: self.winlock.acquire() try: while 1: newdata = self.data_adapter.q.get(block=False) self.win[:-1,:] = self.win[1:,:] self.win[-1,:] = newdata[1:] except Queue.Empty: pass finally: self.winlock.release() self.data_adapter.stop() def stop(self): self.running = False
class WorkerTask(object): """ This class used to represent page loading task """ def __init__(self, args, routine, user_data = None): self._lock_complete = Lock() self.__complete = False self.gathered = False self.args = args if callable(routine): self.routine = routine else: raise AttributeError('<routine> argument should be callable function') self.user_data = user_data self.result = [] self.thread = None def _get_complete(self): """complete property getter""" self._lock_complete.acquire() complete = self.__complete self._lock_complete.release() return complete def _set_complete(self, state): """complete property setter""" if type(state) != types.BooleanType: raise TypeError('state should be boolean') self._lock_complete.acquire() self.__complete = state self._lock_complete.release() complete = property(_get_complete, _set_complete)
class FutureTask(Task): def __init__(self,function,*args,**kwargs): self.function = function self.args = args self.kwargs = kwargs self.lock = Lock() self.e = None self.executing = False; def _execute(self): try: self.lock.acquire() self.executing = True; self.result = self.function(*self.args,**self.kwargs); except Exception as e: self.e = e; finally: self.lock.release() def result(self): try: while True: if self.executing: #检查标志位,避免死锁 self.lock.acquire() if self.e is not None: raise self.e; return self.result finally: self.lock.release()
class _CoreScheduleThread(Thread): def __init__(self,threadpool): self.scheduletasks = []; self.tasklock = Lock(); self.condition = Condition(Lock()) self.threadpool = threadpool Thread.__init__(self) def run(self): while True: self.condition.acquire() if len(self.scheduletasks) == 0: self.condition.wait(); else: task = self.scheduletasks.pop(0) if dates.current_timestamps()>=task.nexttime: self.threadpool.execute(task.function,*task.args,**task.kwargs) task.nexttime = dates.current_timestamps()+task.period; else: self.condition.wait(task.nexttime-dates.current_timestamps()) self.addtask(task) self.condition.release() def addtask(self,task): # copy on write self.tasklock.acquire() tasks = [ t for t in self.scheduletasks ] tasks.append(task) tasks.sort(key=lambda task:task.nexttime) self.scheduletasks = tasks self.tasklock.release()
class ProgressBarLogger: def __init__(self, msg, total): self.msg = msg self.total = total self.status = 0 self.lock = Lock() def log(self, *_): self.lock.acquire() self.status += 1 self._print_progress_bar(self.status, self.total, prefix=self.msg, bar_length=50) self.lock.release() # from here http://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console # Print iterations progress @staticmethod def _print_progress_bar(iteration, total, prefix='', suffix='', decimals=2, bar_length=100): """ Call in a loop to create terminal progress bar @params: iteration - Required : current iteration (Int) total - Required : total iterations (Int) prefix - Optional : prefix string (Str) suffix - Optional : suffix string (Str) decimals - Optional : number of decimals in percent complete (Int) bar_length - Optional : character length of bar (Int) """ filled_length = int(round(bar_length * iteration / float(total))) percents = round(100.00 * (iteration / float(total)), decimals) bar_char = '#' * filled_length + '-' * (bar_length - filled_length) sys.stdout.write('\r%s |%s| %s%s %s' % (prefix, bar_char, percents, '%', suffix)) sys.stdout.flush() if iteration == total: sys.stdout.write('\n') sys.stdout.flush()
def handle(self): #try: data, socket = self.request lock = Lock() lock.acquire() DataOffset = struct.unpack('<H',data[139:141])[0] BrowserPacket = data[82+DataOffset:] ReqType = RequestType(BrowserPacket[0]) Domain = Decode_Name(data[49:81]) Name = Decode_Name(data[15:47]) Role1 = NBT_NS_Role(data[45:48]) Role2 = NBT_NS_Role(data[79:82]) Fprint = WorkstationFingerPrint(data[190:192]) Roles = ParseRoles(data[192:196]) print text("[BROWSER] Request Type : %s" % ReqType) print text("[BROWSER] Address : %s" % self.client_address[0]) print text("[BROWSER] Domain : %s" % Domain) print text("[BROWSER] Name : %s" % Name) print text("[BROWSER] Main Role : %s" % Role1) print text("[BROWSER] 2nd Role : %s" % Role2) print text("[BROWSER] Fingerprint : %s" % Fprint) print text("[BROWSER] Role List : %s" % Roles) RAPThisDomain(self.client_address[0], Domain) lock.release()
class Skeleton(object): def __init__(self, config_file): self.logger = logging.getLogger(self.__class__.__name__) config = Config(config_file) config.get_configs() self.config = config self.lock = Lock() self.queues = dict() def produce_task(self, tasker_name, tasker): while True: try: queue = self.queues.get(tasker_name) self.logger.info("old {0} queue size: {1}".format(tasker_name, queue.qsize())) queue.put(tasker.size) self.logger.info("new {0} queue size: {1}".format(tasker_name, queue.qsize())) except Exception as error: self.logger.exception("{0} {1}".format(tasker_name, error)) finally: time.sleep(self.config.scan_task_interval) return def consume_task(self, n, tasker_name, tasker): while True: handler = Handler() queue = self.queues.get(tasker_name) if queue.empty(): time.sleep(self.config.wait_time) continue try: while not queue.empty(): size = queue.get() self.lock.acquire() handler.human_readable(size) self.lock.release() except Exception as error: self.logger.exception('Thread-{0}: error {1}'.format(n, error)) finally: del(handler) def do_work(self): for tasker_name, tasker in self.config.taskers.items(): self.queues[tasker_name] = Queue() # Spwan produce_task thread t = Thread(target=self.produce_task, args=(tasker_name, tasker)) t.setDaemon(True) t.start() # Spwan consume_task thread for n in range(tasker.max_workers): t = Thread(target=self.consume_task, args=(n, tasker_name, tasker)) t.setDaemon(True) t.start() while True: signal.signal(signal.SIGTERM, sigterm_handler) # Round robin and Sleep some seconds. time.sleep(self.config.scan_task_interval) return
class WaitCursor(Thread): """A waiting cursor for long operation that catch output and flush it after waiting""" def __init__(self): self.state = "WAIT" self.lock = Lock() # Lock used to synchronise IO and cursor stop Thread.__init__(self) def run(self): """Method executed when the thread object start() method is called""" realStdout = sys.stdout # Backup stdout tmpStdout = StringIO() # Store here all data output during waiting state sys.stdout = tmpStdout # Capture stdout cursorState = ("-", "\\", "|", "/") i = 0 self.lock.acquire() while self.state == "WAIT": realStdout.write(cursorState[i % 4]) realStdout.flush() sleep(0.1) realStdout.write("\b") i += 1 # Restore standard output and print temp data sys.stdout = realStdout sys.stdout.write(tmpStdout.getvalue()) sys.stdout.flush() self.lock.release() def stop(self): self.state = "STOP" self.lock.acquire() # Wait end of IO flush before returning
class Tips(object): """ Manage Tips Events. """ def __init__(self, enable): self.enable = enable self._tips = {} self._new_tips = set() self.lock = Lock() if self.enable: self.fetcher = Fetcher(self._tips, self.lock, self._new_tips) self.cleaner = Cleaner(self._tips, self.lock, self._new_tips) self.fetcher.start() self.cleaner.start() def tips(self): return self._tips.values() def new_tips(self): if self._new_tips: wait_free_acquire(self.lock) res = [self._tips[x] for x in self._new_tips] self._new_tips.clear() self.lock.release() return res else: return [] def stop(self): if self.enable: self.fetcher.finnish() self.cleaner.finnish()
class InMemoryItemValue(object): _lock = None """:type _lock Lock""" def __init__(self, value=None, expire_in=None): self._lock = Lock() self._value = value self._expire_in = None self._expire_in_time = None self.update_expire_time(expire_in) @property def value(self): return self._value @value.setter def value(self, val): self._lock.acquire() self._value = val self._expire_in = datetime.now() + timedelta(seconds=float(self._expire_in_time)) if self._expire_in_time else None self._lock.release() def update_expire_time(self, t): self._expire_in_time = t @property def is_expired(self): return (self._expire_in - datetime.now()).days < 0 if self._expire_in else False
class Queue: """Command queue class """ def __init__(self): self.lock = Lock() self.locks = {} def queue(self, command, *args): check = inspect.getargspec(command) cmdname = command.__name__ if command.__name__ else "uknown_cmd" if len(check[0]) != len(args): logging.warn("Queue command '%s' expected %u args, got %u!" % (cmdname, len(check[0]), len(args))) # If we have enough args, try running the command if len(args) >= len(check[0]): args = args[:len(check[0])] # Resize arg list if needed ret = None server = args[0] self.lock.acquire() if not server in self.locks: self.locks[server] = Lock() self.lock.release() self.locks[server].acquire() # Run in an enclosure, so as to be able to release lock if it fails try: ret = command(*args) except Exception as err: logging.warn("Queue command returned error: %s" % err) self.locks[server].release() if ret: return ret return None
def start_manager(self): exit_flags[self.tab_id] = 0 log.info('START | Layers Download Manager') thread_list = ['Alpha', 'Bravo', 'Charlie', 'Delta', 'Echo', 'Foxtrot', 'Golf', 'Hotel', 'India', 'Juliet'] queue_lock = Lock() work_queue = Queue.Queue(len(self.file_paths_and_sizes)) threads = [] for thread_name in thread_list: key = str(uuid.uuid4()) thread = LayerDownloadThread(self.source, thread_name, work_queue, queue_lock, key, self.target_dir, self.tab_id) thread.start() if not threads_map_key in thread_manager_processes: thread_manager_processes[threads_map_key] = {} thread_manager_processes[threads_map_key][key] = thread threads.append(thread) queue_lock.acquire() for word in self.file_paths_and_sizes: work_queue.put(word) queue_lock.release() while not work_queue.empty(): pass exit_flags[self.tab_id] = 1 for t in threads: t.join() log.info('DONE | Layers Download Manager')
class Store: def __init__(self, item_number, person_capacity): self.items_remaining_lock = Lock() self.space_available_lock = Lock() self.item_number = item_number self.person_capacity = person_capacity self.items_remaining = self.item_number self.space_available = self.person_capacity def enter(self): while True: self.space_available_lock.acquire() if self.space_available > 0: self.space_available -= 1 self.space_available_lock.release() break self.space_available_lock.release() def buy(self): self.items_remaining_lock.acquire() if self.items_remaining <= 0: return False self.items_remaining_lock.release() time.sleep(random.randint(5, 10)) self.items_remaining_lock.acquire() if self.items_remaining <= 0: return False self.items_remaining -= 1 self.items_remaining_lock.release() self.space_available_lock.acquire() self.space_available += 1 self.space_available_lock.release() return True
class Manager: def __init__(self): self.networks = set() self.logs = [] self.next_log_id = 0 self.log_lock = Lock() def notifyNetUp(self, pno, net_name): self.networks.add(net_name) sys.stdout.write(">> network: %s is up (%d)\n" % (net_name, len(self.networks))) def getNetworks(self): return self.networks def getLogs(self, since=-1): if since >= 0: return filter(lambda l: l['id'] > since, self.logs) else: return self.logs def putLog(self, host, log): self.log_lock.acquire() l = {'id': self.next_log_id, 'host': host, 'log': log} self.next_log_id = self.next_log_id + 1 sys.stdout.write(">> log: %s\n" % json.dumps(l)) self.logs.append(l) self.log_lock.release()
class shm: def __init__(self, data=None): self._data = data self._lock = Lock() def set(self, data, lock=True): if lock: with self._lock: self._data = data else: self._data = data def get(self, lock=True): if lock: with self._lock: data = self._data return data else: return self._data def acquire(self): self._lock.acquire() def release(self): self._lock.release() data = property(get, set)
class DPMClient(): def __init__(self, uid=None, key=None): self._lock = Lock() self._uid = uid self._key = None if key: self._key = rsa.PublicKey.load_pkcs1(key) def request(self, addr, port, buf): self._lock.acquire() try: return self._request(addr, port, buf) finally: self._lock.release() def _request(self, addr, port, buf): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((addr, port)) try: if self._key: stream = Stream(sock, uid=self._uid, key=self._key) else: stream = Stream(sock) stream.write( buf) if self._key: stream = Stream(sock) _, _, res = stream.readall() return res finally: sock.close()
class Syncer(object): def __init__(self, slave): self.slave = slave self.lock = Lock() self.pb = PostBox() for i in range(int(gconf.sync_jobs)): t = Thread(target=self.syncjob) t.start() def syncjob(self): while True: pb = None while True: self.lock.acquire() if self.pb: pb, self.pb = self.pb, PostBox() self.lock.release() if pb: break time.sleep(0.5) pb.close() pb.wakeup(self.slave.rsync(pb)) def add(self, e): while True: try: self.pb.append(e) return self.pb except BoxClosedErr: pass
class BlaLock(object): """ Simple wrapper class for the thread.lock class which only raises an exception when trying to release an unlocked lock if it's initialized with strict=True. """ def __init__(self, strict=False, blocking=True): self.__strict = strict self.__blocking = blocking self.__lock = Lock() def acquire(self): self.__lock.acquire(self.__blocking) def release(self): try: self.__lock.release() except ThreadError: if self.__strict: raise def locked(self): return self.__lock.locked() def __enter__(self, *args): self.acquire() def __exit__(self, *args): self.release()
class ProcessThread(Thread): # Initialize this thread def __init__(self): Thread.__init__(self) self.stopped = Event() # Cancel Event self.mutex = Lock() self.data = None # Threaded code def run(self): while not self.stopped.isSet(): # Check if status data is available and process it data = None self.mutex.acquire() if self.data: data = self.data self.data = None self.mutex.release() if data: global outputfile try: fp = file(outputfile, 'wu') fp.write(data.encode('utf-8')) fp.close() except Exception, e: console.log(LOG_PYTHON, "Couldn't write status to '%s': %s.\n" % (outputfile, str(e))) self.stopped.wait(0.5)
def _setup_to_do_n_cycles(self, number_of_cycles: int, updates_each_cycle: UpdateCollection=None): """ Sets up the test so that the retriever will only do n cycles. :param number_of_cycles: the number of cycles to do """ if updates_each_cycle is None: updates_each_cycle = UpdateCollection([]) semaphore = Semaphore(0) lock_until_counted = Lock() lock_until_counted.acquire() def increase_counter(*args) -> UpdateCollection: semaphore.release() lock_until_counted.acquire() return updates_each_cycle self.retrieval_manager.update_mapper.get_all_since.side_effect = increase_counter self.retrieval_manager.start() run_counter = 0 while run_counter < number_of_cycles: semaphore.acquire() run_counter += 1 lock_until_counted.release() if run_counter == number_of_cycles: self.retrieval_manager.stop() self.retrieval_manager.update_mapper.get_all_since.side_effect = None
class Promise: def __init__(self): self.value = None self.mutex = Lock() self.callbacks = [] def updateValue(self, new_value): if (self.mutex.acquire(blocking = False) and self.value is None): self.value = new_value for cb in self.callbacks: cb(new_value) self.mutex.release() else: raise RuntimeError("cannot set the value of an already resolved promise") def addCallback(self, cb): self.mutex.acquire(blocking = True) self.callbacks.append(cb) self.mutex.release() return self def map(self, f): fp = Promise() def chain(v): fp.updateValue(f(v)) self.addCallback(chain) return fp def flatMap(self, f): fp = Promise() def chain(v): f(v).addCallback(fp.updateValue) self.addCallback(chain) return fp
class TUNERS: def __init__(self, str): from threading import Lock tuners = "".join(str.split()) # remove white space tuners = tuners.split(',') tuners = [tuple(x.split(':')[0:2]) for x in tuners] # Add priority self.tuner_list = [(i, v[0], v[1]) for i,v in enumerate(tuners)] heapq.heapify(self.tuner_list) self.lock = Lock() def get_tuner(self): self.lock.acquire() try: tuner = heapq.heappop(self.tuner_list) except IndexError: tuner = None finally: self.lock.release() return tuner def put_tuner(self, tuner): self.lock.acquire() heapq.heappush(self.tuner_list, tuner) self.lock.release()
class TfBroadcasterThread(Thread): def __init__(self,child_frame,parent_frame,tf_br=None): Thread.__init__(self) rospy.loginfo("Initializing tf broadcaster with child frame "+child_frame+" and parent frame "+parent_frame) if tf_br is None: self.tf_br = tf.TransformBroadcaster() else: self.tf_br = tf_br self.translation = None self.quaternion = None self.child_frame = child_frame self.parent_frame = parent_frame self.has_transformation=False self.lock=Lock() def set_transformation(self,translation,quaternion): self.lock.acquire() self.translation = translation self.quaternion = quaternion self.lock.release() self.has_transformation =True def run(self): while not rospy.is_shutdown(): try: if self.has_transformation: self.lock.acquire() self.tf_br.sendTransform(self.translation ,self.quaternion , rospy.Time.now(), self.child_frame,self.parent_frame) self.lock.release() except Exception,e: print 'TfBroadcasterThread:',e
class PandoraPool(object): def __init__(self, poolSize, proxy=None, expireTime=3600): self.size = poolSize self.proxy = proxy self.expire = expireTime self.pool = [self.createPandoraAgent() for i in xrange(self.size)] self.mutex = Lock() def createPandoraAgent(self): return PandoraAgent(datetime.now() + timedelta(0, self.expire), self.proxy) def refreshPandoraAgent(self, agent): if agent.isExpired(): agent.authenticate_connection() agent.setExpireDate(datetime.now() + timedelta(0, self.expire)) return agent def getAgent(self): try: return self.refreshPandoraAgent(self.pool.pop()) except IndexError: return self.createPandoraAgent() def hasAvailableConnections(self): return len(self.pool) > 0 def releaseAgent(self, agent): self.mutex.acquire() if len(self.pool) < self.size: self.pool.append(agent) self.mutex.release()
class IndependentWorker(Thread): def __init__(self, id, manager, workerArgs = None): '''A worker needs a task stack(queue), a name (id) and a boss (manager)''' self.id = id self.manager = manager self.completedTasksLock = Lock() self.completedTasks = 0 Thread.__init__(self) def run(self): subject = self.manager.getTask() while subject != None: if self._task(subject): self.completedTasksLock.acquire() self.completedTasks += 1 self.completedTasksLock.release() subject = self.manager.getTask() self._close() sys.exit() def getAndResetCompletedTasks(self): completedTasks = self.completedTasks self.completedTasksLock.acquire() self.completedTasks = 0 self.completedTasksLock.release() return completedTasks def _task(self, subject): print 'You should initialize Worker._task(self, subject)' sys.exit() pass def _close(self): pass
class DebuggingLock: def __init__(self, name): self.lock = Lock() self.name = name def acquire(self, blocking = 1): self.print_tb("Acquire lock") self.lock.acquire(blocking) self.logmsg("===== %s: Thread %s acquired lock\n"% (self.name, currentThread().getName())) def release(self): self.print_tb("Release lock") self.lock.release() def logmsg(self, msg): loglock.acquire() logfile.write(msg + "\n") logfile.flush() loglock.release() def print_tb(self, msg): self.logmsg(".... %s: Thread %s attempting to %s\n"% \ (self.name, currentThread().getName(), msg) + \ "\n".join(traceback.format_list(traceback.extract_stack())))
class Serial: def __init__(self, port='COM5', rate=9600, timeout=10): self._serial = serial.Serial(port, rate, timeout=timeout) self._mutex = Lock() self._mutex.acquire() response = self._serial.readline().strip() print response if response != 'OK': raise Exception("Failed to communicate with the serial device!") self._mutex.release() def _shortCommand(self, command): self._serial.write(command) response = self._serial.readline() return response.strip() def _longCommand(self, command): response = self._shortCommand('RCV ' + str(len(command)) + "\n") if response != 'RDY': return None for i in range(int(math.ceil(len(command) / 128.0))): c = command[128*i:128*(i+1)] response = self._shortCommand(c) return self._serial.readline().strip() def command(self, command): self._mutex.acquire() if len(command) < 128: response = self._shortCommand(command + "\n") else: response = self._longCommand(command) self._mutex.release() return response
class PublisherConsistencyListener(SubscribeListener): """ This class is used to solve the problem that sometimes we create a publisher and then immediately publish a message, before the subscribers have set up their connections. Call attach() to attach the listener to a publisher. It sets up a buffer of outgoing messages, then when a new connection occurs, sends the messages in the buffer. Call detach() to detach the listener from the publisher and restore the original publish methods. After some particular timeout (default to 1 second), the listener stops buffering messages as it is assumed by this point all subscribers will have successfully set up their connections.""" timeout = 1 # Timeout in seconds to wait for new subscribers attached = False def attach(self, publisher): """ Overrides the publisher's publish method, and attaches a subscribe listener to the publisher, effectively routing incoming connections and outgoing publish requests through this class instance """ # Do the attaching self.publisher = publisher publisher.impl.add_subscriber_listener(self) self.publish = publisher.publish publisher.publish = self.publish_override # Set state variables self.lock = Lock() self.established_time = time() self.msg_buffer = [] self.attached = True def detach(self): """ Restores the publisher's original publish method and unhooks the subscribe listeners, effectively finishing with this object """ self.publisher.publish = self.publish if self in self.publisher.impl.subscriber_listeners: self.publisher.impl.subscriber_listeners.remove(self) self.attached = False def peer_subscribe(self, topic_name, topic_publish, peer_publish): """ Called whenever there's a new subscription. If we're still inside the subscription setup window, then we publish any buffered messages to the peer. We also check if we're timed out, but if we are we don't detach (due to threading complications), we just don't propagate buffered messages """ if not self.timed_out(): self.lock.acquire() msgs = copy(self.msg_buffer) self.lock.release() for msg in msgs: peer_publish(msg) def timed_out(self): """ Checks to see how much time has elapsed since the publisher was created """ return time() - self.established_time > self.timeout def publish_override(self, message): """ The publisher's publish method is replaced with this publish method which checks for timeout and if we haven't timed out, buffers outgoing messages in preparation for new subscriptions """ if not self.timed_out(): self.lock.acquire() self.msg_buffer.append(message) self.lock.release() self.publish(message)
class ReSampler: ''' Initializes the resampler particles: The particles to sample from weights: The weights of each particle state_lock: Controls access to particles and weights ''' def __init__(self, particles, weights, state_lock=None): self.particles = particles self.weights = weights # For speed purposes, you may wish to add additional member variable(s) that # cache computations that will be reused in the re-sampling functions # YOUR CODE HERE? if state_lock is None: self.state_lock = Lock() else: self.state_lock = state_lock ''' Performs independently, identically distributed in-place sampling of particles ''' def resample_naiive(self): self.state_lock.acquire() self.weights /= np.sum(self.weights) k = len(self.weights) #resampled_particles= np.zeros((self.particles.shape[0]), dtype= np.float) resampled_particles_indices = np.arange(0, k, 1) resampled_particles_indices = np.random.choice( resampled_particles_indices, size=k, replace=True, p=self.weights) resampled_particles_naiive = self.particles[ resampled_particles_indices] self.particles[:] = resampled_particles_naiive[:] # YOUR CODE HERE self.state_lock.release() ''' Performs in-place, lower variance sampling of particles (As discussed on pg 110 of Probabilistic Robotics) ''' def resample_low_variance(self): self.state_lock.acquire() # YOUR CODE HERE self.weights /= np.sum(self.weights) M = len(self.weights) r = np.random.rand(1) * 1.0 / M c = self.weights[0] U_array = np.zeros(M, dtype=float) last_element = r + ((M - 1.0) / M) U_array = np.linspace(r, last_element, M) #, endpoint=True, retstep= False) particles_resampled = np.zeros((self.particles.shape), dtype=np.float) weights_cumsum = np.cumsum(self.weights, dtype=np.float) particle_bins = np.digitize(U_array, weights_cumsum, right=False) print particle_bins particles_resampled = self.particles[particle_bins] self.particles[:] = particles_resampled[:] ''' reampled_particles= np.zeros((self.particles.shape), dtype= np.float) #print resampled_particles M= len(self.weights) r= np.random.rand(1) * 1.0/M c= self.weights[0] i=0 for m in xrange(M): U= r+ (m* 1.0/M) while U>c: i+=1 c=c + self.weights[i] resampled_particles[m,:]= self.particles[i,:] #print resampled_particles[m,:] self.particles[:] = resampled_particles[:] #print resampled_particles #print self.particles ''' self.state_lock.release()
class spider(): def __init__(self): self.Binary_Search = BinarySearch() # 实例一个二分法过滤器 self.Queue = Queue(100000) # 实例一个队列 self.thread_count = 10 # 线程数 self.lock = Lock() # 线程锁 self.event = Event() def requstGET(self, url): ''' 请求函数 :param url: url :return: text ''' try: response = requests.get(url, headers=headers, timeout=3, verify=False) response.encoding = response.apparent_encoding if response.status_code == 200: return response.text except Exception as e: return None def parser(self, url): ''' 解析网页,取得url保存至队列 :param url: :return: ''' url_hash = self.Binary_Search.hash(url) self.lock.acquire() # 线程锁 if self.Binary_Search.exists(self.Binary_Search.container, url_hash): # 判断url是否在布隆过滤器中 self.lock.release() # 解锁 return None # 在过滤器中直接结束函数 self.Binary_Search.container.append(url_hash) # 不在过滤器中,添加到过滤器中 self.Binary_Search.container.sort() self.lock.release() # 解锁 print(url) text = self.requstGET(url) # 请求url if text: soup = BeautifulSoup(text, 'html.parser') # 解析url for link in soup.find_all('a'): self.Queue.put_nowait(link.get('href')) # 取得新url保存到队列 self.event.wait() def thread_pool(self): ''' 线程池,从队列中取出url,循环开启子线程 :return: ''' while True: task_list = [] for i in range(0, self.thread_count): # 每次开启 thread_count 个线程 try: task_list.append( Thread(target=self.parser, args=(self.Queue.get_nowait(), ))) #创建子线程任务 except Exception as e: print(e) for task in task_list: # 开启线程任务 time.sleep(0.5) task.start() # [task.join() for task in task_list] # 等待这批线程全部结束 time.sleep(5) self.event.set() print(self.Queue.qsize())
class FireflyLedController(): class FireflyPattern(): def __init__(self, board_id, red, green, blue, speed, pattern_str, pattern_name=None): self.board_id = board_id self.red = red self.blue = blue self.green = green self.speed = speed self.pattern_str = pattern_str self.pattern_name = pattern_name if self.board_id is not None: self.packet = create_firefly_packet(self.board_id, [red, green, blue], self.speed, self.pattern_str) else: self.packet = [] def get_minimal(self): return { 'board_id': self.board_id, 'color': [self.red, self.green, self.blue], 'speed': self.speed, 'pattern': self.pattern_str, 'pattern_name': self.pattern_name } def __init__(self): self.createBroadcastSender() # swarms are 1-indexed, because swarm ID 0 means broadcast self.patterns = { board_id : FireflyLedController.FireflyPattern( board_id, 0.5, 0.7, 0.5, 2, "01100110000", "Default") \ for board_id in range(1, NUM_SWARMS + 1) } self.patternLock = Lock() self.is_running = True self.broadcastThread = Thread(target=self.broadcastFireflyPatterns) self.broadcastThread.start() #self.update_firmware() def update_firmware(self): if os.path.exists(FIRMWARE_PATH) and os.path.isfile(FIRMWARE_PATH): with open(FIRMWARE_PATH, 'rb') as fw: data = fw.read() self.fw_hash = hashlib.md5(data).hexdigest() self.have_firmware = True self.firmware_packet = create_firmware_packet( 0, "http://" + self.ipaddr + ":" + str(self.port) + "/firefly_leds/firmware/" + self.fw_hash, self.fw_hash) else: self.have_firmware = False def get_firmware_hash(self): return self.fw_hash if self.have_firmware else None def set_led_pattern(self, board_id, red, green, blue, speed, pattern_str, pattern_name=None): self.patternLock.acquire() self.patterns[board_id] = FireflyLedController.FireflyPattern( board_id, red, green, blue, speed, pattern_str, pattern_name, ) self.patternLock.release() def get_led_patterns(self): self.patternLock.acquire() pattern_copy = [ pattern.get_minimal() for pattern in set(self.patterns.values()) ] self.patternLock.release() return pattern_copy def set_service_addr(self, ipaddr, port): self.ipaddr = ipaddr self.port = port self.update_firmware() # yeah yeah, unexpected side effects. def broadcastFireflyPatterns(self): idx = 0 while self.is_running: print("BROADCAST") #print(f"{self.patterns}") #print(f"{len(self.patterns)}") self.patternLock.acquire() try: for pattern in self.patterns.values(): #print(f"Sending {pattern.packet}") self.sender_socket.sendto(pattern.packet, (MULTICAST_GROUP, LED_CMD_PORT)) time.sleep(0.01) except Exception as e: print(f"Exception {e}") pass # I'm concerned about transient networking issues... # and I really don't know what to do if they happen self.patternLock.release() time.sleep(1.0) # and every 10 seconds, also broadcast the current firmware hash if False: #if idx % 10 == 0: if self.have_firmware: print("FW UPDATE PACKET") self.sender_socket.sendto(self.firmware_packet, (MULTICAST_GROUP, LED_CMD_PORT)) idx += 1 def createBroadcastSender(self, ttl=4): self.sender_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sender_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # self.sender_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.sender_socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl)
class A3CAgent: # Actor-Critic Main Optimization Algorithm def __init__(self, env_name): # Initialization # Environment and PPO parameters self.env_name = env_name self.env = gym.make(env_name) self.action_size = self.env.action_space.n self.EPISODES, self.episode, self.max_average = 20000, 0, -21.0 # specific for pong self.lock = Lock() self.lr = 0.000025 self.ROWS = 80 self.COLS = 80 self.REM_STEP = 4 # Instantiate plot memory self.scores, self.episodes, self.average = [], [], [] self.Save_Path = 'Models' self.state_size = (self.REM_STEP, self.ROWS, self.COLS) if not os.path.exists(self.Save_Path): os.makedirs(self.Save_Path) self.path = '{}_A3C_{}'.format(self.env_name, self.lr) self.Model_name = os.path.join(self.Save_Path, self.path) # Create Actor-Critic network model self.Actor, self.Critic = OurModel(input_shape=self.state_size, action_space=self.action_size, lr=self.lr) # make predict function to work while multithreading self.Actor._make_predict_function() self.Critic._make_predict_function() global graph graph = tf.get_default_graph() def act(self, state): # Use the network to predict the next action to take, using the model prediction = self.Actor.predict(state)[0] action = np.random.choice(self.action_size, p=prediction) return action def discount_rewards(self, reward): # Compute the gamma-discounted rewards over an episode gamma = 0.99 # discount rate running_add = 0 discounted_r = np.zeros_like(reward) for i in reversed(range(0, len(reward))): if reward[ i] != 0: # reset the sum, since this was a game boundary (pong specific!) running_add = 0 running_add = running_add * gamma + reward[i] discounted_r[i] = running_add discounted_r -= np.mean(discounted_r) # normalizing the result discounted_r /= np.std(discounted_r) # divide by standard deviation return discounted_r def replay(self, states, actions, rewards): # reshape memory to appropriate shape for training states = np.vstack(states) actions = np.vstack(actions) # Compute discounted rewards discounted_r = self.discount_rewards(rewards) # Get Critic network predictions value = self.Critic.predict(states)[:, 0] # Compute advantages advantages = discounted_r - value # training Actor and Critic networks self.Actor.fit(states, actions, sample_weight=advantages, epochs=1, verbose=0) self.Critic.fit(states, discounted_r, epochs=1, verbose=0) def load(self, Actor_name, Critic_name): self.Actor = load_model(Actor_name, compile=False) #self.Critic = load_model(Critic_name, compile=False) def save(self): self.Actor.save(self.Model_name + '_Actor.h5') #self.Critic.save(self.Model_name + '_Critic.h5') pylab.figure(figsize=(18, 9)) def PlotModel(self, score, episode): self.scores.append(score) self.episodes.append(episode) self.average.append(sum(self.scores[-50:]) / len(self.scores[-50:])) if str(episode)[-2:] == "00": # much faster than episode % 100 pylab.plot(self.episodes, self.scores, 'b') pylab.plot(self.episodes, self.average, 'r') pylab.ylabel('Score', fontsize=18) pylab.xlabel('Steps', fontsize=18) try: pylab.savefig(self.path + ".png") except OSError: pass return self.average[-1] def imshow(self, image, rem_step=0): cv2.imshow(self.Model_name + str(rem_step), image[rem_step, ...]) if cv2.waitKey(25) & 0xFF == ord("q"): cv2.destroyAllWindows() return def GetImage(self, frame, image_memory): if image_memory.shape == (1, *self.state_size): image_memory = np.squeeze(image_memory) # croping frame to 80x80 size frame_cropped = frame[35:195:2, ::2, :] if frame_cropped.shape[0] != self.COLS or frame_cropped.shape[ 1] != self.ROWS: # OpenCV resize function frame_cropped = cv2.resize(frame, (self.COLS, self.ROWS), interpolation=cv2.INTER_CUBIC) # converting to RGB (numpy way) frame_rgb = 0.299 * frame_cropped[:, :, 0] + 0.587 * frame_cropped[:, :, 1] + 0.114 * frame_cropped[:, :, 2] # convert everything to black and white (agent will train faster) frame_rgb[frame_rgb < 100] = 0 frame_rgb[frame_rgb >= 100] = 255 # converting to RGB (OpenCV way) #frame_rgb = cv2.cvtColor(frame_cropped, cv2.COLOR_RGB2GRAY) # dividing by 255 we expresses value to 0-1 representation new_frame = np.array(frame_rgb).astype(np.float32) / 255.0 # push our data by 1 frame, similar as deq() function work image_memory = np.roll(image_memory, 1, axis=0) # inserting new frame to free space image_memory[0, :, :] = new_frame # show image frame #self.imshow(image_memory,0) #self.imshow(image_memory,1) #self.imshow(image_memory,2) #self.imshow(image_memory,3) return np.expand_dims(image_memory, axis=0) def reset(self, env): image_memory = np.zeros(self.state_size) frame = env.reset() for i in range(self.REM_STEP): state = self.GetImage(frame, image_memory) return state def step(self, action, env, image_memory): next_state, reward, done, info = env.step(action) next_state = self.GetImage(next_state, image_memory) return next_state, reward, done, info def run(self): for e in range(self.EPISODES): state = self.reset(self.env) done, score, SAVING = False, 0, '' # Instantiate or reset games memory states, actions, rewards = [], [], [] while not done: #self.env.render() # Actor picks an action action = self.act(state) # Retrieve new state, reward, and whether the state is terminal next_state, reward, done, _ = self.step( action, self.env, state) # Memorize (state, action, reward) for training states.append(state) action_onehot = np.zeros([self.action_size]) action_onehot[action] = 1 actions.append(action_onehot) rewards.append(reward) # Update current state state = next_state score += reward if done: average = self.PlotModel(score, e) # saving best models if average >= self.max_average: self.max_average = average self.save() SAVING = "SAVING" else: SAVING = "" print( "episode: {}/{}, score: {}, average: {:.2f} {}".format( e, self.EPISODES, score, average, SAVING)) self.replay(states, actions, rewards) # close environemnt when finish training self.env.close() def train(self, n_threads): self.env.close() # Instantiate one environment per thread envs = [gym.make(self.env_name) for i in range(n_threads)] # Create threads threads = [ threading.Thread(target=self.train_threading, daemon=True, args=(self, envs[i], i)) for i in range(n_threads) ] for t in threads: time.sleep(2) t.start() for t in threads: time.sleep(10) t.join() def train_threading(self, agent, env, thread): global graph with graph.as_default(): while self.episode < self.EPISODES: # Reset episode score, done, SAVING = 0, False, '' state = self.reset(env) # Instantiate or reset games memory states, actions, rewards = [], [], [] while not done: action = agent.act(state) next_state, reward, done, _ = self.step(action, env, state) states.append(state) action_onehot = np.zeros([self.action_size]) action_onehot[action] = 1 actions.append(action_onehot) rewards.append(reward) score += reward state = next_state self.lock.acquire() self.replay(states, actions, rewards) self.lock.release() # Update episode count with self.lock: average = self.PlotModel(score, self.episode) # saving best models if average >= self.max_average: self.max_average = average self.save() SAVING = "SAVING" else: SAVING = "" print( "episode: {}/{}, thread: {}, score: {}, average: {:.2f} {}" .format(self.episode, self.EPISODES, thread, score, average, SAVING)) if (self.episode < self.EPISODES): self.episode += 1 env.close() def test(self, Actor_name, Critic_name): self.load(Actor_name, Critic_name) for e in range(100): state = self.reset(self.env) done = False score = 0 while not done: self.env.render() action = np.argmax(self.Actor.predict(state)) state, reward, done, _ = self.step(action, self.env, state) score += reward if done: print("episode: {}/{}, score: {}".format( e, self.EPISODES, score)) break self.env.close()
class TTIR(eg.IrDecoderPlugin): def __init__(self): eg.IrDecoderPlugin.__init__(self, 1) def __close__(self): self.irDecoder.Close() def __start__(self): self.dll = None self.hOpen = None pluginDir = os.path.abspath(os.path.dirname(__file__)) dll = CDLL(os.path.join(pluginDir, "TTUSBIR.dll")) self.cCallback = IRCALLBACKFUNC(self.IrCallback) self.hOpen = dll.irOpen(0, USBIR_MODE_DIV, self.cCallback, 0) if self.hOpen == -1: raise self.Exceptions.DeviceNotFound # self.irGetUniqueCode = dll.ir_GetUniqueCode # self.irGetUniqueCode.restype = DWORD self.dll = dll self.data = [] self.timer = eg.ResettableTimer(self.OnTimeout) self.dataLock = Lock() self.lastTime = clock() self.startByte = 1 self.dll.irSetPowerLED(self.hOpen, 0) self.ledTimer = eg.ResettableTimer( partial(self.dll.irSetPowerLED, self.hOpen, 0)) def OnComputerSuspend(self, dummySuspendType): self.dll.irClose(self.hOpen) def OnComputerResume(self, dummySuspendType): self.hOpen = self.dll.irOpen(0, USBIR_MODE_DIV, self.cCallback, 0) if self.hOpen == -1: raise self.Exceptions.DeviceNotFound def __stop__(self): if self.dll is not None: self.dll.irClose(self.hOpen) self.dll = None self.hOpen = None self.cCallback = None self.timer.Stop() self.ledTimer.Stop() def IrCallback(self, context, buf, length, irMode, hOpen, devIdx): if irMode == USBIR_MODE_DIV: self.dll.irSetPowerLED(self.hOpen, 1) self.ledTimer.Reset(1) self.timer.Reset(80) #self.lastTime = clock() self.dataLock.acquire() #print "---", length / 4, clock() - self.lastTime append = self.data.append for i in xrange(self.startByte, min(length / 4, 500)): value = buf[i] & 0x00ffffff append(value) if value > 12000: self.irDecoder.Decode(self.data, len(self.data)) self.data = [] append = self.data.append self.startByte = 0 self.dataLock.release() def OnTimeout(self): if self.dataLock.acquire(0): data = self.data self.data = [] self.startByte = 1 self.dataLock.release() data.append(10000) self.irDecoder.Decode(data, len(data))
class HuskyMapper: def __init__(self, num_rows, num_cols, meters_per_cell): rospy.init_node('occupancy_grid_mapper', anonymous=True) self.tf_listener = tf.TransformListener() self.odometry_position_noise_std_dev = rospy.get_param( "~odometry_position_noise_std_dev") self.odometry_orientation_noise_std_dev = rospy.get_param( "~odometry_orientation_noise_std_dev") og_origin_in_map_frame = np.array([-20, -10, 0]) self.init_log_odds_ratio = 0 #log(0.5/0.5) self.ogm = OccupancyGridMap(num_rows, num_cols, meters_per_cell, og_origin_in_map_frame, self.init_log_odds_ratio) self.max_laser_range = None self.min_laser_range = None self.max_laser_angle = None self.min_laser_angle = None self.odometry = None # baselink is a frame on the husky robot that corresponds roughly to its body center # baselaser is a frame on the husky robot that corresponds to the base of the LiDAR sensor (the scanned points will be in this coordinate frame) # map is the frame corresponding to the global frame of reference # You need to convert points from the baselaser frame to the map frame for this assignment self.q_map_baselink = None # 4x1 quaternion from husky_1/baselink to map frame self.R_map_baselink = None # 3x3 rotation matrix from husky_1/baselink to map frame self.p_map_baselink = None # 3x1 position of husky_1/baselink in map frame self.q_map_baselaser = None # 4x1 quaternion from husky_1/baselaser to map frame self.R_map_baselaser = None # 3x3 rotation matrix from husky_1/baselaser to map frame self.p_map_baselaser = None # 3x1 position of husky_1/baselaser in map frame self.q_baselink_baselaser = np.array([1.0, 0, 0, 0]) self.R_baselink_baselaser = tr.quaternion_matrix( self.q_baselink_baselaser)[0:3, 0:3] self.p_baselink_baselaser = np.array([0.337, 0.0, 0.308]) self.mutex = Lock() self.occupancy_grid_pub = rospy.Publisher('/husky_1/occupancy_grid', OccupancyGrid, queue_size=1) self.laser_points_marker_pub = rospy.Publisher( '/husky_1/debug/laser_points', Marker, queue_size=1) self.robot_pose_pub = rospy.Publisher('/husky_1/debug/robot_pose', PoseStamped, queue_size=1) self.laser_sub = rospy.Subscriber('/husky_1/scan', LaserScan, self.laser_scan_callback, queue_size=1) self.odometry_sub = rospy.Subscriber('/husky_1/odometry/ground_truth', Odometry, self.odometry_callback, queue_size=1) def odometry_callback(self, msg): self.mutex.acquire() self.odometry = msg # Adds noise to the odometry position measurement according to the standard deviations specified as parameters in the launch file # We should have used noisy measurements from the Gazebo simulator, but it is more complicated to configure # so, we add random noise here ourselves, assuming perfect odometry from the simulator. self.odometry.pose.pose.position.x += random.gauss( 0, self.odometry_position_noise_std_dev) self.odometry.pose.pose.position.y += random.gauss( 0, self.odometry_position_noise_std_dev) # # TODO: populate the quaternion from the husky_1/base_link frame to the map frame # based on the current odometry message. In order to know more about where # these frames are located on the robot, run: rosrun rviz rviz and look at the TF # widget. Pay attention to the following frames: husky_1/base_link which is at the center # of the robot's body, husky_1/base_laser, which is the frame of the laser sensor, # and map, which is where odometry messages are expressed in. In fact, odometry # messages from the Husky are transformations from husky_1/base_link to map # #self.q_map_baselink = np.array([x, y, z, w]) # Corrupting the quaternion with noise in yaw, because we have configured the simulator # to return noiseless orientation measurements. yaw_noise = random.gauss( 0, self.odometry_orientation_noise_std_dev) * pi / 180.0 q_truebaselink_noisybaselink = np.array( [0, 0, np.sin(yaw_noise), np.cos(yaw_noise)]) self.q_map_baselink = tr.quaternion_multiply( self.q_map_baselink, q_truebaselink_noisybaselink) # Computes the rotation matrix from husky_1/base_link to map self.R_map_baselink = tr.quaternion_matrix(self.q_map_baselink)[0:3, 0:3] # # TODO: populate the position of the husky_1/base_link frame in map frame # coordinates based on the current odometry message # # #self.p_map_baselink = np.array([x, y, z]) # # TODO: populate the quaternion from the frame husky_1/base_laser to the map frame # note: you have access to the static quaternion from husky_1/base_laser to # husky_1/base_link #self.q_map_baselaser = tr.quaternion_multiply(? , ?) # # TODO: populate the rotation matrix from the frame husky_1/base_laser to the map frame # note: you have access to the static rotation matrix from husky_1/base_laser to # husky_1/base_link # also note: np.dot(A,B) multiplies numpy matrices A and B, whereas A*B is element-wise # multiplication, which is not usually what you want # #self.R_map_baselaser = ? # # TODO: populate the origin of the frame husky_1/base_laser in coordinates of the map frame # note: you have access to the static rotation matrix from husky_1/base_laser to # husky_1/base_link and also to the origin of the husky_1/base_laser frame in coordinates of # frame husky_1/base_link # also note: np.dot(A,B) multiplies numpy matrices A and B, whereas A*B is element-wise # multiplication, which is not usually what you want # #self.p_map_baselaser = ? self.mutex.release() def from_laser_to_map_coordinates(self, points_in_baselaser_frame): # # The robot's odometry is with respect to the map frame, but the points measured from # the laser are given with respec to the frame husky_1/base_laser. This function convert # the measured points in the laser scan from husky_1/base_laser to the map frame. # points_in_map_frame = [ np.dot(self.R_map_baselaser, xyz_baselaser) + self.p_map_baselaser for xyz_baselaser in points_in_baselaser_frame ] return points_in_map_frame def is_in_field_of_view(self, robot_row, robot_col, robot_theta, row, col): # Returns true iff the cell (row, col) in the grid is in the field of view of the 2D laser of the # robot located at cell (robot_row, robot_col) and having yaw robot_theta in the map frame. # Useful things to know: # 1) self.ogm.meters_per_cell converts cell distances to metric distances # 2) atan2(y,x) gives the angle of the vector (x,y) # 3) atan2(sin(theta_1 - theta_2), cos(theta_1 - theta_2)) gives the angle difference between theta_1 and theta_2 in [-pi, pi] # 4) self.max_laser_range and self.max_laser_angle specify some of the limits of the laser sensor # # TODO: fill this # return False def inverse_measurement_model(self, row, col, robot_row, robot_col, robot_theta_in_map, beam_ranges, beam_angles): alpha = 0.1 beta = 10 * pi / 180.0 p_occupied = 0.999 # # TODO: Find the range r and angle diff_angle of the beam (robot_row, robot_col) ------> (row, col) # r should be in meters and diff_angle should be in [-pi, pi]. Useful things to know are same as above. # #r = ? #diff_angle = ? closest_beam_angle, closest_beam_idx = min((val, idx) for ( idx, val) in enumerate([abs(diff_angle - ba) for ba in beam_angles])) r_cb = beam_ranges[closest_beam_idx] theta_cb = beam_angles[closest_beam_idx] if r > min(self.max_laser_range, r_cb + alpha / 2.0) or abs(diff_angle - theta_cb) > beta / 2.0: return self.init_log_odds_ratio if r_cb < self.max_laser_range and abs(r - r_cb) < alpha / 2.0: return log(p_occupied / (1 - p_occupied)) if r <= r_cb: return log((1 - p_occupied) / p_occupied) return 0.0 def laser_scan_callback(self, msg): self.mutex.acquire() self.min_laser_angle = msg.angle_min self.max_laser_angle = msg.angle_max self.min_laser_range = msg.range_min self.max_laser_range = msg.range_max if self.odometry is None: # ignore the laser message if no odometry has been received self.mutex.release() return N = len(msg.ranges) ranges_in_baselaser_frame = msg.ranges angles_in_baselaser_frame = [ (msg.angle_max - msg.angle_min) * float(i) / N + msg.angle_min for i in xrange(len(msg.ranges)) ] angles_in_baselink_frame = angles_in_baselaser_frame[::-1] # This is because the z-axis of husky_1/base_laser is pointing downwards, while for husky_1/base_link and the map frame # the z-axis points upwards points_xyz_in_baselaser_frame = [ np.array([r * cos(theta), r * sin(theta), 0]) for (r, theta ) in zip(ranges_in_baselaser_frame, angles_in_baselaser_frame) if r < self.max_laser_range and r > self.min_laser_range ] points_xyz_in_map_frame = self.from_laser_to_map_coordinates( points_xyz_in_baselaser_frame) baselaser_x_in_map = self.p_map_baselaser[0] baselaser_y_in_map = self.p_map_baselaser[1] baselaser_row, baselaser_col = self.ogm.cartesian_to_grid_coords( baselaser_x_in_map, baselaser_y_in_map) _, _, yaw_map_baselaser = tr.euler_from_quaternion( self.q_map_baselaser) _, _, yaw_map_baselink = tr.euler_from_quaternion(self.q_map_baselink) # Publishing the pose of the robot as a red arrow in rviz to help you debug ps = self._get_pose_marker(msg.header.stamp, 'map', self.p_map_baselaser, self.q_map_baselaser) # Publishing the points of the laser transformed into the map frame, as green points in rviz, to help you debug pts_marker = self._get_2d_laser_points_marker(msg.header.stamp, 'map', points_xyz_in_map_frame) self.mutex.release() self.robot_pose_pub.publish(ps) self.laser_points_marker_pub.publish(pts_marker) # # This is the main loop in occupancy grid mapping # max_laser_range_in_cells = int( self.max_laser_range / self.ogm.meters_per_cell) + 1 for delta_row in xrange(-max_laser_range_in_cells, max_laser_range_in_cells): for delta_col in xrange(-max_laser_range_in_cells, max_laser_range_in_cells): row = baselaser_row + delta_row col = baselaser_col + delta_col if row < 0 or row >= self.ogm.num_rows or col < 0 or col >= self.ogm.num_cols: continue if self.is_in_field_of_view(baselaser_row, baselaser_col, yaw_map_baselink, row, col): delta_log_odds = self.inverse_measurement_model( row, col, baselaser_row, baselaser_col, yaw_map_baselaser, ranges_in_baselaser_frame, angles_in_baselink_frame) - self.init_log_odds_ratio self.ogm.update_log_odds_ratio_in_grid_coords( row, col, delta_log_odds) self.occupancy_grid_pub.publish( self.ogm.get_map_as_ros_msg(msg.header.stamp, 'map')) def _get_2d_laser_points_marker(self, timestamp, frame_id, pts_in_map): msg = Marker() msg.header.stamp = timestamp msg.header.frame_id = frame_id msg.ns = 'laser_points' msg.id = 0 msg.type = 6 msg.action = 0 msg.points = [Point(pt[0], pt[1], pt[2]) for pt in pts_in_map] msg.colors = [ColorRGBA(0, 1.0, 0, 1.0) for pt in pts_in_map] for pt in pts_in_map: assert ((not np.isnan(pt).any()) and np.isfinite(pt).all()) msg.scale.x = 0.1 msg.scale.y = 0.1 msg.scale.z = 0.1 return msg def _get_pose_marker(self, timestamp, frame_id, p, q): ps = PoseStamped() ps.pose.position.x = p[0] ps.pose.position.y = p[1] ps.pose.position.z = p[2] ps.pose.orientation.x = q[0] ps.pose.orientation.y = q[1] ps.pose.orientation.z = q[2] ps.pose.orientation.w = q[3] ps.header.stamp = timestamp ps.header.frame_id = frame_id return ps def run(self): rate = rospy.Rate(200) while not rospy.is_shutdown(): rate.sleep()
class BundleRegistry(GObject.GObject): """Tracks the available activity bundles""" __gsignals__ = { 'bundle-added': (GObject.SignalFlags.RUN_FIRST, None, ([GObject.TYPE_PYOBJECT])), 'bundle-removed': (GObject.SignalFlags.RUN_FIRST, None, ([GObject.TYPE_PYOBJECT])), 'bundle-changed': (GObject.SignalFlags.RUN_FIRST, None, ([GObject.TYPE_PYOBJECT])), } def __init__(self): logging.debug('STARTUP: Loading the bundle registry') GObject.GObject.__init__(self) self._mime_defaults = self._load_mime_defaults() # Queue of bundles to be installed/upgraded self._install_queue = _InstallQueue(self) # Bundle installation happens in a separate thread, which needs # access to _bundles. Protect all _bundles access with a lock. self._lock = Lock() self._bundles = [] # hold a reference to the monitors so they don't get disposed self._gio_monitors = [] dirs = [env.get_user_activities_path(), env.get_user_library_path()] for data_dir in GLib.get_system_data_dirs(): dirs.append(os.path.join(data_dir, "sugar", "activities")) for activity_dir in dirs: self._scan_directory(activity_dir) directory = Gio.File.new_for_path(activity_dir) monitor = directory.monitor_directory( flags=Gio.FileMonitorFlags.NONE, cancellable=None) monitor.connect('changed', self.__file_monitor_changed_cb) self._gio_monitors.append(monitor) self._favorite_bundles = [] for i in range(desktop.get_number_of_views()): self._favorite_bundles.append({}) settings = Gio.Settings('org.sugarlabs') self._protected_activities = settings.get_strv('protected-activities') try: self._load_favorites() except Exception: logging.exception('Error while loading favorite_activities.') self._hidden_activities = [] self._load_hidden_activities() self._convert_old_favorites() self._scan_new_favorites() self._desktop_model = desktop.get_model() self._desktop_model.connect('desktop-view-icons-changed', self.__desktop_view_icons_changed_cb) def __desktop_view_icons_changed_cb(self, model): number_of_views = desktop.get_number_of_views() if len(self._favorite_bundles) < number_of_views: for i in range(number_of_views - len(self._favorite_bundles)): self._favorite_bundles.append({}) try: self._load_favorites() except Exception: logging.exception('Error while loading favorite_activities.') def __file_monitor_changed_cb(self, monitor, one_file, other_file, event_type): if event_type == Gio.FileMonitorEvent.CREATED or \ event_type == Gio.FileMonitorEvent.ATTRIBUTE_CHANGED: self.add_bundle(one_file.get_path(), set_favorite=True) elif event_type == Gio.FileMonitorEvent.DELETED: self.remove_bundle(one_file.get_path()) for root in GLib.get_system_data_dirs(): root = os.path.join(root, 'sugar', 'activities') try: os.listdir(root) except OSError: logging.debug('Can not find GLib system dir %s', root) continue activity_dir = os.path.basename(one_file.get_path()) try: bundle = bundle_from_dir(os.path.join(root, activity_dir)) except MalformedBundleException: continue if bundle is not None: path = bundle.get_path() if path is not None: self.add_bundle(path) def _load_mime_defaults(self): defaults = {} f = open(os.environ["SUGAR_MIME_DEFAULTS"], 'r') for line in f.readlines(): line = line.strip() if line and not line.startswith('#'): mime = line[:line.find(' ')] handler = line[line.rfind(' ') + 1:] defaults[mime] = handler f.close() return defaults def _get_favorite_key(self, bundle_id, version): """We use a string as a composite key for the favorites dictionary because JSON doesn't support tuples and python won't accept a list as a dictionary key. """ if ' ' in bundle_id: raise ValueError('bundle_id cannot contain spaces') return '%s %s' % (bundle_id, version) def _load_favorites(self): for i in range(desktop.get_number_of_views()): # Special-case 0 for backward compatibility if i == 0: favorites_path = env.get_profile_path('favorite_activities') else: favorites_path = env.get_profile_path( 'favorite_activities_%d' % (i)) if os.path.exists(favorites_path): favorites_data = json.load(open(favorites_path)) favorite_bundles = favorites_data['favorites'] if not isinstance(favorite_bundles, dict): raise ValueError('Invalid format in %s.' % favorites_path) if favorite_bundles: first_key = list(favorite_bundles.keys())[0] if not isinstance(first_key, str): raise ValueError('Invalid format in %s.' % favorites_path) first_value = list(favorite_bundles.values())[0] if first_value is not None and \ not isinstance(first_value, dict): raise ValueError('Invalid format in %s.' % favorites_path) self._favorite_bundles[i] = favorite_bundles def _load_hidden_activities(self): path = os.environ.get('SUGAR_ACTIVITIES_HIDDEN', None) try: with open(path) as file: for line in file.readlines(): bundle_id = line.strip() if bundle_id: self._hidden_activities.append(bundle_id) except IOError: logging.error('Error when loading hidden activities %s', path) def _convert_old_favorites(self): for i in range(desktop.get_number_of_views()): for key in list(self._favorite_bundles[i].keys()): data = self._favorite_bundles[i][key] if data is None: data = {} if 'favorite' not in data: data['favorite'] = True self._favorite_bundles[i][key] = data self._write_favorites_file(i) def _scan_new_favorites(self): for bundle in self: bundle_id = bundle.get_bundle_id() key = self._get_favorite_key(bundle_id, bundle.get_activity_version()) if key not in self._favorite_bundles[_DEFAULT_VIEW]: self._favorite_bundles[_DEFAULT_VIEW][key] = \ {'favorite': bundle_id not in self._hidden_activities} self._write_favorites_file(_DEFAULT_VIEW) def get_bundle(self, bundle_id): """Returns an bundle given his service name""" with self._lock: for bundle in self._bundles: if bundle.get_bundle_id() == bundle_id: return bundle return None def __iter__(self): with self._lock: copy = list(self._bundles) return copy.__iter__() def __len__(self): with self._lock: return len(self._bundles) def _scan_directory(self, path): if not os.path.isdir(path): return # Sort by mtime to ensure a stable activity order bundles = {} for f in os.listdir(path): try: bundle_dir = os.path.join(path, f) if os.path.isdir(bundle_dir): bundles[bundle_dir] = os.stat(bundle_dir).st_mtime except Exception: logging.exception( 'Error while processing installed activity' ' bundle %s:', bundle_dir) bundle_dirs = list(bundles.keys()) bundle_dirs.sort(key=lambda x: bundles[x]) for folder in bundle_dirs: try: self.add_bundle(folder, emit_signals=False) except: # pylint: disable=W0702 logging.exception( 'Error while processing installed activity' ' bundle %s:', folder) def add_bundle(self, bundle_path, set_favorite=False, emit_signals=True, force_downgrade=False): """ Add a bundle to the registry. If the bundle is a duplicate with one already in the registry, the existing one from the registry is returned. Otherwise, the newly added bundle is returned on success, or None on failure. """ try: bundle = bundle_from_dir(bundle_path) except MalformedBundleException: logging.exception('Error loading bundle %r', bundle_path) return None # None is a valid return value from bundle_from_dir helper. if bundle is None: logging.error('No bundle in %r', bundle_path) return None bundle_id = bundle.get_bundle_id() logging.debug('STARTUP: Adding bundle %s', bundle_id) installed = self.get_bundle(bundle_id) if installed is not None: if NormalizedVersion(installed.get_activity_version()) == \ NormalizedVersion(bundle.get_activity_version()): logging.debug("Bundle already known") return installed if not force_downgrade and \ NormalizedVersion(installed.get_activity_version()) >= \ NormalizedVersion(bundle.get_activity_version()): logging.debug('Skip old version for %s', bundle_id) return None else: logging.debug('Upgrade %s', bundle_id) self.remove_bundle(installed.get_path(), emit_signals) if set_favorite: favorite = not self.is_bundle_hidden(bundle.get_bundle_id(), bundle.get_activity_version()) self._set_bundle_favorite(bundle.get_bundle_id(), bundle.get_activity_version(), favorite) with self._lock: self._bundles.append(bundle) if emit_signals: self.emit('bundle-added', bundle) return bundle def remove_bundle(self, bundle_path, emit_signals=True): removed = None self._lock.acquire() for bundle in self._bundles: if bundle.get_path() == bundle_path: self._bundles.remove(bundle) removed = bundle break self._lock.release() if emit_signals and removed is not None: self.emit('bundle-removed', removed) return removed is not None def get_activities_for_type(self, mime_type): result = [] mime = mimeregistry.get_registry() default_bundle_id = mime.get_default_activity(mime_type) default_bundle = None for bundle in self: if not isinstance(bundle, ActivityBundle): continue if mime_type in (bundle.get_mime_types() or []): if bundle.get_bundle_id() == default_bundle_id: default_bundle = bundle elif self.get_default_for_type(mime_type) == \ bundle.get_bundle_id(): result.insert(0, bundle) else: result.append(bundle) if default_bundle is not None: result.insert(0, default_bundle) return result def get_default_for_type(self, mime_type): return self._mime_defaults.get(mime_type) def _find_bundle(self, bundle_id, version): with self._lock: for bundle in self._bundles: if bundle.get_bundle_id() == bundle_id and \ bundle.get_activity_version() == version: return bundle raise ValueError('No bundle %r with version %r exists.' % (bundle_id, version)) def set_bundle_favorite(self, bundle_id, version, favorite, favorite_view=0): changed = self._set_bundle_favorite(bundle_id, version, favorite, favorite_view) if changed: bundle = self._find_bundle(bundle_id, version) self.emit('bundle-changed', bundle) def _set_bundle_favorite(self, bundle_id, version, favorite, favorite_view=0): key = self._get_favorite_key(bundle_id, version) if key not in self._favorite_bundles[favorite_view]: self._favorite_bundles[favorite_view][key] = {} elif favorite == \ self._favorite_bundles[favorite_view][key]['favorite']: return False self._favorite_bundles[favorite_view][key]['favorite'] = favorite self._write_favorites_file(favorite_view) return True def is_bundle_favorite(self, bundle_id, version, favorite_view=0): key = self._get_favorite_key(bundle_id, version) if key not in self._favorite_bundles[favorite_view]: return False return self._favorite_bundles[favorite_view][key]['favorite'] def is_bundle_hidden(self, bundle_id, version): key = self._get_favorite_key(bundle_id, version) if key in self._favorite_bundles[_DEFAULT_VIEW]: data = self._favorite_bundles[_DEFAULT_VIEW][key] return data['favorite'] is False else: return bundle_id in self._hidden_activities def is_activity_protected(self, bundle_id): return bundle_id in self._protected_activities def set_bundle_position(self, bundle_id, version, x, y, favorite_view=0): key = self._get_favorite_key(bundle_id, version) if key not in self._favorite_bundles[favorite_view]: raise ValueError('Bundle %s %s not favorite' % (bundle_id, version)) if 'position' not in self._favorite_bundles[favorite_view][key] or \ [x, y] != \ self._favorite_bundles[favorite_view][key]['position']: self._favorite_bundles[favorite_view][key]['position'] = [x, y] else: return self._write_favorites_file(favorite_view) bundle = self._find_bundle(bundle_id, version) self.emit('bundle-changed', bundle) def get_bundle_position(self, bundle_id, version, favorite_view=0): """Get the coordinates where the user wants the representation of this bundle to be displayed. Coordinates are relative to a 1000x1000 area. """ key = self._get_favorite_key(bundle_id, version) if key not in self._favorite_bundles[favorite_view] or \ 'position' not in self._favorite_bundles[favorite_view][key]: return (-1, -1) else: return \ tuple(self._favorite_bundles[favorite_view][key]['position']) def _write_favorites_file(self, favorite_view): if favorite_view == 0: path = env.get_profile_path('favorite_activities') else: path = env.get_profile_path('favorite_activities_%d' % (favorite_view)) favorites_data = {'favorites': self._favorite_bundles[favorite_view]} json.dump(favorites_data, open(path, 'w'), indent=1) def is_installed(self, bundle): for installed_bundle in self: if bundle.get_bundle_id() == installed_bundle.get_bundle_id() and \ NormalizedVersion(bundle.get_activity_version()) == \ NormalizedVersion(installed_bundle.get_activity_version()): return True return False def install(self, bundle, force_downgrade=False): """ Install a bundle, upgrading or optionally downgrading any existing version. If the same version of the bundle is already installed, this function returns False without doing anything. If the installation succeeded, True is returned. By default, downgrades will be refused (AlreadyInstalledException will be raised) but the force_downgrade flag can override that behaviour and cause the downgrade to happen. The bundle is installed in the user activity directory. System-installed activities cannot be upgraded/downgraded; in such case, the bundle will be installed as a duplicate in the user activity directory. RegistrationException is raised if the bundle cannot be registered after it is installed. """ result = [None] self.install_async(bundle, self._sync_install_cb, result, force_downgrade) while result[0] is None: Gtk.main_iteration() if isinstance(result[0], Exception): raise result[0] return result[0] def _sync_install_cb(self, bundle, result, user_data): # Async callback for install() user_data[0] = result def install_async(self, bundle, callback, user_data, force_downgrade=False): """ Asynchronous version of install(). The result of the installation is presented to a user-defined callback with the following parameters: 1. The bundle that passed to this method 2. The result of the operation (True, False, or an Exception - see the install() docs) 3. The user_data passed to this method The callback is always invoked from main-loop context. """ self._install_queue.enqueue(bundle, force_downgrade, self._bundle_installed_cb, [callback, user_data]) def _bundle_installed_cb(self, bundle, result, data): """ Completion handler for the bundle Install thread. Called in main loop context. Finishes registration and invokes user callback as necessary. """ callback = data[0] user_data = data[1] # Installation didn't happen, or error? if isinstance(result, Exception) or result is False: callback(bundle, result, user_data) return if bundle.get_bundle_id() is not None: registered = self.add_bundle(result, set_favorite=True, force_downgrade=True) if registered is None: callback(bundle, RegistrationException(), user_data) return callback(bundle, True, user_data) def uninstall(self, bundle, force=False, delete_profile=False): """ Uninstall a bundle. If a different version of bundle is found in the activity registry, this function does nothing unless force is True. If the bundle is not found in the activity registry at all, this function simply returns. """ act = self.get_bundle(bundle.get_bundle_id()) if not act: logging.debug("Bundle is not installed") return if not force and \ act.get_activity_version() != bundle.get_activity_version(): logging.warning('Not uninstalling, different bundle present') return if not act.is_user_activity(): logging.debug('Do not uninstall system activity') return install_path = act.get_path() bundle.uninstall(force, delete_profile) self.remove_bundle(install_path) alt_bundles = self.get_system_bundles(act.get_bundle_id()) if alt_bundles: alt_bundles.sort( key=lambda b: NormalizedVersion(b.get_activity_version())) alt_bundles.reverse() new_bundle = alt_bundles[0] self.add_bundle(new_bundle.get_path()) def get_system_bundles(self, bundle_id): """ Searches for system bundles (eg. those in /usr/share/sugar/activities) with a given bundle id. Prams: * bundle_id (string): the bundle id to look for Returns a list of ActivityBundle or ContentBundle objects, or an empty list if there are none found. """ bundles = [] for root in GLib.get_system_data_dirs(): root = os.path.join(root, 'sugar', 'activities') try: dir_list = os.listdir(root) except OSError: logging.debug('Can not find GLib system dir %s', root) continue for activity_dir in dir_list: try: bundle = bundle_from_dir(os.path.join(root, activity_dir)) except MalformedBundleException: continue if bundle.get_bundle_id() == bundle_id: bundles.append(bundle) return bundles
if self.stop_event.is_set(): continue while data != "": ((message_id, ), data) = unpack_helper('<B', data) if message_id == MSG_LEAVE: print("[CHAT] %s has left the chat" % self.name) peers_lock.acquire() try: del peers[self.guid] finally: peers_lock.release() elif message_id == MSG_CHAT_MESSAGE: # Read all data for this message ((chat_len, ), data) = unpack_helper('<I', data) ((message, ), data) = unpack_helper('<%ds' % chat_len, data) print("[CHAT] %s says: %s" % (self.name, message)) elif message_id == MSG_JOIN_NETWORK: ((guid_len, ), data) = unpack_helper('<I', data) ((guid, ), data) = unpack_helper('<%ds' % guid_len, data) ((hostname_len, ), data) = unpack_helper('<I', data) ((hostname, ), data) = unpack_helper('<%ds' % hostname_len, data) ((name_len, ), data) = unpack_helper('<I', data)
class _InstallQueue(object): """ A class to represent a queue of bundles to be installed, and to handle execution of each task in the queue. Only for internal bundleregistry use. The use of a queue means that we serialize all bundle upgrade processing. This is necessary to avoid many difficult corner-cases like: what happens if two users try to asynchronously and simultaenously install different version of the same bundle? We maintain at maximum one thread to do the actual bundle install. When done, the thread enqueues a callback in the main thread (via the GLib main loop). """ def __init__(self, registry): self._lock = Lock() self._queue = [] self._thread_running = False self._registry = registry def enqueue(self, bundle, force_downgrade, callback, user_data): task = _InstallTask(bundle, force_downgrade, callback, user_data) self._lock.acquire() self._queue.append(task) if not self._thread_running: self._thread_running = True Thread(target=self._thread_func).start() self._lock.release() def _thread_func(self): while True: self._lock.acquire() if len(self._queue) == 0: self._thread_running = False self._lock.release() return task = self._queue.pop() self._lock.release() self._do_work(task) def _do_work(self, task): bundle = task.bundle bundle_id = bundle.get_bundle_id() act = self._registry.get_bundle(bundle_id) logging.debug("InstallQueue task %s installed %r", bundle_id, act) if act: # Same version already installed? if act.get_activity_version() == bundle.get_activity_version(): logging.debug('No upgrade needed, same version already ' 'installed.') task.queue_callback(False) return # Would this new installation be a downgrade? if NormalizedVersion(bundle.get_activity_version()) <= \ NormalizedVersion(act.get_activity_version()) \ and not task.force_downgrade: task.queue_callback(AlreadyInstalledException()) return # Uninstall the previous version, if we can if act.is_user_activity(): try: act.uninstall() except: logging.exception('Uninstall failed, still trying to ' 'install newer bundle') else: logging.warning('Unable to uninstall system activity, ' 'installing upgraded version in user ' 'activities') try: task.queue_callback(bundle.install()) except Exception as e: logging.debug("InstallThread install failed: %r", e) task.queue_callback(e)
class LeagueMgrAPIs(object): def __init__(self, league_mgr_addr): self._zmq_context = zmq.Context() self._zmq_context.setsockopt(zmq.TCP_KEEPALIVE, 1) self._zmq_context.setsockopt(zmq.TCP_KEEPALIVE_CNT, 60) self._zmq_context.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 60) self._zmq_context.setsockopt(zmq.TCP_KEEPALIVE_INTVL, 60) self._socket = self._zmq_context.socket(zmq.REQ) ip, port = league_mgr_addr.split(':') self._socket.connect("tcp://%s:%s" % (ip, port)) self._req_lock = Lock() def request_actor_task(self, actor_id, learner_id): self._req_lock.acquire() while True: self._socket.send_string("request_actor_task", zmq.SNDMORE) self._socket.send_pyobj((actor_id, learner_id)) task = self._socket.recv_pyobj() if not isinstance(task, LeagueMgrErroMsg): break time.sleep(10) self._req_lock.release() return task def request_learner_task(self, learner_id): self._req_lock.acquire() while True: self._socket.send_string("request_learner_task", zmq.SNDMORE) self._socket.send_pyobj(learner_id) task = self._socket.recv_pyobj() if not isinstance(task, LeagueMgrErroMsg): break time.sleep(1) self._req_lock.release() return task def query_learner_task(self, learner_id): self._req_lock.acquire() self._socket.send_string("query_learner_task", zmq.SNDMORE) self._socket.send_pyobj(learner_id) task = self._socket.recv_pyobj() self._req_lock.release() if isinstance(task, LeagueMgrErroMsg): return None else: return task def notify_actor_task_begin(self, actor_id): self._req_lock.acquire() self._socket.send_string("notify_actor_task_begin", zmq.SNDMORE) self._socket.send_pyobj(actor_id) assert self._socket.recv_string() == "ok" self._req_lock.release() def notify_actor_task_end(self, actor_id, match_result): self._req_lock.acquire() self._socket.send_string("notify_actor_task_end", zmq.SNDMORE) self._socket.send_pyobj((actor_id, match_result)) assert self._socket.recv_string() == "ok" self._req_lock.release() def notify_learner_task_begin(self, learner_id, learner_task): self._req_lock.acquire() self._socket.send_string("notify_learner_task_begin", zmq.SNDMORE) self._socket.send_pyobj((learner_id, learner_task)) assert self._socket.recv_string() == "ok" self._req_lock.release() def notify_learner_task_end(self, learner_id): self._req_lock.acquire() self._socket.send_string("notify_learner_task_end", zmq.SNDMORE) self._socket.send_pyobj(learner_id) assert self._socket.recv_string() == "ok" self._req_lock.release() def request_add_model(self, model): self._req_lock.acquire() self._socket.send_string("request_add_model", zmq.SNDMORE) self._socket.send_pyobj(model) assert self._socket.recv_string() == "ok" self._req_lock.release()
class Downloader(object): @property def state(self): return self._state @property def connections(self): return len([ c for c in self._download_info["parts"] if c["status"] in [self.states.downloading, self.states.connecting] ]), self._max_connections @property def downloaded(self): return self.__change_units__( sum([ c["current"] - c["start"] for c in self._download_info["parts"] ])) @property def average_speed(self): return self.__change_units__(self._average_speed) @property def speed(self): return self.__change_units__(self._speed) @property def remaining_time(self): if self.speed[0] and self._file_size: t = old_div((self.size[0] - self.downloaded[0]), self.speed[0]) else: t = 0 return time.strftime("%H:%M:%S", time.gmtime(t)) @property def download_url(self): return self.url @property def size(self): return self.__change_units__(self._file_size) @property def progress(self): if self._file_size: return float(self.downloaded[0]) * 100 / float(self._file_size) elif self._state == self.states.completed: return 100 else: return 0 @property def filename(self): return self._filename @property def fullpath(self): return os.path.abspath(filetools.join(self._path, self._filename)) # Funciones def start_dialog(self, title=config.get_localized_string(60200)): from platformcode import platformtools progreso = platformtools.dialog_progress_bg( title, config.get_localized_string(60201)) try: self.start() while self.state == self.states.downloading: time.sleep(0.2) line1 = "%s" % (self.filename) line2 = config.get_localized_string(59983) % ( self.downloaded[1], self.downloaded[2], self.size[1], self.size[2], self.speed[1], self.speed[2], self.connections[0], self.connections[1]) line3 = config.get_localized_string(60202) % ( self.remaining_time) progreso.update(int(self.progress), line1, line2 + line3) self.__update_json() finally: progreso.close() def start(self): self.__update_json(started=False) if self._state == self.states.error: return conns = [] for x in range(self._max_connections): try: conns.append(self.__open_connection__("0", "")) except: self._max_connections = x self._threads = [ Thread(target=self.__start_part__, name="Downloader %s/%s" % (x + 1, self._max_connections)) for x in range(self._max_connections) ] break del conns self._start_time = time.time() - 1 self._state = self.states.downloading self._speed_thread.start() self._save_thread.start() for t in self._threads: t.start() def stop(self, erase=False): if self._state == self.states.downloading: # Detenemos la descarga self._state = self.states.stopped for t in self._threads: if t.isAlive(): t.join() if self._save_thread.isAlive(): self._save_thread.join() if self._seekable: # Guardamos la info al final del archivo self.file.seek(0, 2) try: offset = self.file.tell() except: offset = self.file.seek(0, 1) if not PY3: self.file.write(str(self._download_info)) self.file.write("%0.16d" % offset) else: download_info_dump = jsontools.dump( self._download_info).encode('utf-8') self.file.write(download_info_dump) self.file.write(b"%0.16d" % offset) self.file.close() if erase: os.remove(filetools.join(self._path, self._filename)) def __speed_metter__(self): self._speed = 0 self._average_speed = 0 downloaded = self._start_downloaded downloaded2 = self._start_downloaded t = time.time() t2 = time.time() time.sleep(1) while self.state == self.states.downloading: self._average_speed = old_div( (self.downloaded[0] - self._start_downloaded), (time.time() - self._start_time)) self._speed = old_div( (self.downloaded[0] - self._start_downloaded), (time.time() - self._start_time)) # self._speed = (self.downloaded[0] - downloaded) / (time.time() -t) if time.time() - t > 5: t = t2 downloaded = downloaded2 t2 = time.time() downloaded2 = self.downloaded[0] time.sleep(0.5) # Funciones internas def __init__(self, url, path, filename=None, headers=[], resume=True, max_connections=10, block_size=2**17, part_size=2**24, max_buffer=10, json_path=None): # Parametros self._resume = resume self._path = path self._filename = filename self._max_connections = max_connections self._block_size = block_size self._part_size = part_size self._max_buffer = max_buffer self._json_path = json_path try: import xbmc self.tmp_path = xbmc.translatePath("special://temp/") except: self.tmp_path = os.getenv("TEMP") or os.getenv("TMP") or os.getenv( "TMPDIR") self.states = type( 'states', (), { "stopped": 0, "connecting": 1, "downloading": 2, "completed": 3, "error": 4, "saving": 5 }) self._state = self.states.stopped self._download_lock = Lock() self._headers = { "User-Agent": "Kodi/15.2 (Windows NT 10.0; WOW64) App_Bitness/32 Version/15.2-Git:20151019-02e7013" } self._speed = 0 self._buffer = {} self._seekable = True self._threads = [ Thread(target=self.__start_part__, name="Downloader %s/%s" % (x + 1, self._max_connections)) for x in range(self._max_connections) ] self._speed_thread = Thread(target=self.__speed_metter__, name="Speed Meter") self._save_thread = Thread(target=self.__save_file__, name="File Writer") # Actualizamos los headers self._headers.update(dict(headers)) # Separamos los headers de la url self.__url_to_headers__(url) # Obtenemos la info del servidor self.__get_download_headers__() self._file_size = int(self.response_headers.get("content-length", "0")) if not self.response_headers.get( "accept-ranges") == "bytes" or self._file_size == 0: self._max_connections = 1 self._part_size = 0 self._resume = False # Obtenemos el nombre del archivo self.__get_download_filename__() # Abrimos en modo "a+" para que cree el archivo si no existe, luego en modo "r+b" para poder hacer seek() self.file = filetools.file_open(filetools.join(self._path, self._filename), "a+", vfs=VFS) if self.file: self.file.close() self.file = filetools.file_open(filetools.join(self._path, self._filename), "r+b", vfs=VFS) if not self.file: return if self._file_size >= 2**31 or not self._file_size: try: self.file.seek(2**31, 0) except OverflowError: self._seekable = False logger.info( "Cannot do seek() or tell() in files larger than 2GB") self.__get_download_info__() try: logger.info("Descarga inicializada: Partes: %s | Ruta: %s | Archivo: %s | Tamaño: %s" % \ (str(len(self._download_info["parts"])), self._pathencode('utf-8'), \ self._filenameencode('utf-8'), str(self._download_info["size"]))) except: pass def __url_to_headers__(self, url): # Separamos la url de los headers adicionales self.url = url.split("|")[0] # headers adicionales if "|" in url: self._headers.update( dict([[ header.split("=")[0], urllib.parse.unquote_plus(header.split("=")[1]) ] for header in url.split("|")[1].split("&")])) def __get_download_headers__(self): if self.url.startswith("https"): try: conn = urllib.request.urlopen( urllib.request.Request(self.url.replace("https", "http"), headers=self._headers)) conn.fp._sock.close() self.url = self.url.replace("https", "http") except: pass for x in range(3): try: if not sys.hexversion > 0x0204FFFF: conn = urllib.request.urlopen( urllib.request.Request(self.url, headers=self._headers)) conn.fp._sock.close() else: conn = urllib.request.urlopen(urllib.request.Request( self.url, headers=self._headers), timeout=5) except: self.response_headers = dict() self._state = self.states.error else: self.response_headers = conn.headers self._state = self.states.stopped break def __get_download_filename__(self): # Obtenemos nombre de archivo y extension if "filename" in self.response_headers.get( "content-disposition", "") and "attachment" in self.response_headers.get( "content-disposition", ""): cd_filename, cd_ext = os.path.splitext( urllib.parse.unquote_plus( re.compile( "attachment; filename ?= ?[\"|']?([^\"']+)[\"|']?"). match(self.response_headers.get( "content-disposition")).group(1))) elif "filename" in self.response_headers.get( "content-disposition", "") and "inline" in self.response_headers.get( "content-disposition", ""): cd_filename, cd_ext = os.path.splitext( urllib.parse.unquote_plus( re.compile( "inline; filename ?= ?[\"|']?([^\"']+)[\"|']?").match( self.response_headers.get( "content-disposition")).group(1))) else: cd_filename, cd_ext = "", "" url_filename, url_ext = os.path.splitext( urllib.parse.unquote_plus( filetools.basename(urllib.parse.urlparse(self.url)[2]))) if self.response_headers.get( "content-type", "application/octet-stream") != "application/octet-stream": mime_ext = mimetypes.guess_extension( self.response_headers.get("content-type")) else: mime_ext = "" # Seleccionamos el nombre mas adecuado if cd_filename: self.remote_filename = cd_filename if not self._filename: self._filename = cd_filename elif url_filename: self.remote_filename = url_filename if not self._filename: self._filename = url_filename # Seleccionamos la extension mas adecuada if cd_ext: if not cd_ext in self._filename: self._filename += cd_ext if self.remote_filename: self.remote_filename += cd_ext elif mime_ext: if not mime_ext in self._filename: self._filename += mime_ext if self.remote_filename: self.remote_filename += mime_ext elif url_ext: if not url_ext in self._filename: self._filename += url_ext if self.remote_filename: self.remote_filename += url_ext def __change_units__(self, value): import math units = ["B", "KB", "MB", "GB"] if value <= 0: return 0, 0, units[0] else: return value, old_div(value, 1024.0**int(math.log( value, 1024))), units[int(math.log(value, 1024))] def __get_download_info__(self): # Continuamos con una descarga que contiene la info al final del archivo self._download_info = {} try: if not self._resume: raise Exception() self.file.seek(-16, 2) offset = int(self.file.read()) self.file.seek(offset, 0) data = self.file.read()[:-16] self._download_info = eval(data) if not self._download_info["size"] == self._file_size: raise Exception() self.file.seek(offset, 0) try: self.file.truncate() except: pass if not self._seekable: for part in self._download_info["parts"]: if part["start"] >= 2**31 and part[ "status"] == self.states.completed: part["status"] == self.states.stopped part["current"] == part["start"] self._start_downloaded = sum([ c["current"] - c["start"] for c in self._download_info["parts"] ]) self.pending_parts = set([ x for x, a in enumerate(self._download_info["parts"]) if not a["status"] == self.states.completed ]) self.completed_parts = set([ x for x, a in enumerate(self._download_info["parts"]) if a["status"] == self.states.completed ]) self.save_parts = set() self.download_parts = set() # La info no existe o no es correcta, comenzamos de 0 except: self._download_info["parts"] = [] if self._file_size and self._part_size: for x in range(0, self._file_size, self._part_size): end = x + self._part_size - 1 if end >= self._file_size: end = self._file_size - 1 self._download_info["parts"].append({ "start": x, "end": end, "current": x, "status": self.states.stopped }) else: self._download_info["parts"].append({ "start": 0, "end": self._file_size - 1, "current": 0, "status": self.states.stopped }) self._download_info["size"] = self._file_size self._start_downloaded = 0 self.pending_parts = set( [x for x in range(len(self._download_info["parts"]))]) self.completed_parts = set() self.save_parts = set() self.download_parts = set() self.file.seek(0, 0) try: self.file.truncate() except: pass def __open_connection__(self, start, end): headers = self._headers.copy() if not end: end = "" headers.update({"Range": "bytes=%s-%s" % (start, end)}) if not sys.hexversion > 0x0204FFFF: conn = urllib.request.urlopen( urllib.request.Request(self.url, headers=headers)) else: conn = urllib.request.urlopen(urllib.request.Request( self.url, headers=headers), timeout=5) return conn def __check_consecutive__(self, id): return id == 0 or (len(self.completed_parts) >= id and sorted(self.completed_parts)[id - 1] == id - 1) def __save_file__(self): logger.info("Thread started: %s" % threading.current_thread().name) while self._state == self.states.downloading: if not self.pending_parts and not self.download_parts and not self.save_parts: # Descarga finalizada self._state = self.states.completed self.file.close() continue elif not self.save_parts: continue save_id = min(self.save_parts) if not self._seekable and self._download_info["parts"][save_id][ "start"] >= 2**31 and not self.__check_consecutive__( save_id): continue if self._seekable or self._download_info["parts"][save_id][ "start"] < 2**31: self.file.seek(self._download_info["parts"][save_id]["start"], 0) try: # file = open(os.path.join(self.tmp_path, self._filename + ".part%s" % save_id), "rb") # self.file.write(file.read()) # file.close() # os.remove(os.path.join(self.tmp_path, self._filename + ".part%s" % save_id)) for a in self._buffer.pop(save_id): self.file.write(a) self.save_parts.remove(save_id) self.completed_parts.add(save_id) self._download_info["parts"][save_id][ "status"] = self.states.completed except: import traceback logger.error(traceback.format_exc()) self._state = self.states.error if self.save_parts: for s in self.save_parts: self._download_info["parts"][s]["status"] = self.states.stopped self._download_info["parts"][s][ "current"] = self._download_info["parts"][s]["start"] logger.info("Thread stopped: %s" % threading.current_thread().name) def __get_part_id__(self): self._download_lock.acquire() if len(self.pending_parts): id = min(self.pending_parts) self.pending_parts.remove(id) self.download_parts.add(id) self._download_lock.release() return id else: self._download_lock.release() return None def __set_part_connecting__(self, id): logger.info("ID: %s Establishing connection" % id) self._download_info["parts"][id]["status"] = self.states.connecting def __set_part__error__(self, id): logger.info("ID: %s Download failed" % id) self._download_info["parts"][id]["status"] = self.states.error self.pending_parts.add(id) self.download_parts.remove(id) def __set_part__downloading__(self, id): logger.info("ID: %s Downloading data ..." % id) self._download_info["parts"][id]["status"] = self.states.downloading def __set_part_completed__(self, id): logger.info("ID: %s Download finished!" % id) self._download_info["parts"][id]["status"] = self.states.saving self.download_parts.remove(id) self.save_parts.add(id) while self._state == self.states.downloading and len( self._buffer) > self._max_connections + self._max_buffer: time.sleep(0.1) def __set_part_stopped__(self, id): if self._download_info["parts"][id][ "status"] == self.states.downloading: self._download_info["parts"][id]["status"] = self.states.stopped self.download_parts.remove(id) self.pending_parts.add(id) def __open_part_file__(self, id): #file = open(os.path.join(self.tmp_path, self._filename + ".part%s" % id), "a+") #file = open(os.path.join(self.tmp_path, self._filename + ".part%s" % id), "r+b") self.file = filetools.file_open(filetools.join( self.tmp_path, self._filename + ".part%s" % id), "a+", vfs=VFS) self.file.close() self.file = filetools.file_open(filetools.join( self.tmp_path, self._filename + ".part%s" % id), "r+b", vfs=VFS) file.seek( self._download_info["parts"][id]["current"] - self._download_info["parts"][id]["start"], 0) return file def __start_part__(self): logger.info("Thread Started: %s" % threading.current_thread().name) while self._state == self.states.downloading: id = self.__get_part_id__() if id is None: break self.__set_part_connecting__(id) try: connection = self.__open_connection__( self._download_info["parts"][id]["current"], self._download_info["parts"][id]["end"]) except: self.__set_part__error__(id) time.sleep(5) continue self.__set_part__downloading__(id) # file = self.__open_part_file__(id) if not id in self._buffer: self._buffer[id] = [] speed = [] while self._state == self.states.downloading: try: start = time.time() buffer = connection.read(self._block_size) speed.append( old_div(len(buffer), ((time.time() - start) or 0.001))) except: logger.info("ID: %s Error downloading data" % id) self._download_info["parts"][id][ "status"] = self.states.error self.pending_parts.add(id) self.download_parts.remove(id) break else: if len(buffer) and self._download_info["parts"][id][ "current"] < self._download_info["parts"][id][ "end"]: # file.write(buffer) self._buffer[id].append(buffer) self._download_info["parts"][id]["current"] += len( buffer) if len(speed) > 10: velocidad_minima = old_div( old_div(sum(speed), len(speed)), 3) velocidad = speed[-1] vm = self.__change_units__(velocidad_minima) v = self.__change_units__(velocidad) if velocidad_minima > speed[-1] and velocidad_minima > speed[-2] and \ self._download_info["parts"][id]["current"] < \ self._download_info["parts"][id]["end"]: if connection.fp: connection.fp._sock.close() logger.info( "ID: %s Restarting connection! | Minimum Speed: %.2f %s/s | Speed: %.2f %s/s" % \ (id, vm[1], vm[2], v[1], v[2])) # file.close() break else: self.__set_part_completed__(id) if connection.fp: connection.fp._sock.close() # file.close() break self.__set_part_stopped__(id) logger.info("Thread stopped: %s" % threading.current_thread().name) def __update_json(self, started=True): item = Item().fromjson(filetools.read(self._json_path)) progress = int(self.progress) if started and item.downloadStatus == 0: # stopped logger.info('Download paused') self.stop() elif item.downloadProgress != progress or not started: params = { "downloadStatus": 4, "downloadComplete": 0, "downloadProgress": progress } item.__dict__.update(params) filetools.write(self._json_path, item.tojson())
class MySqlPool(object): __pool = None def __init__(self, db, host, port, user, passwd): # 数据库构造函数,从连接池中取出连接,并生成操作游标 try: self.lock = Lock() self._conn = PooledDB(creator=pymysql, mincached=1, maxcached=20, host=host, port=port, user=user, passwd=passwd, db=db, use_unicode=True, charset='utf8mb4', cursorclass=DictCursor).connection() self._cursor = self._conn.cursor() except Exception as e: print(e) def __query(self, sql, param=None): if param is None: count = self._cursor.execute(sql) else: count = self._cursor.execute(sql, param) return count def __execute(self, sql, param=None): conn = self._conn cursor = conn.cursor() try: self.lock.acquire() cursor.execute(sql, param) conn.commit() self.lock.release() except pymysql.Error as e: conn.rollback() print(e) def excute(self, sql): self.__execute(sql) def __execute_many(self, sql, param=None): conn = self._conn cursor = conn.cursor() try: self.lock.acquire() cursor.executemany(sql, param) conn.commit() self.lock.release() except pymysql.Error as e: conn.rollback() print(e) def update(self, sql, param=None): """ pool = MySqlPool() pool.update("update user_info set name = %s where user_id = %s", param=('Lucy', '001')) """ self.__execute(sql, param) def delete(self, sql, param=None): """ pool = MySqlPool() pool.delete('delete from user_info where name is null') """ self.__execute(sql, param) def insert_one(self, sql, param=None): """ pool = MySqlPool() pool.insert_one("INSERT INTO `user_info` (`user_id`,`name`) VALUES (%s,%s)", ('001', 'Jim')) """ self.__execute(sql, param) def insert_many(self, sql, param=None): """ pool = MySqlPool() pool.insert_many("INSERT INTO `user_info` (`user_id`,`name`) VALUES (%s,%s)", [('001', 'Jim'), ('002', 'Lucius')]) """ self.__execute_many(sql, param) def get_all(self, sql, param=None): conn = self._conn cursor = conn.cursor() count = cursor.execute(sql, param) if count > 0: result = cursor.fetchall() else: result = False return result def get_one(self, sql, param=None): conn = self._conn cursor = conn.cursor() count = cursor.execute(sql, param) if count > 0: result = cursor.fetchone() else: result = False return result def get_many(self, sql, num, param=None): conn = self._conn cursor = conn.cursor() count = cursor.execute(sql, param) if count > 0: result = cursor.fetchmany(num) else: result = False return result def dispose(self): self._conn.close()
class DXLCommunicator(Communicator): """ Communicator class for DXL MX-64 devices. This class implements Dynamixel MX-64 series specific packet communicator (uses DXL Protocol 1.0). Only goal torque actuator commands are currently allowed. """ def __init__( self, idn, baudrate, timeout_connection=0.5, sensor_dt=0.006, device_path='None', use_ctypes_driver=True, ): """ Inits DXLCommunicator class with device and task-specific parameters. Establishes serial connection with a dxl device. If device_path is 'None', the function searches for a serial port address and connects to the first device found. Args: idn: An integer representing the DXL ID number baudrate: An integer representing a baudrate to connect at timeout_connection: A float representing connection timeout parameter in seconds sensor_dt: A float representing the cycle time for polling sensory information and writing torque commands. device_path: A string containing the serial port address (e.g., /dev/ttyACM0 or /dev/ttyUSB0 on linux) use_ctypes_driver: A bool. Use this flag to choose between ctypes based DynamixelSDK or a custom pyserial DXL driver. By default, the ctypes driver is used. """ self.idn = idn self.baudrate = baudrate self.timeout = timeout_connection self.sensor_dt = sensor_dt self.device_path = device_path self.dt_tol = 0.0001 self.read_wait_time = 0.0001 self.use_ctypes_driver = use_ctypes_driver self.read_block = dxl_mx64.MX64.subblock( 'version_0', 'goal_acceleration', ret_dxl_type=use_ctypes_driver) self.write_block = dxl_mx64.MX64.subblock( 'goal_torque', 'goal_torque', ret_dxl_type=use_ctypes_driver) sensor_args = { 'buffer_len': SharedBuffer.DEFAULT_BUFFER_LEN, 'array_len': self.read_block.len(), 'array_type': 'd', 'np_array_type': 'd' } actuator_args = { 'buffer_len': SharedBuffer.DEFAULT_BUFFER_LEN, 'array_len': 1, 'array_type': 'd', 'np_array_type': 'd' } super(DXLCommunicator, self).__init__(use_sensor=True, use_actuator=True, sensor_args=sensor_args, actuator_args=actuator_args) self.just_read = 0 self.just_read_lock = Lock() self.port_lock = Lock() self.read_start_time = self.read_end_time = time.time() self.read_time = 0 self.last_actuation_updated = time.time() self.max_actuation_time = 1 self.torque = 0 def run(self): """ Override base class run method to setup dxl driver. The shared object file should be loaded within the process that sends commands and receives information from the DXL. Hence the dxl_driver is initialized here. This is an issue only when using the ctypes driver. Regular imports work for the pyserial driver. """ if self.use_ctypes_driver: from senseact.devices.dxl import dxl_driver_v1 as dxl_commv1 else: from senseact.devices.dxl import dxl_commv1 self.dxl_driver = dxl_commv1 # Make connection self.port = dxl_commv1.make_connection(self.baudrate, self.timeout, self.device_path) # Prepare for torque control self.write_torque_mode_enable(dxl_commv1, self.port, self.idn, 1, use_ctypes_driver=self.use_ctypes_driver) vals = self.dxl_driver.read_a_block(self.port, self.idn, self.read_block, self.read_wait_time) # Set Return Delay time = 0 if not vals["rtd"] == 0: self.write_return_delay_time( dxl_commv1, self.port, self.idn, 0, use_ctypes_driver=self.use_ctypes_driver) # Set the dynamixel to wheel mode (infinite rotations and mx-64AT encoder positions Є [0, 4095]) if not (vals["angle_limit_cw"] == -pi and vals["angle_limit_ccw"] == -pi): self.set_wheel_mode() # Overwrite torque command written in the preceding expt in case of unsafe exit self.write_torque(0) # catching SIGINT (Ctrl+C) so that we can close dxl safely signal.signal(signal.SIGINT, self._close) super(DXLCommunicator, self).run() def _sensor_handler(self): """ Receives and stores sensory packets from DXL. Waits for packets to arrive from DXL through `dxl_driver.read_a_block_vals` call, communicator sensor cycle time, and stores the packet in `sensor_buffer`. """ self._time_and_wait() self.port_lock.acquire() vals = self.dxl_driver.read_a_block_vals(self.port, self.idn, self.read_block, self.read_wait_time) self.port_lock.release() self.just_read_lock.acquire() self.just_read = 1 self.just_read_lock.release() self.sensor_buffer.write(vals) def _actuator_handler(self): """ Sends actuation commands to DXL. Only torque control allowed now. We send zero torque (i.e., stopping movements) if there is a delay in actuation. """ if time.time() - self.last_actuation_updated > self.max_actuation_time: self.torque = 0 self.just_read_lock.acquire() if self.just_read == 1: self.just_read = 0 self.just_read_lock.release() else: self.just_read_lock.release() time.sleep(0.0001) return if self.actuator_buffer.updated(): self.last_actuation_updated = time.time() recent_actuation, _, _ = self.actuator_buffer.read_update() self.torque = recent_actuation[0] self.write_torque(self.torque) def write_torque(self, torque): """ Writes goal torque commands to the DXL control table. Args: torque: An float between [-1024 and 1024] """ packet = self.dxl_driver.packet_write_buffer( self.idn, self.write_block, self.write_block.data_from_vals([torque])) self.port_lock.acquire() self.dxl_driver.loop_until_written(self.port, self.idn, packet) self.port_lock.release() def _time_and_wait(self): """ Maintains communicator cycle time. If sending a command takes 'x'ms, wait (sensor_dt - x)ms to maintain cycle time. """ self.read_end_time = time.time() self.read_time = self.read_end_time - self.read_start_time if self.read_time > self.sensor_dt: print("Warning: Iteration time exceeded sensor_dt {}ms by {}ms". format(self.sensor_dt * 1000, (self.read_time - self.sensor_dt) * 1000)) else: time.sleep(max(0, self.sensor_dt - self.read_time - self.dt_tol)) self.read_start_time = time.time() def _close(self, *args, **kwargs): """ Safely closes the DXL device. Disables torque before shutting down the script. NOTE: This method currently gets called 3 times due to interactions with gym.core.env the following two lines can be used for debugging if this needs to get fixed Args: *args: Additional args **kwargs: Additional keyword args """ print("Closing Gripper communicator") # Wait for the sensor and actuator threads to close before setting the torque to 0 # This is to ensure there isn't contention for sending messages on the port super(DXLCommunicator, self)._close() self.write_torque_mode_enable(self.dxl_driver, self.port, self.idn, 0, use_ctypes_driver=self.use_ctypes_driver) self.write_torque(0) self.dxl_driver.clear_port(self.port) self.dxl_driver.close(self.port) @staticmethod def write_torque_mode_enable(dxl_driver, port, idn, enable, use_ctypes_driver=True): """ Enables/Disables torque control mode in DXL MX-64 devices. 0 - Disabled - Position control mode 1 - Enabled - Torque control mode Args: dxl_driver: A variable that points to the DXL driver port: Dynamixel portHandler object idn: An integer representing the DXL ID number enable: 0 or 1 use_ctypes_driver: A bool to choose between Ctypes and pyserial driver """ block = dxl_mx64.MX64.subblock('torque_control_mode_enable', 'torque_control_mode_enable', ret_dxl_type=use_ctypes_driver) packet = dxl_driver.packet_write_buffer(idn, block, block.data_from_vals([enable])) dxl_driver.loop_until_written(port, idn, packet) @staticmethod def write_return_delay_time(dxl_driver, port, idn, rtd, use_ctypes_driver=True): """ Writes the return delay time to the DXL control table Return Delay Time is the delay time per data value that takes from the transmission of Instruction Packet until the return of Status Packet. Args: dxl_driver: A variable that points to the DXL driver port: Dynamixel portHandler object idn: An integer representing the DXL ID number rtd: A float representing delay time between packets in milliseconds. use_ctypes_driver: A bool to choose between Ctypes and pyserial driver """ block = dxl_mx64.MX64.subblock('rtd', 'rtd', ret_dxl_type=use_ctypes_driver) packet = dxl_driver.packet_write_buffer(idn, block, block.data_from_vals([rtd])) dxl_driver.loop_until_written(port, idn, packet) @staticmethod def write_to_register(dxl_driver, port, idn, reg_name, val, use_ctypes_driver=True): """ Writes a value to a given register of the DXL device General function to write a value to any given register Args: dxl_driver:A variable that points to the DXL driver port: Dynamixel portHandler object idn: An integer representing the DXL ID number reg_name: A string containing the name of the register to write to val: An int or a float depending on the register use_ctypes_driver: A bool to choose between Ctypes and pyserial driver """ block = dxl_mx64.MX64.subblock(reg_name, reg_name, ret_dxl_type=use_ctypes_driver) packet = dxl_driver.packet_write_buffer(idn, block, block.data_from_vals([val])) dxl_driver.loop_until_written(port, idn, packet) def set_wheel_mode(self): """ Sets the DXL to wheel mode (i.e., infinite turns) This is done by setting angle_limit_cw and angle_limit_ccw registers to zero in the control table """ self.write_to_register(self.dxl_driver, self.port, self.idn, 'angle_limit_cw', -pi, use_ctypes_driver=self.use_ctypes_driver) self.write_to_register(self.dxl_driver, self.port, self.idn, 'angle_limit_ccw', -pi, use_ctypes_driver=self.use_ctypes_driver)
class BTCtl: DATA_IN = 0x82 DATA_OUT = 0x05 def __init__(self, usb): self._usb = usb self._rx_thread = None self._con = False self._con_lock = Lock() self._rx_stopped = Event() self._rx_stopped.clear() self._msg_handler = None @staticmethod def find(): context = usb1.USBContext() handle = context.openByVendorIDAndProductID(UBERTOOTH_VENDOR_ID, UBERTOOTH_PRODUCT_ID, skip_on_error=True) if handle is None: raise Exception("Ubertooth device not found") sys.exit(1) return BTCtl(handle) def register_msg_handler(self, h): assert (self._msg_handler is None) self._msg_handler = h def unregister_msg_handler(self, h): assert (self._msg_handler == h) self._msg_handler = None def _print_console(self, msg): msg = "\x1b[32;1m%s\x1b[0m" % msg.decode() eprint(msg) def _print_debug(self, msg): msg = "\x1b[31;1m%s\x1b[0m" % msg.decode() eprint(msg) def _handle_default(self, t, msg): if t == BTCTL_STATE_RESP: print_state(*msg) else: log.info("No cmd for message %d" % t) def _handle_msg(self, data): t, msg = data[0], data[4:] if t == BTCTL_DEBUG: self._print_console(msg) else: if t == BTCTL_RX_PKT: msg = BTCtlRxPkt.unpack(msg) if self._msg_handler is not None: self._msg_handler(t, msg) else: self._handle_default(t, msg) def _rx_thread_main(self): msg = None msg_size = 0 log.info("Rx thread started") while not self._rx_stopped.isSet(): try: data = self._usb.bulkRead(self.DATA_IN, 64, 100) except usb1.USBErrorTimeout: continue t = data[:1] #log.info ("got data (l:%d, t:%s): %s"%(len(data),repr(t),repr(data))) if t == b'P': self._print_debug(data[1:].strip(b"\x00")) else: if msg is None: assert (t == b'S') msg_size, = unpack("<H", data[2:4]) msg = data[4:] assert (len(data) == min(64, 4 + msg_size)) else: assert (t == b'C') assert (len(data) == min(64, 1 + msg_size - len(msg))) msg += data[1:] if len(msg) == msg_size: self._handle_msg(msg) msg = None log.debug("Rx thread stopped") def connect(self): if self.connected(): log.warning("Already connected") return self._usb.claimInterface(0) self._rx_thread = Thread(target=self._rx_thread_main) self._rx_thread.start() log.info("USB connected") self._con = True self.send_idle_cmd() sleep(1) def connected(self): return self._con def close(self): if not self.connected(): log.warning("Not connected") return self.send_idle_cmd() self._rx_stopped.set() self._rx_thread.join() self._usb.close() self._con = False def _send_usb_bulk(self, bulk): assert (len(bulk) <= 64) self._usb.bulkWrite(self.DATA_OUT, bulk) def _send_usb(self, data): self._con_lock.acquire() usb_hdr = b"S\x00" + pack("<H", len(data)) chunk, data = data[:60], data[60:] self._send_usb_bulk(usb_hdr + chunk) while data: chunk, data = data[:63], data[63:] self._send_usb_bulk(b'C' + chunk) self._con_lock.release() def _send_cmd(self, t, data=b''): hdr = pack("<I", t) self._send_usb(hdr + data) def send_debug_cmd(self, msg): msg = msg[:256] self._send_cmd(BTCTL_DEBUG, msg) def send_idle_cmd(self): log.info("Send idle") self._send_cmd(BTCTL_IDLE_REQ) def send_set_freq_off_cmd(self, off): self._send_cmd(BTCTL_SET_FREQ_OFF_REQ, p16(off)) def send_set_max_ac_errors_cmd(self, max_ac_errors): self._send_cmd(BTCTL_SET_MAX_AC_ERRORS_REQ, p16(max_ac_errors)) def send_set_bdaddr_cmd(self, bdaddr): self._send_cmd(BTCTL_SET_BDADDR_REQ, pack("<Q", bdaddr)) def send_inquiry_cmd(self): self._send_cmd(BTCTL_INQUIRY_REQ) def send_inquiry_scan_cmd(self): self._send_cmd(BTCTL_INQUIRY_SCAN_REQ) def send_paging_cmd(self, bdaddr): self._send_cmd(BTCTL_PAGING_REQ, pack("<Q", bdaddr)) def send_monitor_cmd(self, bdaddr): self._send_cmd(BTCTL_MONITOR_REQ, pack("<Q", bdaddr)) def send_page_scan_cmd(self): self._send_cmd(BTCTL_PAGE_SCAN_REQ) def send_acl_cmd(self, llid, data, flow=1, lt_addr=1, flags=0, bt_type=None): acl = BTCtlACLPkt(llid, data, flow, bt_type) bb_hdr = BBHdr(lt_addr, acl.bt_type, flags) payload = bb_hdr.pack() + acl.pack() log.debug("send bb %s, acl: %s" % (bb_hdr, acl)) self._send_cmd(BTCTL_TX_ACL_REQ, payload) def send_set_eir_cmd(self, eir_data): acl = BTCtlACLPkt(LLID_L2CAP_START, eir_data, 1) bb_hdr = BBHdr(0, acl.bt_type) # send on lt_addr 0 payload = bb_hdr.pack() + acl.pack() log.debug("Set eir %s, acl: %s" % (bb_hdr, acl)) self._send_cmd(BTCTL_SET_EIR_REQ, payload) def send_set_afh_cmd(self, instant, mode, afh_map): cmd = pack("<IB", instant, mode) + afh_map log.debug("send set afh: %s" % hexlify(cmd)) self._send_cmd(BTCTL_SET_AFH_REQ, cmd)
class RobotManager(object): _instance = None def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super(RobotManager, cls).__new__(cls, *args, **kwargs) cls.__initialized = False return cls._instance def init(self): if self.__initialized: return self.__initialized = True log.i('robot manager init') if LAUNCH_NAVIGATION: self.launch_nav = roslauncher.roslauncher(NAVIGATION_LAUNCH_CMD) self.launch_nav.launch() self.robot_speed = Twist() # tempararily stores robot speed self.robot_pose = Pose() # tempararily stores robot pose self.rs = RobotStatus() ### Robot Stauts Lock # - Used to protect all the data in robot_status # - Should always acquire/release this lock when doing something about robot_status. self.robot_status_lock = Lock() # ros rospy.init_node('robot_manager', anonymous=False) rospy.Subscriber('/odom', Odometry, self.odom_cb) self.cmd_vel_pub = rospy.Publisher('/cmd_vel', Twist, queue_size=10) self.init_pose_pub = rospy.Publisher('/initialpose', PoseWithCovarianceStamped, queue_size=10) if ROBOT_POSE_TYPE == ABSOLUTE: self.tf_listener = tf.TransformListener() try: self.tf_listener.waitForTransform('/map', '/base_link', rospy.Time(0), rospy.Duration(30.0)) except (tf.Exception, tf.ConnectivityException, tf.LookupException) as e: log.e('wait for tf transform between /map and /base_link timeout %s'%e) return False # robot status update thread self.rs_update_rountine = Thread(target=self.rs_update, name='robot_status_update') self.rs_update_rountine.setDaemon(True) self.rs_update_rountine.start() if ROBOT_STATUS_DEBUG: self.rs_debug = Thread(target=self.rs_debug_print, name='robot_status_debug') self.rs_debug.setDaemon(True) self.rs_debug.start() return True def spin(self): rospy.spin() def odom_cb(self, odom): self.robot_speed.linear.x = odom.twist.twist.linear.x self.robot_speed.linear.y = odom.twist.twist.linear.y self.robot_speed.angular.z = odom.twist.twist.angular.z if ROBOT_POSE_TYPE == RELATIVE: self.robot_pose = odom.pose.pose def lookup_tf(self): try: (trans, rot) = self.tf_listener.lookupTransform('/map', '/base_link', rospy.Time(0)) except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException): if DEBUG: log.e('fail to lookup tf transformation. tf error') return self.robot_pose.position.x = trans[0] self.robot_pose.position.y = trans[1] self.robot_pose.orientation.x = rot[0] self.robot_pose.orientation.y = rot[1] self.robot_pose.orientation.z = rot[2] self.robot_pose.orientation.w = rot[3] def rs_debug_print(self): now = time() while True: if time()-now > 1.0: if self.robot_status_lock.acquire(): log.d('speed - vx:',self.rs.vx, 'vy:', self.rs.vy, 'vw:', self.rs.vw) log.d('pose - x:', self.rs.x, 'y:', self.rs.y, 'yaw:', self.rs.yaw) self.robot_status_lock.release() now = time() def rs_update(self): ''' Update velocities and pose to robot status ''' interval = 1.0/ROBOT_STATUS_UPDATE_FREQUENCY now = time() while True: if (time()-now)>interval: if self.robot_status_lock.acquire(): # update velocities self.rs.vx = self.robot_speed.linear.x self.rs.vy = self.robot_speed.linear.y self.rs.vw = self.robot_speed.angular.z # update pose if ROBOT_POSE_TYPE == ABSOLUTE: self.lookup_tf() if ROBOT_POSE_TYPE == RELATIVE: pass self.rs.x = self.robot_pose.position.x self.rs.y = self.robot_pose.position.y orientation = self.robot_pose.orientation quat = [orientation.x, orientation.y, orientation.z, orientation.w] self.rs.yaw = tf.transformations.euler_from_quaternion(quat)[-1] self.robot_status_lock.release() log.t("v=[%f, %f, %f] pos=[%f, %f, %f]" %(self.rs.vx, self.rs.vy, self.rs.vw, self.rs.x, self.rs.y, self.rs.yaw)) now = time() def get_speed(self): '''Get the current robot speeds. ''' self.robot_status_lock.acquire() vx = self.rs.vx vy = self.rs.vy vw = self.rs.vw self.robot_status_lock.release() return (vx, vy, vw) def get_pose(self): '''Get the current robot pose. ''' self.robot_status_lock.acquire() x = self.rs.x y = self.rs.y yaw = self.rs.yaw self.robot_status_lock.release() return (x, y, yaw) def set_vel(self, vx, vy, vw): '''Control the robot's speeds by publishing command to /cmd_vel topic.''' cmd = Twist() cmd.linear.x = vx cmd.linear.y = vy cmd.angular.z = vw self.cmd_vel_pub.publish(cmd) def set_pose(self, x, y, yaw): '''Set the robot pose''' pose = PoseWithCovarianceStamped() pose.header.stamp = rospy.Time.now() pose.header.frame_id = 'map' pose.pose.covariance = [0.25, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.25, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06853891945200942] pose.pose.pose.position.x = x pose.pose.pose.position.y = y quat = tf.transformations.quaternion_from_euler(0, 0, yaw) pose.pose.pose.orientation.x = quat[0] pose.pose.pose.orientation.y = quat[1] pose.pose.pose.orientation.z = quat[2] pose.pose.pose.orientation.w = quat[3] self.init_pose_pub.publish(pose) # clear costmap rospy.wait_for_service('/move_base/clear_costmaps') clear = rospy.ServiceProxy('/move_base/clear_costmaps', Empty) try: clear() except rospy.ServiceException, e: print(e)
class WebsocketServerBase(ABC): def __init__(self, args, listen_address, listen_port, mitm_mapper, db_wrapper, routemanagers, device_mappings, auths): self.__current_users = {} self.__listen_adress = listen_address self.__listen_port = listen_port self._send_queue = queue.Queue() self.__received = {} # map with received IDs self.__receivedMutex = Lock() self.__requests = {} # map with ID, event mapping self.__requestsMutex = Lock() self.__nextId = 0 self.__idMutex = Lock() self.args = args self.db_wrapper = db_wrapper self.device_mappings = device_mappings self.routemanagers = routemanagers self.auths = auths self._mitm_mapper = mitm_mapper def start_server(self): log.info("Starting websocket server...") loop = asyncio.new_event_loop() # build list of origin IDs allowed_origins = [] for device in self.device_mappings.keys(): allowed_origins.append(device) log.info("Device mappings: %s" % str(self.device_mappings)) log.info("Allowed origins derived: %s" % str(allowed_origins)) asyncio.set_event_loop(loop) asyncio.get_event_loop().run_until_complete( websockets.serve(self.handler, self.__listen_adress, self.__listen_port, max_size=2**25, origins=allowed_origins, ping_timeout=60, ping_interval=60)) asyncio.get_event_loop().run_forever() async def __unregister(self, websocket): id = str(websocket.request_headers.get_all("Origin")[0]) worker = self.__current_users.get(id, None) if worker is None: return else: # worker[1].stop_worker() self.__current_users.pop(id) async def __register(self, websocket): # await websocket.recv() log.info("Client registering....") try: id = str(websocket.request_headers.get_all("Origin")[0]) except IndexError: log.warning( "Client from %s tried to connect without Origin header" % str(websocket)) # TODO: list IP or whatever... return False if self.auths: try: authBase64 = str( websocket.request_headers.get_all("Authorization")[0]) except IndexError: log.warning( "Client from %s tried to connect without auth header" % str(websocket)) return False if self.__current_users.get(id, None) is not None: log.warning("Worker for %s is already running" % str(id)) return False elif self.auths and authBase64 and not check_auth( authBase64, self.args, self.auths): return False lastKnownState = {} client_mapping = self.device_mappings[id] daytime_routemanager = self.routemanagers[ client_mapping["daytime_area"]].get("routemanager") if client_mapping.get("nighttime_area", None) is not None: nightime_routemanager = self.routemanagers[ client_mapping["nighttime_area"]].get("routemanager", None) else: nightime_routemanager = None devicesettings = client_mapping["settings"] started = False if MadGlobals.sleep is True: # start the appropriate nighttime manager if set if nightime_routemanager is None: pass elif nightime_routemanager.mode in [ "raids_mitm", "mon_mitm", "iv_mitm" ]: Worker = WorkerMITM(self.args, id, lastKnownState, self, daytime_routemanager, nightime_routemanager, self._mitm_mapper, devicesettings, db_wrapper=self.db_wrapper) started = True elif nightime_routemanager.mode in ["raids_ocr"]: from worker.WorkerOcr import WorkerOcr Worker = WorkerOcr(self.args, id, lastKnownState, self, daytime_routemanager, nightime_routemanager, devicesettings, db_wrapper=self.db_wrapper) started = True elif nightime_routemanager.mode in ["pokestops"]: Worker = WorkerQuests(self.args, id, lastKnownState, self, daytime_routemanager, nightime_routemanager, self._mitm_mapper, devicesettings, db_wrapper=self.db_wrapper) started = True else: log.fatal("Mode not implemented") sys.exit(1) if not MadGlobals.sleep or not started: # we either gotta run daytime mode OR nighttime routemanager not set if daytime_routemanager.mode in [ "raids_mitm", "mon_mitm", "iv_mitm" ]: Worker = WorkerMITM(self.args, id, lastKnownState, self, daytime_routemanager, nightime_routemanager, self._mitm_mapper, devicesettings, db_wrapper=self.db_wrapper) elif daytime_routemanager.mode in ["raids_ocr"]: from worker.WorkerOcr import WorkerOcr Worker = WorkerOcr(self.args, id, lastKnownState, self, daytime_routemanager, nightime_routemanager, devicesettings, db_wrapper=self.db_wrapper) elif daytime_routemanager.mode in ["pokestops"]: Worker = WorkerQuests(self.args, id, lastKnownState, self, daytime_routemanager, nightime_routemanager, self._mitm_mapper, devicesettings, db_wrapper=self.db_wrapper) else: log.fatal("Mode not implemented") sys.exit(1) newWorkerThread = Thread(name='worker_%s' % id, target=Worker.start_worker) self.__current_users[id] = [newWorkerThread, Worker, websocket] newWorkerThread.daemon = False newWorkerThread.start() return True def __send(self, id, message): nextMessage = OutgoingMessage(id, message) self._send_queue.put(nextMessage) async def _retrieve_next_send(self): found = None while found is None: try: found = self._send_queue.get_nowait() except: await asyncio.sleep(0.02) return found # return self._send_queue.get(True) async def _producer_handler(self, websocket, path): while True: # retrieve next message from queue to be sent, block if empty next = None while next is None: next = await self._retrieve_next_send() # TODO: next consists of pair <id, message>, split that up and send message to the user with the ID await self.__send_specific(next.id, next.message) # message = await websocket.recv() # log.debug("Recv: %s" % str("Done")) # await asyncio.wait([value[1].send(next.message) for key, value in self.__current_users if key == next.id]) # await websocket.send() async def __send_specific(self, id, message): for key, value in self.__current_users.items(): if key == id and value[2].open: await value[2].send(message) # [value[1].send(next.message) for key, value in self.__current_users if key == next.id] async def _consumer_handler(self, websocket, path): while True: message = None id = str(websocket.request_headers.get_all("Origin")[0]) try: asyncio.wait_for(websocket.recv(), timeout=0.01) message = await websocket.recv() except asyncio.TimeoutError: log.debug('timeout!') await asyncio.sleep(0.02) except websockets.exceptions.ConnectionClosed: log.debug("Connection closed while receiving data") log.debug("Closed connection to %s" % str(id)) worker = self.__current_users.get(id, None) return # TODO: cleanup, stop worker... if message is not None: self.__onMessage(message) async def handler(self, websocket, path): log.debug("Waiting for connections") continueWork = await self.__register(websocket) # newWorkerThread = Thread(name='worker%s' % id, target=self._consumer_handler, args=(websocket, path,)) # newWorkerThread.daemon = False # newWorkerThread.start() if not continueWork: return consumer_task = asyncio.ensure_future( self._consumer_handler(websocket, path)) producer_task = asyncio.ensure_future( self._producer_handler(websocket, path)) done, pending = await asyncio.wait( [producer_task, consumer_task], return_when=asyncio.FIRST_COMPLETED, ) for task in pending: task.cancel() log.info("All done with %s" % str(websocket.request_headers.get_all("Origin")[0])) await self.__unregister(websocket) def __onMessage(self, message): id = -1 response = None if isinstance(message, str): log.debug("Receiving message: %s" % str(message)) splitup = message.split(";") id = int(splitup[0]) response = splitup[1] else: log.debug("Received binary values.") id = int.from_bytes(message[:4], byteorder='big', signed=False) response = message[4:] self.__setResponse(id, response) if not self.__setEvent(id): # remove the response again - though that is kinda stupid self.__popResponse(id) def __getNewMessageId(self): self.__idMutex.acquire() self.__nextId += 1 self.__nextId = int(math.fmod(self.__nextId, 100000)) if self.__nextId == 100000: self.__nextId = 1 toBeReturned = self.__nextId self.__idMutex.release() return toBeReturned def __setRequest(self, id, event): self.__requestsMutex.acquire() self.__requests[id] = event self.__requestsMutex.release() def __setEvent(self, id): self.__requestsMutex.acquire() result = False if id in self.__requests: self.__requests[id].set() result = True else: # the request has already been deleted due to a timeout... result = False self.__requestsMutex.release() return result def __removeRequest(self, id): self.__requestsMutex.acquire() self.__requests.pop(id) self.__requestsMutex.release() def __setResponse(self, id, message): self.__receivedMutex.acquire() self.__received[id] = message self.__receivedMutex.release() def __popResponse(self, id): self.__receivedMutex.acquire() message = self.__received.pop(id) self.__receivedMutex.release() return message def sendAndWait(self, id, message, timeout): log.debug("Sending command: %s" % message) if self.__current_users.get(id, None) is None: raise WebsocketWorkerRemovedException messageId = self.__getNewMessageId() messageEvent = Event() messageEvent.clear() self.__setRequest(messageId, messageEvent) toBeSent = u"%s;%s" % (str(messageId), message) log.debug("Sending:") log.debug("To be sent: %s" % toBeSent) self.__send(id, toBeSent) result = None log.debug("Timeout: " + str(timeout)) if messageEvent.wait(timeout): log.debug("Received anser, popping response") log.debug("Received an answer") # okay, we can get the response.. result = self.__popResponse(messageId) log.debug("Answer: %s" % result) else: # timeout reached log.warning("Timeout reached while waiting for a response...") if self.__current_users.get(id, None) is None: raise WebsocketWorkerRemovedException log.debug("Received response: %s" % str(result)) self.__removeRequest(messageId) log.debug("Returning response to worker.") return result
class SoundSink(SoundPipeline): __gsignals__ = SoundPipeline.__generic_signals__.copy() __gsignals__.update({ "eos": one_arg_signal, }) def __init__(self, sink_type=None, sink_options={}, codecs=get_decoders(), codec_options={}, volume=1.0): if not sink_type: sink_type = get_default_sink() if sink_type not in get_sink_plugins(): raise InitExit(1, "invalid sink: %s" % sink_type) matching = [ x for x in CODEC_ORDER if (x in codecs and x in get_decoders()) ] log("SoundSink(..) found matching codecs %s", matching) if not matching: raise InitExit( 1, "no matching codecs between arguments '%s' and supported list '%s'" % (csv(codecs), csv(get_decoders().keys()))) codec = matching[0] decoder, parser, stream_compressor = get_decoder_elements(codec) SoundPipeline.__init__(self, codec) self.container_format = (parser or "").replace("demux", "").replace("depay", "") self.sink_type = sink_type self.stream_compressor = stream_compressor log("container format=%s, stream_compressor=%s, sink type=%s", self.container_format, self.stream_compressor, self.sink_type) self.levels = deque(maxlen=100) self.volume = None self.src = None self.queue = None self.normal_volume = volume self.target_volume = volume self.volume_timer = 0 self.overruns = 0 self.underruns = 0 self.overrun_events = deque(maxlen=100) self.queue_state = "starting" self.last_data = None self.last_underrun = 0 self.last_overrun = 0 self.refill = True self.last_max_update = monotonic_time() self.last_min_update = monotonic_time() self.level_lock = Lock() pipeline_els = [] appsrc_el = [ "appsrc", #"do-timestamp=1", "name=src", "emit-signals=0", "block=0", "is-live=0", "stream-type=%s" % STREAM_TYPE, "format=%s" % BUFFER_FORMAT ] pipeline_els.append(" ".join(appsrc_el)) if parser: pipeline_els.append(parser) if decoder: decoder_str = plugin_str(decoder, codec_options) pipeline_els.append(decoder_str) pipeline_els.append("audioconvert") pipeline_els.append("audioresample") if QUEUE_TIME > 0: pipeline_els.append(" ".join([ "queue", "name=queue", "min-threshold-time=0", "max-size-buffers=0", "max-size-bytes=0", "max-size-time=%s" % QUEUE_TIME, "leaky=%s" % QUEUE_LEAK ])) pipeline_els.append("volume name=volume volume=0") sink_attributes = SINK_SHARED_DEFAULT_ATTRIBUTES.copy() #anything older than this may cause problems (ie: centos 6.x) #because the attributes may not exist sink_attributes.update(SINK_DEFAULT_ATTRIBUTES.get(sink_type, {})) get_options_cb = DEFAULT_SINK_PLUGIN_OPTIONS.get( sink_type.replace("sink", "")) if get_options_cb: v = get_options_cb() log("%s()=%s", get_options_cb, v) sink_attributes.update(v) sink_attributes.update(sink_options) sink_str = plugin_str(sink_type, sink_attributes) pipeline_els.append(sink_str) if not self.setup_pipeline_and_bus(pipeline_els): return self.volume = self.pipeline.get_by_name("volume") self.src = self.pipeline.get_by_name("src") self.queue = self.pipeline.get_by_name("queue") if self.queue: if QUEUE_SILENT: self.queue.set_property("silent", False) else: self.queue.connect("overrun", self.queue_overrun) self.queue.connect("underrun", self.queue_underrun) self.queue.connect("running", self.queue_running) self.queue.connect("pushing", self.queue_pushing) def __repr__(self): return "SoundSink('%s' - %s)" % (self.pipeline_str, self.state) def cleanup(self): SoundPipeline.cleanup(self) self.cancel_volume_timer() self.sink_type = "" self.src = None def start(self): SoundPipeline.start(self) self.timeout_add(UNMUTE_DELAY, self.start_adjust_volume) def start_adjust_volume(self, interval=100): if self.volume_timer != 0: glib.source_remove(self.volume_timer) self.volume_timer = self.timeout_add(interval, self.adjust_volume) return False def cancel_volume_timer(self): if self.volume_timer != 0: glib.source_remove(self.volume_timer) self.volume_timer = 0 def adjust_volume(self): if not self.volume: self.volume_timer = 0 return False cv = self.volume.get_property("volume") delta = self.target_volume - cv from math import sqrt, copysign change = copysign(sqrt(abs(delta)), delta) / 15.0 gstlog("adjust_volume current volume=%.2f, change=%.2f", cv, change) self.volume.set_property("volume", max(0, cv + change)) if abs(delta) < 0.01: self.volume_timer = 0 return False return True def queue_pushing(self, *_args): gstlog("queue_pushing") self.queue_state = "pushing" self.emit_info() return True def queue_running(self, *_args): gstlog("queue_running") self.queue_state = "running" self.emit_info() return True def queue_underrun(self, *_args): now = monotonic_time() if self.queue_state == "starting" or 1000 * ( now - self.start_time) < GRACE_PERIOD: gstlog("ignoring underrun during startup") return 1 self.underruns += 1 gstlog("queue_underrun") self.queue_state = "underrun" if now - self.last_underrun > 5: #only count underruns when we're back to no min time: qmin = self.queue.get_property("min-threshold-time") // MS_TO_NS clt = self.queue.get_property("current-level-time") // MS_TO_NS gstlog("queue_underrun level=%3i, min=%3i", clt, qmin) if qmin == 0 and clt < 10: self.last_underrun = now self.refill = True self.set_max_level() self.set_min_level() self.emit_info() return 1 def get_level_range(self, mintime=2, maxtime=10): now = monotonic_time() filtered = [ v for t, v in tuple(self.levels) if (now - t) >= mintime and (now - t) <= maxtime ] if len(filtered) >= 10: maxl = max(filtered) minl = min(filtered) #range of the levels recorded: return maxl - minl return 0 def queue_overrun(self, *_args): now = monotonic_time() if self.queue_state == "starting" or 1000 * ( now - self.start_time) < GRACE_PERIOD: gstlog("ignoring overrun during startup") return 1 clt = self.queue.get_property("current-level-time") // MS_TO_NS log("queue_overrun level=%ims", clt) now = monotonic_time() #grace period of recording overruns: #(because when we record an overrun, we lower the max-time, # which causes more overruns!) if now - self.last_overrun > 2: self.last_overrun = now self.set_max_level() self.overrun_events.append(now) self.overruns += 1 return 1 def set_min_level(self): if not self.queue: return now = monotonic_time() elapsed = now - self.last_min_update lrange = self.get_level_range() log("set_min_level() lrange=%i, elapsed=%i", lrange, elapsed) if elapsed < 1: #not more than once a second return if self.refill: #need to have a gap between min and max, #so we cannot go higher than mst-50: mst = self.queue.get_property("max-size-time") // MS_TO_NS mrange = max(lrange + 100, UNDERRUN_MIN_LEVEL) mtt = min(mst - 50, mrange) gstlog( "set_min_level mtt=%3i, max-size-time=%3i, lrange=%s, mrange=%s (UNDERRUN_MIN_LEVEL=%s)", mtt, mst, lrange, mrange, UNDERRUN_MIN_LEVEL) else: mtt = 0 cmtt = self.queue.get_property("min-threshold-time") // MS_TO_NS if cmtt == mtt: return if not self.level_lock.acquire(False): gstlog("cannot get level lock for setting min-threshold-time") return try: self.queue.set_property("min-threshold-time", mtt * MS_TO_NS) gstlog("set_min_level min-threshold-time=%s", mtt) self.last_min_update = now finally: self.level_lock.release() def set_max_level(self): if not self.queue: return now = monotonic_time() elapsed = now - self.last_max_update if elapsed < 1: #not more than once a second return lrange = self.get_level_range(mintime=0) log("set_max_level lrange=%3i, elapsed=%is", lrange, int(elapsed)) cmst = self.queue.get_property("max-size-time") // MS_TO_NS #overruns in the last minute: olm = len([x for x in tuple(self.overrun_events) if now - x < 60]) #increase target if we have more than 5 overruns in the last minute: target_mst = lrange * (100 + MARGIN + min(100, olm * 20)) // 100 #from 100% down to 0% in 2 seconds after underrun: pct = max(0, int((self.last_overrun + 2 - now) * 50)) #use this last_overrun percentage value to temporarily decrease the target #(causes overruns that drop packets and lower the buffer level) target_mst = max(50, int(target_mst - pct * lrange // 100)) mst = (cmst + target_mst) // 2 if self.refill: #temporarily raise max level during underruns, #so set_min_level has more room for manoeuver: mst += UNDERRUN_MIN_LEVEL #cap it at 1 second: mst = min(mst, 1000) log( "set_max_level overrun count=%-2i, margin=%3i, pct=%2i, cmst=%3i, target=%3i, mst=%3i", olm, MARGIN, pct, cmst, target_mst, mst) if abs(cmst - mst) <= max(50, lrange // 2): #not enough difference return if not self.level_lock.acquire(False): gstlog("cannot get level lock for setting max-size-time") return try: self.queue.set_property("max-size-time", mst * MS_TO_NS) log("set_max_level max-size-time=%s", mst) self.last_max_update = now finally: self.level_lock.release() def eos(self): gstlog("eos()") if self.src: self.src.emit('end-of-stream') self.cleanup() return 0 def get_info(self): info = SoundPipeline.get_info(self) if QUEUE_TIME > 0 and self.queue: clt = self.queue.get_property("current-level-time") qmax = self.queue.get_property("max-size-time") qmin = self.queue.get_property("min-threshold-time") info["queue"] = { "min": qmin // MS_TO_NS, "max": qmax // MS_TO_NS, "cur": clt // MS_TO_NS, "pct": min(QUEUE_TIME, clt) * 100 // qmax, "overruns": self.overruns, "underruns": self.underruns, "state": self.queue_state, } return info def can_push_buffer(self): if not self.src: log("no source, dropping buffer") return False if self.state in ("stopped", "error"): log("pipeline is %s, dropping buffer", self.state) return False return True def uncompress_data(self, data, metadata): if not data or not metadata: return data compress = metadata.get("compress") if not compress: return data assert compress in ("lz4", "lzo") v = decompress_by_name(data, compress) #log("decompressed %s data: %i bytes into %i bytes", compress, len(data), len(v)) return v def add_data(self, data, metadata=None, packet_metadata=()): if not self.can_push_buffer(): return data = self.uncompress_data(data, metadata) for x in packet_metadata: self.do_add_data(x) if self.do_add_data(data, metadata): self.rec_queue_level(data) self.set_max_level() self.set_min_level() #drop back down quickly if the level has reached min: if self.refill: clt = self.queue.get_property("current-level-time") // MS_TO_NS qmin = self.queue.get_property( "min-threshold-time") // MS_TO_NS gstlog("add_data: refill=%s, level=%i, min=%i", self.refill, clt, qmin) if qmin > 0 and clt > qmin: self.refill = False self.emit_info() def do_add_data(self, data, metadata=None): #having a timestamp causes problems with the queue and overruns: log("do_add_data(%s bytes, %s) queue_state=%s", len(data), metadata, self.queue_state) buf = gst.Buffer.new_allocate(None, len(data), None) buf.fill(0, data) if metadata: #having a timestamp causes problems with the queue and overruns: #ts = metadata.get("timestamp") #if ts is not None: # buf.timestamp = normv(ts) # log.info("timestamp=%s", ts) d = metadata.get("duration") if d is not None: d = normv(d) if d > 0: buf.duration = normv(d) if self.push_buffer(buf): self.inc_buffer_count() self.inc_byte_count(len(data)) return True return False def rec_queue_level(self, data): q = self.queue if not q: return clt = q.get_property("current-level-time") // MS_TO_NS log("pushed %5i bytes, new buffer level: %3ims, queue state=%s", len(data), clt, self.queue_state) now = monotonic_time() self.levels.append((now, clt)) def push_buffer(self, buf): #buf.size = size #buf.timestamp = timestamp #buf.duration = duration #buf.offset = offset #buf.offset_end = offset_end #buf.set_caps(gst.caps_from_string(caps)) r = self.src.emit("push-buffer", buf) if r != gst.FlowReturn.OK: if self.queue_state != "error": log.error("Error pushing buffer: %s", r) self.update_state("error") self.emit('error', "push-buffer error: %s" % r) return 0 return 1
class Crazyflie(): """The Crazyflie class""" # Called on disconnect, no matter the reason disconnected = Caller() # Called on unintentional disconnect only connection_lost = Caller() # Called when the first packet in a new link is received link_established = Caller() # Called when the user requests a connection connection_requested = Caller() # Called when the link is established and the TOCs (that are not cached) # have been downloaded connected = Caller() # Called if establishing of the link fails (i.e times out) connection_failed = Caller() # Called for every packet received packet_received = Caller() # Called for every packet sent packet_sent = Caller() # Called when the link driver updates the link quality measurement link_quality_updated = Caller() state = State.DISCONNECTED def __init__(self, link=None, ro_cache=None, rw_cache=None): """ Create the objects from this module and register callbacks. ro_cache -- Path to read-only cache (string) rw_cache -- Path to read-write cache (string) """ self.link = link self._toc_cache = TocCache(ro_cache=ro_cache, rw_cache=rw_cache) self.incoming = _IncomingPacketHandler(self) self.incoming.setDaemon(True) self.incoming.start() self.commander = Commander(self) self.log = Log(self) self.console = Console(self) self.param = Param(self) self.link_uri = "" # Used for retry when no reply was sent back self.packet_received.add_callback(self._check_for_initial_packet_cb) self.packet_received.add_callback(self._check_for_answers) self._answer_patterns = {} self._send_lock = Lock() self.connected_ts = None # Connect callbacks to logger self.disconnected.add_callback( lambda uri: logger.info("Callback->Disconnected from [%s]", uri)) self.disconnected.add_callback(self._disconnected) self.link_established.add_callback( lambda uri: logger.info("Callback->Connected to [%s]", uri)) self.connection_lost.add_callback(lambda uri, errmsg: logger.info( "Callback->Connection lost to" " [%s]: %s", uri, errmsg)) self.connection_failed.add_callback(lambda uri, errmsg: logger.info( "Callback->Connected failed to" " [%s]: %s", uri, errmsg)) self.connection_requested.add_callback(lambda uri: logger.info( "Callback->Connection initialized[%s]", uri)) self.connected.add_callback(lambda uri: logger.info( "Callback->Connection setup finished [%s]", uri)) def _disconnected(self, link_uri): """ Callback when disconnected.""" self.connected_ts = None def _start_connection_setup(self): """Start the connection setup by refreshing the TOCs""" logger.info("We are connected[%s], request connection setup", self.link_uri) self.log.refresh_toc(self._log_toc_updated_cb, self._toc_cache) def _param_toc_updated_cb(self): """Called when the param TOC has been fully updated""" logger.info("Param TOC finished updating") self.connected_ts = datetime.datetime.now() self.connected.call(self.link_uri) def _log_toc_updated_cb(self): """Called when the log TOC has been fully updated""" logger.info("Log TOC finished updating") self.param.refresh_toc(self._param_toc_updated_cb, self._toc_cache) def _link_error_cb(self, errmsg): """Called from the link driver when there's an error""" logger.warning("Got link error callback [%s] in state [%s]", errmsg, self.state) if (self.link is not None): self.link.close() self.link = None if (self.state == State.INITIALIZED): self.connection_failed.call(self.link_uri, errmsg) if (self.state == State.CONNECTED or self.state == State.SETUP_FINISHED): self.disconnected.call(self.link_uri) self.connection_lost.call(self.link_uri, errmsg) self.state = State.DISCONNECTED def _link_quality_cb(self, percentage): """Called from link driver to report link quality""" self.link_quality_updated.call(percentage) def _check_for_initial_packet_cb(self, data): """ Called when first packet arrives from Crazyflie. This is used to determine if we are connected to something that is answering. """ self.state = State.CONNECTED self.link_established.call(self.link_uri) self.packet_received.remove_callback(self._check_for_initial_packet_cb) def open_link(self, link_uri): """ Open the communication link to a copter at the given URI and setup the connection (download log/parameter TOC). """ self.connection_requested.call(link_uri) self.state = State.INITIALIZED self.link_uri = link_uri try: self.link = cflib.crtp.get_link_driver(link_uri, self._link_quality_cb, self._link_error_cb) if not self.link: message = "No driver found or malformed URI: {}"\ .format(link_uri) logger.warning(message) self.connection_failed.call(link_uri, message) else: # Add a callback so we can check that any data is comming # back from the copter self.packet_received.add_callback( self._check_for_initial_packet_cb) self._start_connection_setup() except Exception as ex: # pylint: disable=W0703 # We want to catch every possible exception here and show # it in the user interface import traceback logger.error("Couldn't load link driver: %s\n\n%s", ex, traceback.format_exc()) exception_text = "Couldn't load link driver: %s\n\n%s" % ( ex, traceback.format_exc()) if self.link: self.link.close() self.link = None self.connection_failed.call(link_uri, exception_text) def close_link(self): """Close the communication link.""" logger.info("Closing link") if (self.link is not None): self.commander.send_setpoint(0, 0, 0, 0) if (self.link is not None): self.link.close() self.link = None self._answer_patterns = {} self.disconnected.call(self.link_uri) def add_port_callback(self, port, cb): """Add a callback to cb on port""" self.incoming.add_port_callback(port, cb) def remove_port_callback(self, port, cb): """Remove the callback cb on port""" self.incoming.remove_port_callback(port, cb) def _no_answer_do_retry(self, pk, pattern): """Resend packets that we have not gotten answers to""" logger.debug("Resending for pattern %s", pattern) # Set the timer to None before trying to send again self.send_packet(pk, expected_reply=pattern, resend=True) def _check_for_answers(self, pk): """ Callback called for every packet received to check if we are waiting for an answer on this port. If so, then cancel the retry timer. """ self._send_lock.acquire() longest_match = () if len(self._answer_patterns) > 0: data = (pk.header, ) + pk.datat for p in self._answer_patterns.keys(): logger.debug("Looking for pattern match on %s vs %s", p, data) if len(p) <= len(data): if p == data[0:len(p)]: match = data[0:len(p)] if len(match) >= len(longest_match): logger.debug("Found new longest match %s", match) longest_match = match if len(longest_match) > 0: del self._answer_patterns[longest_match] self._send_lock.release() def send_packet(self, pk, expected_reply=(), resend=False): """ Send a packet through the link interface. pk -- Packet to send expect_answer -- True if a packet from the Crazyflie is expected to be sent back, otherwise false """ self._send_lock.acquire() if (self.link is not None): self.link.send_packet(pk) self.packet_sent.call(pk) if len(expected_reply) > 0 and not resend: pattern = (pk.header, ) + expected_reply logger.debug( "Sending packet and expecting the %s pattern back", pattern) new_timer = Timer( 0.2, lambda: self._no_answer_do_retry(pk, pattern)) self._answer_patterns[pattern] = new_timer new_timer.start() elif resend: # Check if we have gotten an answer, if not try again pattern = expected_reply if pattern in self._answer_patterns: logger.debug("We want to resend and the pattern is there") if self._answer_patterns[pattern]: new_timer = Timer( 0.2, lambda: self._no_answer_do_retry(pk, pattern)) self._answer_patterns[pattern] = new_timer new_timer.start() else: logger.debug("Resend requested, but no pattern found: %s", self._answer_patterns) self._send_lock.release()
class SentinelHubDownloadClient(DownloadClient): """ Download client specifically configured for download from Sentinel Hub service """ _CACHED_SESSIONS = {} def __init__(self, *, session=None, **kwargs): """ :param session: An OAuth2 session with Sentinel Hub service :type session: SentinelHubSession or None :param kwargs: Optional parameters from DownloadClient """ super().__init__(**kwargs) if session is not None and not isinstance(session, SentinelHubSession): raise ValueError(f'A session parameter has to be an instance of {SentinelHubSession.__name__} or None, but ' f'{session} was given') self.session = session self.rate_limit = SentinelHubRateLimit(num_processes=self.config.number_of_download_processes) self.lock = Lock() @retry_temporal_errors @fail_user_errors def _execute_download(self, request): """ Executes the download with a single thread and uses a rate limit object, which is shared between all threads """ thread_name = currentThread().getName() while True: sleep_time = self._execute_with_lock(self.rate_limit.register_next) if sleep_time == 0: response = self._do_download(request) self._execute_with_lock(self.rate_limit.update, response.headers) if response.status_code != requests.status_codes.codes.TOO_MANY_REQUESTS: response.raise_for_status() LOGGER.debug('%s: Successful download from %s', thread_name, request.url) return response.content else: LOGGER.debug('%s: Sleeping for %0.2f', thread_name, sleep_time) time.sleep(sleep_time) def _execute_with_lock(self, thread_unsafe_function, *args, **kwargs): """ Executes a function inside a thread lock and handles potential errors """ self.lock.acquire() try: return thread_unsafe_function(*args, **kwargs) finally: self.lock.release() def _do_download(self, request): """ Runs the download """ return requests.request( request.request_type.value, url=request.url, json=request.post_values, headers=self._prepare_headers(request), timeout=self.config.download_timeout_seconds ) def _prepare_headers(self, request): """ Prepares final headers by potentially joining them with session headers """ if not request.use_session: return request.headers if self.session is None: self.session = self._execute_with_lock(self._get_session) return { **self.session.session_headers, **request.headers } def _get_session(self): """ Provides a session object either from cache or it creates a new one """ cache_key = self.config.sh_client_id, self.config.sh_client_secret, self.config.get_sh_oauth_url() if cache_key in SentinelHubDownloadClient._CACHED_SESSIONS: return SentinelHubDownloadClient._CACHED_SESSIONS[cache_key] session = SentinelHubSession(config=self.config) SentinelHubDownloadClient._CACHED_SESSIONS[cache_key] = session return session
class NucleoUSART(AvatarPeripheral, Thread): def read_status_register(self, offset, size): self.lock.acquire(True) ret = self.status_register self.lock.release() return ret def read_data_register(self, offset, size): self.lock.acquire(True) ret = self.data_buf[0] self.data_buf = self.data_buf[1:] if len(self.data_buf) == 0: self.status_register &= ~SR_RXNE self.lock.release() return ret def write_data_register(self, offset, size, value): if self.connected: self.conn.send(bytes((chr(value).encode('utf-8')))) return True def nop_read(self, offset, size): return 0x00 def nop_write(self, offset, size, value): return True def __init__(self, name, address, size, nucleo_usart_port=5656, **kwargs): Thread.__init__(self) AvatarPeripheral.__init__(self, name, address, size) self.port = nucleo_usart_port self.data_buf = bytearray() self.status_register = SR_TXE | SR_TC self.read_handler[0:3] = self.read_status_register self.read_handler[4:7] = self.read_data_register self.write_handler[4:7] = self.write_data_register self.read_handler[8:size] = self.nop_read self.write_handler[8:size] = self.nop_write self.connected = False self.lock = Lock() self._close = Event() self.sock = None self.conn = None self.daemon = True self.start() def shutdown(self): self._close.set() if self.conn: self.conn.close() if self.sock: self.sock.close() def run(self): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.setblocking(0) self.sock.bind(('127.0.0.1', self.port)) self.sock.settimeout(0.1) while not self._close.is_set(): self.sock.listen(1) try: self.conn, addr = self.sock.accept() self.conn.settimeout(0.1) self.connected = True except socket.timeout: continue except OSError as e: if e.errno == 9: # Bad file descriptor error. Only happens when we called # closed on the socket, the continuing the loop will # terminate, which is the desired behaviour continue else: # Something terrible happened raise (e) while not self._close.is_set(): try: chr = self.conn.recv(1) except socket.timeout: continue if not chr: break self.lock.acquire(True) self.data_buf += chr self.status_register |= SR_RXNE self.lock.release() self.connected = False
class DesignState(object): def __init__(self): self.designs = {} self.designs_lock = Lock() self.builds = [] self.builds_lock = Lock() self.tasks = [] self.tasks_lock = Lock() return # TODO Need to lock a design base or change once implementation # has started def get_design(self, design_id): if design_id not in self.designs.keys(): raise DesignError("Design ID %s not found" % (design_id)) return objects.SiteDesign.obj_from_primitive(self.designs[design_id]) def post_design(self, site_design): if site_design is not None: my_lock = self.designs_lock.acquire(blocking=True, timeout=10) if my_lock: design_id = site_design.id if design_id not in self.designs.keys(): self.designs[design_id] = site_design.obj_to_primitive() else: self.designs_lock.release() raise StateError("Design ID %s already exists" % design_id) self.designs_lock.release() return True raise StateError("Could not acquire lock") else: raise DesignError("Design change must be a SiteDesign instance") def put_design(self, site_design): if site_design is not None: my_lock = self.designs_lock.acquire(blocking=True, timeout=10) if my_lock: design_id = site_design.id if design_id not in self.designs.keys(): self.designs_lock.release() raise StateError("Design ID %s does not exist" % design_id) else: self.designs[design_id] = site_design.obj_to_primitive() self.designs_lock.release() return True raise StateError("Could not acquire lock") else: raise DesignError("Design base must be a SiteDesign instance") def get_current_build(self): latest_stamp = 0 current_build = None for b in self.builds: if b.build_id > latest_stamp: latest_stamp = b.build_id current_build = b return deepcopy(current_build) def get_build(self, build_id): for b in self.builds: if b.build_id == build_id: return b return None def post_build(self, site_build): if site_build is not None and isinstance(site_build, SiteBuild): my_lock = self.builds_lock.acquire(block=True, timeout=10) if my_lock: exists = [ b for b in self.builds if b.build_id == site_build.build_id ] if len(exists) > 0: self.builds_lock.release() raise DesignError("Already a site build with ID %s" % (str(site_build.build_id))) self.builds.append(deepcopy(site_build)) self.builds_lock.release() return True raise StateError("Could not acquire lock") else: raise DesignError("Design change must be a SiteDesign instance") def put_build(self, site_build): if site_build is not None and isinstance(site_build, SiteBuild): my_lock = self.builds_lock.acquire(block=True, timeout=10) if my_lock: buildid = site_build.buildid for b in self.builds: if b.buildid == buildid: b.merge_updates(site_build) self.builds_lock.release() return True self.builds_lock.release() return False raise StateError("Could not acquire lock") else: raise DesignError("Design change must be a SiteDesign instance") def get_task(self, task_id): for t in self.tasks: if t.get_id() == task_id or str(t.get_id()) == task_id: return deepcopy(t) return None def post_task(self, task): if task is not None and isinstance(task, tasks.Task): my_lock = self.tasks_lock.acquire(blocking=True, timeout=10) if my_lock: task_id = task.get_id() matching_tasks = [ t for t in self.tasks if t.get_id() == task_id ] if len(matching_tasks) > 0: self.tasks_lock.release() raise StateError("Task %s already created" % task_id) self.tasks.append(deepcopy(task)) self.tasks_lock.release() return True else: raise StateError("Could not acquire lock") else: raise StateError("Task is not the correct type") def put_task(self, task, lock_id=None): if task is not None and isinstance(task, tasks.Task): my_lock = self.tasks_lock.acquire(blocking=True, timeout=10) if my_lock: task_id = task.get_id() t = self.get_task(task_id) if t.lock_id is not None and t.lock_id != lock_id: self.tasks_lock.release() raise StateError("Task locked for updates") task.lock_id = lock_id self.tasks = [ i if i.get_id() != task_id else deepcopy(task) for i in self.tasks ] self.tasks_lock.release() return True else: raise StateError("Could not acquire lock") else: raise StateError("Task is not the correct type") def lock_task(self, task_id): my_lock = self.tasks_lock.acquire(blocking=True, timeout=10) if my_lock: lock_id = uuid.uuid4() for t in self.tasks: if t.get_id() == task_id and t.lock_id is None: t.lock_id = lock_id self.tasks_lock.release() return lock_id self.tasks_lock.release() return None else: raise StateError("Could not acquire lock") def unlock_task(self, task_id, lock_id): my_lock = self.tasks_lock.acquire(blocking=True, timeout=10) if my_lock: for t in self.tasks: if t.get_id() == task_id and t.lock_id == lock_id: t.lock_id = None self.tasks_lock.release() return True self.tasks_lock.release() return False else: raise StateError("Could not acquire lock")
class Highlight(object): def __init__(self, bot): self.bot = bot self.lock = Lock() self.highlights = dataIO.load_json("data/highlight/words.json") def _update_highlights(self, new_obj): self.lock.acquire() try: dataIO.save_json("data/highlight/words.json", new_obj) self.highlights = dataIO.load_json("data/highlight/words.json") finally: self.lock.release() async def _sleep_then_delete(self, msg, time): await asyncio.sleep(time) await self.bot.delete_message(msg) def _get_guild_ids(self): guilds = [list(x) for x in self.highlights['guilds']] return list(itertools.chain.from_iterable(guilds)) # flatten list def _check_guilds(self, guild_id): """returns guild pos in list""" guilds_ids = self._get_guild_ids() if guild_id not in guilds_ids: new_guild = {} users = {} users['users'] = [] new_guild[guild_id] = users self.highlights['guilds'].append(new_guild) self._update_highlights(self.highlights) return next(x for (x, d) in enumerate(self.highlights['guilds']) if guild_id in d) def _is_registered(self, guild_idx, guild_id, user_id): users = self.highlights['guilds'][guild_idx][guild_id]['users'] for user in users: if user_id == user['id']: return [ next(index for (index, d) in enumerate(users) if d["id"] == user['id']), user ] return None @commands.group(name="highlight", pass_context=True, no_pm=True) async def highlight(self, ctx): """Slack-like feature to be notified based on specific words outside of at-mentions""" if ctx.invoked_subcommand is None: await send_cmd_help(ctx) @highlight.command(name="add", pass_context=True, no_pm=True) async def add_highlight(self, ctx, word: str): """Add a word to be highlighted in the current guild""" guild_id = ctx.message.server.id user_id = ctx.message.author.id user_name = ctx.message.author.name guild_idx = self._check_guilds(guild_id) user = self._is_registered(guild_idx, guild_id, user_id) if user is not None: user_idx = user[0] user_add = user[1] if len(user_add['words']) <= 4 and word not in user_add[ 'words']: # user can only have max of 5 words user_add['words'].append(word) self.highlights['guilds'][guild_idx][guild_id]['users'][ user_idx] = user_add self._update_highlights(self.highlights) t_msg = await self.bot.say( "Highlight word added, {}".format(user_name)) await self._sleep_then_delete(t_msg, 2) else: msg = "Sorry {}, you already have 5 words highlighted" msg += ", or you are trying to add a duplicate word" t_msg = await self.bot.say(msg.format(user_name)) await self._sleep_then_delete(t_msg, 5) else: new_user = {} new_user['id'] = ctx.message.author.id new_user['words'] = [word] self.highlights['guilds'][guild_idx][guild_id]['users'].append( new_user) self._update_highlights(self.highlights) t_msg = await self.bot.say( "Registered and highlight word added, {}".format(user_name)) await self._sleep_then_delete(t_msg, 2) await self.bot.delete_message(ctx.message) @highlight.command(name="remove", pass_context=True, no_pm=True) async def remove_highlight(self, ctx, word: str): """Remove a highlighted word in the current guild""" guild_id = ctx.message.server.id user_id = ctx.message.author.id user_name = ctx.message.author.name guild_idx = self._check_guilds(guild_id) user = self._is_registered(guild_idx, guild_id, user_id) if user is not None: user_idx = user[0] user_rm = user[1] if word in user_rm['words']: user_rm['words'].remove(word) self.highlights['guilds'][guild_idx][guild_id]['users'][ user_idx] = user_rm self._update_highlights(self.highlights) t_msg = await self.bot.say( "Highlight word removed, {}".format(user_name)) await self._sleep_then_delete(t_msg, 2) else: t_msg = await self.bot.say( "Sorry {}, you don't have this word highlighted".format( user_name)) await self._sleep_then_delete(t_msg, 5) else: msg = "Sorry {}, you aren't currently registered for highlights." msg += " Add a word to become registered" t_msg = await self.bot.say(msg.format(user_name)) await self._sleep_then_delete(t_msg, 5) await self.bot.delete_message(ctx.message) @highlight.command(name="list", pass_context=True, no_pm=True) async def list_highlight(self, ctx): """List your highighted words for the current guild""" guild_id = ctx.message.server.id user_id = ctx.message.author.id user_name = ctx.message.author.name guild_idx = self._check_guilds(guild_id) user = self._is_registered(guild_idx, guild_id, user_id) if user is not None: user_list = user[1] if len(user_list['words']) > 0: msg = "" for word in user_list['words']: msg += word msg += "\n" embed = discord.Embed(description=msg, colour=discord.Colour.red()) embed.set_author(name=ctx.message.author.name, icon_url=ctx.message.author.avatar_url) t_msg = await self.bot.say(embed=embed) await self._sleep_then_delete(t_msg, 5) else: t_msg = await self.bot.say( "Sorry {}, you have no highlighted words currently".format( user_name)) await self._sleep_then_delete(t_msg, 5) else: msg = "Sorry {}, you aren't currently registered for highlights." msg += " Add a word to become registered" t_msg = await self.bot.say(msg.format(user_name)) await self._sleep_then_delete(t_msg, 5) @highlight.command(name="import", pass_context=True, no_pm=False) async def import_highlight(self, ctx, from_server: str): """Transfer highlights from a different guild to the current guild, OVERWRITING any words in the current guild""" guild_id = ctx.message.server.id user_id = ctx.message.author.id user_name = ctx.message.author.name guild_idx = self._check_guilds(guild_id) user = self._is_registered(guild_idx, guild_id, user_id) guild = discord.utils.get(self.bot.servers, name=from_server) # This is kind of ugly, dont really like it but kind of has to be done like this based on how i have # the highlight data structured, will definitely be revisting this if guild is not None and user is not None: user_idx = user[0] if guild.id in self._get_guild_ids(): impt_guild_idx = self._check_guilds(guild.id) impt_user = self._is_registered(impt_guild_idx, guild.id, user_id) if impt_user is not None: impt_user_idx = impt_user[0] impt = self.highlights['guilds'][impt_guild_idx][ guild.id]['users'][impt_user_idx] self.highlights['guilds'][guild_idx][guild_id]['users'][ user_idx] = impt self._update_highlights(self.highlights) t_msg = await self.bot.say( "Highlight words imported from {0} for {1}".format( from_server, user_name)) await self._sleep_then_delete(t_msg, 3) else: msg = "Sorry {}, the guild you want to import from" msg += " is not registered for highlights, or you are not registered in that guild" t_msg = await self.bot.say(msg.format(user_name)) await self._sleep_then_delete(t_msg, 5) else: msg = "Sorry {}, this bot is not in the guild you want to import from," msg += " or you are not registered in this guild" t_msg = await self.bot.say(msg.format(user_name)) await self._sleep_then_delete(t_msg, 5) async def check_highlights(self, msg): if isinstance(msg.channel, discord.PrivateChannel): return guild_id = msg.server.id user_id = msg.author.id user_name = msg.author.name user_obj = msg.author guild_idx = self._check_guilds(guild_id) # iterate through every users words on the server, and notify all highlights for user in self.highlights['guilds'][guild_idx][guild_id]['users']: for word in user['words']: active = await self._is_active(user['id'], msg.channel, msg) match = self._is_word_match(word, msg.content) if match and not active and user_id != user['id']: hilite_user = msg.server.get_member(user['id']) if hilite_user is None: # Handle case where user is no longer in the server of interest. continue perms = msg.channel.permissions_for(hilite_user) if not perms.read_messages: break await self._notify_user(hilite_user, msg, word) async def _notify_user(self, user, message, word): await asyncio.sleep( 3 ) # possibly pick up messages after trigger that will help with the context msgs = [] async for msg in self.bot.logs_from(message.channel, limit=6, around=message): msgs.append(msg) msg_ctx = sorted(msgs, key=lambda r: r.timestamp) notify_msg = "In {1.channel.mention}, you were mentioned with highlight word **{0}**:\n".format( word, message) embed_msg = "" for msg in msg_ctx: time = msg.timestamp time = time.replace(tzinfo=timezone.utc).astimezone( tz=None).strftime('%H:%M:%S %Z') embed_msg += "[{0}] {1.author.name}#{1.author.discriminator}: {1.content}\n".format( time, msg) embed = discord.Embed(title=user.name, description=embed_msg, colour=discord.Colour.red()) time = message.timestamp.replace(tzinfo=timezone.utc).astimezone( tz=None) footer = "Triggered at | {}".format( time.strftime('%a, %d %b %Y %I:%M%p %Z')) embed.set_footer(text=footer) await self.bot.send_message(user, content=notify_msg, embed=embed) def _is_word_match(self, word, string): regex = r'\b{}\b'.format(word.lower()) return bool(re.search(regex, string.lower())) async def _is_active(self, user_id, channel, message): is_active = False async for msg in self.bot.logs_from(channel, limit=50, before=message): delta_since_msg = message.timestamp - msg.timestamp if msg.author.id == user_id and delta_since_msg <= timedelta( seconds=20): is_active = True break return is_active
class Connection(object): """The RPyC *connection* (AKA *protocol*). :param root: the :class:`~rpyc.core.service.Service` object to expose :param channel: the :class:`~rpyc.core.channel.Channel` over which messages are passed :param config: the connection's configuration dict (overriding parameters from the :data:`default configuration <DEFAULT_CONFIG>`) """ def __init__(self, root, channel, config={}): self._closed = True self._config = DEFAULT_CONFIG.copy() self._config.update(config) if self._config["connid"] is None: self._config["connid"] = "conn%d" % ( next(_connection_id_generator), ) self._HANDLERS = self._request_handlers() self._channel = channel self._seqcounter = itertools.count() self._recvlock = Lock() self._sendlock = Lock() self._recv_event = Condition() self._request_callbacks = {} self._local_objects = RefCountingColl() self._last_traceback = None self._proxy_cache = WeakValueDict() self._netref_classes_cache = {} self._remote_root = None self._send_queue = [] self._local_root = root self._closed = False def __del__(self): self.close() def __enter__(self): return self def __exit__(self, t, v, tb): self.close() def __repr__(self): a, b = object.__repr__(self).split(" object ") return "%s %r object %s" % (a, self._config["connid"], b) def _cleanup(self, _anyway=True): # IO if self._closed and not _anyway: return self._closed = True self._channel.close() self._local_root.on_disconnect(self) self._request_callbacks.clear() self._local_objects.clear() self._proxy_cache.clear() self._netref_classes_cache.clear() self._last_traceback = None self._remote_root = None self._local_root = None # self._seqcounter = None # self._config.clear() del self._HANDLERS def close(self, _catchall=True): # IO """closes the connection, releasing all held resources""" if self._closed: return self._closed = True try: self._async_request(consts.HANDLE_CLOSE) except EOFError: pass except Exception: if not _catchall: raise finally: self._cleanup(_anyway=True) @property def closed(self): # IO """Indicates whether the connection has been closed or not""" return self._closed def fileno(self): # IO """Returns the connectin's underlying file descriptor""" return self._channel.fileno() def ping(self, data=None, timeout=3): # IO """Asserts that the other party is functioning properly, by making sure the *data* is echoed back before the *timeout* expires :param data: the data to send (leave ``None`` for the default buffer) :param timeout: the maximal time to wait for echo :raises: :class:`PingError` if the echoed data does not match :raises: :class:`EOFError` if the remote host closes the connection """ if data is None: data = "abcdefghijklmnopqrstuvwxyz" * 20 res = self.async_request(consts.HANDLE_PING, data, timeout=timeout) if res.value != data: raise PingError("echo mismatches sent data") def _get_seq_id(self): # IO return next(self._seqcounter) def _send(self, msg, seq, args): # IO data = brine.dump((msg, seq, args)) # GC might run while sending data # if so, a BaseNetref.__del__ might be called # BaseNetref.__del__ must call asyncreq, # which will cause a deadlock # Solution: # Add the current request to a queue and let the thread that currently # holds the sendlock send it when it's done with its current job. # NOTE: Atomic list operations should be thread safe, # please call me out if they are not on all implementations! self._send_queue.append(data) # It is crucial to check the queue each time AFTER releasing the lock: while self._send_queue: if not self._sendlock.acquire(False): # Another thread holds the lock. It will send the data after # it's done with its current job. We can safely return. return try: # Can happen if another consumer was scheduled in between # `while` and `acquire`: if not self._send_queue: # Must `continue` to ensure that `send_queue` is checked # after releasing the lock! (in case another producer is # scheduled before `release`) continue data = self._send_queue.pop(0) self._channel.send(data) finally: self._sendlock.release() def _box(self, obj): # boxing """store a local object in such a way that it could be recreated on the remote party either by-value or by-reference""" if brine.dumpable(obj): return consts.LABEL_VALUE, obj if type(obj) is tuple: return consts.LABEL_TUPLE, tuple(self._box(item) for item in obj) elif isinstance(obj, netref.BaseNetref) and obj.____conn__ is self: return consts.LABEL_LOCAL_REF, obj.____id_pack__ else: id_pack = get_id_pack(obj) self._local_objects.add(id_pack, obj) try: cls = obj.__class__ except Exception: # see issue #16 cls = type(obj) if not isinstance(cls, type): cls = type(obj) return consts.LABEL_REMOTE_REF, id_pack def _unbox(self, package): # boxing """recreate a local object representation of the remote object: if the object is passed by value, just return it; if the object is passed by reference, create a netref to it""" label, value = package if label == consts.LABEL_VALUE: return value if label == consts.LABEL_TUPLE: return tuple(self._unbox(item) for item in value) if label == consts.LABEL_LOCAL_REF: return self._local_objects[value] if label == consts.LABEL_REMOTE_REF: id_pack = (str(value[0]), value[1], value[2] ) # so value is a id_pack if id_pack in self._proxy_cache: proxy = self._proxy_cache[id_pack] proxy.____refcount__ += 1 # if cached then remote incremented refcount, so sync refcount else: proxy = self._netref_factory(id_pack) self._proxy_cache[id_pack] = proxy return proxy raise ValueError("invalid label %r" % (label, )) def _netref_factory(self, id_pack): # boxing """id_pack is for remote, so when class id fails to directly match """ if id_pack[0] in netref.builtin_classes_cache: cls = netref.builtin_classes_cache[id_pack[0]] elif id_pack[1] in self._netref_classes_cache: cls = self._netref_classes_cache[id_pack[1]] else: # in the future, it could see if a sys.module cache/lookup hits first cls_methods = self.sync_request(consts.HANDLE_INSPECT, id_pack) cls = netref.class_factory(id_pack, cls_methods) self._netref_classes_cache[id_pack[1]] = cls return cls(self, id_pack) def _dispatch_request(self, seq, raw_args): # dispatch try: handler, args = raw_args args = self._unbox(args) res = self._HANDLERS[handler](self, *args) except Exception: # need to catch old style exceptions too t, v, tb = sys.exc_info() self._last_traceback = tb logger = self._config["logger"] if logger and t is not StopIteration: logger.debug("Exception caught", exc_info=True) if t is SystemExit and self._config["propagate_SystemExit_locally"]: raise if t is KeyboardInterrupt and self._config[ "propagate_KeyboardInterrupt_locally"]: raise self._send(consts.MSG_EXCEPTION, seq, self._box_exc(t, v, tb)) else: self._send(consts.MSG_REPLY, seq, self._box(res)) def _box_exc(self, typ, val, tb): # dispatch? return vinegar.dump( typ, val, tb, include_local_traceback=self._config["include_local_traceback"], include_local_version=self._config["include_local_version"]) def _unbox_exc(self, raw): # dispatch? return vinegar.load( raw, import_custom_exceptions=self._config["import_custom_exceptions"], instantiate_custom_exceptions=self. _config["instantiate_custom_exceptions"], instantiate_oldstyle_exceptions=self. _config["instantiate_oldstyle_exceptions"]) def _dispatch(self, data): # serving---dispatch? msg, seq, args = brine.load(data) if msg == consts.MSG_REQUEST: self._dispatch_request(seq, args) elif msg == consts.MSG_REPLY: obj = self._unbox(args) self._request_callbacks.pop(seq)(False, obj) elif msg == consts.MSG_EXCEPTION: obj = self._unbox_exc(args) self._request_callbacks.pop(seq)(True, obj) else: raise ValueError("invalid message type: %r" % (msg, )) def serve(self, timeout=1, wait_for_lock=True): # serving """Serves a single request or reply that arrives within the given time frame (default is 1 sec). Note that the dispatching of a request might trigger multiple (nested) requests, thus this function may be reentrant. :returns: ``True`` if a request or reply were received, ``False`` otherwise. """ timeout = Timeout(timeout) with self._recv_event: if not self._recvlock.acquire(False): return wait_for_lock and self._recv_event.wait( timeout.timeleft()) try: data = self._channel.poll(timeout) and self._channel.recv() if not data: return False except EOFError: self.close() raise finally: self._recvlock.release() with self._recv_event: self._recv_event.notify_all() self._dispatch(data) return True def poll(self, timeout=0): # serving """Serves a single transaction, should one arrives in the given interval. Note that handling a request/reply may trigger nested requests, which are all part of a single transaction. :returns: ``True`` if a transaction was served, ``False`` otherwise""" return self.serve(timeout, False) def serve_all(self): # serving """Serves all requests and replies for as long as the connection is alive.""" try: while not self.closed: self.serve(None) except (socket.error, select_error, IOError): if not self.closed: raise except EOFError: pass finally: self.close() def serve_threaded(self, thread_count=10): # serving """Serves all requests and replies for as long as the connection is alive.""" def _thread_target(): try: while True: self.serve(None) except (socket.error, select_error, IOError): if not self.closed: raise except EOFError: pass try: threads = [spawn(_thread_target) for _ in range(thread_count)] for thread in threads: thread.join() finally: self.close() def poll_all(self, timeout=0): # serving """Serves all requests and replies that arrive within the given interval. :returns: ``True`` if at least a single transaction was served, ``False`` otherwise """ at_least_once = False timeout = Timeout(timeout) try: while True: if self.poll(timeout): at_least_once = True if timeout.expired(): break except EOFError: pass return at_least_once def sync_request(self, handler, *args): # serving """requests, sends a synchronous request (waits for the reply to arrive) :raises: any exception that the requets may be generated :returns: the result of the request """ timeout = self._config["sync_request_timeout"] return self.async_request(handler, *args, timeout=timeout).value def _async_request(self, handler, args=(), callback=(lambda a, b: None)): # serving seq = self._get_seq_id() self._request_callbacks[seq] = callback try: self._send(consts.MSG_REQUEST, seq, (handler, self._box(args))) except Exception: self._request_callbacks.pop(seq, None) raise def async_request(self, handler, *args, **kwargs): # serving """Send an asynchronous request (does not wait for it to finish) :returns: an :class:`rpyc.core.async_.AsyncResult` object, which will eventually hold the result (or exception) """ timeout = kwargs.pop("timeout", None) if kwargs: raise TypeError("got unexpected keyword argument(s) %s" % (list(kwargs.keys()), )) res = AsyncResult(self) self._async_request(handler, args, res) if timeout is not None: res.set_expiry(timeout) return res @property def root(self): # serving """Fetches the root object (service) of the other party""" if self._remote_root is None: self._remote_root = self.sync_request(consts.HANDLE_GETROOT) return self._remote_root def _check_attr(self, obj, name, perm): # attribute access config = self._config if not config[perm]: raise AttributeError("cannot access %r" % (name, )) prefix = config["allow_exposed_attrs"] and config["exposed_prefix"] plain = config["allow_all_attrs"] plain |= config["allow_exposed_attrs"] and name.startswith(prefix) plain |= config["allow_safe_attrs"] and name in config["safe_attrs"] plain |= config["allow_public_attrs"] and not name.startswith("_") has_exposed = prefix and hasattr(obj, prefix + name) if plain and (not has_exposed or hasattr(obj, name)): return name if has_exposed: return prefix + name if plain: return name # chance for better traceback raise AttributeError("cannot access %r" % (name, )) def _access_attr(self, obj, name, args, overrider, param, default): # attribute access if is_py3k: if type(name) is bytes: name = str(name, "utf8") elif type(name) is not str: raise TypeError("name must be a string") else: if type(name) not in (str, unicode): # noqa raise TypeError("name must be a string") name = str(name) # IronPython issue #10 + py3k issue accessor = getattr(type(obj), overrider, None) if accessor is None: accessor = default name = self._check_attr(obj, name, param) return accessor(obj, name, *args) @classmethod def _request_handlers(cls): # request handlers return { consts.HANDLE_PING: cls._handle_ping, consts.HANDLE_CLOSE: cls._handle_close, consts.HANDLE_GETROOT: cls._handle_getroot, consts.HANDLE_GETATTR: cls._handle_getattr, consts.HANDLE_DELATTR: cls._handle_delattr, consts.HANDLE_SETATTR: cls._handle_setattr, consts.HANDLE_CALL: cls._handle_call, consts.HANDLE_CALLATTR: cls._handle_callattr, consts.HANDLE_REPR: cls._handle_repr, consts.HANDLE_STR: cls._handle_str, consts.HANDLE_CMP: cls._handle_cmp, consts.HANDLE_HASH: cls._handle_hash, consts.HANDLE_INSTANCECHECK: cls._handle_instancecheck, consts.HANDLE_DIR: cls._handle_dir, consts.HANDLE_PICKLE: cls._handle_pickle, consts.HANDLE_DEL: cls._handle_del, consts.HANDLE_INSPECT: cls._handle_inspect, consts.HANDLE_BUFFITER: cls._handle_buffiter, consts.HANDLE_OLDSLICING: cls._handle_oldslicing, consts.HANDLE_CTXEXIT: cls._handle_ctxexit, } def _handle_ping(self, data): # request handler return data def _handle_close(self): # request handler self._cleanup() def _handle_getroot(self): # request handler return self._local_root def _handle_del(self, obj, count=1): # request handler self._local_objects.decref(get_id_pack(obj), count) def _handle_repr(self, obj): # request handler return repr(obj) def _handle_str(self, obj): # request handler return str(obj) def _handle_cmp(self, obj, other, op='__cmp__'): # request handler # cmp() might enter recursive resonance... yet another workaround # return cmp(obj, other) try: return getattr(type(obj), op)(obj, other) except (AttributeError, TypeError): return NotImplemented def _handle_hash(self, obj): # request handler return hash(obj) def _handle_call(self, obj, args, kwargs=()): # request handler return obj(*args, **dict(kwargs)) def _handle_dir(self, obj): # request handler return tuple(dir(obj)) def _handle_inspect(self, id_pack): # request handler return tuple( get_methods(netref.LOCAL_ATTRS, self._local_objects[id_pack])) def _handle_getattr(self, obj, name): # request handler return self._access_attr(obj, name, (), "_rpyc_getattr", "allow_getattr", getattr) def _handle_delattr(self, obj, name): # request handler return self._access_attr(obj, name, (), "_rpyc_delattr", "allow_delattr", delattr) def _handle_setattr(self, obj, name, value): # request handler return self._access_attr(obj, name, (value, ), "_rpyc_setattr", "allow_setattr", setattr) def _handle_callattr(self, obj, name, args, kwargs=()): # request handler obj = self._handle_getattr(obj, name) return self._handle_call(obj, args, kwargs) def _handle_ctxexit(self, obj, exc): # request handler if exc: try: raise exc except Exception: exc, typ, tb = sys.exc_info() else: typ = tb = None return self._handle_getattr(obj, "__exit__")(exc, typ, tb) def _handle_instancecheck(self, obj, other_id_pack): # Create a name pack which would be familiar here and see if there is a hit other_id_pack2 = (other_id_pack[0], other_id_pack[1], 0) if other_id_pack2 in self._netref_classes_cache: cls = self._netref_classes_cache[other_id_pack2] other = cls(self, other_id_pack2) return isinstance(other, obj) else: # might just have missed cache, FIX ME return False def _handle_pickle(self, obj, proto): # request handler if not self._config["allow_pickle"]: raise ValueError("pickling is disabled") return bytes(pickle.dumps(obj, proto)) def _handle_buffiter(self, obj, count): # request handler return tuple(itertools.islice(obj, count)) def _handle_oldslicing(self, obj, attempt, fallback, start, stop, args): # request handler try: # first try __xxxitem__ getitem = self._handle_getattr(obj, attempt) return getitem(slice(start, stop), *args) except Exception: # fallback to __xxxslice__. see issue #41 if stop is None: stop = maxint getslice = self._handle_getattr(obj, fallback) return getslice(start, stop, *args)
class AeromeScentController(object): def __init__(self, compatibility_dummy): self.log = None self.open_valves = None self.state_change_lock = Lock() self.status_changed_callback = None def initialize_controller(self, status_changed_callback): self.log = logging.getLogger("aeromeScentController") self.log.error("Init GPIO controller") GPIO.setmode(GPIO.BOARD) for pin in SCENT_ID_TO_PIN_MAPPING.values(): GPIO.setup(pin, GPIO.OUT) GPIO.setup(FLUSH_VALVE_PIN, GPIO.OUT) self.status_changed_callback = status_changed_callback self.close_all_valves() @staticmethod def get_state(): ret = {} for pin_id, pin in SCENT_ID_TO_PIN_MAPPING.iteritems(): ret[pin_id] = GPIO.input(pin) == GPIO.HIGH return ret def open_valve(self, valve_id): self._set_valve_state(valve_id, GPIO.HIGH) def close_valve(self, valve_id): self._set_valve_state(valve_id, GPIO.LOW) def close(self): self.close_all_valves() GPIO.cleanup() def close_all_valves(self): self.state_change_lock.acquire() try: self.log.error("Closing all") self._set_all_pins_low() finally: self.state_change_lock.release() def _set_valve_state(self, valve_id, state): valve_key = str(valve_id) if valve_key not in SCENT_ID_TO_PIN_MAPPING: self.log.error("Unknown valve %s" % valve_key) return valve_pin = SCENT_ID_TO_PIN_MAPPING[valve_key] self.state_change_lock.acquire() try: self._set_pin_to_state(valve_pin, state) finally: self.state_change_lock.release() def _set_pin_to_state(self, pin, state): if GPIO.input(pin) != state: self.log.error("Setting " + str(pin) + " to " + str(state)) GPIO.output(pin, state) if state == GPIO.HIGH: GPIO.output(FLUSH_VALVE_PIN, state) self.open_valves += 1 else: self.open_valves -= 1 if self.open_valves < 1: self._set_all_pins_low() self.status_changed_callback() else: self.log.error("Pin " + str(pin) + " is already " + str(state)) def _set_all_pins_low(self): GPIO.output(SCENT_ID_TO_PIN_MAPPING.values() + [FLUSH_VALVE_PIN], GPIO.LOW) self.open_valves = 0 self.status_changed_callback()
class PastlyLogger: # error, warn, etc. are file names to open for logging. # If a log level doesn't have a file name given for it, messages destined # for that level cascade down to the next noisiest level. # Example 1: warn=foo.txt, debug=bar.txt # error and warn messages go to foo.txt, all other messages to bar.txt # Example 2: notice=baz.txt # error, warn, and notice messages go to baz.txt, all others are lost # # overwrite is a list of log levels that should overwrite their log file # when opening instead of appending. # Example: notice=a.txt, info=b.txt, overwrite=['info'] # error, warn, and notice messages are appended to a.txt; # b.txt is overwritten and info messages are appended to it; # all debug messages are lost # # log_threads tells the logger whether or not to log thread names # # default tells the logger what level to log at when called with # log('foobar') instead of log.info('foobar') def __init__(self, error=None, warn=None, notice=None, info=None, debug=None, overwrite=[], log_threads=False, default='notice'): self.log_threads = log_threads assert default in ['debug', 'info', 'notice', 'warn', 'error'] self.default_level = default # buffering=1 means line-based buffering if error: self.error_fd = open(error, 'w' if 'error' in overwrite else 'a', buffering=1) self.error_fd_mutex = Lock() else: self.error_fd = None self.error_fd_mutex = None if warn: self.warn_fd = open(warn, 'w' if 'warn' in overwrite else 'a', buffering=1) self.warn_fd_mutex = Lock() else: self.warn_fd = None self.warn_fd_mutex = None if notice: self.notice_fd = open(notice, 'w' if 'notice' in overwrite else 'a', buffering=1) self.notice_fd_mutex = Lock() else: self.notice_fd = None self.notice_fd_mutex = None if info: self.info_fd = open(info, 'w' if 'info' in overwrite else 'a', buffering=1) self.info_fd_mutex = Lock() else: self.info_fd = None self.info_fd_mutex = None if debug: self.debug_fd = open(debug, 'w' if 'debug' in overwrite else 'a', buffering=1) self.debug_fd_mutex = Lock() else: self.debug_fd = None self.debug_fd_mutex = None self.debug('Creating PastlyLogger instance') def __call__(self, *s): if self.default_level == 'debug': return self.debug(*s) elif self.default_level == 'info': return self.info(*s) elif self.default_level == 'notice': return self.notice(*s) elif self.default_level == 'warn': return self.warn(*s) elif self.default_level == 'error': return self.error(*s) def __del__(self): self.debug('Deleting PastlyLogger instance') self.flush() if self.error_fd: self.error_fd.close() if self.warn_fd: self.warn_fd.close() if self.notice_fd: self.notice_fd.close() if self.info_fd: self.info_fd.close() if self.debug_fd: self.debug_fd.close() self.error_fd, self.warn_fd = None, None self.notice_fd, self.info_fd, self.debug_fd = None, None, None if self.error_fd_mutex: if not self.error_fd_mutex.acquire(blocking=False): self.error_fd_mutex.release() if self.warn_fd_mutex: if not self.warn_fd_mutex.acquire(blocking=False): self.warn_fd_mutex.release() if self.notice_fd_mutex: if not self.notice_fd_mutex.acquire(blocking=False): self.notice_fd_mutex.release() if self.info_fd_mutex: if not self.info_fd_mutex.acquire(blocking=False): self.info_fd_mutex.release() if self.debug_fd_mutex: if not self.debug_fd_mutex.acquire(blocking=False): self.debug_fd_mutex.release() def _log_file(fd, lock, log_threads, level, *s): assert fd lock.acquire() ts = datetime.now() if log_threads: fd.write('[{}] [{}] [{}] {}\n'.format( ts, level, current_thread().name, ' '.join([str(_) for _ in s]))) else: fd.write('[{}] [{}] {}\n'.format(ts, level, ' '.join([str(_) for _ in s]))) lock.release() def flush(self): if self.error_fd: self.error_fd.flush() if self.warn_fd: self.warn_fd.flush() if self.notice_fd: self.notice_fd.flush() if self.info_fd: self.info_fd.flush() if self.debug_fd: self.debug_fd.flush() def debug(self, *s, level='debug'): if self.debug_fd: return PastlyLogger._log_file(self.debug_fd, self.debug_fd_mutex, self.log_threads, level, *s) return None def info(self, *s, level='info'): if self.info_fd: return PastlyLogger._log_file(self.info_fd, self.info_fd_mutex, self.log_threads, level, *s) else: return self.debug(*s, level=level) def notice(self, *s, level='notice'): if self.notice_fd: return PastlyLogger._log_file(self.notice_fd, self.notice_fd_mutex, self.log_threads, level, *s) else: return self.info(*s, level=level) def warn(self, *s, level='warn'): if self.warn_fd: return PastlyLogger._log_file(self.warn_fd, self.warn_fd_mutex, self.log_threads, level, *s) else: return self.notice(*s, level=level) def error(self, *s, level='error'): if self.error_fd: return PastlyLogger._log_file(self.error_fd, self.error_fd_mutex, self.log_threads, level, *s) else: return self.warn(*s, level=level)
class _Selection(DOMWidget): """Base class for Selection widgets ``options`` can be specified as a list or dict. If given as a list, it will be transformed to a dict of the form ``{str(value):value}``. When programmatically setting the value, a reverse lookup is performed among the options to set the value of ``selected_label`` accordingly. The reverse lookup uses the equality operator by default, but an other predicate may be provided via the ``equals`` argument. For example, when dealing with numpy arrays, one may set equals=np.array_equal. """ value = Any(help="Selected value") selected_label = Unicode(help="The label of the selected value", sync=True) options = Any( help="""List of (key, value) tuples or dict of values that the user can select. The keys of this list are the strings that will be displayed in the UI, representing the actual Python choices. The keys of this list are also available as _options_labels. """) _options_dict = Dict() _options_labels = Tuple(sync=True) _options_values = Tuple() disabled = Bool(False, help="Enable or disable user changes", sync=True) description = Unicode( help="Description of the value this widget represents", sync=True) def __init__(self, *args, **kwargs): self.value_lock = Lock() self.options_lock = Lock() self.equals = kwargs.pop('equals', lambda x, y: x == y) self.on_trait_change(self._options_readonly_changed, [ '_options_dict', '_options_labels', '_options_values', '_options' ]) if 'options' in kwargs: self.options = kwargs.pop('options') DOMWidget.__init__(self, *args, **kwargs) self._value_in_options() def _make_options(self, x): # If x is a dict, convert it to list format. if isinstance(x, (OrderedDict, dict)): return [(k, v) for k, v in x.items()] # Make sure x is a list or tuple. if not isinstance(x, (list, tuple)): raise ValueError('x') # If x is an ordinary list, use the option values as names. for y in x: if not isinstance(y, (list, tuple)) or len(y) < 2: return [(i, i) for i in x] # Value is already in the correct format. return x def _options_changed(self, name, old, new): """Handles when the options tuple has been changed. Setting options implies setting option labels from the keys of the dict. """ if self.options_lock.acquire(False): try: self.options = new options = self._make_options(new) self._options_dict = {i[0]: i[1] for i in options} self._options_labels = [i[0] for i in options] self._options_values = [i[1] for i in options] self._value_in_options() finally: self.options_lock.release() def _value_in_options(self): # ensure that the chosen value is one of the choices if self._options_values: if self.value not in self._options_values: self.value = next(iter(self._options_values)) def _options_readonly_changed(self, name, old, new): if not self.options_lock.locked(): raise TraitError( "`.%s` is a read-only trait. Use the `.options` tuple instead." % name) def _value_changed(self, name, old, new): """Called when value has been changed""" if self.value_lock.acquire(False): try: # Reverse dictionary lookup for the value name for k, v in self._options_dict.items(): if self.equals(new, v): # set the selected value name self.selected_label = k return # undo the change, and raise KeyError self.value = old raise KeyError(new) finally: self.value_lock.release() def _selected_label_changed(self, name, old, new): """Called when the value name has been changed (typically by the frontend).""" if self.value_lock.acquire(False): try: self.value = self._options_dict[new] finally: self.value_lock.release()
class Communicator: UPDATE_INTERVAL = 0.4 def __init__(self, websocket_handler, worker_id: str, worker_instance_ref, command_timeout: float): # Throws ValueError if unable to connect! # catch in code using this class self.worker_id: str = worker_id self.worker_instance_ref = worker_instance_ref self.websocket_handler = websocket_handler self.__command_timeout: float = command_timeout self.__sendMutex = Lock() def cleanup_websocket(self): logger.info( "Communicator of {} acquiring lock to cleanup worker in websocket", str(self.worker_id)) self.__sendMutex.acquire() try: logger.info("Communicator of {} calling cleanup", str(self.worker_id)) self.websocket_handler.clean_up_user(self.worker_id, self.worker_instance_ref) finally: self.__sendMutex.release() def __runAndOk(self, command, timeout) -> bool: return self.__run_and_ok_bytes(command, timeout) def __run_and_ok_bytes(self, message, timeout: float, byte_command: int = None) -> bool: self.__sendMutex.acquire() try: result = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, message, timeout, byte_command=byte_command) return result is not None and "OK" == result.strip() finally: self.__sendMutex.release() def install_apk(self, filepath: str, timeout: float) -> bool: # TODO: check if file exists... with open(filepath, "rb") as file: # opening for [r]eading as [b]inary data = file.read( ) # if you only wanted to read 512 bytes, do .read(512) return self.__run_and_ok_bytes(message=data, timeout=timeout, byte_command=1) def startApp(self, package_name): return self.__runAndOk("more start {}\r\n".format(package_name), self.__command_timeout) def stopApp(self, package_name): if not self.__runAndOk("more stop {}\r\n".format(package_name), self.__command_timeout): logger.error( "Failed stopping {}, please check if SU has been granted", package_name) return False else: return True def passthrough(self, command): response = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "passthrough {}".format(command), self.__command_timeout) return response def reboot(self) -> bool: return self.__runAndOk("more reboot now\r\n", self.__command_timeout) def restartApp(self, package_name) -> bool: return self.__runAndOk("more restart {}\r\n".format(package_name), self.__command_timeout) def resetAppdata(self, package_name) -> bool: return self.__runAndOk("more reset {}\r\n".format(package_name), self.__command_timeout) def clearAppCache(self, package_name) -> bool: return self.__runAndOk("more cache {}\r\n".format(package_name), self.__command_timeout) def magisk_off(self, package_name) -> bool: return self.passthrough( "su -c magiskhide --rm {}".format(package_name)) def magisk_on(self, package_name) -> bool: return self.passthrough( "su -c magiskhide --add {}".format(package_name)) def turnScreenOn(self) -> bool: return self.__runAndOk("more screen on\r\n", self.__command_timeout) def click(self, x, y) -> bool: return self.__runAndOk( "screen click {} {}\r\n".format(str(int(round(x))), str(int(round(y)))), self.__command_timeout) def swipe(self, x1, y1, x2, y2): return self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "touch swipe {} {} {} {}\r\n".format(str(int(round(x1))), str(int(round(y1))), str(int(round(x2))), str(int(round(y2)))), self.__command_timeout) def touchandhold(self, x1, y1, x2, y2, time: int = 3000) -> bool: return self.__runAndOk( "touch swipe {} {} {} {} {}".format(str(int(round(x1))), str(int(round(y1))), str(int(round(x2))), str(int(round(y2))), str(int(time))), self.__command_timeout) def getscreensize(self) -> str: response = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "screen size", self.__command_timeout) return response def uiautomator(self) -> str: response = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "more uiautomator", self.__command_timeout) return response def get_screenshot( self, path, quality: int = 70, screenshot_type: ScreenshotType = ScreenshotType.JPEG) -> bool: if quality < 10 or quality > 100: logger.error("Invalid quality value passed for screenshots") return False screenshot_type_str: str = "jpeg" if screenshot_type == ScreenshotType.PNG: screenshot_type_str = "png" self.__sendMutex.acquire() try: encoded = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "screen capture {} {}\r\n".format(screenshot_type_str, quality), self.__command_timeout) finally: self.__sendMutex.release() if encoded is None: return False elif isinstance(encoded, str): logger.debug("Screenshot response not binary") if "KO: " in encoded: logger.error( "get_screenshot: Could not retrieve screenshot. Make sure your RGC is updated." ) return False elif "OK:" not in encoded: logger.error("get_screenshot: response not OK") return False return False else: logger.debug("Storing screenshot...") with open(path, "wb") as fh: fh.write(encoded) logger.debug("Done storing, returning") return True def backButton(self) -> bool: return self.__runAndOk("screen back\r\n", self.__command_timeout) def homeButton(self) -> bool: return self.__runAndOk("touch keyevent 3", self.__command_timeout) def enterButton(self) -> bool: return self.__runAndOk("touch keyevent 61", self.__command_timeout) def sendText(self, text): return self.__runAndOk("touch text " + str(text), self.__command_timeout) def isScreenOn(self) -> bool: self.__sendMutex.acquire() try: state = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "more state screen\r\n", self.__command_timeout) if state is None: return False return "on" in state finally: self.__sendMutex.release() def isPogoTopmost(self) -> bool: self.__sendMutex.acquire() try: topmost = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "more topmost app\r\n", self.__command_timeout) if topmost is None: return False return "com.nianticlabs.pokemongo" in topmost finally: self.__sendMutex.release() def topmostApp(self) -> str: self.__sendMutex.acquire() try: topmost = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "more topmost app\r\n", self.__command_timeout) if topmost is None: return False return topmost finally: self.__sendMutex.release() def setLocation(self, lat, lng, alt): self.__sendMutex.acquire() try: response = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "geo fix {} {} {}\r\n".format(lat, lng, alt), self.__command_timeout) return response finally: self.__sendMutex.release() def terminate_connection(self): self.__sendMutex.acquire() try: response = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "exit\r\n", 5) return response finally: self.__sendMutex.release() # coords need to be float values # speed integer with km/h ####### # This blocks! ####### def walkFromTo(self, startLat, startLng, destLat, destLng, speed): with self.__sendMutex: # calculate the time it will take to walk and add it to the timeout! distance = get_distance_of_two_points_in_meters( startLat, startLng, destLat, destLng) # speed is in kmph, distance in m # we want m/s -> speed / 3.6 speed_meters = speed / 3.6 seconds_traveltime = distance / speed_meters response = self.websocket_handler.send_and_wait( self.worker_id, self.worker_instance_ref, "geo walk {} {} {} {} {}\r\n".format(startLat, startLng, destLat, destLng, speed), self.__command_timeout + seconds_traveltime) return response
class RDMTestThread(Thread): """The RDMResponder tests are closely coupled to the Wrapper (yuck!). So we need to run this all in a separate thread. This is all a bit of a hack and you'll get into trouble if multiple things are running at once... """ RUNNING, COMPLETED, ERROR = range(3) TESTS, COLLECTOR = range(2) def __init__(self, pid_store, logs_directory): super(RDMTestThread, self).__init__() self._pid_store = pid_store self._logs_directory = logs_directory self._terminate = False self._request = None # guards _terminate and _request self._cv = Condition() self._wrapper = None self._test_state_lock = Lock() # guards _test_state self._test_state = {} def Stop(self): self._cv.acquire() self._terminate = True self._cv.notify() self._cv.release() def ScheduleTests(self, universe, uid, test_filter, broadcast_write_delay, inter_test_delay, dmx_frame_rate, slot_count): """Schedule the tests to be run. Callable from any thread. Callbable by any thread. Returns: An error message, or None if the tests were scheduled. """ if not self._CheckIfConnected(): return 'Lost connection to OLAD' self._cv.acquire() if self._request is not None: self._cv.release() return 'Existing request pending' self._request = lambda: self._RunTests( universe, uid, test_filter, broadcast_write_delay, inter_test_delay, dmx_frame_rate, slot_count) self._cv.notify() self._cv.release() return None def ScheduleCollector(self, universe, skip_queued_messages): """Schedule the collector to run on a universe. Callable by any thread. Returns: An error message, or None if the collection was scheduled. """ if not self._CheckIfConnected(): return 'Lost connection to OLAD' self._cv.acquire() if self._request is not None: self._cv.release() return 'Existing request pending' self._request = lambda: self._RunCollector(universe, skip_queued_messages) self._cv.notify() self._cv.release() return None def Stat(self): """Check the state of the tests. Callable by any thread. Returns: The status of the tests. """ self._test_state_lock.acquire() state = dict(self._test_state) self._test_state_lock.release() return state def run(self): self._wrapper = ClientWrapper() self._collector = ModelCollector(self._wrapper, self._pid_store) while True: self._cv.acquire() if self._terminate: logging.info('quitting test thread') self._cv.release() return if self._request is not None: request = self._request self._request = None self._cv.release() request() continue # nothing to do, go into the wait self._cv.wait() self._cv.release() def _UpdateStats(self, tests_completed, total_tests): self._test_state_lock.acquire() self._test_state['tests_completed'] = tests_completed self._test_state['total_tests'] = total_tests self._test_state_lock.release() def _RunTests(self, universe, uid, test_filter, broadcast_write_delay, inter_test_delay, dmx_frame_rate, slot_count): self._test_state_lock.acquire() self._test_state = { 'action': self.TESTS, 'tests_completed': 0, 'total_tests': None, 'state': self.RUNNING, 'duration': 0, } start_time = datetime.now() self._test_state_lock.release() runner = TestRunner.TestRunner(universe, uid, broadcast_write_delay, inter_test_delay, self._pid_store, self._wrapper) for test in TestRunner.GetTestClasses(TestDefinitions): runner.RegisterTest(test) dmx_sender = None if dmx_frame_rate > 0 and slot_count > 0: logging.info( 'Starting DMXSender with slot count %d and FPS of %d' % (slot_count, dmx_frame_rate)) dmx_sender = DMXSender(self._wrapper, universe, dmx_frame_rate, slot_count) try: tests, device = runner.RunTests(test_filter, False, self._UpdateStats) except Exception as e: self._test_state_lock.acquire() self._test_state['state'] = self.ERROR self._test_state['exception'] = str(e) self._test_state['traceback'] = traceback.format_exc() self._test_state_lock.release() return finally: if dmx_sender is not None: dmx_sender.Stop() timestamp = int(time()) end_time = datetime.now() test_parameters = { 'broadcast_write_delay': broadcast_write_delay, 'inter_test_delay': inter_test_delay, 'dmx_frame_rate': dmx_frame_rate, 'dmx_slot_count': slot_count, } log_saver = TestLogger.TestLogger(self._logs_directory) logs_saved = True try: log_saver.SaveLog(uid, timestamp, end_time, tests, device, test_parameters) except TestLogger.TestLoggerException: logs_saved = False self._test_state_lock.acquire() # we can't use total_seconds() since it requires Python 2.7 time_delta = end_time - start_time self._test_state['duration'] = (time_delta.seconds + time_delta.days * 24 * 3600) self._test_state['state'] = self.COMPLETED self._test_state['tests'] = tests self._test_state['logs_saved'] = logs_saved self._test_state['timestamp'] = timestamp self._test_state['uid'] = uid self._test_state_lock.release() def _RunCollector(self, universe, skip_queued_messages): """Run the device model collector for a universe.""" logging.info('Collecting for %d' % universe) self._test_state_lock.acquire() self._test_state = { 'action': self.COLLECTOR, 'state': self.RUNNING, } self._test_state_lock.release() try: output = self._collector.Run(universe, skip_queued_messages) except Exception as e: self._test_state_lock.acquire() self._test_state['state'] = self.ERROR self._test_state['exception'] = str(e) self._test_state['traceback'] = traceback.format_exc() self._test_state_lock.release() return self._test_state_lock.acquire() self._test_state['state'] = self.COMPLETED self._test_state['output'] = output self._test_state_lock.release() def _CheckIfConnected(self): """Check if the client is connected to olad. Returns: True if connected, False otherwise. """ # TODO(simon): add this check, remember it needs locking. return True
class Log(object): __metaclass__ = Singleton def __init__(self): super(Log, self).__init__() self.thread_lock = Lock() self.print_require_level = 0 self.file_require_level = 0 def log(self, level, msg, args): try: self.thread_lock.acquire() if args and isinstance(args, tuple): msg = msg % tuple( (arg.encode("utf8") if isinstance(arg, unicode) else arg for arg in args)) if level >= self.print_require_level: try: print msg except: pass if level >= self.file_require_level: file_name = path.join( BASE_DIR, 'log', ''.join([ 'log_', strftime('%Y-%m-%d', localtime()), '_', str(getpid()), '.txt' ])) f = open(file_name, 'a') f.write(get_string(msg)) f.write('\n') f.close() except Exception as e: print '(' * 6 print msg print args print format_exc() print ')' * 6 finally: self.thread_lock.release() def debug(self, msg, *args): self.log(1, ''.join([strftime('[%H:%M:%S]debug :', localtime()) + msg]), args) def info(self, msg, *args): self.log(2, ''.join([strftime('[%H:%M:%S]info :', localtime()) + msg]), args) def trace(self, msg, *args): self.log(3, ''.join([strftime('[%H:%M:%S]trace :', localtime()) + msg]), args) def warn(self, msg, *args): self.log(4, ''.join([strftime('[%H:%M:%S]warn :', localtime()) + msg]), args) def error(self, msg, *args): self.log(5, ''.join([strftime('[%H:%M:%S]error :', localtime()) + msg]), args) def critical(self, msg, *args): self.log(6, ''.join([strftime('[%H:%M:%S]critical:', localtime()) + msg]), args)