class ConcurrentQueue(object): def __init__(self): self.en = Lock() self.de = Lock() self.queue = deque() self.de.acquire() def enqueue(self, ele): self.en.acquire() self.queue.append(ele) self.en.release() if self.de.locked(): self.de.release() print(f'enqueue size {len(self.queue)}, {threading.get_ident()}') def dequeue(self): self.de.acquire() val = self.queue.popleft() if len(self.queue): self.de.release() if self.en.locked(): self.en.release() print(f'dequeue size {len(self.queue)}, {threading.get_ident()}') return val
class BoundedBlockingQueue(object): def __init__(self, capacity: int): self.que = [] self.producer = Lock() self.customer = Lock() self.customer.acquire() self.mutex = Lock() self.n = capacity def enqueue(self, element: int) -> None: self.producer.acquire() self.que.append(element) if self.size() != self.n: self.producer.release() if self.customer.locked(): self.customer.release() def dequeue(self) -> int: self.customer.acquire() ret = self.que.pop(0) if self.size(): self.customer.release() if self.producer.locked(): self.producer.release() return ret def _size(self): return len(self.que) def size(self) -> int: ret = None with self.mutex: ret = self._size() return ret
class BoundedBlockingQueue(object): def __init__(self, capacity: int): self.enque_lock = Lock() self.deque_lock = Lock() self.cap = capacity self.q = SimpleQueue() self.deque_lock.acquire() def enqueue(self, element: int) -> None: self.enque_lock.acquire() self.q.put(element) if self.q.qsize() < self.cap: self.enque_lock.release() if self.deque_lock.locked(): self.deque_lock.release() def dequeue(self) -> int: self.deque_lock.acquire() val = None if self.q.qsize() > 0: val = self.q.get() if self.q.qsize(): self.deque_lock.release() if val and self.enque_lock.locked(): self.enque_lock.release() return val def size(self) -> int: return self.q.qsize()
class BoundedBlockingQueue(object): def __init__(self, capacity: int): self.en = Lock() self.de = Lock() self.q = deque() self.capacity = capacity self.de.acquire() def enqueue(self, element: int) -> None: self.en.acquire() self.q.append(element) if len(self.q) < self.capacity: self.en.release() if self.de.locked(): self.de.release() def dequeue(self) -> int: self.de.acquire() val = self.q.popleft() if len(self.q): self.de.release() if val and self.en.locked(): self.en.release() return val def size(self) -> int: return len(self.q)
class TrafficLight: def __init__(self): self.directions12 = (1, 2) self.dir12_lock = Lock() self.dir34_lock = Lock() self.dir12_lock.acquire() def carArrived( self, carId: int, # ID of the car roadId: int, # ID of the road the car travels on. Can be 1 (road A) or 2 (road B) direction: int, # Direction of the car turnGreen: 'Callable[[], None]', # Use turnGreen() to turn light to green on current road crossCar: 'Callable[[], None]' # Use crossCar() to make car cross the intersection ) -> None: if direction in self.directions12: if self.dir34_lock.locked(): # Other roda is green self.dir34_lock.release() self.dir12_lock.acquire() turnGreen() else: if self.dir12_lock.locked(): self.dir12_lock.release() self.dir34_lock.acquire() turnGreen() crossCar()
class KeyEventQueue: def __init__(self): self.__lock = Lock() self.__queue = [] def append(self, kevt: KeyEvent): try: self.__lock.acquire() self.__queue.append(kevt) except: raise finally: if self.__lock.locked(): self.__lock.release() def take(self) -> KeyEvent: try: self.__lock.acquire() v = self.__queue.pop(0) if self.__queue else None except: raise finally: if self.__lock.locked(): self.__lock.release() return v def clear(self): self.__queue.clear()
def useLock(): lock = Lock() with lock: print(lock.locked()) print("lock is available") # print(lock.acquire(timeout=3)) print(lock.locked())
class Condition(object): "Pipe-based condition uses poll to mimic build-in Condition" MSECSMAX = 0x7FFFFFFF def __init__ (self): self._lock = Lock() self._lockset = set() self._poll = select.poll() self._readfd, self._writefd = os.pipe() self._poll.register(self._readfd, select.POLLIN) super(Condition, self).__init__() def acquire(self): return self._lock.acquire() def release(self): return self._lock.release() def locked(self): return self._lock.locked() def wait(self, timeout=None): assert self._lock.locked(), 'wait called on un-acquire()d lock' waitlock = allocate_lock() waitlock.acquire() self._lockset.add(waitlock) self._lock.release() try: if timeout is None: self._waituntil(waitlock) else: self._waitupto(timeout, waitlock) finally: self._lock.acquire() if waitlock.acquire(0): os.read(self._readfd, 1) self._lockset.remove(waitlock) def _waitupto(self, timeout, waitlock): startuptime = uptime.secs() while not timeout < 0: msecs = min(timeout * 1000, self.MSECSMAX) readable = self._pollreadable(msecs) if waitlock.locked(): curuptime = uptime.secs() lapsetime = curuptime - startuptime timeout = timeout - lapsetime else: break else: return False return True def _waituntil(self, waitlock): while waitlock.locked(): self._pollreadable() def _pollreadable(self, timeout = None): try: return self._poll.poll(timeout) except select.error, error: msglog.log('broadway', msglog.types.WARN, 'Notifier ignoring %s' % (error,)) return False
def _lock(): # 锁 ll = Lock() ll.acquire() ll.release() ll.locked() rl = RLock() rl.acquire() rl.release()
class StandardServo: max_degree = 180 min_freq = 1100 max_freq = 1900 def __init__(self, pin): self.control = None self.pin = pin self._motor_initialize() self.running = False self.angle = None self.write_lock = Lock() self.write_thread = None self.start() def _motor_initialize(self): self.control = kit.servo[self.pin] # self.control.actuation_range = self.max_degree # self.control.set_pulse_width_range(self.min_freq, self.max_freq) # self.change_angle(0) def start(self): if self.running: print('Motor is already running') return None else: self.running = True self.angle = 0 self.write_thread = Thread(target=self._motor_thread) self.write_thread.start() return self def _motor_thread(self): while self.running: self.write_lock.acquire() self.control.angle = self.angle def change_angle(self, angle): if angle != self.angle: self.angle = angle if self.write_lock.locked(): self.write_lock.release() def stop(self): print(self.pin, "motor is shutting down...") if self.running: self.running = False if self.write_lock.locked(): self.write_lock.release() self.write_thread.join() print(self.pin, "motor is shut down...") else: print(self.pin, "motor is already shut down...")
class market_thread(market, Thread): def __init__(self, sync = True,data_path = '', animation_speed = 1.0, random=False,length=0, graph=False, graph_span=50, cos_mrkt_data = None, type = None): '''/* * sync: True: wait for all agent finish to step into next unit of time * number_of_agent : in case you want more than one agent * animation_speed: if not sync, the market will move forward animation_speed unit per second * eg: if animation_speed = 0.5 , then market will move forward 1 unit of time every 2 seconds. * graph: enable real time market grpah * graph_span: How many past units of time are included in the graph. 0 : Entire history * cos_mrkt_data: import customized mrkt_data */''' if(cos_mrkt_data is not None): mrkt_data.__init__ = cos_mrkt_data print("market type is:", type) market.__init__(self, random=random, length=length,data_path = data_path, type = type) Thread.__init__(self) self.sync = sync #self.next_time_status = False self.next_time_mutex = Lock() self.exit = False self.animation_speed = animation_speed def set_next_time_status(self, value=True): self.next_time_mutex.acquire() while (self.next_time_mutex.locked()): pass #self.next_time_status = value def set_exit(self, value=True): # used to end the thread self.exit = value def run(self) -> None: while True: if self.ended: # if the simulation runs out break if self.exit: # be terminated break if(self.sync): if(self.next_time_mutex.locked()): self.next_time() self.next_time_mutex.release() else: time.sleep(1/self.animation_speed) self.next_time()
class LockableClass(object): def __init__(self): self._lock = Lock() def test_method(self): lock_state = self._lock.locked() # pylint: disable=no-member return lock_state @synchronized def sync_test_method(self): lock_state = self._lock.locked() # pylint: disable=no-member return lock_state
class SuccessLock: def __init__(self): self.lock = Lock() self.pause = Lock() self.code = 0L self.success = False self.finished = True def reset(self): self.success = False self.finished = False def set(self): self.lock.acquire() if not self.pause.locked(): self.pause.acquire() self.first = True self.code += 1L self.lock.release() return self.code def trip(self, code, s=False): self.lock.acquire() try: if code == self.code and not self.finished: r = self.first self.first = False if s: self.finished = True self.success = True return r finally: self.lock.release() def give_up(self): self.lock.acquire() self.success = False self.finished = True self.lock.release() def wait(self): self.pause.acquire() def unwait(self, code): if code == self.code and self.pause.locked(): self.pause.release() def isfinished(self): self.lock.acquire() x = self.finished self.lock.release() return x
class SuccessLock: def __init__(self): self.lock = Lock() self.pause = Lock() self.code = 0L self.success = False self.finished = True def reset(self): self.success = False self.finished = False def set(self): self.lock.acquire() if not self.pause.locked(): self.pause.acquire() self.first = True self.code += 1L self.lock.release() return self.code def trip(self, code, s = False): self.lock.acquire() try: if code == self.code and not self.finished: r = self.first self.first = False if s: self.finished = True self.success = True return r finally: self.lock.release() def give_up(self): self.lock.acquire() self.success = False self.finished = True self.lock.release() def wait(self): self.pause.acquire() def unwait(self, code): if code == self.code and self.pause.locked(): self.pause.release() def isfinished(self): self.lock.acquire() x = self.finished self.lock.release() return x
class MonitoringThread(Thread, Button): def __init__(self, pin: int): Thread.__init__(self) Button.__init__(self, pin, pull_up=None, active_state=False) self.when_activated = self.contact self.when_deactivated = self.released self.lock_active = Lock() self.lock_inactive = Lock() self.lock_active.acquire() self.exit_event = Event() def contact(self): self.lock_active.release() self.lock_inactive.acquire() def released(self): self.lock_inactive.release() self.lock_active.acquire() def exit(self): self.exit_event.set() if self.lock_active.locked(): self.lock_active.release() if self.lock_inactive.locked(): self.lock_inactive.release() def print_part(self, s: str): sys.stdout.write(s) sys.stdout.flush() def run(self): print( f'Starting impulse counter at {self.pin} at {datetime.now():%Y-%m-%d %H:%M:%S}' ) contacts = 0 while not self.exit_event.is_set(): self.print_part('waiting...') self.lock_active.acquire() if self.exit_event.is_set(): break contacts += 1 mark = datetime.now() self.print_part( f'{BCK*10}[{contacts:03d}] {mark:%Y-%m-%d %H:%M:%S} ') self.lock_active.release() self.lock_inactive.acquire() print(f'{(datetime.now() - mark).total_seconds():3.4} seconds') self.lock_inactive.release()
class Store: def __init__(self): self.items = {} self.__lock__ = Lock() def lock(self): if self.locked(): return self.__lock__.acquire() def release(self): if not self.locked(): return self.__lock__.release() def locked(self): return self.__lock__.locked() def add(self, instance): from resources.models import Resource self.items.update({ instance.id : RespaOutlookManager(instance) }) res = Resource.objects.get(pk=instance.resource.id) if res: res.configuration = instance res.save()
class GenericMindNode: def __init__(self, name="GenericMindNode", node_type=node_types.GENERIC_NODE): self.name = name self.node_type = node_type self.client_CommandConcept = None self.subscriber_PerceptionConcept = None self.sensor_cb_lock = Lock() self.sensor_cb_thread = None def sensor_callback(self, perception_concept: msg.PerceptionConcept): rospy.loginfo("From SensorInterpreter: %s:%s" % (perception_concept.symbol, perception_concept.modifier)) def _sensor_callback_thread(self, perception_concept: msg.PerceptionConcept): rospy.loginfo("sensor_callback thread") if self.sensor_cb_lock.acquire(blocking=False): self.sensor_callback(perception_concept) self.sensor_cb_lock.release() def _sensor_callback(self, perception_concept: msg.PerceptionConcept): if self.sensor_cb_lock.locked(): rospy.loginfo("sensor_callback is busy!") else: self.sensor_cb_thread = Thread(target=self._sensor_callback_thread, args=(perception_concept,), daemon=True) self.sensor_cb_thread.start() def emotion_callback(self, params: msg.EmotionParams): params_dict = json.loads(params.params_json) rospy.loginfo(f"Params: ${str(params_dict)}") def _init_communication(self): rospy.logdebug("Client \'%s\' is starting..." % self.name) self.client_CommandConcept = rospy.ServiceProxy(zros.services.CONCEPT_TO_COMMAND, srv.CommandConcept) self.subscriber_PerceptionConcept = zros.get.subscriber(topic_name=zros.topics.MAIN_SENSOR_INTERPRETER, data_class=msg.PerceptionConcept, callback=self._sensor_callback) self.subscriber_EmotionParams = zros.get.subscriber(topic_name=zros.topics.EMOTION_PARAMS, data_class=msg.EmotionParams, callback=self.emotion_callback) rospy.loginfo("Client \'%s\' is ready" % self.name) def start(self): rospy.logdebug("Node \'%s\' is starting..." % self.name) rospy.init_node(self.name, anonymous=False) self._init_communication() rospy.loginfo("[ DONE ] Node \'%s\' is ready..." % self.name) def to_will_cb(self, symbol: str, modifier: tuple = ()) -> str: """ Sends concept requests from callbacks """ rospy.loginfo("The mind is willing: %s:%s" % (symbol, str(modifier))) try: resp = self.client_CommandConcept(self.node_type, symbol, str(modifier)) rospy.logwarn("Response for %s: %s" % (symbol, resp)) return resp.result except rospy.ServiceException as e: print("Service call failed: %s" % e) def __call__(self): self.start()
class BoundedBlockingQueue(object): def __init__(self, capacity: int): self.locke, self.lockd = Lock(), Lock() self.dq = deque() self.capacity = capacity self.lockd.acquire() # Lock dequeue first because it's empty def enqueue(self, element: int) -> None: self.locke.acquire() self.dq.append(element) if len(self.dq) < self.capacity: self.locke.release() if self.lockd.locked( ): # if dequeue locked, release it since now we have element self.lockd.release() def dequeue(self) -> int: self.lockd.acquire() res = self.dq.popleft() if len(self.dq): self.lockd.release() if res and self.locke.locked( ): # if enqueue locked, release it since now we have element self.locke.release() return res def size(self) -> int: return len(self.dq)
def __init__(self, username, key, throttle_seconds=None, **kwargs): """ :param username: Your Adafruit username :type username: str :param key: Your Adafruit IO key :type key: str :param throttle_seconds: If set, then instead of sending the values directly over ``send`` the plugin will first collect all the samples within the specified period and then dispatch them to Adafruit IO. You may want to set it if you have data sources providing a lot of data points and you don't want to hit the throttling limitations of Adafruit. :type throttle_seconds: float """ global data_throttler_lock super().__init__(**kwargs) self._username = username self._key = key self.aio = Client(username=username, key=key) self.throttle_seconds = throttle_seconds if not data_throttler_lock: data_throttler_lock = Lock() if self.throttle_seconds and not data_throttler_lock.locked(): self._get_redis() self.logger.info('Starting Adafruit IO throttler thread') data_throttler_lock.acquire(False) self.data_throttler = Thread(target=self._data_throttler()) self.data_throttler.start()
class ConcurrentPopulations: def __init__(self): self._lock = Lock() self._generator = None self._discriminator = None def lock(self): self._lock.acquire() def unlock(self): self._lock.release() @property def generator(self): return self._generator @generator.setter def generator(self, value): self._generator = value @property def discriminator(self): return self._discriminator @discriminator.setter def discriminator(self, value): self._discriminator = value def locked(self): return self._lock.locked()
def __init__(self, path: str, lock: Lock, who: str = ""): self.conn = sqlite3.connect(path) if who: self.whoami = who assert lock.locked() self.lock = lock print("start connection for {}".format(self.whoami))
def worker(q: Queue, stop: Lock): ex = {'client': 'WORKER'} while not stop.locked(): # For exit thread in stopping simulation. try: task_id = q.get(timeout=QUEUE_GET_TIMEOUT) except Empty: continue logger.info('New task - {}'.format(TASK_TYPES[task_id]), extra=ex) search_time = uniform(*TIME_SEARCH_EDUCATIONAL) \ if task_id == TASK_EDUCATIONAL else uniform(*TIME_SEARCH_TECHNICAL) sleep(search_time) logger.info('Task completed in {:.2f} min'.format(search_time), extra=ex) output_time = uniform(*TIME_RESULT_OUTPUT) sleep(output_time) logger.info('Results output in {:.2f} min'.format(output_time), extra=ex) q.task_done()
class FrameBuffer(object): def initialize(self, width, height, dtype): self._shape = (height, width) self._buffers = [ np.zeros(self._shape, dtype=dtype), np.zeros(self._shape, dtype=dtype) ] self._write_index = 0 self._read_index = 1 self._lock = Lock() def write(self, data): source = data.reshape(self._shape) destination = self._buffers[self._write_index] np.copyto(dst=destination, src=source) self._swap_buffers() def read(self): assert self._lock.locked() return self._buffers[self._read_index] def read_lock(self): return self._lock def _swap_buffers(self): with self._lock: self._write_index, self._read_index = (self._read_index, self._write_index)
class SingleThread(): def __init__(self, progressbar): self.lock = Lock() self.progressbar = progressbar def run_with_progressbar(self, method, args=None, text=None, no_thread=False): if no_thread: if method and args: method(args) if method: method() else: self.progressbar.start(text) self._run(method, args) def _run(self, method, args=None): if not self.lock.locked(): self.lock.acquire() thread.start_new_thread(self._thread_task, (method, args,)) else: logging.warning("Thread not finished" + str(method) + str(args)) def _thread_task(self, method, args): try: if method and args: method(args) elif method: method() time.sleep(0.1) except Exception, e: logging.error(method.__name__ + "(" + str(args) + "):" + str(e)) finally:
class Fork(): """ Fork is a shared resource among Philosophers. A Fork can only be held by one Philosopher at a time """ counter = 0 timeout = 2 # in seconda def __init__(self): self.lock = Lock() self.id = Fork.counter Fork.counter += 1 def is_used(self): return self.lock.locked() def pickup(self): start = time.time() got_lock = False while time.time() - start < self.timeout and not got_lock: got_lock = self.lock.acquire(0) if not got_lock: logging.error('Deadlock') exit() def putdown(self): self.lock.release()
def manage_data(): mutex = Lock() try: # Calculate what was unix time 6 months ago seconds_in_six_months = 180 * 60 * 60 * 24 unix_time_now = time.time() unix_time_six_months_ago = unix_time_now - seconds_in_six_months karma_logs_db_conn = sqlite3.connect('karma_logs.db', check_same_thread=False) karma_logs_db_cursor = karma_logs_db_conn.cursor() mutex.acquire() # delete the record of comments that are of older than 6 months karma_logs_db_cursor.execute( "DELETE FROM comments WHERE submission_created_utc <= '{}'".format(unix_time_six_months_ago)) # commit karma_logs_db_conn.commit() mutex.release() user_database_obj.archive_data() user_database_obj.erase_data() print("Old data deleted " + time.strftime('%I:%M %p %Z')) except Exception as main_loop_exception: if mutex.locked(): mutex.release() tb = traceback.format_exc() print(main_loop_exception) try: send_message_to_discord(tb) except Exception as discord_exception: print("Error sending message to discord", str(discord_exception))
class LoginScreenVM: def __init__(self): self.__auth = SpotifyAuth.get_instance() self.success = False self.lock = Lock() def login(self, write_func): if self.lock.locked(): write_func("Please only attempt one login at a time.") return self.lock.acquire() old_stderr = sys.stderr sys.stderr = open(os.devnull, "w") if self.success: write_func("You are already logged in!") return try: if self.__auth.establish_connection(): self.success = True write_func(f"Welcome, {self.__auth.current_user}!") else: self.success = False write_func("Login cancelled by user") except Exception: self.success = False write_func("An unknown login error occurred.") finally: self.lock.release() sys.stderr = old_stderr
class Answer(list): """Safe storage of partial answers from server. When client try to acquire for reading, it will be locked until full response form client has been received. The answer is automatically locked at instantation by the current, so the only way that a client could consume the data is by releasing by the server side when answer is full filled. This is intended for Answers to be created in the network mainloop that will be process the partial responses from server. """ def __init__(self, *args, **kw): self._lock = Lock() self.call_args = None self.error = None self.return_code = None super(Answer, self).__init__(*args, **kw) self.acquire() def acquire(self, timeout=-1): "Try to lock the answer to fill it up or use it." return self._lock.acquire(timeout=timeout) def release(self): "Release the lock, usually done by server thread." if self._lock.locked(): self._lock.release() else: print('Warning: trying to release an unlocked mutex') @property def values(self): "Convert the values to a ordinary list" return list(self)
class Locker: """ Allow the connected_objects to be use by only one thread, but also to be kill if another thread need it """ def __init__(self): self.mutex = Lock() self.mutex_killer = Lock() self.killer = False def lock(self): if not (self.killer) and self.mutex.locked(): self.kill() self.mutex.acquire() self.mutex_killer.acquire() self.killer = False self.mutex_killer.release() def unlock(self): self.mutex.release() def test(self): return self.killer def kill(self): self.mutex_killer.acquire() self.killer = True self.mutex_killer.release()
def terminal(q: Queue, task_type: int, stop: Lock): ex = {'client': '{} TERMINAL'.format(TASK_TYPES[task_type])} while not stop.locked(): # For exit thread in stopping simulation. try: task_id = q.get(timeout=QUEUE_GET_TIMEOUT) except Empty: continue logger.debug('New request', extra=ex) prepare_time = uniform(*TIME_REQUEST_PREPARE) sleep(prepare_time) logger.debug('Results prepared in {:.2f} min'.format(prepare_time), extra=ex) search_time = uniform(*TIME_SEARCH_EDUCATIONAL) \ if task_id == TASK_EDUCATIONAL else uniform(*TIME_SEARCH_TECHNICAL) sleep(search_time) logger.debug('Task completed in {:.2f} min'.format(search_time), extra=ex) output_time = uniform(*TIME_RESULT_OUTPUT) sleep(output_time) logger.debug('Results output in {:.2f} min'.format(output_time), extra=ex) q.task_done()
class BlaLock(object): """ Simple wrapper class for the thread.lock class which only raises an exception when trying to release an unlocked lock if it's initialized with strict=True. """ def __init__(self, strict=False, blocking=True): self.__strict = strict self.__blocking = blocking self.__lock = Lock() def acquire(self): self.__lock.acquire(self.__blocking) def release(self): try: self.__lock.release() except ThreadError: if self.__strict: raise def locked(self): return self.__lock.locked() def __enter__(self, *args): self.acquire() def __exit__(self, *args): self.release()
class TaskQueue(Thread): def __init__(self, batch_processor: BatchProcessor, queue_capacity: int = 0, max_wait_time: float = 0.5): super().__init__() self._interval = max_wait_time self._batch_processor = batch_processor self._stop = False self._queue = Queue(maxsize=queue_capacity) self._execution_lock = Lock() self._count_trigger = Lock() def _size_overflow(self): return self._queue.qsize() >= self._batch_processor.get_batch_size() def stop(self): self._stop = True def async_submit(self, data) -> Future: if not self._stop: data_pack = DataPack(data) self._queue.put(data_pack) locked = self._execution_lock.acquire(blocking=False) if locked: if self._size_overflow() and self._count_trigger.locked(): self._count_trigger.release() self._execution_lock.release() # else means failed to get lock, batch processor is running return data_pack else: raise Exception("Task queue was stopped to accept task.") def _process(self): log.debug("[BP] Process starting, waiting for lock") with self._execution_lock: log.debug("[BP] Lock acquired") self._batch_processor.process([ self._queue.get() for _ in range( min(self._queue.qsize(), self._batch_processor.get_batch_size())) ]) self._count_trigger.acquire(blocking=False) def run(self) -> None: log.info("Batch service started.") while not (self._stop and self._queue.qsize() == 0): if self._stop: log.info("Exit flag set, cleaning queue...") try: if self._queue.qsize() < self._batch_processor.get_batch_size( ) and not self._stop: self._count_trigger.acquire(timeout=self._interval) if self._queue.qsize() > 0: self._process() except Exception as ex: log.fatal("Some critical error happened while doing batch", exc_info=ex) time.sleep(1) log.info("Queue cleaned, exiting batch process.")
class Channel: """ Channel allows communication/synchronization between threads""" def __init__(self): self.message = {} self.write_mutex = Lock() self.read_mutex = Lock() self.read_mutex.acquire() def write(self, msg): """ Write allows a message to be sent down the channel""" self.write_mutex.acquire() self.message = msg self.read_mutex.release() def read(self): """ read allows a message to be read from the channel """ self.read_mutex.acquire() msg = self.message self.message = {} self.write_mutex.release() return msg def is_full(self): """ return true if we can't write to the channel because it hasn't been read from yet """ return self.write_mutex.locked()
class Terminal: lock = None app = None def __init__(self, app): self.lock = Lock() self.lock.acquire() self.app = app self.app.debug_print = lambda cmd: eval(cmd) def show_terminal(self): if self.lock.locked(): self.lock.release() def pause_terminal(self): if not self.lock.locked(): self.lock.acquire() def run_terminal(self): with self.lock: printc("\n\nInput commands to RCAT below. Type help for list for commands.", "blue") while 1: self.lock.acquire() sys.stdout.write(ansi_codes["green"] + "[rcat]: ") line = sys.stdin.readline() if line.startswith("quit"): printc("Quitting RCAT....", "yellow") sys.exit(0) if line.startswith("help"): printc( "quit: (Force) quits RCAT\nprint arg: Runs a 'print arg' in the application. Application represented by 'app' (e.g. print app)", "endc", ) if line.startswith("restart"): self.app.declare_game_end() if line.startswith("print"): try: cmd = line.split() if len(cmd) > 2: logging.warn("Only one variable at a time.") else: app = self.app print eval(cmd[1]) except Exception, e: logging.error("Could not read variable.") print e self.show_terminal()
class Renderer(Thread): def __init__(self): Thread.__init__(self) self.x = 0 self.snapshot = 0 self.param_update = Lock() print_(self.param_update.locked()) self.param_ready = Lock() self.param_ready.acquire() # not ready self.ss_update = Lock() # not wanted self.ss_ready = Lock() self.ss_ready.acquire() # not ready self.param_lock = Lock() self.print_ = get_print("Rnd", Fore.YELLOW) def set_param(self, x): self.x = x def draw(self): sleep_time = randint(0,5)/10. self.print_("drawing") # if self.param_update.locked(): # # if it is updating, wait until it is ready # self.param_ready.acquire() self.param_update.acquire() self.print_("using param - %d" % self.x) sleep(sleep_time) # rendering to frame buffer self.snapshot = self.x # swap frame buffer self.print_("frame buffer - %d" % self.snapshot) if self.ss_update.locked(): self.save_snapshot() self.ss_ready.release() self.param_update.release() sleep(0.5 - sleep_time) def save_snapshot(self): self.print_("saving snapshot - %d - %d"%(self.snapshot, self.x)) def run(self): while True: self.draw()
class Locker(metaclass=Singleton): def __init__(self): self.lock = Lock() def set_acquire(self): self.lock.acquire() def set_release(self): self.lock.release() def is_locked(self): return self.lock.locked() def printID(self): return id(self.lock)
class SingleThread(): def __init__(self, progressbar=None): self.lock = Lock() self.progressbar = progressbar def run_with_progressbar(self, method, args=None, text='', no_thread=False, with_lock=True): #with_lock - shows, does it necessarily to do a lock or not if no_thread: if method and args: method(args) if method: method() else: self._run(method, args, text, with_lock) def _run(self, method, args=None, text='', with_lock=True): if not self.lock.locked(): self.lock.acquire() if self.progressbar: self.progressbar.start(text) thread.start_new_thread(self._thread_task, (method, args,)) else: logging.warning("Previous thread not finished " + str(method) + " " + str(args)) if not with_lock: logging.info("Try to run method without progress bar") thread.start_new_thread(self._thread_task, (method, args)) def _thread_task(self, method, args, with_lock=True): try: if method and args: method(args) elif method: method() except Exception, e: logging.error(str(e)) exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout) finally:
class BaseFoobnixControls(): def __init__(self): self.vk_service = VKService(FC().access_token, FC().user_id) self.count_errors = 0 self.is_scrobbled = False self.start_time = None self.cache_text = None self.play_lock = Lock() def check_for_media(self, args): dirs = [] files = [] for arg in args: if os.path.isdir(arg): dirs.append(arg) elif os.path.isfile(arg) and get_file_extension(arg) in FC().all_support_formats: files.append(arg) if dirs: self.on_add_folders(dirs) elif files: self.on_add_files(files) try: self.play_first_added(files) except: logging.error("Can't to play first added file") def play_first_added(self, added_files): tree = self.notetabs.get_current_tree() model = tree.get_model() number = len(model) - len(added_files) if (number) > -1: iter = model.get_iter_from_string(str(number)) bean = tree.get_bean_from_model_iter(model, iter) tree.set_play_icon_to_bean(bean) self.play(bean) def love_this_tracks(self, beans=None): if not beans: return map(self.lastfm_service.love, beans) def add_to_my_playlist(self, beans=None): if not beans: return map(self.vk_service.add, beans) def copy_link(self, beans=None): if not beans: return if hasattr(beans[0], 'path'): pyperclip.copy(beans[0].path) if FC().notifier: notification = Notify.Notification.new("In clipboard", beans[0].path, "") notification.set_urgency(Notify.Urgency.LOW) notification.set_timeout(FC().notify_time) notification.show() def show_google_results(self, query): return [FModel('"%s" not found' % query)] def get_active_bean(self): tree = self.notetabs.get_current_tree() if tree: return tree.get_selected_or_current_bean() def play_selected_song(self): current = self.get_active_bean() tree = self.notetabs.get_current_tree() if not current: try: current = tree.get_bean_under_pointer_icon() except AttributeError: return if not current: return None logging.debug("play current bean is %s" % str(current.text)) if current and current.is_file: tree.set_play_icon_to_bean(current) self.play(current) def check_path(self, path): if path: if not path.startswith("http://"): if os.path.exists(path): return True else: try: u = urlopen(path, timeout=5) # @UnusedVariable if "u" not in vars(): return False return True except: return False return False def save_beans_to(self, beans): return None def on_chage_player_state(self, state, bean): logging.debug("bean state %s" % state) self.set_dbus_state(state, bean) if not FC().system_icons_dinamic: return None if state == STATE_STOP: self.trayicon.set_image_from_path(FC().stop_icon_entry) elif state == STATE_PAUSE: self.trayicon.set_image_from_path(FC().pause_icon_entry) elif state == STATE_PLAY: self.trayicon.set_image_from_path(FC().play_icon_entry) if bean and bean.type: logging.debug("bean state and type %s %s" % (state, bean.type)) if bean.type == FTYPE_RADIO: return self.trayicon.set_image_from_path(FC().radio_icon_entry) @idle_task def set_dbus_state(self, state, bean): if self.dbus: self.dbus._update_info(bean) if state is STATE_PLAY: self.dbus._set_state_play() elif state is STATE_PAUSE: self.dbus._set_state_pause() else: self.dbus._set_state_stop() def on_add_folders(self, paths=None): if not paths: paths = directory_chooser_dialog(_("Choose folders to open"), FC().last_dir) if not paths: return tree = self.notetabs.get_current_tree() FC().last_dir = os.path.dirname(paths[0]) if tree.is_empty(): if len(paths) > 1: tabname = os.path.basename(FC().last_dir) else: tabname = os.path.basename(paths[0]) self.notetabs.rename_tab(tree.scroll, tabname) tree.append(paths) def on_add_files(self, paths=None, tab_name=None): if not paths: paths = file_chooser_dialog(_("Choose file to open"), FC().last_dir) if not paths: return tree = self.notetabs.get_current_tree() FC().last_dir = os.path.dirname(paths[0]) if tree.is_empty(): tabname = os.path.split(os.path.dirname(paths[0]))[1] self.notetabs.rename_tab(tree.scroll, tabname) tree.append(paths) def set_playlist_tree(self): self.notetabs.set_playlist_tree() def set_playlist_plain(self): self.notetabs.set_playlist_plain() def load_music_tree(self): tabs = len(FCache().cache_music_tree_beans) tabhelper = self.perspectives.get_perspective('fs').get_tabhelper() for tab in xrange(tabs - 1, -1, -1): tabhelper._append_tab(FCache().tab_names[tab], rows=FCache().cache_music_tree_beans[tab]) if not FCache().cache_music_tree_beans[tab]: self.perspectives.get_perspective('fs').show_add_button() else: self.perspectives.get_perspective('fs').hide_add_button() logging.info("Tree loaded from cache") if FC().update_tree_on_start: def cycle(): for n in xrange(len(FCache().music_paths)): tab_child = tabhelper.get_nth_page(n) tree = tab_child.get_child() self.update_music_tree(tree, n) GLib.idle_add(cycle) def update_music_tree(self, tree, number_of_page=0): logging.info("Update music tree" + str(FCache().music_paths[number_of_page])) tree.clear_tree() # safe method FCache().cache_music_tree_beans[number_of_page] = {} all = get_all_music_by_paths(FCache().music_paths[number_of_page], self) try: self.perspectives.get_perspective('fs').hide_add_button() except AttributeError: logging.warn("Object perspective not exists yet") if not all: try: self.perspectives.get_perspective('fs').show_add_button() except AttributeError: logging.warn("Object perspective not exists yet") tree.append_all(all) # safe method tree.ext_width = tree.ext_column.get_width() GLib.idle_add(tree.save_rows_from_tree, FCache().cache_music_tree_beans[number_of_page]) #GLib.idle_add(self.tabhelper.on_save_tabs) # for true order @idle_task def set_visible_video_panel(self, flag): return #FC().is_view_video_panel = flag #if flag: # self.movie_window.show() #else: # self.movie_window.hide() @idle_task def volume_up(self): self.volume.volume_up() @idle_task def volume_down(self): self.volume.volume_down() @idle_task def mute(self): self.volume.mute() @idle_task def hide(self): self.main_window.hide() @idle_task def show_hide(self): self.main_window.show_hide() @idle_task def show(self): self.main_window.show() @idle_task def play_pause(self): if self.media_engine.get_state() == STATE_PLAY: self.media_engine.state_pause() elif self.media_engine.get_state() == STATE_STOP: self.state_play(True) else: self.media_engine.state_play() @idle_task def seek_up(self): self.media_engine.seek_up() @idle_task def seek_down(self): self.media_engine.seek_down() @idle_task def windows_visibility(self): visible = self.main_window.get_property('visible') if visible: GLib.idle_add(self.main_window.hide) else: GLib.idle_add(self.main_window.show) @idle_task def state_play(self, under_pointer_icon=False): if self.media_engine.get_state() == STATE_PAUSE: self.media_engine.state_play() self.statusbar.set_text(self.media_engine.bean.info) elif under_pointer_icon: tree = self.notetabs.get_current_tree() bean = tree.get_bean_under_pointer_icon() self.play(bean) else: self.play_selected_song() @idle_task def show_preferences(self): self.preferences.show() @idle_task def state_pause(self): self.media_engine.state_pause() @idle_task_priority(priority=GLib.PRIORITY_HIGH_IDLE) def state_stop(self, remember_position=False): self.record.hide() self.media_engine.state_stop(remember_position) if not remember_position: self.statusbar.set_text(_("Stopped")) self.seek_bar.clear() @idle_task def state_play_pause(self): self.media_engine.state_play_pause() bean = self.media_engine.bean if self.media_engine.get_state() == STATE_PLAY: self.statusbar.set_text(bean.info) else: self.statusbar.set_text(_("Paused | ") + str(bean.info)) def state_is_playing(self): return self.media_engine.get_state() == STATE_PLAY def fill_bean_from_vk(self, bean): if bean.type and bean.type == FTYPE_RADIO: return False vk = self.vk_service.find_one_track(bean.get_display_name()) if vk: bean.path = vk.path bean.time = vk.time return True else: return False def fill_bean_by_vk_aid(self, bean): if not bean.vk_audio_id: return False if bean.type and bean.type == FTYPE_RADIO: return False track = self.vk_service.find_track_by_id(bean.vk_audio_id) if track: bean.path = track.path bean.time = track.time return True return False @idle_task def play(self, bean): if not bean or not bean.is_file: return self.play_lock.acquire() self.seek_bar.clear() ## TODO: Check for GTK+3.4 (Status icon doesn't have a set_tooltip method) self.statusbar.set_text(bean.info) self.trayicon.set_text(bean.text) #self.movie_window.set_text(bean.text) if bean.type == FTYPE_RADIO: self.record.show() self.seek_bar.progressbar.set_fraction(0) self.seek_bar.set_text(_("Radio ") + bean.text.capitalize()) else: self.record.hide() self.main_window.set_title(bean.text) thread.start_new_thread(self._one_thread_play, (bean,)) def _one_thread_play(self, bean): try: self._play(bean) finally: if self.play_lock.locked(): self.play_lock.release() def _play(self, bean): if not bean.path: bean.path = get_bean_posible_paths(bean) if not self.check_path(bean.path): if bean.iso_path and os.path.exists(bean.iso_path): logging.info("Try to remount " + bean.iso_path) mount_tmp_iso(bean.iso_path) elif bean.vk_audio_id: self.fill_bean_by_vk_aid(bean) elif not bean.path or ("userapi" in bean.path) or ("vk.me" in bean.path): self.fill_bean_from_vk(bean) else: resource = bean.path if bean.path else bean.text logging.error("Resourse " + resource + " not found") self.media_engine.state_stop(show_in_tray=False) self.statusbar.set_text(_("Resource not found")) self.seek_bar.set_text(_("Resource not found")) self.count_errors += 1 time.sleep(2) if self.count_errors < 4: if self.play_lock.locked(): self.play_lock.release() self.next() else: self.seek_bar.set_text(_("Stopped. No resources found")) return elif os.path.isdir(bean.path): return self.count_errors = 0 self.media_engine.play(bean) self.is_scrobbled = False self.start_time = False if bean.type != FTYPE_RADIO: self.update_info_panel(bean) self.set_visible_video_panel(False) @idle_task def notify_playing(self, pos_sec, dur_sec, bean): if not bean.type or bean.type != FTYPE_RADIO: self.seek_bar.update_seek_status(pos_sec, dur_sec) else: self.seek_bar.fill_seekbar() if pos_sec == 2 or (pos_sec > 2 and (pos_sec % 20) == 0): self.net_wrapper.execute(self.lastfm_service.report_now_playing, bean) if not self.start_time: self.start_time = str(int(time.time())) if not self.is_scrobbled and bean.type != FTYPE_RADIO: ## song should be scrobbled if 90% has been played or played greater than 5 minutes if pos_sec > (dur_sec * 0.9) or pos_sec > (60 * 5): self.is_scrobbled = True self.net_wrapper.execute(self.lastfm_service.report_scrobbled, bean, self.start_time, dur_sec) """download music""" if FC().automatic_online_save and bean.path and bean.path.startswith("http://"): self.dm.append_task(bean) @idle_task def notify_title(self, bean, raw_text): logging.debug("Notify title: " + raw_text) text = raw_text.partition("||")[0] if not self.cache_text: self.cache_text = text self.statusbar.set_text(raw_text.replace("||", "|")) text = normalize_text(text) self.seek_bar.set_text(text) t_bean = bean.create_from_text(text) self.update_info_panel(t_bean) self.set_dbus_state(STATE_PLAY, t_bean) if FC().enable_radio_scrobbler: start_time = str(int(time.time())) self.net_wrapper.execute(self.lastfm_service.report_now_playing, t_bean) if " - " in text and self.cache_text != text: c_bean = copy.copy(bean) prev_bean = c_bean.create_from_text(self.cache_text) self.net_wrapper.execute(self.lastfm_service.report_scrobbled, prev_bean, start_time, 200) self.cache_text = text @idle_task def notify_error(self, msg): logging.error("notify error " + msg) self.seek_bar.set_text(msg) self.perspectives.get_perspective('info').clear() @idle_task def notify_eos(self): self.next() @idle_task def player_seek(self, percent): self.media_engine.seek(percent) @idle_task def player_volume(self, percent): self.media_engine.volume(percent) def search_vk_page_tracks(self, vk_ulr): logging.debug("Search vk_service page tracks") results = self.vk_service.find_tracks_by_url(vk_ulr) all = [] p_bean = FModel(vk_ulr).add_font("bold") all.append(p_bean) for i, bean in enumerate(results): bean.tracknumber = i + 1 bean.parent(p_bean).add_is_file(True) all.append(bean) self.notetabs.append_tab(vk_ulr, all) def search_all_tracks(self, query): def search_all_tracks_task(): analytics.action("SEARCH_search_all_tracks") results = self.vk_service.find_tracks_by_query(query) if not results: results = [] all = [] p_bean = FModel(query).add_font("bold") all.append(p_bean) for i, bean in enumerate(results): bean.tracknumber = i + 1 bean.parent(p_bean).add_is_file(True) all.append(bean) if not results: all = self.show_google_results(query) self.notetabs.append_tab(query, all) self.in_thread.run_with_spinner(search_all_tracks_task, no_thread=True) def search_top_tracks(self, query): def search_top_tracks_task(query): analytics.action("SEARCH_search_top_tracks") results = self.lastfm_service.search_top_tracks(query) if not results: results = [] all = [] parent_bean = FModel(query) all.append(parent_bean) for i, bean in enumerate(results): bean.tracknumber = i + 1 bean.parent(parent_bean).add_is_file(True) all.append(bean) if not results: all = self.show_google_results(query) self.notetabs.append_tab(query, all) self.in_thread.run_with_spinner(search_top_tracks_task, query) def search_top_albums(self, query): def search_top_albums_task(query): analytics.action("SEARCH_search_top_albums") results = self.lastfm_service.search_top_albums(query) if not results: results = [] self.notetabs.append_tab(query, None) albums_already_inserted = [] for album in results[:15]: all = [] if album.album.lower() in albums_already_inserted: continue album.is_file = False tracks = self.lastfm_service.search_album_tracks(album.artist, album.album) for i, track in enumerate(tracks): track.tracknumber = i + 1 track.album = album.album track.parent(album).add_is_file(True) all.append(track) if len(all) > 0: all = [album] + all albums_already_inserted.append(album.album.lower()) self.notetabs.append_all(all) if not results: all = self.show_google_results(query) self.notetabs.append_all(all) self.in_thread.run_with_spinner(search_top_albums_task, query) def search_top_similar(self, query): def search_top_similar_task(query): analytics.action("SEARCH_search_top_similar") results = self.lastfm_service.search_top_similar_artist(query) if not results: results = [] self.notetabs.append_tab(query, None) for artist in results[:15]: all = [] artist.is_file = False all.append(artist) tracks = self.lastfm_service.search_top_tracks(artist.artist) for i, track in enumerate(tracks): track.tracknumber = i + 1 track.parent(artist).add_is_file(True) all.append(track) self.notetabs.append_all(all) if not results: all = self.show_google_results(query) #inline(query) self.in_thread.run_with_spinner(search_top_similar_task, query) def search_top_tags(self, query): def search_top_tags_task(query): analytics.action("SEARCH_search_top_tags") results = self.lastfm_service.search_top_tags(query) if not results: logging.debug("tag result not found") results = [] self.notetabs.append_tab(query, None) for tag in results[:15]: all = [] tag.is_file = False all.append(tag) tracks = self.lastfm_service.search_top_tag_tracks(tag.text) for i, track in enumerate(tracks): track.tracknumber = i + 1 track.parent(tag).add_is_file(True) all.append(track) self.notetabs.append_all(all) if not results: all = self.show_google_results(query) self.notetabs.append_all(all) #inline(query) self.in_thread.run_with_spinner(search_top_tags_task, query) @idle_task def update_info_panel(self, bean): self.perspectives.get_perspective('info').update(bean) @idle_task def append_to_new_notebook(self, text, beans, optimization=False): self.notetabs._append_tab(text, beans, optimization) @idle_task def append_to_current_notebook(self, beans): self.notetabs.append_all(beans) @idle_task def next(self): bean = self.notetabs.next() if not bean: return gap = FC().gap_secs time.sleep(gap) logging.debug("play current bean is %s" % str(bean.text)) self.play(bean) @idle_task def prev(self): bean = self.notetabs.prev() if not bean: return self.play(bean) def quit(self, *a): self.state_stop() self.main_window.hide() self.trayicon.hide() logging.info("Controls - Quit") for element in self.__dict__: if isinstance(self.__dict__[element], Quitable): self.__dict__[element].on_quit() FC().save() GLib.idle_add(Gtk.main_quit) # wait for complete stop task def check_version(self): uuid = FCBase().uuid current_version = FOOBNIX_VERSION system = "not_set" try: import platform system = platform.system() except: pass try: from socket import gethostname f = urlopen("http://www.foobnix.com/version?uuid=" + uuid + "&host=" + gethostname() + "&version=" + current_version + "&platform=" + system, timeout=7) #f = urllib2.urlopen("http://localhost:8080/version?uuid=" + uuid + "&host=" + gethostname() + "&v=" + current_version) except Exception as e: logging.error("Check version error: " + str(e)) return None new_version_line = f.read() logging.info("version " + current_version + "|" + new_version_line + "|" + str(uuid)) f.close() if FC().check_new_version and compare_versions(current_version, new_version_line) == 1: info_dialog_with_link_and_donate(new_version_line) def on_load(self): """load controls""" for element in self.__dict__: if isinstance(self.__dict__[element], LoadSave): init = time.time() self.__dict__[element].on_load() logging.debug("%f LOAD ON START %s" % (time.time() - init, str(self.__dict__[element]))) """load others""" #self.movie_window.hide_all() self.main_window.show() self.search_progress.stop() """base layout""" self.layout.on_load() """check for new version""" if os.name == 'nt': self.check_version() else: pass #GLib.idle_add(self.check_version) @idle_task_priority(GLib.PRIORITY_LOW) def play_first_file_in_playlist(self): active_playlist_tree = self.notetabs.get_current_tree() filter_model = active_playlist_tree.get_model() current_model = filter_model.get_model() def play_item(iter, active_playlist_tree, filter_model, current_model): bean = active_playlist_tree.get_bean_from_model_iter(current_model, iter) if not bean: return if bean.font != 'bold': self.play(bean) tree_selection = active_playlist_tree.get_selection() filter_iter = filter_model.convert_child_iter_to_iter(iter) if filter_iter[0]: GLib.idle_add(tree_selection.select_iter, filter_iter[1]) active_playlist_tree.set_play_icon_to_bean_to_selected() else: iter = current_model.iter_next(iter) play_item(iter, active_playlist_tree, filter_model, current_model) iter = current_model.get_iter_first() play_item(iter, active_playlist_tree, filter_model, current_model) def on_save(self): for element in self.__dict__: if isinstance(self.__dict__[element], LoadSave): logging.debug("SAVE " + str(self.__dict__[element])) self.__dict__[element].on_save() def download(self): self.dm.append_task(bean=self.notetabs.get_current_tree().get_current_bean_by_UUID())
class NaptConnection(object): def __init__(self, client, server): Utils.expects_type(socket.socket, client, 'client') Utils.expects_type(socket.socket, server, 'server', True) self.lock = Lock() self.id = 0 self.client = NaptSocket(self, client, True) self.server = NaptSocket(self, server, False) self.is_connecting = False self.is_connected = False self.is_closed = False self.tag = None self.tls = False self.debug = True self.bind_port = 0 self.connected = Event() self.closed = Event() self.client_closing = Event() self.server_closing = Event() self.client_closed = Event() self.server_closed = Event() self.client_recieved= Event() self.server_recieved= Event() def __str__(self): return 'NaptConnection{ %s }' % ', '.join([ 'id=%d' % self.id, 'client=%s' % str(self.client), 'server=%s' % str(self.server), 'is_connecting=%s' % str(self.is_connecting), 'is_connected=%s' % str(self.is_connected)]) # public def connect(self, endpoint): Utils.assertion(self.lock.locked(), 'need lock') if self.is_connecting: raise Exception() # InvalidOperationException self.is_connecting = True self.server.status = NaptSocketStatus.Connecting threading.Thread(target = self.do_connect, args = (endpoint,), name = self.__class__.__name__).start() # private def do_connect(self, endpoint): try: self.server.connect(endpoint) # blocking #self.client.socket.settimeout(5.0) #self.server.socket.settimeout(5.0) with self.lock: if self.is_closed: # todo close return self.is_connected = True #print('INVOKE: on_connected') self.on_connected(None) except Exception as ex: print(' endpoint: %s' % str(endpoint)) Utils.print_exception(ex) self.close() # public def close(self): with self.lock: #if self.is_closed: # return self.close_client() self.close_server() self.is_closed = True if self.debug: print('NaptConnection.close: %s' % str(self)) self.on_closed(None) # protected virtual def on_connected(self, e): self.connected(self, e) # protected virtual def on_closed(self, e): self.closed(self, e) # protected virtual def on_client_closing(self, e): self.client_closing(self, e) # protected virtual def on_server_closing(self, e): self.server_closing(self, e) # protected virtual def on_client_closed(self, e): self.client_closed(self, e) # protected virtual def on_server_closed(self, e): self.server_closed(self, e) # protected virtual def on_client_recieved(self, e): # NaptConnectionEventArgs self.client_recieved(self, e) # protected virtual def on_server_recieved(self, e): # NaptConnectionEventArgs self.server_recieved(self, e) # internal def recv(self, so): Utils.expects_type(NaptSocket, so, 'so') if so.is_client: self.recv_client() else: self.recv_server() # internal def error(self, so): Utils.expects_type(NaptSocket, so, 'so') # todo error # private def recv_client(self): try: #self.client.socket.settimeout(5.0) #self.server.socket.settimeout(5.0) data= self.client.socket.recv(4096) e = NaptConnectionEventArgs(self, data, 0, len(data)) if len(data) == 0: # closed #self.close_client(); self.close() return #print(' DATA: %s' % str(data), flush=True) self.on_client_recieved(e) self.server.push(data, 0, len(data)) except Exception as ex: # SocketException Utils.print_exception(ex) self.close() # private def recv_server(self): try: #self.client.socket.settimeout(5.0) #self.server.socket.settimeout(5.0) data= self.server.socket.recv(4096) e = NaptConnectionEventArgs(self, data, 0, len(data)) if len(data) == 0: # closed #self.close_server() self.close() return #print(' DATA: %s' % str(data), flush=True) self.on_server_recieved(e) self.client.push(data, 0, len(data)) except Exception as ex: # SocketException Utils.print_exception(ex) self.close() # private def close_client(self): #if self.debug: # print(' NaptConnection.close_client: %s' % str(self.client)) try: self.on_client_closing(None) except Exception as ex: Utils.print_exception(ex) try: if self.client.close(): self.on_client_closed(None) except Exception as ex: Utils.print_exception(ex) # private void def close_server(self): #if self.debug: # print(' NaptConnection.close_server: %s' % str(self.server)) try: self.on_server_closing(None) except Exception as ex: Utils.print_exception(ex) try: if self.server.close(): self.on_server_closed(None); except Exception as ex: Utils.print_exception(ex)
class ConfigDialog(object): __xml = None __dialog = None __mode = 0 __button = BUTTON_1 MODES = [u'0', u'A', u'B', u'AB'] __modes = [] __buttons = [] __actions = [] __act = None __cbAct = None __vbActConfig = None __ulock = None def __init__(self, config): self.__conf = config self.__xml = glade.XML(resource('config.glade')) self.__xml.signal_autoconnect(self) self.__dialog = self.__xml.get_widget('gdConfig') self.__buttons = [self.__xml.get_widget('b%i'%(btn+1)) for btn in range(BUTTON_1, BUTTON_9+1)] self.__modes = [None, self.__xml.get_widget('tbModeA'), self.__xml.get_widget('tbModeB')] self.__actions = sorted(list(actionObjs()), (lambda a,b: cmp(a.label, b.label))) self.__cbAct = self.__xml.get_widget('cbAction') set_model_from_list(self.__cbAct, map((lambda a: a.label), self.__actions)) self.__vbActConfig = self.__xml.get_widget('vbActConfig') self.__ulock = Lock() self.__set_action(self.__conf[self.__mode, self.__button][0]) self.__update(True) def __name(self): FORMAT = u"%(num)i (%(mode)s)" mode = self.MODES[self.__mode] return FORMAT % {'mode': mode, 'num': self.__button + 1} def __set_action(self, action): if self.__act is not None: for name, label, conf in self.__act.config: conf.destroy() if isinstance(action, int): i = action action = self.__actions[i] else: print "change", self.__actions, action i = self.__actions.index(action) self.__cbAct.set_active(i) self.__act = action self.__vbActConfig.forall(self.__vbActConfig.remove) # Get rid of previous children ops = self.__conf[self.__mode, self.__button][1] if ops is None: ops = {} print "load ops", ops for name, label, conf in action.config: print "nlc:", name, label, conf ui = conf.createInput(self.__save) if name in ops: conf.setValue(ops[name]) self.__vbActConfig.add(ui) self.__vbActConfig.show_all() def __save(self): if self.__ulock.locked(): return # Don't save when updating ops = {} for name, label, conf in self.__act.config: ops[name] = conf.getValue() print "Save:", self.__act, ops self.__conf[self.__mode, self.__button] = (self.__act, ops) print "saved:", self.__conf[self.__mode, self.__button] def __update(self, first=False): if not self.__ulock.acquire(False): return try: self.__xml.get_widget('lbldButton').set_label(self.__name()) if bool(self.__mode & MODE_A) != bool(self.__modes[MODE_A].get_active()): self.__modes[MODE_A].set_active(bool(num & MODE_A)) if bool(self.__mode & MODE_B) != bool(self.__modes[MODE_B].get_active()): self.__modes[MODE_B].set_active(bool(self.__mode & MODE_B)) conf = self.__conf[self.__mode, self.__button] self.__set_action(conf[0] or Action()) finally: self.__ulock.release() def buttonChange(self, widget, data=None): num = self.__buttons.index(widget) print "buttonChange", widget, num self.__save() if num != self.__button: self.__button = num self.__update() def modeChange(self, widget, data=None): num = self.__modes.index(widget) print "modeChange", widget, num self.__save() self.__mode = (self.__mode & ~num) | (num if widget.get_active() else 0) self.__update() def actionChange(self, widget, data=None): print "actionChange", widget, data i = widget.get_active() self.__set_action(i) self.__save() def __del__(self): self.__dialog.destroy() self.__dialog = None def run(self): rv = self.__dialog.run() self.__dialog.hide() return rv def grab(self): return self.__dialog.present()
class RSS2AlarmManager(object): implements(IRSS2Document) adapts(IAlarmManager) def __new__(klass, manager): syndicator_name = '__' + klass.__name__ syndicator = getattr(manager, syndicator_name, None) if syndicator is None: print 'Creating new RSS2AlarmManager' syndicator = super(RSS2AlarmManager, klass).__new__(klass) setattr(manager, syndicator_name, syndicator) syndicator.initialize(manager) return syndicator def initialize(self, manager): self.debug = 0 self.manager = manager self.cache_lock = Lock() self.cache_ttl = 30 self.closed_event_ttl = 3 * 60 self.caches = {} self.close_events = {} self.event_queue = EventQueue() self.hostname = AlarmEvent.LOCALORIGIN self.uri_base = 'http://' + self.hostname if self.uri_base[-1] == '/': self.uri_base = self.uri_base[0:-1] self.categories = { 'RAISED': rss.Category("Raised"), 'INACTIVE': rss.Category("Inactive"), 'ACCEPTED': rss.Category("Accepted"), 'CLEARED': rss.Category("Cleared"), 'CLOSED': rss.Category("Closed") } self.last_pub_time = None self.updatesub = self.manager.register_for_type( self.initialize_caches, FormationUpdated) self.subscription = self.manager.register_for_type( self.handle_alarm_update, StateEvent, None, True) self.initialize_caches() def initialize_caches(self, *args): if self.debug: tstart = time.time() print 'RSS2 Syndic initializing caches because: %s' % (args,) events = [] self.cache_lock.acquire() try: self.event_queue.popqueue() for group in map(Alarm.get_events, self.manager.get_alarms()): events.extend(group) events.extend(self.manager.get_remote_events()) events.extend(self.event_queue.popqueue()) self.caches = {None: ItemCache()} cache = self.process_events(events) finally: self.cache_lock.release() if self.debug: tend = time.time() tlapse = tend - tstart print 'RSS2 cache init of %s events took %s seconds.' % (len(events), tlapse) return cache def process_events(self, events): if not self.cache_lock.locked(): raise Exception('Process events cannot be called unless locked.') cache = {} if events: guids = map(Event.get_guid, events) items = map(self.item_from_event, events) cache.update(zip(guids, items)) for existing in self.caches.values(): existing.update(cache) return cache def handle_alarm_update(self, event): if self.debug: tstart = time.time() if isinstance(event, StateEvent): event = event.source if event.is_state('closed'): self.close_events[event.GUID] = uptime.secs() self.event_queue.enqueue(event) if self.cache_lock.acquire(0): try: self.trim_expired_caches() self.process_events(self.event_queue.popqueue()) finally: self.cache_lock.release() else: print 'Alarm update not processing queue; locked.' if self.debug: tend = time.time() tlapse = tend - tstart print 'Took RSS2 Syndic %s secs to handle alarm event.' % tlapse return def render(self, request_path = None, cache_id = None): if request_path is None: request_path = '/syndication' xmldoc = self.setup_xmldoc() channel = self.setup_channel(request_path) xmldoc.root_element.channel = channel self.cache_lock.acquire() try: self.trim_expired_caches() queue = self.event_queue.popqueue() if queue: self.process_events(queue) finally: self.cache_lock.release() items = self.get_items(cache_id) map(channel.items.append, items) return str(xmldoc) def setup_xmldoc(self): xmldoc = rss.XMLDoc() xmldoc.root_element = rss.RSS() return xmldoc def setup_channel(self, request_path): publish_time = time.time() channel = rss.Channel() channel.title = rss.Title("Network Building Mediator Alarms") if request_path[0] != '/': request_path = '/' + request_path url = self.uri_base + request_path channel.link = rss.Link(url) channel.description = rss.Description("RSS 2.0 feed of Network " "Building Mediator alarms.") if self.last_pub_time is not None: channel.last_build_date = rss.LastBuildDate(self.last_pub_time) self.last_pub_time = publish_time channel.generator = rss.Generator("Network Building " "Mediator Alarm Syndication") channel.docs = rss.Docs('http://blogs.law.harvard.edu/tech/rss') return channel def get_items(self, cid): """ Get Alarm Event Items that have not been returned to client with ID "cid". If client ID is None, return all. """ count = None if cid: count = 250 if cid not in self.caches: self.caches[cid] = ItemCache(self.caches[None]) cache = self.caches[cid] return cache.read(count) def trim_expired_caches(self): if not self.cache_lock.locked(): raise Exception('Must be locked to trim caches.') removed = [] now = uptime.secs() allitems = self.caches[None] for guid,closed in self.close_events.items(): if (now - closed) > self.closed_event_ttl: if guid in allitems: del(allitems[guid]) del(self.close_events[guid]) for cid,cache in self.caches.items(): if cid and (cache.since_touched() > self.cache_ttl): del(self.caches[cid]) removed.append(cid) if self.debug and removed: print 'Cache trim trimmed the following IDs: %s.' % (removed,) return removed def item_from_event(self, event): if isinstance(event, StateEvent): event = event.source return IRSS2ItemElement(event).as_item()
class Downloader: def __init__(self, url, dlhash, rawserver, failed_func, max_errors = 10): if DEBUG: log('dd-downloader::__init__: url', url, 'hash', binascii.hexlify(dlhash)) self.url = url self.rawserver = rawserver self.failed_func = failed_func self.final_url = None self.storage = None self.lock = Lock() self.measure = Measure(10.0) self.errors = 0 self.max_errors = max_errors self.seek = None self.shutdown_flag = False self.running = False self.log_prefix = 'dd-downloader::' + binascii.hexlify(dlhash) + ':' def predownload(self, callback, timeout = 10): if self.lock.locked(): self.seek = pos return t = Thread(target=self._predownload, args=[callback, timeout]) t.setName('dd-downloader-predownload-' + t.getName()) t.setDaemon(True) t.start() def _predownload(self, callback, timeout): self.lock.acquire() self.running = True try: if DEBUG: log(self.log_prefix + '_predownload: url', self.url, 'timeout', timeout) stream = urlOpenTimeout(self.url, timeout=timeout) content_type = stream.info().getheader('Content-Type') content_length = stream.info().getheader('Content-Length') if DEBUG: log(self.log_prefix + '_predownload: request finished: content_type', content_type, 'content_length', content_length) data = '' while True: if self.shutdown_flag: if DEBUG: log(self.log_prefix + '_predownload: got shutdown flag while reading: url', self.url) break buf = stream.read(524288) if not buf: if DEBUG: log(self.log_prefix + '_predownload: eof: url', self.url) break data += buf if DEBUG: log(self.log_prefix + '_predownload: read chunk: url', self.url, 'read_len', len(data)) stream.close() if not self.shutdown_flag: if DEBUG: log(self.log_prefix + '_predownload: finished, run callback: url', self.url, 'content_type', content_type, 'content_length', content_length, 'data_len', len(data)) callback(content_type, data) except Exception as e: if DEBUG: print_exc() self.failed_func(e) finally: self.running = False self.lock.release() def init(self, callback = None, timeout = 10): if callback is None: return self._init() t = Thread(target=self._init, args=[callback, timeout]) t.setName('dd-downloader-init-' + t.getName()) t.setDaemon(True) t.start() def _init(self, callback = None, timeout = 10): try: scheme, host, path = self.parse_url(self.url) redirects = 0 connection = HTTPConnection(host) while True: connection.request('HEAD', path, None, {'Host': host, 'User-Agent': USER_AGENT}) r = connection.getresponse() if r.status == 200: break elif r.status == 301 or r.status == 302: redirect_url = r.getheader('Location', None) if DEBUG: log(self.log_prefix + 'init: got redirect: url', self.url, 'redirect', redirect_url) scheme, rhost, path = self.parse_url(redirect_url) redirects += 1 if redirects > MAX_REDIRECTS: raise Exception('Too much redirects') if rhost != host: connection.close() connection = HTTPConnection(rhost) host = rhost else: raise Exception('Bad http status: ' + str(r.status)) mime = r.getheader('Content-Type', None) length = r.getheader('Content-Length', None) connection.close() if length is None: raise Exception('No content-length in response') if mime is None: raise Exception('No content-type in response') length = int(length) self.final_url = scheme + '://' + host + path if DEBUG: log(self.log_prefix + 'init: got response: length', length, 'mime', mime, 'final_url', self.final_url) if callback is None: return (length, mime) callback(length, mime) except Exception as e: if DEBUG: print_exc() if callback is None: raise e else: self.failed_func(e) def set_storage(self, storage): self.storage = storage def start(self, pos = 0): if self.storage is None: raise Exception('Storage is not set') if self.final_url is None: raise Exception('Final url is not set') if self.lock.locked(): self.seek = pos return t = Thread(target=self._request, args=[pos]) t.setName('dd-downloader-' + t.getName()) t.setDaemon(True) t.start() def _request(self, pos): self.lock.acquire() self.running = True try: while True: if self.shutdown_flag: if DEBUG: log(self.log_prefix + '_request: got shutdown flag before read: url', self.url) break pos = self.storage.get_unfinished_pos(pos) if pos is None: if DEBUG: log(self.log_prefix + '_request: no unfinished pos, break: url', self.url) break self._read(pos) if self.seek is not None: pos = self.seek self.seek = None continue break except ReadErrorException: if DEBUG: log(self.log_prefix + '_request: read error, retry immediatelly: url', self.url, 'pos', pos) start_lambda = lambda : self.start(pos) self.rawserver.add_task(start_lambda, 0.1) except FatalErrorException as e: if DEBUG: log(self.log_prefix + '_request: fatal error, exit: url', self.url, 'pos', pos) self.failed_func(e) except Exception as e: self.errors += 1 if DEBUG: print_exc() if self.errors > self.max_errors: if DEBUG: log(self.log_prefix + '_request: non-fatal error, max errors reached: errors', self.errors, 'max', self.max_errors) self.failed_func(e) else: retry_in = 5 * (1 + self.errors / 10) if DEBUG: log(self.log_prefix + '_request: non-fatal error: url', self.url, 'pos', pos, 'errors', self.errors, 'retry_in', retry_in) start_lambda = lambda : self.start(pos) self.rawserver.add_task(start_lambda, retry_in) finally: self.running = False self.lock.release() def is_running(self): return self.running def _read(self, pos): scheme, host, path = self.parse_url(self.final_url) request_range = str(pos) + '-' connection = HTTPConnection(host) connection.request('GET', path, None, {'Host': host, 'User-Agent': USER_AGENT, 'Range': 'bytes=%s' % request_range}) r = connection.getresponse() if DEBUG: log(self.log_prefix + '_read: url', self.url, 'final', self.final_url, 'pos', pos, 'status', r.status) if r.status != 200 and r.status != 206: if DEBUG: log(self.log_prefix + '_read: bad http status: url', self.url, 'status', r.status) connection.close() if 400 <= r.status < 500: raise FatalErrorException, 'http status ' + str(r.status) else: raise NonFatalErrorException, 'http status ' + str(r.status) request_size = r.getheader('Content-Length', None) if request_size is None: if DEBUG: log(self.log_prefix + '_read: missing content length: url', self.url) connection.close() return try: request_size = int(request_size) except: if DEBUG: print_exc() connection.close() return if DEBUG: log(self.log_prefix + '_read: url', self.url, 'request_range', request_range, 'request_size', request_size) total_read = 0 read_size = 16384 while True: chunk = r.read(read_size) if not chunk: if total_read != request_size: if DEBUG: log(self.log_prefix + '_read: no data, raise read error: url', self.url, 'pos', pos, 'total_read', total_read, 'request_size', request_size) raise ReadErrorException() if DEBUG: log(self.log_prefix + '_read: no data, exit: url', self.url, 'pos', pos) break chunk_len = len(chunk) total_read += chunk_len if DEBUG: log('>>>> ' + self.log_prefix + '_read: got chunk: pos', pos, 'chunk_len', chunk_len, 'total_read', total_read) self.measure.update_rate(chunk_len) if chunk_len != read_size and total_read != request_size: if DEBUG: log(self.log_prefix + '_read: bad data len, raise read error: url', self.url, 'pos', pos, 'total_read', total_read, 'request_size', request_size, 'chunk_len', chunk_len, 'read_size', read_size) raise ReadErrorException() if self.shutdown_flag: if DEBUG: log(self.log_prefix + '_read: got shutdown flag on read: url', self.url) break try: t = time.time() updated_len = self.storage.write(pos, chunk) if DEBUG: log('%%%%' + self.log_prefix + '_read: write to storage: pos', pos, 'len', chunk_len, 'time', time.time() - t) if updated_len == 0: if DEBUG: log(self.log_prefix + '_read: data exists in storage: url', self.url, 'pos', pos, 'len', chunk_len, 'seek_flag', self.seek) if self.seek is None: self.seek = self.storage.get_unfinished_pos(pos) if self.seek is None: if DEBUG: log(self.log_prefix + '_read: no unfinished data, exit: url', self.url, 'pos', pos) break except: if DEBUG: print_exc() log(self.log_prefix + '_read: cannot write, exit: url', self.url) raise FatalErrorException, 'cannot write to storage' if self.seek is not None: log(self.log_prefix + '_read: got seek: url', self.url, 'seek', self.seek) break pos += chunk_len connection.close() def parse_url(self, url): scheme, host, path, pars, query, fragment = urlparse(url) if scheme != 'http': raise ValueError('Unsupported scheme ' + scheme) if len(host) == 0: raise ValueError('Empty host') if len(path) == 0: path = '/' if len(pars) > 0: path += ';' + pars if len(query) > 0: path += '?' + query if len(fragment) > 0: path += '#' + fragment return (scheme, host, path) def shutdown(self): if DEBUG: log(self.log_prefix + 'shutdown: ---') self.shutdown_flag = True
class IOSXR(object): """ Establishes a connection with the IOS-XR device via SSH and facilitates the communication through the XML agent. """ _XML_SHELL = 'xml' _XML_MODE_PROMPT = r'XML>' _READ_DELAY = 0.1 # at least 0.1, corresponding to 600 max loops (60s timeout) _XML_MODE_DELAY = 1 # should be able to read within one second _ITERATOR_ID_ERROR_MSG = ( 'Non supported IteratorID in Response object.' 'Turn iteration off on your XML agent by configuring "xml agent [tty | ssl] iteration off".' 'For more information refer to' 'http://www.cisco.com/c/en/us/td/docs/ios_xr_sw/iosxr_r4-1/xml/programming/guide/xl41apidoc.pdf, 7-99.' 'Please turn iteration off for the XML agent.' ) def __init__(self, hostname, username, password, port=22, timeout=60, logfile=None, lock=True): """ IOS-XR device constructor. :param hostname: (str) IP or FQDN of the target device :param username: (str) Username :param password: (str) Password :param port: (int) SSH Port (default: 22) :param timeout: (int) Timeout (default: 60 sec) :param logfile: File-like object to save device communication to or None to disable logging :param lock: (bool) Auto-lock config upon open() if set to True, connect without locking if False (default: True) """ self.hostname = str(hostname) self.username = str(username) self.password = str(password) self.port = int(port) self.timeout = int(timeout) self.logfile = logfile self.lock_on_connect = lock self.locked = False self._cli_prompt = None self._xml_agent_locker = Lock() def __getattr__(self, item): """ Dynamic getter to translate generic show commands. David came up with this dynamic method. It takes calls with show commands encoded in the name. I'll replace the underscores for spaces and issues the show command on the device... pretty neat! non keyword params for show command: all non keyword arguments is added to the command to allow dynamic parameters: eg: .show_interface("GigabitEthernet0/0/0/0") keyword params for show command: config=True/False : set True to run show command in config mode eg: .show_configuration_merge(config=True) """ def _getattr(*args, **kwargs): cmd = item.replace('_', ' ') for arg in args: cmd += " %s" % arg if kwargs.get("config"): response = self._execute_config_show(cmd) else: response = self._execute_show(cmd) match = re.search(".*(!! IOS XR Configuration.*)</Exec>", response, re.DOTALL) if match is not None: response = match.group(1) return response if item.startswith('show'): return _getattr else: raise AttributeError("type object '%s' has no attribute '%s'" % (self.__class__.__name__, item)) def make_rpc_call(self, rpc_command): """ Allow a user to query a device directly using XML-requests. :param rpc_command: (str) rpc command such as: <Get><Operational><LLDP><NodeTable></NodeTable></LLDP></Operational></Get> """ result = self._execute_rpc(rpc_command) return ET.tostring(result) def open(self): """ Open a connection to an IOS-XR device. Connects to the device using SSH and drops into XML mode. """ try: self.device = ConnectHandler(device_type='cisco_xr', ip=self.hostname, port=self.port, username=self.username, password=self.password) self.device.timeout = self.timeout except NetMikoTimeoutException as t_err: raise ConnectError(t_err.message) except NetMikoAuthenticationException as au_err: raise ConnectError(au_err.message) self._cli_prompt = self.device.find_prompt() # get the prompt self._enter_xml_mode() def _timeout_exceeded(self, start=None, msg='Timeout exceeded!'): if not start: return False # reference not specified, noth to compare => no error if time.time() - start > self.timeout: # it timeout exceeded, throw TimeoutError raise TimeoutError(msg, self) return False def _lock_xml_agent(self, start=None): while (not self._xml_agent_locker.acquire(False) and not self._timeout_exceeded(start, 'Waiting to acquire the XML agent!')): # will wait here till the XML agent is ready to receive new requests # if stays too much, _timeout_exceeded will raise TimeoutError pass # do nothing, just wait return True # ready to go now def _unlock_xml_agent(self): if self._xml_agent_locker.locked(): self._xml_agent_locker.release() def _send_command_timing(self, command): return self.device.send_command_timing(command, delay_factor=self._READ_DELAY, max_loops=self._XML_MODE_DELAY/self._READ_DELAY, strip_prompt=False, strip_command=False) def _in_cli_mode(self): out = self._send_command_timing('\n') if not out: return False if self._cli_prompt in out: return True return False def _enter_xml_mode(self): self._unlock_xml_agent() # release - other commands should not have anyway access to the XML agent # when not in XML mode self._lock_xml_agent() # make sure it won't collide with other parallel requests out = self._send_command_timing(self._XML_SHELL) # send xml shell command if '0x24319600' in out: # XML agent is not enabled raise ConnectError('XML agent is not enabled. Please configure `xml agent tty iteration off`!', self) self._unlock_xml_agent() if self.lock_on_connect: self.lock() def _send_command(self, command, delay_factor=None, start=None, expect_string=None, read_output=None, receive=False): if not expect_string: expect_string = self._XML_MODE_PROMPT if read_output is None: read_output = '' if not delay_factor: delay_factor = self._READ_DELAY if not start: start = time.time() output = read_output last_read = '' if not read_output and not receive: # because the XML agent is able to process only one single request over the same SSH session at a time # first come first served self._lock_xml_agent(start) try: max_loops = self.timeout / delay_factor last_read = self.device.send_command_expect(command, expect_string=expect_string, strip_prompt=False, strip_command=False, delay_factor=delay_factor, max_loops=max_loops) output += last_read except IOError as ioe: if ((not last_read and self._in_cli_mode()) or (self._cli_prompt in output and "% Invalid input detected at '^' marker." in output)): # something happened # e.g. connection with the XML agent died while reading # netmiko throws error and the last output read is empty (ofc) # and in CLI mode # # OR # # Sometimes the XML agent simply exits and all issued commands provide the following output # (as in CLI mode) # <? # ^ # % Invalid input detected at '^' marker. # RP/0/RSP1/CPU0:edge01.dus01#<xml version="1.0" encoding="UTF-8"? # ^ # % Invalid input detected at '^' marker. # RP/0/RSP1/CPU0:edge01.dus01#<xml version # # Which of course does not contain the XML and netmiko throws the not found error # therefore we need to re-enter in XML mode self._enter_xml_mode() # and let's issue the command again if still got time if not self._timeout_exceeded(start=start): # if still got time # reiterate the command from the beginning return self._send_command(command, expect_string=expect_string, delay_factor=delay_factor) else: output += self._netmiko_recv() # try to read some more if '0xa3679e00' in output: # when multiple parallel request are made, the device throws the error: # ERROR: 0xa3679e00 'XML Service Library' detected the 'fatal' condition # 'Multiple concurrent requests are not allowed over the same session. # A request is already in progress on this session.' # we could use a mechanism similar to NETCONF and push the requests in queue and serve them sequentially # BUT we are not able to assign unique IDs and identify the request-reply map # so will throw an error that does not help too much :( raise XMLCLIError('XML agent cannot process parallel requests!', self) if not output.strip().endswith('XML>'): if '0x44318c06' in output or (self._cli_prompt and expect_string != self._cli_prompt and \ (output.startswith(self._cli_prompt) or output.endswith(self._cli_prompt))): # sometimes the device throws a stupid error like: # ERROR: 0x44318c06 'XML-TTY' detected the 'warning' condition # 'A Light Weight Messaging library communication function returned an error': No such device or address # and the XML agent connection is closed, but the SSH connection is fortunately maintained # OR sometimes, the device simply exits from the XML mode without any clue # In both cases, we need to re-enter in XML mode... # so, whenever the CLI promt is detected, will re-enter in XML mode # unless the expected string is the prompt self._unlock_xml_agent() self._enter_xml_mode() # however, the command could not be executed properly, so we need to raise the XMLCLIError exception raise XMLCLIError('Could not properly execute the command. Re-entering XML mode...', self) if not output.strip(): # empty output, means that the device did not start delivering the output # but for sure is still in XML mode as netmiko did not throw error if not self._timeout_exceeded(start=start): return self._send_command(command, receive=True, start=start) # let's try receiving more raise XMLCLIError(output.strip(), self) self._unlock_xml_agent() return str(output.replace('XML>', '').strip()) def _netmiko_recv(self): output = '' for tmp_output in self.device.receive_data_generator(): output += tmp_output return output # previous module function __execute_rpc__ def _execute_rpc(self, command_xml, delay_factor=.1): xml_rpc_command = '<?xml version="1.0" encoding="UTF-8"?><Request MajorVersion="1" MinorVersion="0">' \ + command_xml + '</Request>' response = self._send_command(xml_rpc_command, delay_factor=delay_factor) try: root = ET.fromstring(response) except ET.XMLSyntaxError as xml_err: if 'IteratorID="' in response: raise IteratorIDError(self._ITERATOR_ID_ERROR_MSG, self) raise InvalidXMLResponse('Unable to process the XML Response from the device!', self) if 'IteratorID' in root.attrib: raise IteratorIDError(self._ITERATOR_ID_ERROR_MSG, self) childs = [x.tag for x in list(root)] result_summary = root.find('ResultSummary') if result_summary is not None and int(result_summary.get('ErrorCount', 0)) > 0: if 'CLI' in childs: error_msg = root.find('CLI').get('ErrorMsg') or '' elif 'Commit' in childs: error_msg = root.find('Commit').get('ErrorMsg') or '' error_code = root.find('Commit').get('ErrorCode') or '' if error_code == '0x41866c00': # yet another pointless IOS-XR error: # if the config DB was changed by another process, # while the current SSH connection is established and alive, # we won't be able to commit and the device will throw the following error: # 'CfgMgr' detected the 'warning' condition # 'One or more commits have occurred from other configuration sessions since this session started # or since the last commit was made from this session.' # dumb. # in this case we need to re-open the connection with the XML agent _candidate_config = self.get_candidate_config(merge=True) self.discard_config() # discard candidate config try: # exiting from the XML mode self._send_command('exit', expect_string=self._cli_prompt) except XMLCLIError: pass # because does not end with `XML>` self._enter_xml_mode() # re-entering XML mode self.load_candidate_config(config=_candidate_config) return self.commit_config() elif error_code == '0x41864e00' or error_code == '0x43682c00': # raises this error when the commit buffer is empty raise CommitError('The target configuration buffer is empty.', self) else: error_msg = root.get('ErrorMsg') or '' error_msg += '\nOriginal call was: %s' % xml_rpc_command raise XMLCLIError(error_msg, self) if 'CLI' in childs: cli_childs = [x.tag for x in list(root.find('CLI'))] if 'Configuration' in cli_childs: output = root.find('CLI').find('Configuration').text elif 'Exec' in cli_childs: output = root.find('CLI').find('Exec').text if output is None: output = '' elif 'Invalid input detected' in output: raise InvalidInputError('Invalid input entered:\n%s' % output, self) return root # previous module function __execute_show__ def _execute_show(self, show_command): """ Executes an operational show-type command. """ rpc_command = '<CLI><Exec>{show_command}</Exec></CLI>'.format( show_command=escape_xml(show_command) ) response = self._execute_rpc(rpc_command) raw_response = response.xpath('.//CLI/Exec')[0].text return raw_response.strip() if raw_response else '' # previous module function __execute_config_show__ def _execute_config_show(self, show_command, delay_factor=.1): """ Executes a configuration show-type command. """ rpc_command = '<CLI><Configuration>{show_command}</Configuration></CLI>'.format( show_command=escape_xml(show_command) ) response = self._execute_rpc(rpc_command, delay_factor=delay_factor) raw_response = response.xpath('.//CLI/Configuration')[0].text return raw_response.strip() if raw_response else '' def close(self): """ Close the connection to the IOS-XR device. Clean up after you are done and explicitly close the router connection. """ if self.lock_on_connect or self.locked: self.unlock() self._unlock_xml_agent() self.device.remote_conn.close() def lock(self): """ Lock the config database. Use if Locking/Unlocking is not performaed automatically by lock=False """ if not self.locked: rpc_command = '<Lock/>' try: self._execute_rpc(rpc_command) except XMLCLIError: raise LockError('Unable to enter in configure exclusive mode!', self) self.locked = True def unlock(self): """ Unlock the IOS-XR device config. Use if Locking/Unlocking is not performaed automatically by lock=False """ if self.locked: rpc_command = '<Unlock/>' try: self._execute_rpc(rpc_command) except XMLCLIError: raise UnlockError('Unable to unlock the config!', self) self.locked = False def load_candidate_config(self, filename=None, config=None): """ Load candidate confguration. Populate the attribute candidate_config with the desired configuration and loads it into the router. You can populate it from a file or from a string. If you send both a filename and a string containing the configuration, the file takes precedence. :param filename: Path to the file containing the desired configuration. By default is None. :param config: String containing the desired configuration. """ configuration = '' if filename is None: configuration = config else: with open(filename) as f: configuration = f.read() rpc_command = '<CLI><Configuration>{configuration}</Configuration></CLI>'.format( configuration=escape_xml(configuration) # need to escape, otherwise will try to load invalid XML ) try: self._execute_rpc(rpc_command) except InvalidInputError as e: self.discard_config() raise InvalidInputError(e.message, self) def get_candidate_config(self, merge=False, formal=False): """ Retrieve the configuration loaded as candidate config in your configuration session. :param merge: Merge candidate config with running config to return the complete configuration including all changed :param formal: Return configuration in IOS-XR formal config format """ command = "show configuration" if merge: command += " merge" if formal: command += " formal" response = self._execute_config_show(command) match = re.search(".*(!! IOS XR Configuration.*)$", response, re.DOTALL) if match is not None: response = match.group(1) return response def compare_config(self): """ Compare configuration to be merged with the one on the device. Compare executed candidate config with the running config and return a diff, assuming the loaded config will be merged with the existing one. :return: Config diff. """ _show_merge = self._execute_config_show('show configuration merge') _show_run = self._execute_config_show('show running-config') diff = difflib.unified_diff(_show_run.splitlines(1)[2:-2], _show_merge.splitlines(1)[2:-2]) return ''.join([x.replace('\r', '') for x in diff]) def compare_replace_config(self): """ Compare configuration to be replaced with the one on the device. Compare executed candidate config with the running config and return a diff, assuming the entire config will be replaced. :return: Config diff. """ diff = self._execute_config_show('show configuration changes diff') return ''.join(diff.splitlines(1)[2:-2]) def commit_config(self, label=None, comment=None, confirmed=None): """ Commit the candidate config. :param label: Commit comment, displayed in the commit entry on the device. :param comment: Commit label, displayed instead of the commit ID on the device. :param confirmed: Commit with auto-rollback if new commit is not made in 30 to 300 sec """ rpc_command = '<Commit' if label: rpc_command += ' Label="%s"' % label if comment: rpc_command += ' Comment="%s"' % comment if confirmed: if 30 <= int(confirmed) <= 300: rpc_command += ' Confirmed="%d"' % int(confirmed) else: raise InvalidInputError('confirmed needs to be between 30 and 300 seconds', self) rpc_command += '/>' self._execute_rpc(rpc_command) def commit_replace_config(self, label=None, comment=None, confirmed=None): """ Commit the candidate config to the device, by replacing the existing one. :param comment: User comment saved on this commit on the device :param label: User label saved on this commit on the device :param confirmed: Commit with auto-rollback if new commit is not made in 30 to 300 sec """ rpc_command = '<Commit Replace="true"' if label: rpc_command += ' Label="%s"' % label if comment: rpc_command += ' Comment="%s"' % comment if confirmed: if 30 <= int(confirmed) <= 300: rpc_command += ' Confirmed="%d"' % int(confirmed) else: raise InvalidInputError('confirmed needs to be between 30 and 300 seconds', self) rpc_command += '/>' self._execute_rpc(rpc_command) def discard_config(self): """ Clear uncommited changes in the current session. Clear previously loaded configuration on the device without committing it. """ rpc_command = '<Clear/>' self._execute_rpc(rpc_command) def rollback(self, rb_id=1): """ Rollback the last committed configuration. :param rb_id: Rollback a specific number of steps. Default: 1 """ rpc_command = '<Unlock/><Rollback><Previous>{rb_id}</Previous></Rollback><Lock/>'.format(rb_id=rb_id) self._execute_rpc(rpc_command)
class Exec_lock: """A type of locking object for locking execution of relax.""" def __init__(self, fake_lock=False): """Set up the lock-like object. @keyword fake_lock: A flag which is True will allow this object to be debugged as the locking mechanism is turned off. @type fake_lock: bool """ # Store the arg. self._fake_lock = fake_lock # Init a threading.Lock object. self._lock = Lock() # The status container. self._status = Status() # The name and mode of the locker. self._name = [] self._mode = [] # Script nesting level. self._nest = 0 # Auto-analysis from script launch. self._auto_from_script = False # Debugging. if self._fake_lock: self.log = open('lock.log', 'w') def acquire(self, name, mode='script'): """Simulate the Lock.acquire() mechanism. @param name: The name of the locking code. @type name: str @keyword mode: The mode of the code trying to obtain the lock. This can be one of 'script' for the scripting interface or 'auto-analysis' for the auto-analyses. @type mode: str """ # Debugging. if self._status.debug: sys.stdout.write("debug> Execution lock: Acquisition by '%s' ('%s' mode).\n" % (name, mode)) # Store the new name and mode. self._name.append(name) self._mode.append(mode) # Nested locking. if self.locked(): # Increment the nesting counter. self._nest += 1 # Debugging. if self._fake_lock: self.log.write("Nested by %s (to level %s)\n" % (name, self._nest)) self.log.flush() # Return without doing anything. return # Debugging. if self._fake_lock: self.log.write("Acquired by %s\n" % self._name[-1]) self.log.flush() return # Acquire the real lock. lock = self._lock.acquire() # Notify observers. status = Status() status.observers.exec_lock.notify() # Return the real lock. return lock def locked(self): """Simulate the Lock.locked() mechanism.""" # Debugging (pseudo-locking based on _name). if self._fake_lock: if len(self._name): return True else: return False # Call the real method. return self._lock.locked() def release(self): """Simulate the Lock.release() mechanism.""" # Debugging. if self._status.debug: sys.stdout.write("debug> Execution lock: Release by '%s' ('%s' mode).\n" % (self._name[-1], self._mode[-1])) # Pop the name and mode. self._name.pop(-1) self._mode.pop(-1) # Nested locking. if self._nest: # Debugging. if self._fake_lock: self.log.write("Nested locking decrement (%s -> %s)\n" % (self._nest, self._nest-1)) self.log.flush() # Decrement. self._nest -= 1 # Return without releasing the lock. return # Debugging. if self._fake_lock: # Main text. text = 'Release' # Test suite info. if hasattr(self, 'test_name'): text = text + 'd by %s' % self.test_name # Write out, flush, and exit the method. self.log.write("%s\n\n" % text) self.log.flush() return # Release the real lock. release = self._lock.release() # Notify observers. status = Status() status.observers.exec_lock.notify() # Return the status. return release
class Optimizer(Thread): def __init__(self, rendis): Thread.__init__(self) # only when green light is unlocked will the optimization continue self.green_light = Lock() if not rendis: raise RuntimeError("Renderer dispatcher is not provided") self.rendis = rendis self.rendis.register(self._on_renderer_reboot) self.renderer = rendis.acquire() renderer = self.renderer self._optim_method = lambda *x: None self._energy_list = [] self.set_param = renderer.set_param self.get_param = renderer.get_param self._target_img = None self._target_scores = {} self._finished_callback = lambda *args, **kwds: None self._finished_callback_args = [] self._iter_callback = lambda *args: None self._iter_callback_args = [] self.line_search_first = False self._method_name = "unset" self._eval_term_records = () self._eval_sum_record = None self.result = None self._stop_sign = False self.stopable = False @log.task_log @plotting.plot_task def run(self): # check params if self._target_img == None: raise RuntimeError("Target image missing") if len(self._energy_list) == 0: raise RuntimeError("No energy function selected") if self._method_name == "unset": raise RuntimeError("Method unset") if self.renderer == None: raise RuntimeError("Renderer missing") x = self.rendis.acquire().get_param() self.renderer = self.rendis.acquire_new( energy_terms=[n for f, n, w in self._energy_list], penalty_terms=[self.penalty_name], target_image=self._target_img, atb_controls=False, ) print "after reboot" self.renderer.set_param(x) print "after setting x" # build optimizer err_func = self._wrap_eval(self.energy_func) if self.line_search_first: self.line_search_init_param(err_func) else: pass # wrap optimizer with green_light self.result = self._optim_method(x=x, f=err_func) self._finished_callback(*self._finished_callback_args) print "finished!" return def line_search_init_param(self, func): x = self.renderer.get_param() jac = _get_jac(func=func, delta=0.005, x0=x) search_direction = -func(x) / jac(x) res = optimize.line_search(f=func, myfprime=jac, xk=x, pk=search_direction) alpha = res[0] x_new = x + alpha * search_direction self.renderer.set_param(x_new) def _wrap_eval(self, func): """ wrapping the evaluation function to be controllable by locks """ def wrapped(*x): self.green_light.acquire() self.green_light.release() return func(*x) return wrapped def switch(self): if self.green_light.locked(): self.green_light.release() else: self.green_light.acquire() def play(self): if self.green_light.locked(): self.green_light.release() def pause(self): if not self.green_light.locked(): self.green_light.acquire() """ method_dic is a mapping between optimizing method to a closure that takes the name (in some case like CMA, name doesn't affect anything) and returns the uniformed optimizing interface fmin(f, x0) """ method_dic = { "CMA": cma_optimize, "Nelder-Mead": scipy_optimize, "Powell": scipy_optimize, "CG": scipy_optimize_jac, "BFGS": scipy_optimize_jac, "Newton-CG": scipy_optimize_jac, "L-BFGS-B": scipy_optimize_jac, "TNC": scipy_optimize_jac, "COBYLA": scipy_optimize, "SLSQP": scipy_optimize_jac # ,"dogleg": scipy_optimize_jac, # needs hessian, maybe later # "trust-ncg": scipy_optimize_jac } linear_search_list = ["CG", "BFGS"] # TODO: finish the list def set_method(self, name): # this is the interface for manager if name == "CMA": self._optim_method = self._explict_CMA self.stopable = True else: self._optim_method = Optimizer.method_dic[name](name) self._method_name = name def _explict_CMA(self, f, x): # it is very inappropriate to call them solutions as they are guesses # I'd rather call it samples or guesses but I yield def generate(solutions): for x in solutions: try: yield f(x) except Exception as e: # like encountering an all-black error print e.message self.renderer = self.rendis.acquire_new() print "renderer reboot" yield f(x) sigma_0 = 1 ftarget = 1.5e-4 opts = cma.CMAOptions() opts["ftarget"] = ftarget es = cma.CMAEvolutionStrategy(x, sigma_0, opts) while not es.stop() and not self._stop_sign: solutions = es.ask() fvals = [y for y in generate(solutions)] es.tell(solutions, fvals) res = es.result() self.best_x = es.best.x self.best_f = es.best.f return res[0], res[1] def stop(self): """ Only works under using CMAES. Called to terminate the current optimization thread. Note that the termination is not instant, but will wait for the last round of CMA to finish. """ self._stop_sign = True @plotting.init_plot def set_energy(self, func_names, weights): if len(func_names) != len(weights): err_msg = "energy function number (%d) doesn't match weight number (%d)" % (len(func_names), len(weights)) raise RuntimeError(err_msg) self._energy_list = zip( [Optimizer.energy_dic[name](self._target_img, self._target_img_path) for name in func_names], func_names, weights, ) self._eval_term_records = tuple([] for i in func_names) self._eval_sum_record = [] def set_target(self, image_path): # TODO: finish setting the target image self._target_img_path = image_path self._target_img = Image.open(image_path) self._target_img = self._target_img.convert("L") # used to have other pre-computation when setting an target image def set_finished_callback(self, callback, *args): self._finished_callback = callback self._finished_callback_args = args def set_iter_callback(self, callback, *args): self._iter_callback = callback self._iter_callback_args = args # TODO: penalty could be listed penalty_weight = 1e-3 penalty_name = "penalty" def penalty(self, x): res = sum(x ** 2) * self.penalty_weight self.renderer.set_penalty_value(self.penalty_name, res) return res def get_mat_model2snapshot(self, img): mat_shaj = self.renderer.shadow.shaject_mat mat_view = self.renderer.cam_cap.view_mat mat_proj = self.renderer.cam_cap.proj_mat w, h = img.size # mat_c2s is projection from clip coordinate to image coordinate mat_c2s = np.array( [[(w - 1) / 2, 0, 0, (w - 1) / 2], [0, (1 - h) / 2, 0, (h - 1) / 2], [0, 0, 0, 0], [0, 0, 0, 1]] ) mat_c2s *= np.array(mat_proj * mat_view * mat_shaj).T # cgkit.mat4 is column major return mat_c2s def model_center_penalty_closure(self, img): # presuming the camera for capturing, light and receiver won't move # or otherwise all the mat shall be computed on the fly mat_c2s = self.get_mat_model2snapshot(img) def model_center_penalty(x): positions = self._strip_positions(x) return 0 return model_center_penalty def _strip_positions(self, x): return [tuple(x[0:3]), tuple(x[6:9]), tuple(x[12:15])] def _on_renderer_reboot(self): self.renderer = self.rendis.acquire() # obsolete def _record_term(f): # decorator that records each term of each evaluation @wraps(f) def inner(cls, x): res = f(cls, x) for record, res_term in zip(cls._eval_term_records, res): y, w, n = res_term record.append(y) return res return inner # obsolete def _record_sum(f): # decorator that records sum of all terms of each evaluation @wraps(f) def inner(cls, x): res = f(cls, x) cls._eval_sum_record.append(res) return res return inner def _sum(f): @log.energy_sum_log @wraps(f) def inner(cls, x): res = f(cls, x) for y, w, n in res: cls.renderer.set_energy_value(name=n, val=y * w) total = sum([w * y for y, w, n in res]) + cls.penalty(x) cls.renderer.set_total(total) return total return inner @plotting.plot_sum @_sum @log.energy_term_log @plotting.plot_terms def energy_func(self, x): self.renderer.set_param(x) img = self.renderer.acquire_snapshot() self._iter_callback(*self._iter_callback_args) return [(func(img), weight, name) for func, name, weight in self._energy_list] """ energy_dic is a static attribute of optimizer, which maps energy function names to a callable instance. The mapped callable is a closure that takes the target image as input, and return a method calculating the energy function value based on input image. """ energy_dic = { "XOR comparison": xor_closure, "first moments (normalized)": sq_diff_closure(get_fst_moments), "secondary moments (normalized)": sq_diff_closure(get_sec_moments), "uncovered pixels": uncovered_closure, "distance field": distance_field_closure, }
class SQL_Panel(ScrollView): ''' The logback panel is used to turn logs on and off as well as adjust the error level of each one and the default error level. ''' grid = ObjectProperty(None) active = BooleanProperty(False) query = "Select * from logs where parent in (select parent from logs where line like '%org.hibernate.SQL%') ORDER BY parent" get_parents = "select parent from logs where line like '%org.hibernate.SQL%'" get_exclusive_parents = "select parent from logs where line like '%org.hibernate.SQL%' and parent not in ({0})" parent_query = "Select * from logs where parent is {0}" def setServer(self,server): ''' Save the local variables needed. Server specifically, but also this method spins up an instance of the Logback model to reference. ''' self.server = server self.logback = model.LogbackFile(server.con) loggers = self.logback.getLoggers() self.loggers = [] for i in loggers: if i.name == 'org.hibernate.SQL': self.ohsql = i if i.name == 'org.hibernate.type': self.ohtype = i self.active = not (self.ohsql.commented and self.ohtype.commented) self.load_queries() self.bind(on_touch_down=self.on_touch_down) def load_queries(self): self.logFile = model.gstore.logManager.openLog( '/usr/local/tomcat/logs/portal/portal.log',self.server.con) self.query_list_hashes = [] self.query_list = {} self.query_nodes = {} self.updating = Lock() self.parents = [] Clock.schedule_interval(self.thread_update,1) def thread_update(self,*args): Logger.debug("SQL Panel: Is there a running updater thread? {0}".format(self.updating.locked())) if self.updating.locked() == False: Logger.debug("SQL Panel: Preparing a new thread") t = Thread(target=self.update,args=args) t.start() Logger.debug("SQL Panel: Dispached thread") Logger.debug("SQL Panel: Done checking the updater.") def update(self, *args): #this makes sure only one update process is running at a time. Logger.debug("SQL Panel: Is an update process alread running?: {0}".format(self.updating.locked())) if self.updating.acquire(False) == False: Logger.debug("SQL Panel: Lock not acquired: {0}".format(self.updating.locked())) return Logger.debug("SQL Panel: Acquired lock:".format(self.updating.locked())) #Get the list of parents into a string form. p_list = ','.join([str(i) for i in self.parents]) #query for the queries we don't already have parents = self.logFile.query(self.get_exclusive_parents.format(p_list)) #save those to the raw parents list. May get ride of this later. self.parents_raw = parents #Process the new queries: for p in parents: self.parents.append(p[0]) qhash, query = self.process_query(p[0]) if qhash in self.query_list_hashes: #self.query_list[qhash].ids.append(p[0]) self.query_list[qhash].add_instance(p[0]) else: self.query_list_hashes.append(qhash) self.query_list[qhash] = SQL_Query(qhash,query,p[0],self) self.updating.release() Logger.debug("SQL Panel: Done updating, toggling self.update to: {0}".format(self.updating.locked())) def process_query(self,parent): ''' Take the given number, find all the parts of the query, and then string them together into one piece. This also returns the hash for that query to make use in dictionaries easier. ''' query_raw = self.logFile.query(self.parent_query.format(parent)) query = '' query = '\n'.join([i[2] for i in query_raw[1:]]) q_hash = hash(query) return (q_hash,query) def get_from(self,query): ''' I mostly made this to get a snippet to make the query preview short. ''' q_split = query.split('\n') for i,j in enumerate(q_split): if "from" in j: return q_split[0] + " " + j + " " + q_split[i+1] + "..." elif "update" in j: return j + " " + q_split[i+1] + "..." elif "into" in j: return j + " " + q_split[i+1] + "..." return q_split[1] def load_logger(self,logger): ''' Load the logger and create it's view. ''' self.logger = logger self.name = logger.name if logger.commented: self.active = False else: self.active = True self.level = logger.level def toggle_me(self): ''' called whenever the status of the activity toggle changes. This method makes sure that the underlying logger get's the memo. ''' self.active = self.ids['logback_switch'].active if self.active: self.ohsql.commented = False self.ohtype.commented = False self.logback.save() else: self.ohsql.commented = True self.ohtype.commented = True self.logback.save() return self.active def on_touch_down(self, touch): if touch.is_double_tap: try: Clipboard.copy(self.query_tree.selected_node.text) except AttributeError: Logger.debug("SQL Panel: Object didn't have text.") ScrollView.on_touch_down(self, touch)
class IMAPServer: """Initializes all variables from an IMAPRepository() instance Various functions, such as acquireconnection() return an IMAP4 object on which we can operate. Public instance variables are: self.: delim The server's folder delimiter. Only valid after acquireconnection() """ GSS_STATE_STEP = 0 GSS_STATE_WRAP = 1 def __init__(self, repos): self.ui = getglobalui() self.repos = repos self.config = repos.getconfig() self.tunnel = repos.getpreauthtunnel() self.usessl = repos.getssl() self.username = None if self.tunnel else repos.getuser() self.password = None self.passworderror = None self.goodpassword = None self.hostname = None if self.tunnel else repos.gethost() self.port = repos.getport() if self.port == None: self.port = 993 if self.usessl else 143 self.sslclientcert = repos.getsslclientcert() self.sslclientkey = repos.getsslclientkey() self.sslcacertfile = repos.getsslcacertfile() if self.sslcacertfile is None: self.verifycert = None # disable cert verification self.delim = None self.root = None self.maxconnections = repos.getmaxconnections() self.availableconnections = [] self.assignedconnections = [] self.lastowner = {} self.semaphore = BoundedSemaphore(self.maxconnections) self.connectionlock = Lock() self.reference = repos.getreference() self.idlefolders = repos.getidlefolders() self.gss_step = self.GSS_STATE_STEP self.gss_vc = None self.gssapi = False def getpassword(self): """Returns the server password or None""" if self.goodpassword != None: # use cached good one first return self.goodpassword if self.password != None and self.passworderror == None: return self.password # non-failed preconfigured one # get 1) configured password first 2) fall back to asking via UI self.password = self.repos.getpassword() or \ self.ui.getpass(self.repos.getname(), self.config, self.passworderror) self.passworderror = None return self.password def getroot(self): """Returns this server's folder root. Can only be called after one or more calls to acquireconnection.""" return self.root def releaseconnection(self, connection, drop_conn=False): """Releases a connection, returning it to the pool. :param drop_conn: If True, the connection will be released and not be reused. This can be used to indicate broken connections.""" if connection is None: return #noop on bad connection self.connectionlock.acquire() self.assignedconnections.remove(connection) # Don't reuse broken connections if connection.Terminate or drop_conn: connection.logout() else: self.availableconnections.append(connection) self.connectionlock.release() self.semaphore.release() def md5handler(self, response): challenge = response.strip() self.ui.debug('imap', 'md5handler: got challenge %s' % challenge) passwd = self.getpassword() retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest() self.ui.debug('imap', 'md5handler: returning %s' % retval) return retval def plainauth(self, imapobj): self.ui.debug('imap', 'Attempting plain authentication') imapobj.login(self.username, self.getpassword()) def gssauth(self, response): data = base64.b64encode(response) try: if self.gss_step == self.GSS_STATE_STEP: if not self.gss_vc: rc, self.gss_vc = kerberos.authGSSClientInit('imap@' + self.hostname) response = kerberos.authGSSClientResponse(self.gss_vc) rc = kerberos.authGSSClientStep(self.gss_vc, data) if rc != kerberos.AUTH_GSS_CONTINUE: self.gss_step = self.GSS_STATE_WRAP elif self.gss_step == self.GSS_STATE_WRAP: rc = kerberos.authGSSClientUnwrap(self.gss_vc, data) response = kerberos.authGSSClientResponse(self.gss_vc) rc = kerberos.authGSSClientWrap(self.gss_vc, response, self.username) response = kerberos.authGSSClientResponse(self.gss_vc) except kerberos.GSSError as err: # Kerberos errored out on us, respond with None to cancel the # authentication self.ui.debug('imap', '%s: %s' % (err[0][0], err[1][0])) return None if not response: response = '' return base64.b64decode(response) def acquireconnection(self): """Fetches a connection from the pool, making sure to create a new one if needed, to obey the maximum connection limits, etc. Opens a connection to the server and returns an appropriate object.""" self.semaphore.acquire() self.connectionlock.acquire() curThread = currentThread() imapobj = None if len(self.availableconnections): # One is available. # Try to find one that previously belonged to this thread # as an optimization. Start from the back since that's where # they're popped on. imapobj = None for i in range(len(self.availableconnections) - 1, -1, -1): tryobj = self.availableconnections[i] if self.lastowner[tryobj] == curThread.ident: imapobj = tryobj del(self.availableconnections[i]) break if not imapobj: imapobj = self.availableconnections[0] del(self.availableconnections[0]) self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj self.connectionlock.release() # Release until need to modify data """ Must be careful here that if we fail we should bail out gracefully and release locks / threads so that the next attempt can try... """ success = 0 try: while not success: # Generate a new connection. if self.tunnel: self.ui.connecting('tunnel', self.tunnel) imapobj = imaplibutil.IMAP4_Tunnel(self.tunnel, timeout=socket.getdefaulttimeout()) success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) fingerprint = self.repos.get_ssl_fingerprint() imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname, self.port, self.sslclientkey, self.sslclientcert, self.sslcacertfile, self.verifycert, timeout=socket.getdefaulttimeout(), fingerprint=fingerprint ) else: self.ui.connecting(self.hostname, self.port) imapobj = imaplibutil.WrappedIMAP4(self.hostname, self.port, timeout=socket.getdefaulttimeout()) if not self.tunnel: try: # Try GSSAPI and continue if it fails if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss: self.connectionlock.acquire() self.ui.debug('imap', 'Attempting GSSAPI authentication') try: imapobj.authenticate('GSSAPI', self.gssauth) except imapobj.error as val: self.gssapi = False self.ui.debug('imap', 'GSSAPI Authentication failed') else: self.gssapi = True kerberos.authGSSClientClean(self.gss_vc) self.gss_vc = None self.gss_step = self.GSS_STATE_STEP #if we do self.password = None then the next attempt cannot try... #self.password = None self.connectionlock.release() if not self.gssapi: if 'STARTTLS' in imapobj.capabilities and not\ self.usessl: self.ui.debug('imap', 'Using STARTTLS connection') imapobj.starttls() if 'AUTH=CRAM-MD5' in imapobj.capabilities: self.ui.debug('imap', 'Attempting CRAM-MD5 authentication') try: imapobj.authenticate('CRAM-MD5', self.md5handler) except imapobj.error as val: self.plainauth(imapobj) else: # Use plaintext login, unless # LOGINDISABLED (RFC2595) if 'LOGINDISABLED' in imapobj.capabilities: raise OfflineImapError("Plaintext login " "disabled by server. Need to use SSL?", OfflineImapError.ERROR.REPO) self.plainauth(imapobj) # Would bail by here if there was a failure. success = 1 self.goodpassword = self.password except imapobj.error as val: self.passworderror = str(val) raise # update capabilities after login, e.g. gmail serves different ones typ, dat = imapobj.capability() if dat != [None]: imapobj.capabilities = tuple(dat[-1].upper().split()) if self.delim == None: listres = imapobj.list(self.reference, '""')[1] if listres == [None] or listres == None: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] if listres == [None] or listres == None: # No Folders were returned. This occurs, e.g. if the # 'reference' prefix does not exist on the mail # server. Raise exception. err = "Server '%s' returned no folders in '%s'" % \ (self.repos.getname(), self.reference) self.ui.warn(err) raise Exception(err) self.delim, self.root = \ imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) self.connectionlock.acquire() self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj except Exception as e: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" self.semaphore.release() if(self.connectionlock.locked()): self.connectionlock.release() severity = OfflineImapError.ERROR.REPO if type(e) == gaierror: #DNS related errors. Abort Repo sync #TODO: special error msg for e.errno == 2 "Name or service not known"? reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ "ver name correctly and that you are online."%\ (self.hostname, self.repos) raise OfflineImapError(reason, severity) elif isinstance(e, SSLError) and e.errno == 1: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service if self.port != 993: reason = "Could not connect via SSL to host '%s' and non-s"\ "tandard ssl port %d configured. Make sure you connect"\ " to the correct port." % (self.hostname, self.port) else: reason = "Unknown SSL protocol connecting to host '%s' for"\ "repository '%s'. OpenSSL responded:\n%s"\ % (self.hostname, self.repos, e) raise OfflineImapError(reason, severity) elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: # "Connection refused", can be a non-existing port, or an unauthorized # webproxy (open WLAN?) reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ "network." % (self.hostname, self.port, self.reposname) raise OfflineImapError(reason, severity) # Could not acquire connection to the remote; # socket.error(last_error) raised if str(e)[:24] == "can't open socket; error": raise OfflineImapError("Could not connect to remote server '%s' "\ "for repository '%s'. Remote does not answer." % (self.hostname, self.repos), OfflineImapError.ERROR.REPO) else: # re-raise all other errors raise def connectionwait(self): """Waits until there is a connection available. Note that between the time that a connection becomes available and the time it is requested, another thread may have grabbed it. This function is mainly present as a way to avoid spawning thousands of threads to copy messages, then have them all wait for 3 available connections. It's OK if we have maxconnections + 1 or 2 threads, which is what this will help us do.""" self.semaphore.acquire() self.semaphore.release() def close(self): # Make sure I own all the semaphores. Let the threads finish # their stuff. This is a blocking method. with self.connectionlock: # first, wait till all connections had been released. # TODO: won't work IMHO, as releaseconnection() also # requires the connectionlock, leading to a potential # deadlock! Audit & check! threadutil.semaphorereset(self.semaphore, self.maxconnections) for imapobj in self.assignedconnections + self.availableconnections: imapobj.logout() self.assignedconnections = [] self.availableconnections = [] self.lastowner = {} # reset kerberos state self.gss_step = self.GSS_STATE_STEP self.gss_vc = None self.gssapi = False def keepalive(self, timeout, event): """Sends a NOOP to each connection recorded. It will wait a maximum of timeout seconds between doing this, and will continue to do so until the Event object as passed is true. This method is expected to be invoked in a separate thread, which should be join()'d after the event is set.""" self.ui.debug('imap', 'keepalive thread started') while not event.isSet(): self.connectionlock.acquire() numconnections = len(self.assignedconnections) + \ len(self.availableconnections) self.connectionlock.release() threads = [] for i in range(numconnections): self.ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections)) if len(self.idlefolders) > i: # IDLE thread idler = IdleThread(self, self.idlefolders[i]) else: # NOOP thread idler = IdleThread(self) idler.start() threads.append(idler) self.ui.debug('imap', 'keepalive: waiting for timeout') event.wait(timeout) self.ui.debug('imap', 'keepalive: after wait') for idler in threads: # Make sure all the commands have completed. idler.stop() idler.join() self.ui.debug('imap', 'keepalive: all threads joined') self.ui.debug('imap', 'keepalive: event is set; exiting') return def verifycert(self, cert, hostname): '''Verify that cert (in socket.getpeercert() format) matches hostname. CRLs are not handled. Returns error message if any problems are found and None on success. ''' errstr = "CA Cert verifying failed: " if not cert: return ('%s no certificate received' % errstr) dnsname = hostname.lower() certnames = [] # cert expired? notafter = cert.get('notAfter') if notafter: if time.time() >= cert_time_to_seconds(notafter): return '%s certificate expired %s' % (errstr, notafter) # First read commonName for s in cert.get('subject', []): key, value = s[0] if key == 'commonName': certnames.append(value.lower()) if len(certnames) == 0: return ('%s no commonName found in certificate' % errstr) # Then read subjectAltName for key, value in cert.get('subjectAltName', []): if key == 'DNS': certnames.append(value.lower()) # And finally try to match hostname with one of these names for certname in certnames: if (certname == dnsname or '.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]): return None return ('%s no matching domain name found in certificate' % errstr)
class SuccessLock(): def __init__(self, infohash = None): self.lock = Lock() self.pause = Lock() self.ready = Lock() self.code = 0L self.success = False self.finished = True self.log_prefix = 'rerequester:successlock::' if infohash is not None: self.log_prefix += binascii.hexlify(infohash) + ':' def start(self): if DEBUG_LOCK: log(self.log_prefix + 'start: acquire ready lock: thread', currentThread().name) self.ready.acquire() if DEBUG_LOCK: log(self.log_prefix + 'start: acquire ready lock done: thread', currentThread().name) self.success = False self.finished = False def finish(self): if DEBUG_LOCK: log(self.log_prefix + 'finish: release ready lock: thread', currentThread().name) self.ready.release() def isready(self): locked = self.ready.locked() if DEBUG_LOCK: log(self.log_prefix + 'isready: ready lock status: locked', locked, 'thread', currentThread().name) return not locked def set(self): if DEBUG_LOCK: log(self.log_prefix + 'set: acquire lock: thread', currentThread().name) self.lock.acquire() if DEBUG_LOCK: log(self.log_prefix + 'set: acquire lock done: thread', currentThread().name) if not self.pause.locked(): if DEBUG_LOCK: log(self.log_prefix + 'set: pause is not locked, acquire: thread', currentThread().name) self.pause.acquire() if DEBUG_LOCK: log(self.log_prefix + 'set: pause acquire done: thread', currentThread().name) elif DEBUG_LOCK: log(self.log_prefix + 'set: pause is locked: thread', currentThread().name) self.first = True self.finished = False self.success = False self.code += 1L self.lock.release() if DEBUG_LOCK: log(self.log_prefix + 'set: release lock and return: first', self.first, 'code', self.code, 'thread', currentThread().name) return self.code def trip(self, code, s = False): if DEBUG_LOCK: log(self.log_prefix + 'trip: acquire lock: code', code, 's', s, 'self.code', self.code, 'self.finished', self.finished, 'thread', currentThread().name) self.lock.acquire() if DEBUG_LOCK: log(self.log_prefix + 'trip: acquire lock done: code', code, 's', s, 'self.code', self.code, 'self.finished', self.finished, 'thread', currentThread().name) try: if code == self.code and not self.finished: r = self.first self.first = False if s: self.finished = True self.success = True if DEBUG_LOCK: log(self.log_prefix + 'trip: got match: code', code, 's', s, 'self.code', self.code, 'self.finished', self.finished, 'self.success', self.success, 'r', r, 'thread', currentThread().name) return r if DEBUG_LOCK: log(self.log_prefix + 'trip: no match: code', code, 'self.code', self.code, 'self.finished', self.finished, 'thread', currentThread().name) finally: self.lock.release() def give_up(self): self.lock.acquire() self.success = False self.finished = True if DEBUG_LOCK: log(self.log_prefix + 'give_up: self.success', self.success, 'self.finished', self.finished, 'thread', currentThread().name) self.lock.release() def wait(self): if DEBUG_LOCK: log(self.log_prefix + 'wait: acquire pause: thread', currentThread().name) self.pause.acquire() if DEBUG_LOCK: log(self.log_prefix + 'wait: acquire pause done: thread', currentThread().name) def unwait(self, code): if code == self.code and self.pause.locked(): if DEBUG_LOCK: log(self.log_prefix + 'unwait: release pause: code', code, 'self.code', self.code, 'thread', currentThread().name) self.pause.release() elif DEBUG_LOCK: log(self.log_prefix + 'unwait: do not release pause: code', code, 'self.code', self.code, 'thread', currentThread().name) def isfinished(self): self.lock.acquire() x = self.finished self.lock.release() if DEBUG_LOCK: log(self.log_prefix + 'isfinished: x', x, 'thread', currentThread().name) return x
class GeneralLogger(object): def __init__(self, redirect_to, job_backend=None): self.job_backend = job_backend self.buffer = '' self.last_timer = None self.last_messages = '' self.logger = redirect_to or sys.__stdout__ self.lock = Lock() self.attach_last_messages = {} self.buffer_disabled = False def disable_buffer(self): self.buffer_disabled = True self.buffer = '' def clear_buffer(self): self.buffer = '' def fileno(self): return self.logger.fileno() def isatty(self): return self.logger.isatty() def flush(self): self.logger.flush() self.send_buffer() def send_buffer(self): self.last_timer = None if self.buffer: if self.job_backend: if self.job_backend.write_log(self.buffer): self.buffer = '' def attach(self, buffer, read_line=None): """ Read buffer until end (read() returns '') and sends it to self.logger and self.job_backend. :param buffer: a buffer instance with block read() or readline() method :param read_line: callable or True to read line per line. If callable is given, it will be executed per line and ignores does not redirect the line to stdout/logger when callable returns False. """ bid = id(buffer) self.attach_last_messages[bid] = b'' def reader(): current_line = b'' def handle_line(buf): if chunk == b'': return if read_line and callable(read_line): res = read_line(buf) if res is False: return False elif res is not None: buf = res if hasattr(buf, 'encode'): buf = buf.encode('utf-8') self.attach_last_messages[bid] += buf if len(self.attach_last_messages[bid]) > 21 * 1024: self.attach_last_messages[bid] = self.attach_last_messages[bid][-20 * 1024:] self.write(buf) flush_char = b'\n' while True: try: # needs to be 1 so we fetch data in near real-time chunk = buffer.read(1) if chunk == b'': if current_line: handle_line(current_line) return current_line += chunk while flush_char in current_line: pos = current_line.find(flush_char) line = current_line[:pos+1] current_line = current_line[pos+1:] handle_line(line) # todo, periodically flush by '\r' only (progress bars for example) # and make sure only necessary data is sent (by applying \r and \b control characters) except (KeyboardInterrupt, SystemExit): raise except Exception: # we need to make sure, we continue to read otherwise the process of this buffer # will block and we have a stuck process. sys.__stderr__.write(traceback.format_exc() + '\n') sys.__stderr__.flush() thread = Thread(target=reader) thread.daemon = True thread.start() def wait(): thread_join_non_blocking(thread) self.send_buffer() return wait def write(self, message): try: self.lock.acquire() if b'' == message: return if hasattr(message, 'decode'): # don't decode string again # necessary for Python3 message = message.decode('utf-8', errors='replace') self.logger.write(message) self.logger.flush() self.last_messages += message if len(self.last_messages) > 20 * 1024: self.last_messages = self.last_messages[-20 * 1024:] if not self.buffer_disabled: for char in message: if '\b' == char and self.buffer: self.buffer = self.buffer[:-1] else: self.buffer += char if not self.last_timer: self.last_timer = Timer(1.0, self.send_buffer) self.last_timer.start() except (KeyboardInterrupt, SystemExit): raise except Exception: sys.__stderr__.write(traceback.format_exc() + '\n') sys.__stderr__.flush() finally: if self.lock.locked(): self.lock.release()
class _Selection(DOMWidget): """Base class for Selection widgets ``options`` can be specified as a list or dict. If given as a list, it will be transformed to a dict of the form ``{str(value):value}``. When programmatically setting the value, a reverse lookup is performed among the options to set the value of ``selected_label`` accordingly. The reverse lookup uses the equality operator by default, but an other predicate may be provided via the ``equals`` argument. For example, when dealing with numpy arrays, one may set equals=np.array_equal. """ value = Any(help="Selected value") selected_label = Unicode(help="The label of the selected value").tag(sync=True) options = Any(help="""List of (key, value) tuples or dict of values that the user can select. The keys of this list are the strings that will be displayed in the UI, representing the actual Python choices. The keys of this list are also available as _options_labels. """) _options_dict = Dict() _options_labels = Tuple().tag(sync=True) _options_values = Tuple() disabled = Bool(False, help="Enable or disable user changes").tag(sync=True) description = Unicode(help="Description of the value this widget represents").tag(sync=True) def __init__(self, *args, **kwargs): self.value_lock = Lock() self.options_lock = Lock() self.equals = kwargs.pop('equals', lambda x, y: x == y) self.observe(self._options_readonly_changed, names=['_options_dict', '_options_labels', '_options_values', '_options']) if 'options' in kwargs: self.options = kwargs.pop('options') DOMWidget.__init__(self, *args, **kwargs) self._value_in_options() def _make_options(self, x): # If x is a dict, convert it to list format. if isinstance(x, (OrderedDict, dict)): return [(k, v) for k, v in x.items()] # Make sure x is a list or tuple. if not isinstance(x, (list, tuple)): msg = ( "Attempted to set options for '{}' widget with an object of type `{}`\n" "Expected a list, tuple or OrderdDict." ) raise ValueError(msg.format(self.__class__.__name__, type(x))) # If x is an ordinary list, use the option values as names. for y in x: if not isinstance(y, (list, tuple)) or len(y) < 2: return [(i, i) for i in x] # Value is already in the correct format. return x def _options_changed(self, name, old, new): """Handles when the options tuple has been changed. Setting options implies setting option labels from the keys of the dict. """ if self.options_lock.acquire(False): try: self.options = new options = self._make_options(new) self._options_dict = {i[0]: i[1] for i in options} self._options_labels = [i[0] for i in options] self._options_values = [i[1] for i in options] self._value_in_options() finally: self.options_lock.release() def _value_in_options(self): # ensure that the chosen value is one of the choices if self._options_values: if self.value not in self._options_values: self.value = next(iter(self._options_values)) def _options_readonly_changed(self, change): if not self.options_lock.locked(): raise TraitError("`.%s` is a read-only trait. Use the `.options` tuple instead." % change['name']) def _value_changed(self, name, old, new): """Called when value has been changed""" if self.value_lock.acquire(False): try: # Reverse dictionary lookup for the value name for k, v in self._options_dict.items(): if self.equals(new, v): # set the selected value name self.selected_label = k return # undo the change, and raise KeyError self.value = old raise KeyError(new) finally: self.value_lock.release() def _selected_label_changed(self, name, old, new): """Called when the value name has been changed (typically by the frontend).""" if self.value_lock.acquire(False): try: self.value = self._options_dict[new] finally: self.value_lock.release()
class EmergencyNode: def __init__(self): # subscribe to the laser data rospy.Subscriber("/scan", LaserScan, self.emergency_callback) # subscribe to the move commands and execute them as long as not in an emergency rospy.Subscriber("/racecar/move_commands",AckermannDriveStamped, self.move_callback) # advertise that we'll publish on the racecar/emergency_stop topic for notifications self.emergency_pub = rospy.Publisher("racecar/emergency_stop", Bool, queue_size = 1) # advertise that we'll publish on the cmd_vel topic for ackermann_mux_cmd self.cmd_vel_pub = rospy.Publisher("vesc/ackermann_cmd_mux/input/safety", AckermannDriveStamped, queue_size=10) # global variables self.safety_distance = 0.5 self.emergency_time_delta = rospy.Duration.from_sec(0.5) self.emergency_lock = Lock() self.last_move_command = None # stop command msg = AckermannDriveStamped() msg.header.stamp = rospy.Time.now() msg.drive.steering_angle = 0.0 msg.drive.speed = 0.0 self.stop_command = msg # reverse command msg = AckermannDriveStamped() msg.header.stamp = rospy.Time.now() msg.drive.steering_angle = 0.0 msg.drive.speed = -1.0 self.reverse_command = msg # See if obstacle is within cone of +/-30 degrees def emergency_callback(self, LaserScan_msg): if self.emergency_lock.acquire(False): # Laser index 0:1080, using 420:660 for +/- 30 degrees cone = list(LaserScan_msg.ranges[420:660]) print min(cone) if min(cone) < self.safety_distance: dangerIndicies = [(cone.index(i) + 420) for i in cone if i <= self.safety_distance] #There might be dirt on the laser, so check if over threshold size if len(dangerIndicies) > 5: #publish emergency and stop the car for time delta self.emergency_pub.publish(1) self.cmd_vel_pub.publish(self.reverse_command) rospy.Timer(self.emergency_time_delta,self.emergency_over,1) else: self.emergency_pub.publish(0) self.emergency_lock.release() else: self.emergency_lock.release() # release the lock and allow move commands because out of danger def emergency_over(self,event): self.emergency_pub.publish(0) if not (self.last_move_command == None): self.cmd_vel_pub.publish(self.last_move_command) else: self.cmd_vel_pub.publish(self.stop_command) self.emergency_lock.release() # Continue to publish move commands unless in an emergency def move_callback(self, move_command): if self.emergency_lock.locked(): self.last_move_command = move_command else: self.cmd_vel_pub.publish(move_command)
class BaseFoobnixControls(): def __init__(self): self.vk_service = VKService(FC().access_token, FC().user_id) self.count_errors = 0 self.is_scrobbled = False self.start_time = None self.chache_text = None self.play_lock = Lock() def check_for_media(self, args): dirs = [] files = [] for arg in args: if os.path.isdir(arg): dirs.append(arg) elif os.path.isfile(arg) and get_file_extension(arg) in FC().all_support_formats: files.append(arg) if dirs: self.on_add_folders(dirs) gobject.idle_add(self.play_first_file_in_playlist) elif files: self.on_add_files(files) gobject.idle_add(self.play_first_file_in_playlist) def love_this_tracks(self, beans=None): if not beans: return map(self.lastfm_service.love, beans) def show_google_results(self, query): beans = [] beans.append(FModel('"%s" not found trying Google search' % query)) g_results = google_search_results(query) for line in g_results: beans.append(FModel(line).add_is_file(True)) if not g_results: beans.append(FModel('Google not found %s' % query)) return beans def get_active_bean(self): tree = self.notetabs.get_current_tree() if tree: return tree.get_selected_or_current_bean() def play_selected_song(self): current = self.get_active_bean() tree = self.notetabs.get_current_tree() if not current: try: current = tree.get_bean_under_pointer_icon() except AttributeError: return if not current: return None logging.debug("play current bean is %s" % str(current.text)) if current and current.is_file: tree.set_play_icon_to_bean(current) """play radio, do not check VK""" if current.type and current.type == FTYPE_RADIO: self.play(current) return if current.path and current.path.startswith("http://"): if not self.check_path(current.path): res = self.net_wrapper.execute(self.vk_service.find_one_track, current.get_display_name()) if not res: return path = res.path if path: current.path = path """play song""" self.play(current) def check_path(self, path): if path: if not path.startswith("http://"): if os.path.exists(path): return True else: try: """Timeout not compatible with python 2.5""" #u = urlopen(bean.path, timeout = 7) #@UnusedVariable u = urlopen(path) #@UnusedVariable if not vars().has_key("u"): return False return True except: return False return False def save_beans_to(self, beans): return None def on_chage_player_state(self, state, bean): logging.debug("bean state %s" % (state)) if not FC().system_icons_dinamic: return None if state == STATE_STOP: self.trayicon.set_image_from_path(FC().stop_icon_entry) elif state == STATE_PAUSE: self.trayicon.set_image_from_path(FC().pause_icon_entry) elif state == STATE_PLAY: self.trayicon.set_image_from_path(FC().play_icon_entry) if bean and bean.type: logging.debug("bean state and type %s %s" % (state, bean.type)) if bean.type == FTYPE_RADIO: return self.trayicon.set_image_from_path(FC().radio_icon_entry) def on_add_folders(self, paths=None): if not paths: paths = directory_chooser_dialog(_("Choose folders to open"), FC().last_dir) if paths: def task(): path = paths[0] list = path.split("/") FC().last_dir = path[:path.rfind("/")] name = list[len(list) - 1] parent = FModel(name) self.append_to_new_notebook(name, []) all_beans = [] all_beans.append(parent) for bean in get_all_music_by_paths(paths, self): if not bean.is_file: bean.parent(parent).add_is_file(False) all_beans.append(bean) if all_beans: self.append_to_current_notebook(all_beans) else: self.append([self.SearchCriteriaBeen(_("Nothing found to play in the folder(s)") + paths[0])]) self.in_thread.run_with_progressbar(task) def on_add_files(self, paths=None, tab_name=None): if not paths: paths = file_chooser_dialog(_("Choose file to open"), FC().last_dir) copy_paths = copy.deepcopy(paths) for i, path in enumerate(copy_paths): if path.lower().endswith(".m3u") or path.lower().endswith(".m3u8"): paths[i:i + 1] = m3u_reader(path) if len(copy_paths) == 1: ext = os.path.splitext(path)[1] tab_name = os.path.basename(path)[:-len(ext)] break if paths: if paths[0]: if isinstance(paths[0], list): path = paths[0][0] else: path = paths[0] else: if isinstance(path, list): path = paths[1][0] else: path = paths[1] if path: list_path = path.split("/") name = list_path[len(list_path) - 2] if not tab_name: tab_name = os.path.split(os.path.dirname(path))[1] FC().last_dir = path[:path.rfind("/")] self.append_to_new_notebook(tab_name, []) parent = FModel(name) self.append_to_current_notebook([parent]) else: self.append_to_new_notebook(tab_name, []) parent = FModel(tab_name) self.append_to_current_notebook([parent]) beans = [] for path in paths: text = None if isinstance(path, list): text = path[1] path = path[0] bean = FModel(path, path).add_is_file(True) else: bean = FModel(path, path).parent(parent).add_is_file(True) if text: bean.text = text beans.append(bean) if not beans: self.append_to_current_notebook([FModel(_("Nothing found to play in the file(s)") + paths[0])]) else: self.append_to_current_notebook(beans) def set_playlist_tree(self): self.notetabs.set_playlist_tree() def set_playlist_plain(self): self.notetabs.set_playlist_plain() def load_music_tree(self): self.perspective.hide_add_button() if not FCache().cache_music_tree_beans[0] and len(FCache().cache_music_tree_beans) == 1: self.perspective.show_add_button() self.tree.is_empty = True if FCache().tab_names[0]: self.tabhelper.label.set_label(FCache().tab_names[0] + " ") else: tabs = len(FCache().cache_music_tree_beans) self.tree.simple_append_all(FCache().cache_music_tree_beans[tabs - 1]) self.tabhelper.label.set_label(FCache().tab_names[tabs - 1] + " ") for tab in xrange(tabs - 2, -1, -1): tree = NavigationTreeControl(self) tree.simple_append_all(FCache().cache_music_tree_beans[tab]) self.tabhelper._append_tab(FCache().tab_names[tab], navig_tree=tree) if not FCache().cache_music_tree_beans[tab]: tree.is_empty = True self.perspective.show_add_button() logging.info("Tree loaded from cache") if FC().update_tree_on_start: def cycle(): for n in xrange(len(FCache().music_paths)): tab_child = self.tabhelper.get_nth_page(n) tree = tab_child.get_child() self.update_music_tree(tree, n) gobject.idle_add(cycle) def update_music_tree(self, tree=None, number_of_page=0): if not tree: tree = self.tree logging.info("Update music tree" + str(FCache().music_paths[number_of_page])) tree.clear_tree() FCache().cache_music_tree_beans[number_of_page] = [] all = [] all = get_all_music_by_paths(FCache().music_paths[number_of_page], self) for bean in all: FCache().cache_music_tree_beans[number_of_page].append(bean) try: self.perspective.hide_add_button() except AttributeError: logging.warn("Object perspective not exists yet") if not all: tree.is_empty = True try: self.perspective.show_add_button() except AttributeError: logging.warn("Object perspective not exists yet") all.append(FModel(_("Music not found in folder(s):"))) for path in FCache().music_paths[number_of_page]: all.append(FModel(path).add_is_file(True)) else: tree.is_empty = False tree.append_all(all) tree.ext_width = tree.ext_column.get_width() def set_visible_video_panel(self, flag): FC().is_view_video_panel = flag if flag: self.movie_window.show() else: self.movie_window.hide() def volume_up(self): self.volume.volume_up() def volume_down(self): self.volume.volume_down() def mute(self): self.volume.mute() def hide(self): self.main_window.hide() def show_hide(self): self.main_window.show_hide() def show(self): self.main_window.show() def play_pause(self): if self.media_engine.get_state() == STATE_PLAY: self.media_engine.state_pause() else: self.media_engine.state_play() def seek_up(self): self.media_engine.seek_up() def seek_down(self): self.media_engine.seek_down() def windows_visibility(self): visible = self.main_window.get_property('visible') if visible: self.main_window.hide() else: self.main_window.show() def state_play(self, remember_position=False, under_pointer_icon=False): if self.media_engine.get_state() == STATE_PAUSE and not remember_position: self.media_engine.state_play() self.statusbar.set_text(self.media_engine.bean.info) elif under_pointer_icon: tree = self.notetabs.get_current_tree() bean = tree.get_bean_under_pointer_icon() self.play(bean) else: self.play_selected_song() def show_preferences(self): self.preferences.show() def state_pause(self): self.media_engine.state_pause() def state_stop(self, remember_position=False): self.record.hide() self.media_engine.state_stop(remember_position) if not remember_position: self.statusbar.set_text(_("Stopped")) self.seek_bar.clear() def state_play_pause(self): self.media_engine.state_play_pause() bean = self.media_engine.bean if self.media_engine.get_state() == STATE_PLAY: self.statusbar.set_text(bean.info) else: self.statusbar.set_text(_("Paused | ") + str(bean.info)) def state_is_playing(self): return self.media_engine.get_state() == STATE_PLAY def fill_bean_from_vk(self, bean): vk = self.vk_service.find_one_track(bean.get_display_name()) if vk: bean.path = vk.path bean.time = vk.time return True else: return False def play(self, bean): if not bean or not bean.is_file: return self.play_lock.acquire() if bean.type == FTYPE_RADIO: self.record.show() else: self.record.hide() def task(): self.seek_bar.clear() self.statusbar.set_text(bean.info) self.trayicon.set_text(bean.text) self.movie_window.set_text(bean.text) self.main_window.set_title(bean.text) gobject.idle_add(task) thread.start_new_thread(self._one_thread_play, (bean,)) #self._play(bean) def _one_thread_play(self,bean): try: self._play(bean) finally: if self.play_lock.locked(): self.play_lock.release() def _play(self, bean): self.count_errors = 0 if not bean.path: bean.path = get_bean_posible_paths(bean) if bean.path and bean.type != FTYPE_RADIO and bean.path.startswith("http"): if not url_utils.is_exists(bean.path): bean.path = None if not bean.path: if not self.fill_bean_from_vk(bean): def post_task(): self._play(bean) if self.vk_service.is_show_authorization(post_task): return None if self.count_errors < 4: time.sleep(0.5) self.count_errors += 1 if self.play_lock.locked(): self.play_lock.release() self.next() if bean.path: if not os.path.exists(bean.path): if bean.iso_path and os.path.exists(bean.iso_path): logging.info("Try to remount " + bean.iso_path) mount_tmp_iso(bean.iso_path) elif not bean.path.startswith("http"): logging.error("File " + bean.path + " not found") elif os.path.isdir(bean.path): return None self.media_engine.play(bean) self.is_scrobbled = False self.start_time = False if not get_file_extension(bean.path) in FC().video_formats: if bean.type != FTYPE_RADIO: self.update_info_panel(bean) self.set_visible_video_panel(False) def notify_playing(self, pos_sec, dur_sec, bean, sec): self.seek_bar.update_seek_status(pos_sec, dur_sec) sec = int(sec) if sec > 10 and sec % 11 == 0: self.net_wrapper.execute(self.lastfm_service.report_now_playing, bean) if not self.start_time: self.start_time = str(int(time.time())) if not self.is_scrobbled: if sec > dur_sec / 2 or sec > 60: self.is_scrobbled = True self.net_wrapper.execute(self.lastfm_service.report_scrobbled, bean, self.start_time, dur_sec) """download music""" if FC().automatic_online_save: self.dm.append_task(bean) def notify_title(self, text): logging.debug("Notify title" + text) self.statusbar.set_text(text) text = normalize_text(text) self.seek_bar.set_text(text) t_bean = FModel(text).add_type(FTYPE_RADIO).create_from_text(text) self.update_info_panel(t_bean) if FC().enable_radio_scrobbler: start_time = str(int(time.time())) self.net_wrapper.execute(self.lastfm_service.report_now_playing, t_bean) if " - " in text and self.chache_text != text: text = self.chache_text self.net_wrapper.execute(self.lastfm_service.report_scrobbled, t_bean, start_time, 200) def notify_error(self, msg): logging.error("notify error " + msg) self.seek_bar.set_text(msg) self.info_panel.clear() def notify_eos(self): self.next() def player_seek(self, percent): self.media_engine.seek(percent) def player_volume(self, percent): self.media_engine.volume(percent) def search_vk_page_tracks(self, vk_ulr): logging.debug("Search vk_service page tracks") results = self.vk_service.find_tracks_by_url(vk_ulr) all = [] p_bean = FModel(vk_ulr).add_font("bold") all.append(p_bean) for i, bean in enumerate(results): bean.tracknumber = i + 1 bean.parent(p_bean).add_is_file(True) all.append(bean) self.notetabs.append_tab(vk_ulr, all) def search_all_videos(self, query): def inline(): results = self.vk_service.find_videos_by_query(query) all = [] p_bean = FModel(query).add_font("bold") all.append(p_bean) for i, bean in enumerate(results): bean.tracknumber = i + 1 bean.parent(p_bean).add_is_file(True) all.append(bean) if not results: all = self.show_google_results(query) self.notetabs.append_tab(query, all) self.in_thread.run_with_progressbar(inline) def search_all_tracks(self, query): def inline(): results = self.vk_service.find_tracks_by_query(query) if not results: results = [] all = [] p_bean = FModel(query).add_font("bold") all.append(p_bean) for i, bean in enumerate(results): bean.tracknumber = i + 1 bean.parent(p_bean).add_is_file(True) all.append(bean) if not results: all = self.show_google_results(query) self.notetabs.append_tab(query, all) self.in_thread.run_with_progressbar(inline, no_thread=True) def search_top_tracks(self, query): def inline(query): results = self.lastfm_service.search_top_tracks(query) if not results: results = [] all = [] parent_bean = FModel(query) all.append(parent_bean) for i, bean in enumerate(results): bean.tracknumber = i + 1 bean.parent(parent_bean).add_is_file(True) all.append(bean) if not results: all = self.show_google_results(query) self.notetabs.append_tab(query, all) self.in_thread.run_with_progressbar(inline, query) def search_top_albums(self, query): def inline(query): results = self.lastfm_service.search_top_albums(query) if not results: results = [] self.notetabs.append_tab(query, None) albums_already_inserted = [] for album in results[:15]: all = [] if (album.album.lower() in albums_already_inserted): continue album.is_file = False tracks = self.lastfm_service.search_album_tracks(album.artist, album.album) for i, track in enumerate(tracks): track.tracknumber = i + 1 track.album = album.album track.parent(album).add_is_file(True) all.append(track) if (len(all) > 0): all = [album] + all albums_already_inserted.append(album.album.lower()) self.notetabs.append_all(all) if not results: all = self.show_google_results(query) self.notetabs.append_all(all) self.in_thread.run_with_progressbar(inline, query) def search_top_similar(self, query): def inline(query): results = self.lastfm_service.search_top_similar_artist(query) if not results: results = [] self.notetabs.append_tab(query, None) for artist in results[:15]: all = [] artist.is_file = False all.append(artist) tracks = self.lastfm_service.search_top_tracks(artist.artist) for i, track in enumerate(tracks): track.tracknumber = i + 1 track.parent(artist).add_is_file(True) all.append(track) self.notetabs.append_all(all) if not results: all = self.show_google_results(query) #inline(query) self.in_thread.run_with_progressbar(inline, query) def search_top_tags(self, query): def inline(query): results = self.lastfm_service.search_top_tags(query) if not results: logging.debug("tag result not found") results = [] self.notetabs.append_tab(query, None) for tag in results[:15]: all = [] tag.is_file = False all.append(tag) tracks = self.lastfm_service.search_top_tag_tracks(tag.text) for i, track in enumerate(tracks): track.tracknumber = i + 1 track.parent(tag).add_is_file(True) all.append(track) self.notetabs.append_all(all) if not results: all = self.show_google_results(query) self.notetabs.append_all(all) #inline(query) self.in_thread.run_with_progressbar(inline, query) def update_info_panel(self, bean): self.info_panel.update(bean) def append_to_new_notebook(self, text, beans, optimization=False): self.notetabs._append_tab(text, beans, None, optimization) def append_to_current_notebook(self, beans): self.notetabs.append_all(beans) def next(self): bean = self.notetabs.next() if not bean: return gap = FC().gap_secs time.sleep(gap) logging.debug("play current bean is %s" % str(bean.text)) if bean.path: if os.path.isdir(bean.path): return None if bean.path.startswith("http://"): if not self.check_path(bean.path): path = self.net_wrapper.execute(self.vk_service.find_one_track, bean.get_display_name()).path if path: bean.path = path self.play(bean) def prev(self): bean = self.notetabs.prev() if not bean: return if bean.path: if os.path.isdir(bean.path): return None if bean.path.startswith("http://"): if not self.check_path(bean.path): path = self.net_wrapper.execute(self.vk_service.find_one_track, bean.get_display_name()).path if path: bean.path = path self.play(bean) def filter_by_folder(self, value): tree = self.tabhelper.get_current_tree() tree.filter_by_folder(value) self.radio.filter_by_folder(value) self.virtual.filter_by_folder(value) def filter_by_file(self, value): tree = self.tabhelper.get_current_tree() tree.filter_by_file(value) self.radio.filter_by_file(value) self.virtual.filter_by_file(value) self.vk_integration.filter_by_folder(query=value, expand=False) def quit(self, *a): self.state_stop() self.main_window.hide() self.trayicon.hide() logging.info("Controls - Quit") self.notetabs.on_quit() self.virtual.on_quit() self.info_panel.on_quit() self.radio.on_quit() self.my_radio.on_quit() FC().save() gtk.main_quit() def check_version(self): uuid = FCBase().uuid current_version = FOOBNIX_VERSION system = "not_set" try: import platform system = platform.system() except: pass try: from socket import gethostname f = urllib2.urlopen("http://www.foobnix.com/version?uuid=" + uuid + "&host=" + gethostname() + "&version=" + current_version + "&platform=" + system) #f = urllib2.urlopen("http://localhost:8080/version?uuid=" + uuid + "&host=" + gethostname() + "&v=" + current_version) except Exception, e: logging.error("Check version error: " + str(e)) return None new_version_line = f.read() logging.info("version " + current_version + "|" + new_version_line + "|" + str(uuid)) f.close() if FC().check_new_version and compare_versions(current_version, new_version_line) == 1: info_dialog_with_link_and_donate(new_version_line)
class LocationController(object): IDEALBACKLOG = 5 MAXPARALLELCHANNELS = 10 MAXTRANSPERCHANNEL = 250 ZOMBIETIMEOUT = 900 def __init__(self, monitor, host, port, protocol = 'http', debug = 0): self.monitor = monitor self.host = host self.port = port self.protocol = protocol self.debug = debug self.free_channels = [] self.active_channels = Dictionary() self.inactive_channels = Dictionary() self.parallel_channels = Counter() self.pending_transactions = [] self.inflight_transactions = Dictionary() self.state_lock = Lock() self.debugout('Instantiated', 2) def is_transaction_pending(self,sid): for tran in self.pending_transactions: if(tran.sid == sid): return(True) return(False) def recruit_and_send(self): self.state_lock.acquire() self.debugout('Pending Transactions in recruit and send are :%d' %(len(self.pending_transactions))) try: openchannel = self._recruit_channel() finally: self.state_lock.release() if openchannel: self.debugout('Initiating a transaction by recruit and send' ) self.initiate_next_transaction() else: self.debugout('Recruit channel returns None. Unable to init a transaction by recruit and send') def add_transaction(self, transaction): self.state_lock.acquire() self.debugout('Pending Transactions are :%d' %(len(self.pending_transactions))) try: self.debugout('[%s] Adding transaction', 3) transaction.set_manager(self) openchannel = self._recruit_channel() self.pending_transactions.append(transaction) finally: self.state_lock.release() if openchannel: self.initiate_next_transaction() else: self.debugout('Recruit channel returns None. Pending transactions =%d' %(len(self.pending_transactions))) def initiate_next_transaction(self): self.state_lock.acquire() try: channel = self._activate_channel() if channel is not None: transaction = self._activate_transaction() finally: self.state_lock.release() if channel is None: message = 'Activate channel returned None to initiate next.' message += '\n\t\t-- %r\n' self.debugout(message, 1, self) else: transaction.set_channel(channel) message = '[%s] Initiating transaction.' self.debugout(message, 3) if channel.response_count() > self.MAXTRANSPERCHANNEL: message = '[%s] Configuring transaction to close channel:' message += '\n\t%r\n\t%r\n' self.debugout(message, 1, transaction, channel) transaction.close_when_done() transaction.initiate() def handle_completed_transaction(self, transaction): self.debugout('[%s] Handling completed transaction', 3) self.state_lock.acquire() try: self._deactivate_transaction(transaction) self._deactivate_channel(transaction.channel) if self.pending_transactions: initiatenext = (self.inactive_channels or self._recruit_channel()) else: initiatenext = False self._free_channel(transaction.channel) finally: self.state_lock.release() if self.debug > 1: message = '[%s] completed: ' % self message += '%s' % transaction.stats()[1:-1] self.msglog(message, msglog.types.DB, False) if initiatenext: self.initiate_next_transaction() def handle_failed_transaction(self, transaction): try: self.state_lock.acquire() try: # Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None) if(transaction.channel != None ): transaction.channel.accepting_requests(False) finally: self.state_lock.release() message = 'Handling failed transaction %r' % transaction self.msglog(message, msglog.types.WARN) finally: self.handle_completed_transaction(transaction) def _activate_transaction(self): assert self.state_lock.locked() transaction = self.pending_transactions.pop(0) tid = transaction.transaction_number self.inflight_transactions[tid] = transaction return transaction def _deactivate_transaction(self, transaction): assert self.state_lock.locked() tid = transaction.transaction_number try: self.inflight_transactions.pop(tid) except KeyError: self.debugout('Transaction %r not in active list', 1, transaction) return False else: return True def _recruit_channel(self): assert self.state_lock.locked() if not self.free_channels: self._manage_channels() channel = None if self.free_channels: channel = self.free_channels.pop() cid = channel.channel_number self.inactive_channels[cid] = channel return channel def _free_channel(self, channel): assert self.state_lock.locked() cid = channel.channel_number if self.inactive_channels.has_key(cid): self.inactive_channels.pop(cid) if channel.accepting_requests(): self.free_channels.append(channel) def _activate_channel(self): assert self.state_lock.locked() if not self.inactive_channels: channel = self._recruit_channel() else: cid, channel = self.inactive_channels.popitem() if channel is not None: self.active_channels[channel.channel_number] = channel return channel def _deactivate_channel(self, channel): assert self.state_lock.locked() # Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None) if(channel == None): return(False) cid = channel.channel_number try: self.inactive_channels[cid] = self.active_channels.pop(cid) except KeyError: self.debugout('Channel %r not in active list', 1, channel) return False else: if not channel.accepting_requests(): self.inactive_channels.pop(cid) self.parallel_channels.decrement() if not channel.is_closed(): channel.close() self.debugout('Channel %r decommissioned.', 1, channel) return True def _manage_channels(self): assert self.state_lock.locked() for cid, channel in self.active_channels.items(): message = 'Location channel management removing %r from active.' if not channel.accepting_requests(): self.debugout(message % channel, 1) self.active_channels.pop(cid) self.parallel_channels.decrement() maxchannels = self.MAXPARALLELCHANNELS numpending = len(self.pending_transactions) numchannels = self.parallel_channels.value if numchannels == 0: createchannel = True elif numchannels >= maxchannels: createchannel = False elif (numpending / numchannels) > self.IDEALBACKLOG: createchannel = True else: createchannel = False if createchannel: channel = Channel(self.monitor, self.debug) channel.setup_connection(self.host, self.port, self.protocol) self.free_channels.append(channel) self.parallel_channels.increment() return def msglog(self, message, mtype = msglog.types.INFO, autoprefix = False): if autoprefix: message = '%s %s' % (self, message) msglog.log('broadway', mtype, message) def debugout(self, message, debuglevel = 1, *args): if debuglevel <= self.debug: if self not in args: if message.count('%') == len(args): message = '%s ' + message args = (self,) + args return self.msglog(message % args, msglog.types.DB) def __repr__(self): status = ['Controller'] status.append('[%s:%d]' % (self.host, self.port)) channels = self.parallel_channels.value active = len(self.active_channels) inactive = len(self.inactive_channels) free = len(self.free_channels) pending = len(self.pending_transactions) inflight = len(self.inflight_transactions) channelmsg = '%d Channels (%dA %dI %dF)' status.append(channelmsg % (channels, active, inactive, free)) status.append('Transactions (%dA %dP)' % (inflight, pending)) return '<%s>' % (' '.join(status)) def __str__(self): classname = self.__class__.__name__ channels = self.parallel_channels.value status = '(%d => %s:%d)' % (channels, self.host, self.port) return '%s %s' % (classname, status)
class NaptListener(object): def __init__(self, bindaddr): super().__init__() self.lock = Lock() self.bindaddr = bindaddr self.sockets = {} # Dictionary<int, Socket> self.status = NaptListenerStatus.Stopped self.port = 0 self.accepted = Event2('NaptListenerEventArgs') def __str__(self): return 'NaptListener{ %s }' %', '.join([ 'bindaddr=%s' % str(self.bindaddr)]) # public def add_port(self, port): with self.lock: if port in self.sockets: raise Exception() # ArgumentException(nameof(port)) self.listen(port) # private def listen(self, port): # todo ここでBindしないでStart時のBind,Stop時にCloseする so = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) bindaddr= '0.0.0.0' if self.bindaddr == 'any' else self.bindaddr endpoint= (bindaddr, port) self.port = port print(' %s' % str(endpoint)) so.bind(endpoint) so.listen(10) SocketPoller.get_instance().register(so, SocketPollFlag.ReadError, self.polling_callback) # public def remove_port(self, port): with self.lock: so = self.sockets[port] SocketPoller.get_instance().unregister(so) self.sockets.pop(port) so.close() # protected virtual def on_accepted(self, e): # NaptListenerEventArgs self.accepted(self, e) # private def polling_callback_thread(self, so, flag): if 0 != (flag & SocketPollFlag.Read): self.do_recv(so) if 0 != (flag & SocketPollFlag.Write): self.do_send(so) if 0 != (flag & SocketPollFlag.Error): self.do_error(so) def polling_callback(self, so, flag): t = threading.Thread(target=self.polling_callback_thread, args=([so, flag])) t.start() t.join(60.0) if t.is_alive() == True : # Timeout so.close() # protected override def do_recv(self, so): Utils.expects_type(socket.socket, so, 'so') try: so.settimeout(3.0) so_accepted, remote= so.accept(); so_accepted.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 0, 0)) e = NaptListenerEventArgs(so_accepted, so, self.port); self.on_accepted(e) except Exception as ex: print('do_recv: Exception', flush=True) Utils.print_exception(ex) print('', flush=True) try: key = [k for k, v in self.sockets.items() if v == so][0] self.remove_port(key) except Exception as ex: pass #raise ex # protected override def do_send(self, so): Utils.expects_type(socket.socket, so, 'so') # protected override def do_error(self, so): Utils.expects_type(socket.socket, so, 'so') # todo error # protected override def update_select_sockets(self): Utils.assertion(self.lock.locked(), 'need lock') self.read_sockets.extend(self.sockets.values()) self.error_sockets.extend(self.sockets.values())
class RelevantCrawler(APIV3Crawler): """Relevant list crawler bases on YouTube API V3 crawler.""" def __init__(self): APIV3Crawler.__init__(self) self._mutex_update = Lock() self._setup_logger('metadatacrawler') # Reading developer_key from developer.key file from conf, roll over when exceed quota try: with open('conf/developer.key', 'r') as keys: developer_keys = keys.readlines() self.set_keys(developer_keys) self.set_key_index(0) except: self.logger.error('**> The keys file developer.key does not exist! It should be in the conf folder.') sys.exit(1) self._youtube = None self._update_youtube() def _update_youtube(self): self._youtube = discovery.build(self._api_service, self._api_version, developerKey=self._keys[self._key_index], cache_discovery=False) def _search_relevant_list(self, vid, pageToken=None): """Finds the relevant list about a specifies videoId from youtube and returns video id associated with it.""" # start_time = time.time() if self._key_index >= len(self._keys): self.logger.error('Key index out of range, exit.') os._exit(0) else: current_key_index = self._key_index # Call the search().list method to retrieve results matching the specified video term. try: search_response = self._youtube.search().list(part="snippet", relatedToVideoId=vid, type='video', maxResults=50, pageToken=pageToken).execute() except errors.HttpError as error: if error.resp.status == 403: if (not self._mutex_update.locked()) and current_key_index == self._key_index: self._mutex_update.acquire() self.logger.error('The request cannot be completed because quota exceeded.') self.logger.error('Current developer key index {0}.'.format(current_key_index)) if self._key_index < len(self._keys): self.set_key_index(self._key_index+1) self.logger.error('Updated developer key index {0}.'.format(self._key_index)) self._mutex_update.release() time.sleep(random.random()) self._update_youtube() return self._search_relevant_list(vid, pageToken=pageToken) else: raise error # Check to get empty responses handled properly try: if len(search_response['items']) == 0: # self.logger.debug('Request for {0} request number was empty.'.format(vid)) return else: res = [] for video in search_response['items']: rid = video['id']['videoId'] res.append(str(rid)) if 'nextPageToken' in search_response: next_pagetoken = search_response['nextPageToken'] # recursively request next page if next_pagetoken: next_page = self._search_relevant_list(vid, pageToken=next_pagetoken) if next_page: res.extend(next_page) except Exception as e: self.logger.error('Request for {0} failed with error {1} while processing.'.format(vid, str(e))) return return res def _youtube_search_with_exponential_backoff(self, vid): """ Implements Exponential backoff on youtube search.""" for i in xrange(0, 5): try: return self._search_relevant_list(vid) except errors.HttpError as error: if error.resp.status == 500 or error.resp.status == 503: time.sleep((2 ** i) + random.random()) else: self.logger.error('Request for {0} failed with error code {1} at invocation.'.format(vid, error.resp.status)) break self.logger.error('Request for {0} request has an error and never succeeded.'.format(vid)) def start(self, input_file, output_dir): self.logger.warning('**> Outputting result to files...') # define the two queues: one for working jobs, one for results. to_process = Queue() to_write = Queue() # action executed by the worked threads # they input lines from the working queue, process them, obtain the JSON object and put it into writing queue def worker(): while True: # make sure that our workers don't die on us :) try: vid = to_process.get() # process the file only if it was not already done relevant_list = self._youtube_search_with_exponential_backoff(vid) if relevant_list: relevant_list.insert(0, vid) to_write.put(','.join(relevant_list)) except Exception as exc: self.logger.error('[Input Queue] Error in writing: {0}.'.format(str(exc))) to_process.task_done() def writer(): """Function to take values from the output queue and write it to a file """ output_path = '{0}/relevant_list.txt' video_metadata = open(output_path.format(output_dir), 'w') while True: try: jobj = to_write.get() # check for file termination object if jobj == 0: self.logger.warning('**> Termination object received and wait for termination...') video_metadata.close() elif jobj is not None: video_metadata.write('{0}\n'.format(jobj)) except Exception as e: self.logger.error('[Output Queue] Error in writing: {0}.'.format(str(e))) # in any case, mark the current item as done to_write.task_done() # start the working threads - 10 of them for i in range(self._num_threads): t = Thread(target=worker) t.daemon = True t.start() # start the writer thread w = Thread(target=writer) w.daemon = True w.start() # all is good, start the work # opening vid file and reading the video id file to retrieve data with open(input_file, mode='r') as datafile: initial_time = time.time() for line in datafile: vid = line.rstrip().split()[0] to_process.put(vid) # wait for jobs to be done to_process.join() # give the termination object and wait for termination to_write.put(0) to_write.join() self.logger.warning('**> Total time for requests {0:.4f} secs.'.format(time.time() - initial_time)) sys.exit(0)
class ROSImageSubscriber(Thread): ''' Generic tool for ros images ''' def __init__(self,topics,queue_size=1,use_compression=True,loop_rate = 1.0): Thread.__init__(self) self.node_name = 'img_subscriber' try: rospy.init_node(self.node_name, anonymous=True) except: pass self.caller_id = rospy.get_name() self.caller_id = self.caller_id[len(self.node_name)+1:] if not isinstance(topics, list): topics = [topics] self.topics = topics self.pid = os.getpid() #self.daemon = False if(use_compression): self.start_compression_threads() print(str(self)+' subscribing to : '+str(self.topics)) self.mutex = [Lock()]*len(topics) self.has_received_first = [False]*len(topics) self.should_register_mouse = [False]*len(topics) self.mouse_function = [self.wtf]*len(topics) self.bridge = CvBridge() self.images=[] self._stop = threading.Event() self.lock_ = Lock() self.loopy = rospy.Rate(loop_rate) if len(self.topics) == 1: rospy.Subscriber(self.topics[0], Image,self.callback,queue_size=queue_size) self.images.append(np.array([])) else: sub=[] for topic in self.topics: sub.append(message_filters.Subscriber(topic, Image)) self.images.append(np.array([])) ts = message_filters.TimeSynchronizer(sub, queue_size) ts.registerCallback(self.callback) def lock(self): self.lock_.acquire() def release(self): self.lock_.release() def locked(self): return self.lock_.locked() def start_compression_threads(self): self.comp=[] for i in xrange(0,len(self.topics)): isdepth = self.topics[i].find('/depth') >=0 if not isdepth: self.comp.append(CompressTopic(self.topics[i],self.caller_id,compress_mode='compressed')) else: self.comp.append(CompressTopic(self.topics[i],self.caller_id,compress_mode='compressedDepth')) for i in xrange(0,len(self.topics)): self.topics[i] = self.comp[i].topic_out self.comp[i].start() def register_mouse_callback(self,function): for i in xrange(len(self.topics)): self.mouse_function[i] = function self.should_register_mouse[i] = True def mouse_callback_spin_once(self): for i in xrange(len(self.topics)): if self.should_register_mouse[i]: window = self.topics[i]+str(i) function = self.mouse_function[i] print 'Registering',function,'for window name',window cv2.setMouseCallback(window,function) self.should_register_mouse[i] = False def get_window_name(self): if len(self.topics)==1: window = self.topics[0]+str(0) else: window = [] for i in xrange(len(self.topics)): window.append(self.topics[i]+str(i)) return window def stop(self): self._stop.set() print self,' stopped' def wtf(self): for i in xrange(0,len(self.topics)): if self.has_received_first[i]: print 'I have received at least 1 image from : ',self.topics[i] else: print "I'm still waiting for : ",self.topics[i] def run(self): rospy.spin() try: while not rospy.is_shutdown(): self.loopy.sleep() except KeyboardInterrupt: self.stop() for i in xrange(len(self.images)): self.mutex[i].acquire() #cv2.destroyAllWindows()#(self.topics[i]+str(i)) cv2.destroyWindow(self.topics[i]+str(i)) self.mutex[i].release() def callback(self,*msg): if self.locked(): #print "locked" return for i in xrange(len(self.topics)): try: enc = msg[i].encoding if enc=='rgb8': # Strange kinect exception enc='bgr8' if enc=='32FC1': enc='passthrough' if not self.has_received_first[i]: self.has_received_first[i] = True self.mutex[i].acquire() try: self.images[i] = self.bridge.imgmsg_to_cv2(msg[i],enc) except Exception,e: print "Warning, using old cv brdige : ",e self.images[i] = np.array(self.bridge.imgmsg_to_cv(msg[i],enc)) self.mutex[i].release() except CvBridgeError, e: print e
class GstGain: def __init__(self, uri, ref_level): self.__lock = Lock() self.uri = uri self.ref_level = ref_level self.result = (False, 0, 0, uri) self.gain_pipe = None # Create a pipeline with a fake audio output and get the gain levels def gain(self): pipe = 'uridecodebin uri="{0}" ! audioconvert ! rganalysis \ reference-level={1} ! fakesink'.format(self.uri, self.ref_level) self.gain_pipe = Gst.parse_launch(pipe) gain_bus = self.gain_pipe.get_bus() gain_bus.add_signal_watch() gain_bus.connect("message", self._on_message) logging.info('REPLY-GAIN:: started ' + str(self.uri)) self.gain_pipe.set_state(Gst.State.PLAYING) # Block here until EOS self.__lock.acquire(False) self.__lock.acquire() # Reset the pipe self.gain_pipe = None # Return the computation result return self.result def stop(self): if self.gain_pipe is not None: self.gain_pipe.send_event(Gst.Event.new_eos()) def _on_message(self, bus, message): try: if message.type == Gst.MessageType.EOS and self.__lock.locked(): self.__lock.release() elif message.type == Gst.MessageType.TAG: tags = message.parse_tag() tag = tags.get_double(Gst.TAG_TRACK_GAIN) peak = tags.get_double(Gst.TAG_TRACK_PEAK) if tag[0] and peak[0]: self.gain_pipe.set_state(Gst.State.NULL) self.result = (True, tag[1], peak[1], self.uri) if self.__lock.locked(): self.__lock.release() elif message.type == Gst.MessageType.ERROR: logging.debug('REPLY-GAIN:: ' + str(message.parse_error())) self.gain_pipe.set_state(Gst.State.NULL) if self.__lock.locked(): self.__lock.release() except Exception: if self.__lock.locked(): self.__lock.release()