def test_process_queue_consumer_timeout_join_if_task_not_done(self):
with ProcessPoolExecutor(max_workers=1) as executor:
manager = Manager()
exit_event = manager.Event()
work_queue = manager.Queue()
# make sure we take time, but don't timeout on our own
# the join should cause the timeout
consumer = ProcessQueueConsumer(
lambda item: time.sleep(0.5),
work_queue,
exit_event,
2,
1
)
work_queue.put("sleep")
future = executor.submit(consumer.run)
#print("here")
time.sleep(0.1)
self.assertTrue(not future.done())
# have the worker finish
work_queue.put(None)
def get_degrees(self, start_uri=GOOD_MEN, target_uri=KEVIN_BACON):
validation = self.validate_urls([start_uri, target_uri])
worker_pool = Pool(cpu_count())
worker_manager = Manager()
event = worker_manager.Event()
self.q.put([start_uri])
self.qsize.put(1)
# Start up all the workers in the pool
for i in range(cpu_count()):
worker_pool.apply_async(self.explore, (target_uri, event))
event.wait()
worker_pool.terminate()
try:
path = self.deg.get(timeout=self.TIMEOUT)
except Empty:
return [], -1
degrees = len(path) - 1
return path, degrees
def run_prepare(file_paths, num_processes):
"""
Run PrePARE on each file in the submission. Any failures are reported
as an error with the logging and an exception is raised at the end of
processing if one or more files has failed.
:param list file_paths: The paths of the files in the submission's
directory.
:param int num_processes: The number of processes to use in parallel.
:raises SubmissionError: at the end of checking if one or more files has
failed PrePARE's checks.
"""
logger.debug('Starting PrePARE on {} files'.format(len(file_paths)))
jobs = []
manager = Manager()
params = manager.Queue()
file_failed = manager.Event()
if num_processes != 1:
for i in range(num_processes):
p = Process(target=_run_prepare, args=(params, file_failed))
jobs.append(p)
p.start()
for item in itertools.chain(file_paths, (None, ) * num_processes):
params.put(item)
if num_processes == 1:
_run_prepare(params, file_failed)
else:
for j in jobs:
j.join()
if file_failed.is_set():
logger.error('Not all files passed PrePARE')
raise SubmissionError()
logger.debug('All files successfully checked by PrePARE')
class CControlCenter(CProcess):
const_command_queue_process_result_empty = 1
const_command_queue_process_result_notify_terminal = 2
__shared_control_center_info_locker__: Lock = None
__shared_control_center_info__: dict = None
__command_queue__: Queue = None
__control_center_manager__: Manager = None
__control_center_params: dict = None
__control_center_objects__: dict = None
__control_center_objects_locker__: Lock = None
__sentinel_manager__: Manager = None
__sentinel_process__: Process = None
__sentinel_queue__: Queue = None
__sentinel_stop_event__: Event = None
def __init__(self, cmd_queue, shared_control_center_info_locker,
shared_control_center_info):
super().__init__()
self.__command_queue__ = cmd_queue
self.__shared_control_center_info_locker__ = shared_control_center_info_locker
self.__shared_control_center_info__ = shared_control_center_info
def wait_child(self, signum, frame):
CLogger().info('收到SIGCHLD消息')
try:
while True:
# -1 表示任意子进程
# os.WNOHANG 表示如果没有可用的需要 wait 退出状态的子进程，立即返回不阻塞
cpid, status = os.waitpid(-1, os.WNOHANG)
if cpid == 0:
CLogger().info('没有子进程可以处理了.')
break
exitcode = status >> 8
CLogger().info('子进程{0}已退出, 退出码为{1}'.format(cpid, exitcode))
except OSError as e:
if e.errno == errno.ECHILD:
CLogger().info('当前进程没有等待结束的子进程了.')
else:
raise
CLogger().info('处理SIGCHLD消息结束...')
def run(self):
if not CUtils.equal_ignore_case(CSys.get_platform_system_name(),
self.OS_Windows):
base_signal.signal(base_signal.SIGCHLD, self.wait_child)
CLogger().info('控制中心进程[{0}]启动运行...'.format(self.pid))
# 此时, 才在controlCenter的进程中
host_settings_dict = CUtils.dict_value_by_name(
self.__shared_control_center_info__, self.NAME_CMD_SETTINGS, None)
settings.application.load_obj(host_settings_dict)
settings.application.init_sys_path()
CLogger().info('控制中心进程[{0}]启动哨兵值守进程...'.format(self.pid))
self.__control_center_manager__ = Manager()
self.__control_center_params = self.__control_center_manager__.dict()
self.__control_center_objects__ = self.__control_center_manager__.dict(
)
self.__control_center_objects_locker__ = self.__control_center_manager__.Lock(
)
self.__sentinel_manager__ = Manager()
self.__sentinel_queue__ = self.__sentinel_manager__.Queue()
self.__sentinel_stop_event__ = self.__sentinel_manager__.Event()
self.__control_center_params[
self.NAME_CMD_SETTINGS] = host_settings_dict
self.__sentinel_process__ = CSentinel(
self.__control_center_objects_locker__,
self.__control_center_objects__, self.__sentinel_stop_event__,
self.__sentinel_queue__, self.__control_center_params)
self.__sentinel_process__.daemon = True
self.__sentinel_process__.start()
while True:
CLogger().info('控制中心进程[{0}]开始检查接收任务队列...'.format(self.pid))
if self.process_queue_command(
) == self.const_command_queue_process_result_notify_terminal:
break
CLogger().info('控制中心进程[{0}]开始检查哨兵反馈的消息...'.format(self.pid))
self.process_queue_sentinel()
CLogger().info('控制中心进程[{0}]开始同步控制中心对外公布的数据...'.format(self.pid))
# 同步一下控制中心的对外公布数据
self.sync_shared_control_center_info()
# 休息一下
time.sleep(3)
CLogger().info('控制中心进程[{0}]开始进行退出前的准备工作, 该过程会比较复杂漫长...'.format(
self.pid))
# 开始处理退出工作
self.before_stop()
def process_queue_command(self) -> int:
"""
处理命令队列中的所有任务
:return:
const_command_queue_process_result_empty: 队列已经处理完毕, 队列清空
const_command_queue_process_result_notify_terminal: 队列处理中发现了关闭退出的命令, 执行该命令
"""
while True:
try:
command = self.__command_queue__.get(False)
except:
CLogger().info('控制中心进程[{0}]处理完所有命令队列, 将退出...'.format(self.pid))
return self.const_command_queue_process_result_empty
else:
# 收到关闭消息后，退出循环
if command is None:
CLogger().info('控制中心进程[{0}]接收到退出通知, 将退出...'.format(
self.pid))
return self.const_command_queue_process_result_notify_terminal
else:
# 收到正常消息，开始处理
CLogger().info('控制中心进程[{0}]接收到正常通知, 将开始处理通知...'.format(
self.pid))
self.process_command(command)
def process_queue_sentinel(self):
"""
处理哨兵发来的消息队列中的任务
:return:
"""
while True:
try:
sentinel_message = self.__sentinel_queue__.get(False)
except:
CLogger().info('控制中心进程[{0}]处理完所有哨兵反馈的消息, 将退出...'.format(
self.pid))
return
else:
# 收到关闭消息后，退出循环
if sentinel_message is not None:
# 收到正常消息，开始处理
CLogger().info(
'控制中心进程[{0}]开始检查哨兵反馈的消息, 将开始处理该消息...'.format(self.pid))
self.process_sentinel_message(sentinel_message)
else:
CLogger().info('控制中心进程[{0}]发现哨兵反馈来停止的消息, 将退出...'.format(
self.pid))
return
def process_command(self, command):
CLogger().info('控制中心进程[{0}]开始处理指令{1}...'.format(self.pid, command))
cmd_type = command.get(self.NAME_CMD_COMMAND, self.CMD_START)
cmd_title = command.get(self.NAME_CMD_TITLE, '')
if CUtils.equal_ignore_case(cmd_type, self.CMD_START):
CLogger().info('控制中心进程[{0}]收到了启动调度的指令...'.format(self.pid))
self.command_start(command)
elif CUtils.equal_ignore_case(cmd_type, self.CMD_STOP):
CLogger().info('控制中心进程[{0}]收到了停止调度的指令...'.format(self.pid))
self.command_stop(command)
elif CUtils.equal_ignore_case(cmd_type, self.CMD_FORCE_STOP):
CLogger().info('控制中心进程[{0}]收到了强制停止调度的指令...'.format(self.pid))
self.command_force_stop(command)
def process_sentinel_message(self, sentinel_message):
"""
处理子进程哨兵进程反馈的消息
:param sentinel_message:
:return:
"""
cmd_id = sentinel_message.get(self.NAME_CMD_ID)
cmd_title = sentinel_message.get(self.NAME_CMD_TITLE)
if cmd_id is None:
return
CLogger().info('哨兵进程发现调度{0}.{1}的某一个进程无故销毁! '.format(cmd_id, cmd_title))
def sync_shared_control_center_info(self):
pass
def command_start(self, command):
cmd_id = command.get(self.NAME_CMD_ID, '')
cmd_title = command.get(self.NAME_CMD_TITLE, '')
cmd_algorithm = command.get(self.NAME_CMD_ALGORITHM, '')
cmd_trigger = command.get(self.NAME_CMD_TRIGGER, '')
cmd_params = command.get(self.NAME_CMD_PARAMS, '')
CLogger().info('调度{0}.{1}.{2}的调度参数为[{3}]...'.format(
cmd_id, cmd_title, cmd_algorithm, cmd_params))
cmd_parallel_count = super().params_value_by_name(
cmd_params, self.Name_Parallel_Count, 1)
CLogger().info('调度{0}.{1}.{2}的调度并行个数为{3}...'.format(
cmd_id, cmd_title, cmd_algorithm, cmd_parallel_count))
if cmd_parallel_count <= 0:
CLogger().info('调度{0}.{1}.{2}的目标并行数量为0, 系统直接进行调度的停止操作...'.format(
cmd_id, cmd_title, cmd_algorithm))
self.command_stop(command)
return
control_center_object = self.__control_center_objects__.get(cmd_id)
if control_center_object is not None:
CLogger().info('调度{0}.{1}.{2}进程池已经启动, 系统不再继续...'.format(
cmd_id, cmd_title, cmd_algorithm))
return
CLogger().info('调度{0}.{1}.{2}将改为{3}个并行进程...'.format(
cmd_id, cmd_title, cmd_algorithm, cmd_parallel_count))
params = self.__control_center_manager__.dict()
params[self.NAME_CMD_ID] = cmd_id
params[self.NAME_CMD_TITLE] = cmd_title
params[self.NAME_CMD_ALGORITHM] = cmd_algorithm
params[self.NAME_CMD_TRIGGER] = cmd_trigger
params[self.NAME_CMD_PARAMS] = cmd_params
params[self.NAME_CMD_SETTINGS] = settings.application.json_obj
stop_event = self.__control_center_manager__.Event()
subprocess_list = self.__control_center_manager__.list()
control_center_object = dict()
control_center_object[self.NAME_PARAMS] = params
control_center_object[self.NAME_STOP_EVENT] = stop_event
control_center_object[self.NAME_SUBPROCESS_LIST] = subprocess_list
for i in range(cmd_parallel_count):
proc = CWorker(stop_event, params)
proc.start()
subprocess_list.append(proc.pid)
CLogger().info('调度{0}.{1}.{2}成功启动了子进程{3}...'.format(
cmd_id, cmd_title, cmd_algorithm, proc.pid))
# 加入对象dict中记录！
self.__control_center_objects__[cmd_id] = control_center_object
CLogger().info('调度{0}.{1}.{2}成功启动了{3}个并行进程...'.format(
cmd_id, cmd_title, cmd_algorithm, cmd_parallel_count))
def command_stop(self, command):
"""
处理发来的停止命令
:param command:
:return:
"""
cmd_id = command.get(self.NAME_CMD_ID, '')
cmd_title = command.get(self.NAME_CMD_TITLE, '')
CLogger().info('控制中心开始停止调度[{0}.{1}]...'.format(cmd_id, cmd_title))
if self.__control_center_objects__ is None:
CLogger().info('控制中心对象为None...')
return
if len(self.__control_center_objects__) == 0:
CLogger().info('控制中心对象里没有任何池记录...')
return
try:
self.__control_center_objects_locker__.acquire()
control_center_object = self.__control_center_objects__.get(cmd_id)
if control_center_object is None:
CLogger().warning('调度{0}进程池已经不存在, 无需停止...'.format(cmd_id))
return
# 给池里的所有进程，发送关闭信号！
stop_event = control_center_object.get(self.NAME_STOP_EVENT, None)
if stop_event is not None:
stop_event.set()
CLogger().info(
'调度{0}.{1}的进程池中所有进程退出的信号已发出, 进程池的检查和关闭, 将在哨兵进程中处理...'.
format(cmd_id, cmd_title))
else:
CLogger().info(
'调度{0}.{1}的进程池退出信号灯已经无效, 进程池的检查和关闭, 将在哨兵进程中处理...'.format(
cmd_id, cmd_title))
finally:
self.__control_center_objects_locker__.release()
CLogger().info('控制中心已经停止调度[{0}.{1}]'.format(cmd_id, cmd_title))
def before_stop(self):
CLogger().info('控制中心开始进行调度的清理工作...')
CLogger().info('首先停止哨兵值守进程检查调度...')
self.__sentinel_stop_event__.set()
CLogger().info('等待哨兵值守进程退出...')
self.__sentinel_process__.join()
CLogger().info('哨兵值守进程已经停止...')
if self.__control_center_objects__ is None:
CLogger().info('控制中心对象为None...')
return
if len(self.__control_center_objects__) == 0:
CLogger().info('控制中心对象里没有任何池记录...')
return
CLogger().info('开始处理{0}个调度对象的进程池...'.format(
len(self.__control_center_objects__)))
try:
self.__control_center_objects_locker__.acquire()
control_center_object_key_list = self.__control_center_objects__.keys(
)
for control_center_object_key in control_center_object_key_list:
CLogger().info(
'已经提取出{0}进程池...'.format(control_center_object_key))
control_center_object = self.__control_center_objects__.get(
control_center_object_key)
if control_center_object is None:
CLogger().info('参数对象为None...')
self.__control_center_objects__.pop(
control_center_object_key)
continue
command = control_center_object.get(self.NAME_PARAMS, None)
CLogger().info('已经提取出参数对象...')
if command is None:
CLogger().info('参数对象为None...')
self.__control_center_objects__.pop(
control_center_object_key)
continue
cmd_id = command.get(self.NAME_CMD_ID, '')
cmd_title = command.get(self.NAME_CMD_TITLE, '')
CLogger().info('控制中心开始清理调度[{0}.{1}]的所有进程...'.format(
cmd_id, cmd_title))
if cmd_id == '':
continue
# 给池里的所有进程，发送关闭信号！
stop_event = control_center_object.get(self.NAME_STOP_EVENT,
None)
if stop_event is None:
continue
stop_event.set()
CLogger().info('调度{0}.{1}的进程池中所有进程退出的信号已发出...'.format(
cmd_id, cmd_title))
# 等待进程池全部退出
while True:
CLogger().info('检查调度{0}.{1}的进程池中所有进程是否都已经退出...'.format(
cmd_id, cmd_title))
all_subprocess_closed = True
subprocess_list = control_center_object.get(
self.NAME_SUBPROCESS_LIST)
if subprocess_list is None:
break
for subproc_id in subprocess_list:
if CProcessUtils.process_id_exist(subproc_id):
CLogger().info(
'检查调度{0}.{1}的进程池中进程{2}仍然在运行...'.format(
cmd_id, cmd_title, subproc_id))
all_subprocess_closed = False
if all_subprocess_closed:
CLogger().info('检查调度{0}.{1}的进程池中所有进程都已经不在...'.format(
cmd_id, cmd_title))
break
else:
time.sleep(1)
CLogger().info(
'检查调度{0}.{1}的进程池中还有进程正在运行, 等待中...'.format(
cmd_id, cmd_title))
CLogger().info('调度{0}.{1}的进程池中所有进程均已退出...'.format(
cmd_id, cmd_title))
self.__control_center_objects__.pop(control_center_object_key)
finally:
self.__control_center_objects_locker__.release()
CLogger().info('控制中心的所有调度清理工作已完成，控制中心进程将关闭...')
def command_force_stop(self, command):
"""
处理发来的强制停止命令
:param command:
:return:
"""
cmd_id = command.get(self.NAME_CMD_ID, '')
cmd_title = command.get(self.NAME_CMD_TITLE, '')
CLogger().info('控制中心开始停止调度[{0}.{1}]...'.format(cmd_id, cmd_title))
if self.__control_center_objects__ is None:
CLogger().info('控制中心对象为None...')
return
if len(self.__control_center_objects__) == 0:
CLogger().info('控制中心对象里没有任何池记录...')
return
try:
self.__control_center_objects_locker__.acquire()
control_center_object = self.__control_center_objects__.get(cmd_id)
if control_center_object is None:
CLogger().warning('调度{0}进程池已经不存在, 无需停止...'.format(cmd_id))
return
# 给池里的所有进程，发送关闭信号！
stop_event = control_center_object.get(self.NAME_STOP_EVENT, None)
if stop_event is not None:
stop_event.set()
CLogger().info('调度{0}.{1}的进程池中所有进程退出的信号已发出...'.format(
cmd_id, cmd_title))
# 直接杀死进程池中的全部子进程
subprocess_list = control_center_object.get(
self.NAME_SUBPROCESS_LIST)
if subprocess_list is not None:
for subproc_id in subprocess_list:
CProcessUtils.process_kill(subproc_id)
CLogger().info('调度{0}.{1}的进程池中所有进程均已退出...'.format(
cmd_id, cmd_title))
self.__control_center_objects__.pop(cmd_id)
finally:
self.__control_center_objects_locker__.release()
CLogger().info('控制中心已经停止调度[{0}.{1}]'.format(cmd_id, cmd_title))
def __init__(self,
client,
group,
topic,
auto_commit=True,
auto_commit_every_n=AUTO_COMMIT_MSG_COUNT,
auto_commit_every_t=AUTO_COMMIT_INTERVAL,
num_procs=1,
partitions_per_proc=0,
**simple_consumer_options):
# Initiate the base consumer class
super(MultiProcessConsumer,
self).__init__(client,
group,
topic,
partitions=None,
auto_commit=auto_commit,
auto_commit_every_n=auto_commit_every_n,
auto_commit_every_t=auto_commit_every_t)
# Variables for managing and controlling the data flow from
# consumer child process to master
manager = MPManager()
self.queue = manager.Queue(
1024) # Child consumers dump messages into this
self.events = Events(
start=manager.Event(), # Indicates the consumers to start fetch
exit=manager.Event(), # Requests the consumers to shutdown
pause=manager.Event()) # Requests the consumers to pause fetch
self.size = manager.Value(
'i', 0) # Indicator of number of messages to fetch
# dict.keys() returns a view in py3 + it's not a thread-safe operation
# http://blog.labix.org/2008/06/27/watch-out-for-listdictkeys-in-python-3
# It's safer to copy dict as it only runs during the init.
partitions = list(self.offsets.copy().keys())
# By default, start one consumer process for all partitions
# The logic below ensures that
# * we do not cross the num_procs limit
# * we have an even distribution of partitions among processes
if partitions_per_proc:
num_procs = len(partitions) / partitions_per_proc
if num_procs * partitions_per_proc < len(partitions):
num_procs += 1
# The final set of chunks
chunks = [partitions[proc::num_procs] for proc in range(num_procs)]
self.procs = []
for chunk in chunks:
options = {'partitions': list(chunk)}
if simple_consumer_options:
simple_consumer_options.pop('partitions', None)
options.update(simple_consumer_options)
args = (client.copy(), self.group, self.topic, self.queue,
self.size, self.events)
proc = Process(target=_mp_consume, args=args, kwargs=options)
proc.daemon = True
proc.start()
self.procs.append(proc)
class ReplicatorManager(Replicator):
def __init__(self,
name,
ip,
port,
log_server_path,
log_server_lock,
n_replicators=3):
super().__init__(name=name,
ip=ip,
port=port,
log_server_path=log_server_path,
log_server_lock=log_server_lock,
server=False)
# Replicators
self.replicators = []
for i in range(1, n_replicators + 1):
r = Replicator(name=f'Replicator_{i}',
ip=self.ip,
port=self.port + i,
log_server_path=self.log_server_path,
log_server_lock=self.log_server_lock)
self.replicators.append(r)
r.start()
# Variables
#self.request_queue = Queue()
self.manager = Manager()
self.request_queue = self.manager.list()
self.request_answer = self.manager.dict()
# URL
self.add_endpoint(endpoint='/',
handler=self.get_request,
methods=['POST'])
# Process to solve requests
self.solver_process = Process(target=self.solve_request)
self.solver_process.start()
return
def run(self):
super().run()
#Kill childs
self.solver_process.terminate()
for r in self.replicators:
r.terminate()
return
def get_request(self):
data, status = self.get_data(keys=['request', 'timestamp', 'send_id'])
if status != 200:
self.log(data)
return data, status
event_wait = self.manager.Event()
self.log(
f"[RECIVE][REQUEST][{data['send_id']}][{data['request']['type'].upper()}]: {data['request']['data']}"
)
#self.request_queue.put(
self.request_queue.append(
(data['timestamp'], data['send_id'], data['request'], event_wait))
request_answer_key = (data['send_id'], data['timestamp'])
event_wait.wait()
answer = self.request_answer[request_answer_key]
return answer[0], answer[1]
def make_request(self, replicator, client_name, data, rq_type,
request_answer_key):
data['send_id'] = self.name
self.log(
f'[SEND][REQUEST][{replicator.name}]: from {client_name} > {data}')
answer = requests.post(
f'http://{replicator.ip}:{replicator.port}/{rq_type}_file',
json=data)
self.log(
f'[RECIVE][RESPONSE][{replicator.name}][{answer.status_code}]: {answer.text}'
)
if answer.status_code != 200:
self.request_answer[request_answer_key] = (answer.text,
answer.status_code)
return
def solve_request(self):
while True:
#if not self.request_queue.empty():
if self.request_queue:
# Sort requests by timestamp
self.request_queue.sort()
# Picking request
client_timestamp, client_name, client_request, event_wait = self.request_queue.pop(
0)
self.log(
f'[EXECUTE][({client_timestamp},{client_name})]: {client_request}'
)
client_request['data']['send_id'] = client_name
if client_request['type'] == 'create':
answer = self.create_file(data=client_request['data'])
elif client_request['type'] == 'update':
answer = self.update_file(data=client_request['data'])
elif client_request['type'] == 'append':
answer = self.append_file(data=client_request['data'])
elif client_request['type'] == 'delete':
answer = self.delete_file(data=client_request['data'])
elif client_request['type'] == 'get':
answer = self.get_file(data=client_request['data'])
request_answer_key = (client_name, client_timestamp)
self.request_answer[request_answer_key] = answer
# If an Error occurs
if answer[1] != 200:
event_wait.set()
continue
# Send request to relicators
replicators_request = []
for replicator in self.replicators:
#print(f"Encaminhando {client_request} para {replicator.name}")
replicators_request.append(
Process(target=self.make_request,
args=(
replicator,
client_name,
client_request['data'],
client_request['type'],
request_answer_key,
)))
for rq in replicators_request:
rq.start()
# Waiting for replicators
for rq in replicators_request:
rq.join()
event_wait.set()
continue
class PSProfile(Plugin):
name = 'psprofile'
score = 1
def options(self, parser, env=os.environ):
logger.debug("options(self, parser, env = os.environ):...")
parser.add_option(
"--psprofile-file",
action="store",
default=env.get("NOSE_PSP_FILE", "psprofile.json"),
dest="psp_file",
metavar="FILE",
help=
"By Default a psprofile.json is generated in the current working directory"
)
Plugin.options(self, parser, env)
def configure(self, options, conf):
logger.debug("configure(self, options, conf):...")
super(PSProfile, self).configure(options, conf)
if not self.enabled:
logger.debug("plugin not enabled")
return
self.__profile_data = {}
self.__psp_report = options.psp_file
def prepareTestCase(self, test):
pid = os.getpid()
self.testname = test.test._testMethodName
logger.debug("prepareTestCase(self, test):... %s [%s]" %
(self.testname, pid))
# setup the multiprocessing proxy objects
self.__process_manager = Manager()
self.__process_event = self.__process_manager.Event()
self.__process_data = self.__process_manager.dict()
# LBYP approach here
self.__process_data['ioc'] = []
self.__process_data['fds'] = []
self.__process_data['mem'] = []
self.__process_data['time'] = []
self.__process_data['cpu'] = []
self.__process = Process(target=PSProfile.profile,
args=(pid, self.__process_data,
self.__process_event))
def startTest(self, test):
self.__process_event.clear()
self.__process.start()
logger.debug("startTest(self, test):... %s" % self.testname)
def stopTest(self, test):
logger.debug("stopTest(self, test):... %s" % self.testname)
self.__process_event.set()
self.__process.join()
self.__profile_data[self.testname] = dict(self.__process_data)
def report(self, stream):
logger.debug("report(self, stream):...")
json_report = json.dumps(self.__profile_data)
self.write_report(json_report)
if self.conf.verbosity > 4:
stream.writeln(str(json_report))
def write_report(self, json_report):
""" write a report, a .json file with json data
"""
report_file_path = self.__psp_report
with open(report_file_path, 'w') as report_file:
report_file.write(json_report)
print "-" * 70
print "PROFILE: %s" % self.__psp_report
print "-" * 70
def finalize(self, result):
logger.debug("finalize(self, result):...")
@staticmethod
def profile(pid, data, event, interval=1):
"""
PSProfile.profile is executed in a multiprocessing context, the method
can be executed or not within a nose environment.
data -- is a DictProxy type object instance
event -- is a flag to stop or not the process
The data is written by proxy using objects once the process is about
to finish.
If the process to profile isn't alive, it will finish the process
"""
# the method code can be improved, but DictProxy isn't very flexible
ps = psutil.Process(pid)
ioc = []
fds = []
mem = []
stamp = []
cpu = []
while not event.is_set():
if ps.status():
ioc.append(ps.io_counters())
fds.append(ps.num_fds())
mem.append(ps.memory_info())
stamp.append(time.time())
cpu.append(ps.cpu_percent(interval=interval))
else:
event.set()
logger.debug(
"process to profile is not alive, about to finish this process [%s]"
% os.getpid())
data['ioc'] = ioc
data['fds'] = fds
data['mem'] = mem
data['time'] = stamp
data['cpu'] = cpu
# the dictionary between parallel processes
return rdict
# ------------------------------------------------------------------------------
# main
# ------------------------------------------------------------------------------
if __name__ == '__main__':
# event handling (using .is_set and set)
# event = Event()
# shared data between processes
manager = Manager()
# just a dictionary
return_dict = manager.dict()
event = manager.Event()
letters = string.ascii_lowercase
# number of runs = number_groups * number_cores
# = 3 * 100
# loop around groups (number of times to run the set of parallel jobs)
for it in range(GROUPS):
params_per_process = []
jobs = []
# these are the cores (set ridiculously high)
# - these all start in parallel
for jt in range(CORES):
for kit in range(SUBGROUPS):
if SUBGROUPS < len(letters):
app.register_blueprint(main)
app.register_blueprint(stats)
app.register_blueprint(files)
app.config.update(RECORD_EVENT=record_event,
SHARED_DICT=shared_dict,
CAMERA_DICT=camera_dict)
return app
if __name__ == "__main__":
manager = Manager()
record_event = manager.Event()
running_event = manager.Event()
shared_dict = manager.dict()
camera_dict = manager.dict()
running_event.set()
motion = MotionSensor(
shared_data_dict=shared_dict,
running_event=running_event) # create a sensor process
motion.setup()
motion.start()
gpsp = GpsPoller(shared_data_dict=shared_dict, running_event=running_event)
gpsp.setup()
gpsp.start() # start it up
class Pipeline:
def __init__(
self,
max_init_workers: Optional[int] = None,
max_queues_size: int = MAX_QUEUES_SIZE,
):
"""
:param max_init_workers: Number of workers to use for concurrent initialization of stages, default the number of CPUs
:param max_queues_size: Maximum size of any queue instanced for the pipeline (stage input and output queues)
"""
self._containers = LastOrderedDict()
self._error_manager = ErrorManager()
self._max_init_workers = max_init_workers
self._init_executor = None
self._wait_previous_executor = None
self._pipeline_executor = None
self._max_queues_size = max_queues_size
self._out_queue = None
self._enqueue_source = False
self._sync_manager = None
# an empty source, on which we can only occasionally send items
self._source_container = SourceContainer()
self._count = 0
self._executors_ready = False
def _new_mp_queue(self) -> ItemsQueue:
"""
Construct queue for multiprocessing communication
"""
if self._sync_manager is None:
self._sync_manager = Manager()
return self._sync_manager.Queue(maxsize=self._max_queues_size)
def _new_queue(self) -> ItemsQueue:
"""
Construct queue for communication
"""
return Queue(maxsize=self._max_queues_size)
def _new_mp_event(self) -> Event:
"""
Construct synchronization event for multiprocessing
"""
if self._sync_manager is None:
self._sync_manager = Manager()
return self._sync_manager.Event()
@staticmethod
def _new_event() -> Event:
"""
Construct synchronization event
"""
return Event()
def _new_mp_counter(self) -> ProcessCounter:
"""
Construct a safe counter for multiprocessing
"""
if self._sync_manager is None:
self._sync_manager = Manager()
return ProcessCounter(self._sync_manager)
@staticmethod
def _new_counter() -> ThreadCounter:
"""
Construct a safe counter for threads
"""
return ThreadCounter()
def _wait_executors(self, wait_seconds: float = CONCURRENCY_WAIT):
"""
Wait for all containers to start
:param wait_seconds: Recurrently wait these seconds for all stage initializers to finish
"""
if self._executors_ready:
return
if self._init_executor is not None:
self._init_executor.shutdown(wait=True)
self._init_executor = None
while not all(self._containers.values()):
time.sleep(wait_seconds)
self._wait_previous_executor.shutdown(wait=True)
for name, container in self._containers.items():
if isinstance(
container,
(ConcurrentStageContainer, BatchConcurrentStageContainer)):
container.run()
# finalize initialization of the error manager shared by this and other stage threads
self._error_manager.on_start()
self._executors_ready = True
_logger.debug("Pipeline ready to run")
def shutdown(self):
if self._out_queue is not None:
self._out_queue.join()
# if self._init_executor is not None:
# self._init_executor.shutdown()
# FIXME stage shutdown may raise exception, the executor gets stuck
# for name, stage in self._containers.items():
# if isinstance(stage, (ConcurrentStageContainer, BatchConcurrentStageContainer)):
# stage.shutdown()
if self._sync_manager is not None:
self._sync_manager.shutdown()
def __del__(self):
self.shutdown()
def build(self) -> Pipeline:
"""
Pipeline builder method
"""
if not any(self._containers):
raise ValueError("Must append at least a stage")
_logger.debug(f"Building the pipeline on stages: {self._log_stages()}")
self._wait_executors()
return self
def run(self) -> Generator[DataItem, None, None]:
"""
Run the pipeline given a source and a concatenation of stages.
Get the sequence of items through iteration
:return: Iterator over processed items
:raises ValueError: When a source has not been set for the pipeline
"""
if not self._source_container.is_set():
raise ValueError("Set the data source for this pipeline")
_logger.debug(f"Running the pipeline on stages: {self._log_stages()}")
counter = 0
last_stage_name = self._last_stage_name()
terminator_thread = None
source_thread = None
# in case the first stage is concurrent
if self._enqueue_source:
source_thread = Thread(
target=self._source_container.pop_into_queue)
source_thread.start()
while True:
for name, container in self._containers.items():
try:
# concurrent stages run by themselves in threads/processes
if not isinstance(
container,
(ConcurrentStageContainer,
BatchConcurrentStageContainer),
):
container.process()
# but me must periodically check for errors
else:
container.check_errors()
except Exception as e:
self.stop()
# TODO in case of errors we loose pending items!
self._terminate_all(force=True)
self.shutdown()
self._count += 1
raise e
# retrieve finally processed items from the last stage
if name == last_stage_name:
for _ in range(container.size if isinstance(
container, BatchStageContainer) else 1):
item = container.get_processed()
if item is not None:
if not isinstance(item, Stop):
yield item
counter += 1
self._count += 1
# if a stop is finally signaled, start termination of all containers
elif (not self._all_terminated()
and terminator_thread is None):
terminator_thread = Thread(
target=self._terminate_all)
terminator_thread.start()
# an item is None if the final output queue is empty
else:
break
# exit the loop only when all items have been returned
if self._all_empty() and counter >= self._source_container.count():
if source_thread is not None:
source_thread.join()
if terminator_thread is not None:
terminator_thread.join()
self.shutdown()
return
@property
def count(self) -> int:
"""
Get the number of items processed by all executed runs, also for items which have failed
:return: Count of processed items
"""
return self._count
def _terminate_all(self,
force: bool = False,
wait_seconds: float = CONCURRENCY_WAIT):
"""
Terminate all running containers
:param force: If True do not wait for a container to process all items produced by the source
:param wait_seconds: Time to wait before pinging again a container for its termination
"""
_logger.debug("Terminating the pipeline")
# scroll the pipeline by its order and terminate stages after the relative queues are empty
for container in self._containers.values():
if not force:
# ensure the stage has processed all source items
while container.count() < self._source_container.count():
time.sleep(wait_seconds)
container.terminate()
if isinstance(container, ConcurrentStageContainer):
if force:
# empty the queues, losing pending items
container.empty_queues()
while not container.is_terminated():
time.sleep(wait_seconds)
container.queues_join()
while not container.queues_empty():
time.sleep(wait_seconds)
_logger.debug("Termination done")
def _all_terminated(self) -> bool:
"""
Check if all containers have been alerted for termination and are exited
"""
return all(container.is_terminated()
for container in self._containers.values())
def _all_empty(self) -> bool:
"""
Check if all containers are terminated and there are not items left in the queues
"""
return self._all_terminated() and all(
container.queues_empty()
for container in self._containers.values()
if isinstance(container, (ConcurrentStageContainer,
BatchConcurrentStageContainer)))
def process(self, item: DataItem) -> DataItem:
"""
Process a single item synchronously (no concurrency) through the pipeline
"""
_logger.debug(f"Processing {item} on stages: {self._log_stages()}")
last_stage_name = self._containers.last_key()
self._source_container.prepend_item(item)
for name, container in self._containers.items():
container.process()
if name == last_stage_name:
return container.get_processed(block=True)
def process_async(self,
item: DataItem,
callback: Optional[Callable[[DataItem], Any]] = None):
"""
Process a single item asynchronously through the pipeline, stages may run concurrently.
The call returns immediately, processed items are retrieved with :meth:`.Pipeline.get_item`
:param callback: A function to call after a successful process of the item
"""
_logger.debug(
f"Processing asynchronously {item} on stages: {self._log_stages()}"
)
if callback is not None:
item.set_callback(callback)
self._source_container.prepend_item(item)
self._start_pipeline_executor()
def stop(self):
"""
Tell the source to stop to generate items and consequently the pipeline
"""
self._source_container.stop()
def get_item(self, block: bool = True) -> DataItem:
"""
Get a single item from the asynchronous execution of the pipeline on single items from :meth:`.Pipeline.process_async`
:param block: If True wait indefinitely for the next processed item
:raises ValueError: When there is not output queue set, the pipeline is not running asynchronously
:raises queue.Empty: When we do not block and the queue is empty
"""
if self._out_queue is not None:
item = self._out_queue.get(block)
self._out_queue.task_done()
return item
else:
raise ValueError(
"No pipeline is running asynchronously, not item can be retrieved from the output queue"
)
def set_source(self, source: Source) -> Pipeline:
"""
Set the source of the pipeline: a subclass of :class:`.stage.Source`
"""
self._source_container.set(source)
return self
def set_error_manager(self, error_manager: ErrorManager) -> Pipeline:
"""
Set the error manager for handling errors from each stage item processing
"""
self._error_manager = error_manager
for container in self._containers.values():
container.set_error_manager(self._error_manager)
return self
def _last_stage_name(self) -> str:
if self._containers:
return self._containers.last_key()
def _last_container(self) -> BaseContainer:
if self._containers:
return self._containers[self._last_stage_name()]
else:
return self._source_container
def _wait_for_previous(
self,
container: ConnectedStageMixin,
last_stage_name: str,
wait_seconds: float = CONCURRENCY_WAIT,
):
"""
Given a container we want to append to the pipeline, wait for the last one (added to the pipeline) to be created
:param container: A container to add to the pipeline
:param last_stage_name: Name of the last stage currently in the pipeline
:param wait_seconds: Time to recurrently wait the construction of the container relative to the last stage in the pipeline
"""
def _waiter():
if last_stage_name is not None:
while self._containers[last_stage_name] is None:
time.sleep(wait_seconds)
container.set_previous(self._containers[last_stage_name])
else:
container.set_previous(self._source_container)
executor = self._get_wait_previous_executor()
executor.submit(_waiter)
def _build_container(self, name: str, stage: StageType, concurrency: int,
parallel: bool) -> BaseContainer:
"""
Get a new container instance according to the pipeline configuration
:param name: Stage name
:param stage: A stage instance
:param concurrency: Number of concurrent stage executions, if 0 then just create the non-concurrent containers
:param parallel: If True use multiprocessing, otherwise threads
"""
if concurrency <= 0:
constructor = (BatchStageContainer if isinstance(
stage, BatchStage) else StageContainer)
# if not concurrent we must explicitly finalize initialization of this single stage object
stage.on_start()
return constructor(name, stage, self._error_manager)
else:
constructor = (BatchConcurrentStageContainer if isinstance(
stage, BatchStage) else ConcurrentStageContainer)
if parallel:
return constructor(
name,
stage,
self._error_manager,
self._new_mp_queue,
self._new_mp_counter,
self._new_mp_event,
concurrency,
parallel,
)
else:
# if the stage is executed on multiple threads we must finalize initialization once,
# while on multiprocessing each process executor calls it for its own copy of the stage
stage.on_start()
return constructor(
name,
stage,
self._error_manager,
self._new_queue,
self._new_counter,
self._new_event,
concurrency,
parallel,
)
def get_stage(self, name: str) -> StageType:
"""
Get a stage instance by its name
"""
return self._containers.get(name).stage
def append_stage(
self,
name: str,
stage: StageType,
concurrency: int = 0,
parallel: bool = False,
) -> Pipeline:
"""
Append a stage to the pipeline just after the last one appended, or after the source if it is the first stage
:param name: Name for identify the stage in the pipeline, it is also set in the stage and it must be unique in the pipeline
:param stage: Instance of a stage
:param concurrency: Number of concurrent stage executions, if 0 then threads/processes won't be involved for this stage
:param parallel: If True use multiprocessing, otherwise threads
"""
self._executors_ready = False
# FIXME here we force a BatchStage to run on a thread, but we would leave it on the main thread
if concurrency < 1 and isinstance(stage, BatchStage):
parallel = False
concurrency = 1
self._check_stage_name(name)
container = self._build_container(name, stage, concurrency, parallel)
if concurrency > 0:
# if it is concurrent and it is the first stage, make the source working on a output queue
if not self._containers:
self._enqueue_source = True
self._wait_for_previous(
container,
self._last_stage_name()) # wait that previous stage is initialized
self._containers[name] = container
return self
def append_stage_concurrently(
self,
name: str,
stage_class: Callable,
args: Sequence = None,
kwargs: Mapping = None,
concurrency: int = 0,
parallel: bool = False,
) -> Pipeline:
"""
Append a stage class to the pipeline just after the last one appended, or after the source if it is the first stage.
The stage construction will be executed concurrently respect to the general pipeline construction
:param name: Name for identify the stage in the pipeline, it is also set in the stage and it must be unique in the pipeline
:param stage_class: Class of a stage
:param args: List of arguments for the stage constructor
:param kwargs: Dictionary of keyed arguments for the stage constructor
:param concurrency: Number of concurrent stage executions, if 0 then threads/processes won't be involved for this stage
:param parallel: If True use multiprocessing, otherwise threads
"""
self._executors_ready = False
# FIXME here we force a BatchStage to run on a thread, but we would leave it on the main thread
if concurrency < 1 and issubclass(stage_class, BatchStage):
parallel = False
concurrency = 1
if kwargs is None:
kwargs = {}
if args is None:
args = []
self._check_stage_name(name)
# if it is concurrent and it is the first stage, make the source working on a output queue
if concurrency > 0 and not self._containers:
self._enqueue_source = True
last_stage_name = self._last_stage_name()
# set it immediately so the order of the calls of this method is followed in `_containers`
self._containers[name] = None
future = self._get_init_executor(parallel).submit(
stage_class, *args, **kwargs)
def append_stage(stage_future: Future):
stage = stage_future.result()
container = self._build_container(name, stage, concurrency,
parallel)
self._wait_for_previous(container, last_stage_name)
self._containers[name] = container
future.add_done_callback(append_stage)
return self
def _get_init_executor(self, parallel: bool = False) -> Executor:
"""
Get a pool executor for concurrent stage initialization
:param parallel: True if the executor uses multiprocessing, otherwise treads
"""
if self._init_executor is None:
executor = ThreadPoolExecutor if not parallel else ProcessPoolExecutor
self._init_executor = executor(max_workers=self._max_init_workers)
return self._init_executor
def _get_wait_previous_executor(self) -> Executor:
"""
Get a pool executor for the function that will recurrently wait for a container to be ready
"""
if self._wait_previous_executor is None:
self._wait_previous_executor = ThreadPoolExecutor()
return self._wait_previous_executor
def _start_pipeline_executor(self) -> Thread:
"""
Get a thread where to run a pipeline that accepts asynchronous processing of single items
"""
if self._pipeline_executor is None:
self._init_out_queue()
def pipeline_runner():
for item in self.run():
item.callback()
self._out_queue.put(item)
self._pipeline_executor = Thread(target=pipeline_runner,
daemon=True)
self._pipeline_executor.start()
return self._pipeline_executor
def _check_stage_name(self, name: str):
"""
Check if a stage name is not already defined in the pipeline
:raises ValueError: Stage name is already defined in the pipeline
"""
if name in self._containers:
raise ValueError(
f"The stage name {name} is already used in this pipeline")
def _init_out_queue(self):
"""
Get the internal output pipeline for asynchronous processing of single items
"""
self._out_queue = self._new_queue()
def _log_stages(self):
return ", ".join(self._containers.keys())
def main(self):
t0 = timeit.default_timer()
# shared resources manager
m = Manager()
# original screenshots of the game window
raw_frames = m.Queue(maxsize=3)
# environment data
env_data = m.Queue(maxsize=2)
# output images
output_frames = m.Queue(maxsize=3)
# end event
end = m.Event()
# proc's pool
pool = self.__init_mp_pool(pool_size=16)
# need to get sub-process (external game) pid to
# for graceful shutdown
game_pid = m.Value('pid', None)
game = Game(rom_path=ROM_PATH,
emulator_path=EMULATOR_PATH,
rom_name=ROM_NAME,
pid=game_pid)
processes = [pool.apply_async(game.run, args=(end,))] + \
self.init_environment(pool, end, q_source=raw_frames, q_target=env_data) + \
self.init_agent(pool, end, q_source=env_data, q_target=output_frames) + \
self.init_grabber(pool, end, q_target=raw_frames) + \
self.init_gui(pool, end, q_source=output_frames)
fin_processes = []
try:
while True:
for proc in processes:
if (proc.ready() and proc not in fin_processes):
fin_processes.append(proc)
if len(fin_processes) == len(processes):
break
except KeyboardInterrupt:
self.log.info('\nCaught Ctrl+C, terminating workers.')
game.stop(game_pid.value)
pool.terminate()
pool.join()
except Exception as err:
self.log.error('\nMain process err: %s ' % err)
pool.terminate()
pool.join()
else:
pool.close()
pool.join()
finally:
m.shutdown()
self.log.info('Finished processing.\n' +
'Main process worked for %.2f seconds' %
(timeit.default_timer() - t0))
class IntegrationManager:
def __init__(self, db_path, nmergers, batch_size):
# Establish database connection
self.db_path = db_path
self.engine = create_engine(self.db_path)
self.session = sessionmaker(bind=self.engine)()
self.batch_size = batch_size
# Establish process management
self.nmergers = nmergers
self.mergers = []
self.manager = Manager()
self.sequenceq = self.manager.Queue(batch_size * 2)
self.mergedq = self.manager.Queue(batch_size * 10)
self.term_sig = self.manager.Event()
self.hold_sigs = [self.manager.Event() for m in range(nmergers)]
def gather_sequences(self):
print("-- Gathering sequences to parse --")
return [
q[0]
for q in self.session.query(PSM.base_sequence).distinct().all()
]
def start_mergers(self):
print("-- Creating and starting {} mergers --".format(self.nmergers))
for i in range(self.nmergers):
m = PSMMerger(self.db_path, self.sequenceq, self.mergedq,
self.term_sig, self.hold_sigs[i])
self.mergers.append(Process(target=m.run))
[m.start() for m in self.mergers]
def flush_to_database(self):
results = []
while True:
try:
results.append(self.mergedq.get(True, 1))
except Empty:
break
try:
self.session.add_all(results)
self.session.commit()
except OperationalError:
raise
self.session.rollback()
except IntegrityError:
self.session.rollback()
for pep in results:
print(pep)
for mod in pep.modifications:
print(mod)
print("-- Got {} results --".format(len(results)))
def shutdown(self):
print("-- Attempting shutdown -- ")
self.term_sig.set()
for m in self.mergers:
m.join()
self.manager.shutdown()
def run(self):
sequences = self.gather_sequences()
self.start_mergers()
for batchn, ind in enumerate(range(0, len(sequences),
self.batch_size)):
print("-- Processing batch {} --".format(batchn))
for seq in sequences[ind:min(len(sequences), ind +
self.batch_size)]:
self.sequenceq.put(seq)
processes_held = [False]
while not all(processes_held) and not self.sequenceq.empty():
processes_held = [h.is_set() for h in self.hold_sigs]
print("-- Flushing batch {} --".format(batchn))
self.flush_to_database()
self.shutdown()
class Server:
def __init__(self):
self.static_routes: dict = {}
self.dynamic_routes = []
self.pool: Optional[ProcessPoolExecutor] = None
self.proc_env = ProcEnv()
template_404 = SimpleTemplate("<h1>Path or file not found: {}</h1>")
template_500 = SimpleTemplate("<h1>Server error in {}</h1><p>{}")
template_503 = SimpleTemplate(
"<h1>Server timed out after {} seconds in {}</h1>")
def error_404(self, request: Request) -> Response:
"""
Built-in 404: Not Found error handler.
"""
return Response(self.template_404.render(request.path), code=404)
def error_500(self, request: Request, error: Exception) -> Response:
"""
Built-in 500: Server Error handler.
"""
return Response(
self.template_500.render(request.path, str(error)),
code=500,
)
def error_503(self, request: Request) -> Response:
"""
Built-in 503: Server Timeout handler.
"""
return Response(
self.template_503.render(DEFAULT_TIMEOUT, request.path),
code=503,
)
def route(
self,
path: str,
route_type: RouteType = RouteType.pool,
action: Union[Iterable, str] = "GET",
before=None,
after=None,
):
"""
Route decorator, used to assign a route to a function handler by wrapping the function. Accepts a `path`, an optional `route_type`, and an optional list of HTTP verbs (or a single verb string, default "GET") as arguments.
"""
parameters = []
route_regex = None
path_match = re.finditer(path_re, path)
for n in path_match:
parameters.append(n.group(0)[1:-1])
if parameters:
route_regex = re.compile(re.sub(path_re_str, "(.*?)", path))
if isinstance(action, str):
action = [action]
def decorator(callback):
if route_regex:
for _ in action:
self.add_dynamic_route(route_regex, _, callback,
route_type, parameters)
else:
for _ in action:
self.add_route(path, _, callback, route_type)
return callback
return decorator
def add_route(
self,
path: str,
action: str,
callback: Callable,
route_type: RouteType = RouteType.pool,
):
"""
Assign a static route to a function handler.
"""
route = (callback, route_type)
if not self.static_routes.get(path):
self.static_routes[path] = {action: route}
else:
self.static_routes[path][action] = route
def add_dynamic_route(
self,
regex_pattern,
action: str,
callback: Callable,
route_type: RouteType = RouteType.pool,
parameters: list = None,
):
"""
Assign a dynamic route (with wildcards) to a function handler.
"""
self.dynamic_routes.append(
(regex_pattern, action, callback, route_type, parameters))
@classmethod
def run_route_pool(cls, raw_env: bytes, func: Callable, *a, **ka):
"""
Execute a function synchronously in the local environment. A copy of the HTTP request data is passed automatically to the handler as its first argument.
"""
local_env = Request(raw_env)
result = func(local_env, *a, **ka)
if isinstance(result, Response):
return result.as_bytes()
return result
@classmethod
def run_route_pool_stream(cls, remote_queue: Queue, signal, raw_env: bytes,
func: Callable, *a, **ka):
"""
Execute a function synchronously in the process pool, and return results from it incrementally.
"""
local_env = Request(raw_env)
for _ in func(local_env, *a, **ka):
if signal.is_set():
raise ParentProcessConnectionAborted
remote_queue.put(_)
remote_queue.put(None)
async def start_server(self, host: str, port: int):
"""
Launch the asyncio server with the master connection handler.
"""
self.srv = await asyncio.start_server(self.connection_handler, host,
port)
async with self.srv: # type: ignore
_e(f"Listening on {host}:{port}")
await self.srv.serve_forever()
def run(
self,
host: str = "localhost",
port: int = 8000,
workers: Union[bool, int, None] = True,
):
"""
Run pixie_web on the stated hostname and port.
"""
_e("Pixie-web 0.1")
if workers is not None:
if workers is True:
self.use_process_pool()
elif workers is False:
pass
else:
self.use_process_pool(int(workers))
try:
asyncio.run(self.start_server(host, port))
except KeyboardInterrupt:
_e("Closing server with ctrl-C")
except asyncio.CancelledError:
_e("Closing due to internal loop shutdown")
@classmethod
def pool_start(cls):
"""
Launched at the start of each pooled process. This modifies the environment data in the process to let any routes running in the process know that it's in a pool, not in the main process.
"""
proc_env.proc_type = ProcessType.pool
def use_process_pool(self, workers: Optional[int] = None):
"""
Set up the process pool and ensure it's running correctly.
"""
self.mgr = Manager()
self.pool = ProcessPoolExecutor(max_workers=workers,
initializer=Server.pool_start)
from concurrent.futures.process import BrokenProcessPool
try:
self.pool.submit(dummy).result()
except (OSError, RuntimeError, BrokenProcessPool):
_e("'run()' function must be invoked from within 'if __name__ == \"__main__\"' block to invoke multiprocessing. Defaulting to single-process pool."
)
self.pool = None
else:
_e(f"Using {self.pool._max_workers} processes") # type: ignore
self.proc_env.pool = self.pool
def close_server(self):
if self.srv is None:
raise Exception(
"No server to close on this instance. Use `ProcessType.main_async` to route the close operation to the main server."
)
self.srv.close()
self.srv = None
def application(self, environ, start_response):
path = environ["REQUEST_URI"]
verb = environ["REQUEST_METHOD"]
try:
handler, route_type = self.static_routes[path][verb]
except KeyError:
for route in self.dynamic_routes:
if verb != route[1]:
continue
route_match = route[0].fullmatch(path)
if route_match:
handler, route_type = route[2:4]
parameters = route_match.groups()
if (not self.dynamic_routes) or (not route_match):
response = Response(f"Not found: {path}", code=404)
start_response(*response.start_response())
return [response.body]
else:
parameters = []
if route_type is RouteType.sync:
result: Union[bytes, Response,
str] = handler(Request(environ), *parameters)
if result is None:
start_response("200 OK", [("Content-Type", "text/plain")])
return [b""]
elif isinstance(result, Response):
start_response(*result.start_response())
return [result.body]
elif isinstance(result, SimpleResponse):
head, body = result.split(b"\r\n\r\n", 1)
response_type, content_type = head.split(b"\r\n", 2)
protocol, code = response_type.split(b" ", 1)
content_type = content_type.split(b": ", 1)
start_response(code, [("Content-Type", content_type[1])])
return [body]
elif isinstance(result, bytes):
raise NotImplementedError(
"Raw bytestream not supported for WSGI; emit Header() first or use Response or SimpleResponse"
)
else:
# iterable, check for first item as header
# for now, not allowed
raise NotImplementedError(
"Iterable not yet supported for WSGI; use Response or SimpleResponse"
)
# have Response object:
# start_response()
# result_iter()
async def connection_handler(self, reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
"""
Reeads the data from the network connection, and attempts to find an appropriate route for it.
"""
readline = reader.readline
get_loop = asyncio.get_event_loop
write = writer.write
drain = writer.drain
at_eof = reader.at_eof
wait_for = asyncio.wait_for
close = writer.close
AsyncTimeout = asyncio.TimeoutError
run_in_executor = get_loop().run_in_executor
while True:
action = raw_data = signal = content_length = None
while True:
_ = await readline()
if at_eof():
close()
return
if raw_data is None:
raw_data = bytearray(_)
action = _.decode("utf-8").split(" ")
continue
else:
raw_data.extend(_)
if _ in (b"\r\n", b"\n"):
break
if _.startswith(b"Content-Length:"):
content_length = int(_.decode("utf-8").split(":")[1])
if content_length:
raw_data.extend(await reader.read(content_length))
path = action[1].split("?", 1)[0]
verb = action[0]
try:
handler, route_type = self.static_routes[path][verb]
except KeyError:
for route in self.dynamic_routes:
if verb != route[1]:
continue
route_match = route[0].fullmatch(path)
if route_match:
handler, route_type = route[2:4]
parameters = route_match.groups()
if (not self.dynamic_routes) or (not route_match):
write(self.error_404(Request(raw_data)).as_bytes())
await drain()
continue
else:
parameters = []
try:
# Run with no pooling or async, in default process.
# Single-threaded, potentially blocking.
if route_type is RouteType.sync:
result = handler(Request(raw_data), *parameters)
# Run a sync function in an async thread (cooperative multitasking)
elif route_type is RouteType.sync_thread:
result = await run_in_executor(None, handler,
Request(raw_data),
*parameters)
# Run async function in default process.
# Single-threaded, nonblocking.
elif route_type is RouteType.asnc:
result = await handler(Request(raw_data), *parameters)
# Run non-async code in process pool.
# Multi-processing, not blocking.
# Note that we pass `Server.run_route_pool`, not `self.run_route_pool`, because otherwise we can't correctly pickle the object. So we just use the class method that exists in the pool instance, since it doesn't need `self` anyway. If we DID need `self` over there, we could always get the server instance from the module-local server obj.
elif route_type is RouteType.pool:
result = await wait_for(
run_in_executor(
self.pool,
Server.run_route_pool,
raw_data,
handler,
*parameters,
),
DEFAULT_TIMEOUT,
)
# Run incremental stream, potentially blocking, in process pool
elif route_type is RouteType.stream:
job_queue = self.mgr.Queue()
signal = self.mgr.Event()
job = self.pool.submit(
Server.run_route_pool_stream,
job_queue,
signal,
raw_data,
handler,
*parameters,
)
writer.transport.set_write_buffer_limits(0)
# We can't send an async queue object to the subprocess,
# so we use a manager queue and poll it every .1 sec
while True:
while True:
try:
_ = job_queue.get_nowait()
except EmptyQueue:
await asyncio.sleep(0.1)
continue
else:
break
if _ is None:
break
write(_)
await drain()
writer.close()
return
except FileNotFoundError:
result = self.error_404(Request(raw_data))
except AsyncTimeout:
result = self.error_503(Request(raw_data))
except Exception as err:
result = self.error_500(Request(raw_data), err)
try:
if result is None:
write(simple_response(b""))
elif isinstance(result, Response):
write(result.as_bytes())
elif isinstance(result, SimpleResponse):
write(result)
elif isinstance(result, bytes):
write(result)
else:
for _ in result:
write(_)
await drain()
writer.close()
return
await drain()
except ConnectionAbortedError:
if signal:
signal.set()
writer.close()
return
def start_crawler(master_browser=initialise_remote_browser,
child_browser=initialise_remote_browser):
global ARGS
######################################################
# Attaining arguments and configurations for crawler #
######################################################
LOGGER.info('Reading arguments and configurations for crawl session...')
ARGS = read_arguments()
config = get_config()
downloader_count = int(config['MULTIPROCESSING']['Downloaders'])
extractor_count = int(config['MULTIPROCESSING']['Extractors'])
crawl_id = init_crawl_id()
crawlers = init_enabled_crawlers()
# Validate configurations
if downloader_count <= 0 or extractor_count <= 0:
LOGGER.error('Number of processes should be more than 0!')
raise ValueError('Number of processes should be more than 0!')
return
if not crawlers:
LOGGER.error('No crawlers enabled, ending crawl...')
return
LOGGER.info('Arguments and configurations initialised!')
notification = {'crawl_id': crawl_id.hex, 'type': ARGS.type}
oneway_publish('crawler', notification, 'crawler.event.start')
#######################################################
# Initialise sessions needed for authenticated crawls #
#######################################################
LOGGER.info('Preparing sessions for authenticated crawls...')
browser = master_browser()
# Get cookies for crawlers
cookies = {}
try:
for crawler in crawlers:
if crawler.signin(browser):
sleep(
3
) # Quick hack: Wait for all activities on browser to complete
auth_cookies = crawler.extract_auth_cookies(browser)
for key in auth_cookies.keys():
cookies[key] = auth_cookies[key]
browser.close()
browser.quit()
except Exception as e:
raise e
LOGGER.info('Sessions for authenticated crawls prepared!')
#############################
# Preparing crawl processes #
#############################
LOGGER.info('Initialising processes and variables for crawl session...')
try:
patch_autoproxy()
## MANAGER START ##
manager = Manager()
fqdn_metadata = manager.dict()
rate_limiters = manager.dict()
db_conn_init()
for crawler in crawlers:
# Add to queued links
crawler.crawl_id = crawl_id.hex
add_queued_links(crawler)
for fqdn in crawler.fqdns_in_scope:
# Initialise variables
fqdn_metadata[fqdn] = manager.dict({
'class': crawler.__class__,
'robots_txt': None
})
rate_limiters[fqdn] = manager.Event()
if crawler.fqdn == fqdn:
if crawler.robots_url:
robots = RobotFileParser(crawler.robots_url)
robots.read()
fqdn_metadata[fqdn]['robots_txt'] = robots
# Initialise rate limiters
rl = RateLimiterThread(crawler.requests_per_sec,
rate_limiters[fqdn])
rl.daemon = True
rl.start()
db_conn_kill()
# Synchronisation events
start_event = manager.Event()
terminate_time = datetime.now() + timedelta(seconds=ARGS.duration * 60)
# Processes
downloader_procs = []
for _ in range(downloader_count):
downloader = Downloader(crawl_id, child_browser, terminate_time,
cookies, fqdn_metadata, rate_limiters)
downloader.start()
downloader_procs.append(downloader)
extractor_procs = []
for _ in range(extractor_count):
extractor = Extractor(crawl_id, start_event, terminate_time,
fqdn_metadata)
extractor.start()
extractor_procs.append(extractor)
for downloader in downloader_procs:
downloader.browser_started.wait()
processes = downloader_procs + extractor_procs
except Exception as e:
raise e
LOGGER.info('Processes and variables initialised for crawl session!')
##########################################
# Start crawl and sleep until time is up #
##########################################
LOGGER.info(
f'Crawl {crawl_id.hex} starting, come back in {ARGS.duration} minute(s)!'
)
start_event.set()
sleep((terminate_time - datetime.now()).total_seconds() + 5)
LOGGER.info(f'Crawl {crawl_id.hex} completed! Cleaning up...')
##########################################
# Clean up and wait for processes to end #
##########################################
# Give processes time to finish
sleep(15)
# Clean up database
db_conn_init()
DownloadedDocument.objects(crawl_id=crawl_id.hex).delete()
QueuedLink.objects(crawl_id=crawl_id.hex).delete()
db_conn_kill()
# Forcefully terminate if required
forceful_terminations = 0
for process in processes:
try:
process.browser.quit()
process.browser.close()
except:
pass
if process.is_alive():
forceful_terminations += 1
process.terminate()
process.join()
process.close()
LOGGER.info(
f'{forceful_terminations} process(es) had to be forcefully terminated.'
)
oneway_publish('crawler', notification, 'crawler.event.end')
def _init_camera_instance(self):
manager = Manager()
self.info_queue = manager.Queue()
self.event_cap_save = manager.Event()
self.event_cap_show = manager.Event()
self.event_cap_close = manager.Event()
self.shm_show_lists = []
self._cur_save_dict = manager.dict()
self.cap_cfgs = self.camera_cfg.values()
self.event_cap_close.clear()
self.event_cap_save.clear()
self.event_cap_show.clear()
pool = Pool(self._camera_cnt)
for idx, cap_cfg in enumerate(self.cap_cfgs):
manager_dict = {
"info_queue": self.info_queue,
'save_cfg': self.video_cfg,
'event_cap_save': self.event_cap_save,
'event_cap_show': self.event_cap_show,
'event_cap_close': self.event_cap_close,
'shm_list': None,
'save_dict': self._cur_save_dict,
}
if cap_cfg.type == 'USBRSYNC':
assert isinstance(cap_cfg.instance_idx, list)
instance_cnt = len(cap_cfg.instance_idx)
for _ in range(instance_cnt):
self.shm_show_lists.append(manager.list())
manager_dict['shm_list'] = self.shm_show_lists[-instance_cnt:]
elif cap_cfg.type in ['USB', 'RTSP']:
# assert isinstance(cap_cfg.instance_idx, int)
self.shm_show_lists.append(manager.list())
manager_dict['shm_list'] = self.shm_show_lists[-1]
else:
raise ValueError(cap_cfg.type)
pool.apply_async(video_cap_thread, args=(idx, cap_cfg, manager_dict))
pool.close()
self.pool = pool
# create thread to display
self.th_list = []
# for i in range(self._camera_cnt):
qpid_cnt = 0
for _, cap_cfg in enumerate(self.cap_cfgs):
flip = cap_cfg['flip']
scale_ratio = cap_cfg['scale_ratio']
if cap_cfg.type == 'USBRSYNC':
instance_cnt = len(cap_cfg.instance_idx)
elif cap_cfg.type in ['USB', 'RTSP']:
instance_cnt = 1
else:
instance_cnt = 0
for _ in range(instance_cnt):
th = QThreadDisplay(qpid_cnt, self.shm_show_lists[qpid_cnt], self.event_cap_close, self.info_queue, scale_ratio, flip)
th.dis_signal.connect(eval(f"self.label_img{qpid_cnt}").setPixmap)
self.th_list.append(th)
qpid_cnt += 1
for th in self.th_list:
th.start()
def run(self):
workers = []
if self.agent.id_supporters:
manager = Manager()
if self.agent.id_leader == None:
self.energy_list = manager.list(range(self.config.num_agents))
for i in range(0, self.config.num_agents):
self.energy_list[i] = 0
self.stop_event = manager.Event()
stop_event = manager.Event()
for i in range(0, self.config.num_sup):
if self.agent.id_leader == None:
workers.append(
WorkerProcess(self.agent.id_supporters[i],
self.support_recv, self.support_send[i],
self.agent_div_recv,
self.support_div_recv[i],
self.support_reset_recv[i],
self.support_reset_send[i], self.config,
self.sequence, self.hist_obj,
self.energy_list, stop_event,
self.results_path))
else:
workers.append(
WorkerProcess(self.agent.id_supporters[i],
self.support_recv, self.support_send[i],
self.root_div_send,
self.support_div_recv[i],
self.support_reset_recv[i],
self.support_reset_send[i], self.config,
self.sequence, self.hist_obj,
self.energy_list, stop_event,
self.results_path))
for worker in workers:
worker.start()
else:
if self.agent.id_leader == None:
self.energy_list = [0]
self.stop_event = Event()
jump_radius_aux = self.config.test_jump_dist
self.agent.current.init_solution(self.hist_obj)
self.agent.update()
print 'WorkerProcess %d: \n%s' % (self.id, self.agent)
start_process_time = datetime.datetime.now()
self.agent.generation = 1
best_energy = self.agent.pockets[0].energy_value
gens_without_improve = 0
gens_convergence = self.config.test_noimprove
gens_start = 0
restart_successed = True
restarts_failed = 0
energy_calls = sum(self.energy_list)
self.agent.status_log_append(
datetime.datetime.now() - start_process_time, energy_calls)
while (self.stop_event.is_set() == False):
# Crossover it isn't allowed to execute on agent 0
if self.agent.id_leader != None:
if self.agent.leader_pockets[0] != None:
index_pocket_leader_agent = self.fitness_roulette_selection(
self.agent.leader_pockets)
index_pocket_self_agent = self.select_rand_solution(
self.agent.pockets)
self.agent.crossover(
self.agent.leader_pockets[index_pocket_leader_agent],
self.agent.pockets[index_pocket_self_agent],
self.config.crossover_prob)
else:
index_pocket_self_agent = self.select_rand_solution(
self.agent.pockets)
self.agent.current = copy.deepcopy(
self.agent.pockets[index_pocket_self_agent])
# Local search
time_ls_start = datetime.datetime.now()
self.agent.simulated_annealing(self.config.ls_prob_ss,
self.config.test_ls_fact,
self.config.test_jump_prob,
jump_radius_aux,
self.config.test_temp_init,
self.hist_obj)
self.agent.time_ls += datetime.datetime.now() - time_ls_start
jump_radius_aux = jump_radius_aux * self.config.test_jump_fact
updated = self.agent.update()
# Update pockets with supporter data
if self.agent.id_supporters:
while not self.support_recv.empty():
self.receive_solution_pickle(self.support_recv, True)
print '>> WorkerProcess %d receive a pocket from supporters, pocket list: %s' % (
self.id, self.agent.pockets)
# Update pocket_leader with leader data
if self.agent.id_leader != None:
if not self.leader_recv.empty():
self.receive_solution_pickle(self.leader_recv, False)
print '>> WorkerProcess %d receive a list of pockets from leader %d' % (
self.id, self.agent.id_leader)
if updated or self.agent.update():
# Send pocket_leader with leader data
if self.agent.id_supporters:
for i in range(0, self.config.num_sup):
if not self.support_send[i].full():
print '> WorkerProcess %d send a list of pockets to supporter %d' % (
self.id, self.agent.id_supporters[i])
self.send_solution_pickle(self.agent.pockets,
self.support_send[i])
# Send pockets with supporter data
if self.agent.id_leader != None:
if self.agent.pockets[0].energy_value < best_energy:
if not self.leader_send.full():
print '> WorkerProcess %d send a pocket to leader %d with energy: %d' % (
self.id, self.agent.id_leader,
self.agent.pockets[0].energy_value)
self.send_solution_pickle(self.agent.pockets[0],
self.leader_send)
if self.config.calculate_div_density:
# Diversity density calculations
time_div_start = datetime.datetime.now()
if self.agent.id_leader == None:
for i in range(0, self.config.num_agents - 1):
if not self.agent_div_recv[i].empty():
buff = self.agent_div_recv[i].get()
agent_pockets = pickle.loads(buff)
j = 0
for p in agent_pockets:
if p != None:
self.agent.population_pockets[i][
j] = copy.deepcopy(p)
else:
break
j += 1
if self.agent.id_supporters:
for i in range(0, self.config.num_sup):
if not self.support_div_recv[i].empty():
buff = self.support_div_recv[i].get()
supporter_pockets = pickle.loads(buff)
j = 0
for p in supporter_pockets:
if p != None:
self.agent.supporter_pockets[i][
j] = copy.deepcopy(p)
else:
break
j += 1
else:
if not self.root_div_send[self.id - 1].full():
buff = pickle.dumps(self.agent.pockets, 2)
self.root_div_send[self.id - 1].put(buff)
if not self.leader_div_send.full():
buff = pickle.dumps(self.agent.pockets, 2)
self.leader_div_send.put(buff)
if self.agent.id_supporters:
for i in range(0, self.config.num_sup):
if not self.support_div_recv[i].empty():
buff = self.support_div_recv[i].get()
supporter_pockets = pickle.loads(buff)
j = 0
for p in supporter_pockets:
if p != None:
self.agent.supporter_pockets[i][
j] = copy.deepcopy(p)
else:
break
j += 1
self.agent.calculate_densities()
self.agent.time_div += datetime.datetime.now() - time_div_start
self.agent.generation += 1
# Reset control
if self.config.if_reset:
if self.agent.id_leader == None:
if self.agent.pockets[0].energy_value == best_energy:
gens_without_improve += 1
else:
gens_without_improve = 0
if gens_without_improve == gens_convergence:
if self.agent.id_supporters:
for i in range(0, self.config.num_sup):
self.support_reset_send[i].put(0)
for i in range(0, self.config.num_sup):
last_solution = pickle.loads(
self.support_reset_recv[i].get())
if last_solution.energy_value < best_energy:
restarts_failed += 1
restart_successed = False
self.agent.update(last_solution)
best_energy = self.agent.pockets[
0].energy_value
gens_without_improve = 0
if restart_successed:
print '\n***Restart successed***\n'
self.event_restart.set()
if self.agent.id_supporters:
for i in range(0, self.config.num_sup):
self.support_reset_send[i].put(True)
else:
print '\n***Restart failed: %d***\n' % restarts_failed
if self.agent.id_supporters:
for i in range(0, self.config.num_sup):
self.support_reset_send[i].put(False)
restart_successed = True
else:
if not self.leader_reset_recv.empty():
self.leader_reset_recv.get()
if self.agent.id_supporters:
for i in range(0, self.config.num_sup):
self.support_reset_send[i].put(0)
for i in range(0, self.config.num_sup):
self.receive_solution_pickle(
self.support_reset_recv[i], True)
self.send_solution_pickle(self.agent.pockets[0],
self.leader_reset_send)
restart_successed = self.leader_reset_recv.get()
if restart_successed:
self.event_restart.set()
if self.agent.id_supporters:
for i in range(0, self.config.num_sup):
self.support_reset_send[i].put(True)
else:
if self.agent.id_supporters:
for i in range(0, self.config.num_sup):
self.support_reset_send[i].put(False)
restart_successed = True
# Is event restart set?
if self.event_restart.is_set():
if self.agent.id_leader == None:
# Only the root leader can keep the best solution
self.agent.pockets = [self.agent.pockets[0]] + [
None for i in range(1, self.config.num_pockets)
]
self.agent.population_pockets = [[
None for i in range(0, self.config.num_pockets)
] for i in range(1, self.config.num_agents)]
for i in range(0, self.config.num_agents - 1):
if not self.agent_div_recv[i].empty():
self.agent_div_recv[i].get()
else:
self.agent.pockets = [
None for i in range(0, self.config.num_pockets)
]
self.agent.leader_pockets = [
None for i in range(0, self.config.num_pockets)
]
if not self.leader_recv.empty():
self.leader_recv.get()
if self.agent.id_supporters:
while not self.support_recv.empty():
self.support_recv.get()
self.agent.supporter_pockets = [[
None for i in range(0, config.num_pockets)
] for i in range(1, config.num_sup + 1)]
for i in range(0, self.config.num_sup):
if not self.support_div_recv[i].empty():
self.support_div_recv[i].get()
self.agent.restarts += 1
print 'RESTARTING %3d - WorkerProcess %2d - %s' % (
self.agent.restarts, self.id, self.agent)
self.agent.current.init_solution(self.hist_obj)
self.agent.update()
jump_radius_aux = self.config.test_jump_dist
gens_convergence = self.config.test_noimprove + self.agent.generation - gens_convergence - gens_start
gens_start = self.agent.generation
gens_without_improve = 0
self.event_restart.clear()
print 'RESTARTED %3d - WorkerProcess %2d - %s' % (
self.agent.restarts, self.id, self.agent)
self.energy_list[self.id] = self.agent.current.energy_calls
energy_calls = sum(self.energy_list)
self.agent.status_log_append(
datetime.datetime.now() - start_process_time, energy_calls)
if self.agent.id_leader == None:
if energy_calls > self.config.energy_limit:
self.stop_event.set()
best_energy = self.agent.pockets[0].energy_value
if self.agent.id_supporters:
stop_event.set()
if self.agent.id_leader != None:
self.leader_send.cancel_join_thread()
self.root_div_send[self.id - 1].cancel_join_thread()
self.leader_div_send.cancel_join_thread()
self.save_results()
if self.agent.id_supporters:
for worker in workers:
worker.join()
print '\n************ WorkerProcess %d done ************\n' % (self.id)
def __run_merge(self):
"""
Runs the merge.
:return:
"""
self.__set_status(self.RUNNING)
self.__master = None
selected_scans = self.selected_ids
matched_scans = self.__matched_scans
output_files = self.summary()
master_f = output_files['master']
del output_files['master']
scans = {
scan_id: {
'image': matched_scans[scan_id]['image'],
'output': output_files[scan_id]
}
for scan_id in selected_scans
}
print('Merging scan IDs : {}.' ''.format(', '.join(self.selected_ids)))
try:
manager = Manager()
self.__term_evt = term_evt = manager.Event()
self.__shared_progress = mp_sharedctypes.RawArray(
ctypes.c_int32, len(scans))
master_f = os.path.join(self.__output_dir, master_f)
if not self.__overwrite:
mode = 'w-'
else:
mode = 'w'
# trying to access the file (erasing it if necessary)
with XsocsH5.XsocsH5MasterWriter(master_f, mode=mode):
pass
if self.__n_proc is None:
n_proc = cpu_count()
else:
n_proc = self.__n_proc
def init(term_evt_, shared_progress_):
global g_term_evt
global g_shared_progress
g_term_evt = term_evt_
g_shared_progress = shared_progress_
# setting progress to 0
np.frombuffer(self.__shared_progress, dtype='int32')[:] = 0
pool = Pool(n_proc,
initializer=init,
initargs=(term_evt, self.__shared_progress),
maxtasksperchild=2)
def callback(result_):
scan, finished, info = result_
print('{0} finished.'.format(scan))
if not finished:
term_evt.set()
results = {}
self.__proc_indices = proc_indices = {}
for proc_idx, (scan_id, infos) in enumerate(scans.items()):
args = (scan_id, proc_idx, self.__spec_h5, self.__output_dir,
infos['output'], infos['image'], self.beam_energy,
self.chan_per_deg, self.center_chan, self.compression)
results[scan_id] = pool.apply_async(_add_edf_data,
args,
callback=callback)
proc_indices[scan_id] = proc_idx
pool.close()
self.__proc_indices = proc_indices
pool.join()
proc_results = [result.get() for result in results.values()]
proc_codes = np.array(
[proc_result[1] for proc_result in proc_results])
rc = self.DONE
if not np.all(proc_codes == self.DONE):
if self.ERROR in proc_codes:
rc = self.ERROR
elif self.CANCELED in proc_codes:
rc = self.CANCELED
else:
raise ValueError('Unknown return code.')
if rc == self.DONE:
with XsocsH5.XsocsH5MasterWriter(master_f, mode='a') as m_h5f:
items = scans.items()
for proc_idx, (scan_id, infos) in enumerate(items):
entry_fn = infos['output']
entry = entry_fn.rpartition('.')[0]
m_h5f.add_entry_file(entry, entry_fn)
self.__set_status(rc)
except Exception as ex:
self.__set_status(self.ERROR, str(ex))
else:
self.__results = master_f
self.prefix = None
self.__master = master_f
# TODO : catch exception?
if self.__callback:
self.__callback()
return self.__results
def add_routes(app):
app.router.add_static('/static', path=PROJECT_ROOT)
app.router.add_get('/remote', store_handlers.get_remote_manifest)
app.router.add_get('/local', store_handlers.get_local_manifest)
app.router.add_get('/modules/{module}/{version}/readme',
store_handlers.get_module_readme)
app.router.add_post('/install', install_module)
app.router.add_post('/uninstall', store_handlers.uninstall_module)
app.router.add_get('/installstream', get_install_stream)
if __name__ == '__main__':
manager = Manager()
INSTALL_STATE = manager.dict()
SSE_UPDATE_CONDITION = manager.Condition()
SSE_UPDATE_EVENT = manager.Event()
INSTALL_STATE['stage'] = ''
INSTALL_STATE['message'] = ''
INSTALL_STATE['module_name'] = ''
INSTALL_STATE['module_version'] = ''
INSTALL_STATE['cur_chunk'] = 0
INSTALL_STATE['total_chunks'] = 0
INSTALL_STATE['cur_size'] = 0
INSTALL_STATE['total_size'] = 0
INSTALL_STATE['update_time'] = time.time()
install_worker = Process(target=install_from_queue,
args=(INSTALL_QUEUE, INSTALL_STATE,
SSE_UPDATE_EVENT))
install_worker.start()
app = web.Application()
class WorkProcessor:
"""
Work processor responsible for starting PoW generation on different threads
and processing the results. Runs on its own thread.
"""
def __init__(self, process_count, work_lock, work_units):
self.process_count = int(process_count)
self.work_lock = work_lock
self.work_units = work_units
self.work_to_generate = OrderedDict()
self.work_block_hash_pools = OrderedDict()
self.manager = None
self.pool = None
def update(self):
"""
Perform a single round of updates:
* Check which work units need to be solved
* Check which work units have been solved
* Start generating PoW for pending work units
"""
self.update_pending_blocks()
self.update_completed_blocks()
self.start_work()
def shutdown(self):
self.shutdown_pool()
def create_pool(self):
self.manager = Manager()
self.pool = concurrent.futures.ThreadPoolExecutor(
max_workers=self.process_count,
thread_name_prefix="work_local_worker")
def shutdown_pool(self):
self.stop_all_workers()
if not self.pool:
# Pool might already be inactive because no PoW was being worked on
return
self.pool.shutdown()
self.pool = None
self.manager.shutdown()
self.manager = None
def update_pending_blocks(self):
"""
Check the owork units for any unfinished blocks
"""
# Collect pending blocks
with self.work_lock:
for work_unit in self.work_units.values():
if not work_unit.solved:
work_block_hash = work_unit.work_block_hash
self.work_to_generate[work_block_hash] = work_unit
def get_all_active_workers(self):
workers = []
for work_block_hash in self.work_block_hash_pools.keys():
workers += self.get_active_workers(work_block_hash)
return workers
def get_active_workers(self, work_block_hash):
if not self.work_block_hash_pools.get(work_block_hash, None):
return []
return [
worker for worker in self.work_block_hash_pools[work_block_hash]
["workers"] if not worker.done()
]
def get_block_hash_workers(self, work_block_hash):
if not self.work_block_hash_pools.get(work_block_hash, None):
return []
return [
worker for worker in self.work_block_hash_pools[work_block_hash]
["workers"]
]
def start_worker(self, work_unit):
"""
Start a single worker for a work unit
:param work_unit: Pending work unit
:type work_unit: siliqua.work.WorkUnit
"""
work_block_hash = work_unit.work_block_hash
if not self.work_block_hash_pools.get(work_block_hash, None):
self.work_block_hash_pools[work_block_hash] = {
"work_unit": work_unit,
"shutdown_flag": self.manager.Event(),
"workers": []
}
work_info = self.work_block_hash_pools[work_block_hash]
if not self.pool:
self.create_pool()
work_info["workers"].append(
self.pool.submit(process_work,
work_unit=work_info["work_unit"],
shutdown_flag=work_info["shutdown_flag"]))
def stop_all_workers(self):
"""
Stop all running workers for all work units
"""
# Handle all block hash workers simultaneously, since calling
# 'stop_workers' individually could cause this method to block
# for a long time before all workers finally exit
active_workers = self.get_all_active_workers()
for work_info in self.work_block_hash_pools.values():
work_info["shutdown_flag"].set()
for worker in active_workers:
# Wait until all threads complete
worker.result()
return True
def stop_workers(self, work_block_hash):
"""
Stop all running workers for the given work block hash
:param str work_block_hash: Work block hash
"""
if not self.work_block_hash_pools.get(work_block_hash, None):
return
work_info = self.work_block_hash_pools[work_block_hash]
work_info["shutdown_flag"].set()
# Shutdown flag has been set; wait until all workers exit
for worker in work_info["workers"]:
worker.result()
del self.work_block_hash_pools[work_block_hash]
def update_completed_blocks(self):
"""
Check if work has been completed. Push any completed work into the
queue.
"""
try:
with self.work_lock:
remaining_block_hashes = list(
self.work_block_hash_pools.keys())
for work_block_hash in remaining_block_hashes:
workers = self.get_block_hash_workers(work_block_hash)
found_work = False
done_workers, _ = concurrent.futures.wait(
workers,
timeout=0,
return_when=concurrent.futures.FIRST_COMPLETED)
for worker in done_workers:
workers.remove(worker)
exception = worker.exception(timeout=0)
if exception:
logger.warning("Work thread died unexpectedly.")
else:
completed_work_unit = worker.result(timeout=0)
try:
work_unit = self.work_units[work_block_hash]
except KeyError:
# If the work unit no longer exists, the main
# thread already picked up the finished work
# If so, clear the workers as normal
found_work = True
break
work_unit.work = completed_work_unit.work
logger.info(
"Generated PoW for block %s in account %s",
completed_work_unit.block_hash,
completed_work_unit.account_id)
found_work = True
if found_work:
# If we found work, shutdown all workers for this block hash
del self.work_to_generate[work_block_hash]
self.stop_workers(work_block_hash)
logger.debug("%s PoW(s) left to generate",
len(self.work_to_generate))
except Exception as exc:
logger.info("Error: {} {}".format(str(exc), type(exc)))
def start_work(self):
"""
Check the pending work units and either start worker threads if
work is available, or shutdown the pool entirely if no
work is available
"""
if not self.work_to_generate:
if self.pool:
logger.info("No PoW to generate at this time, "
"shutting down worker pool.")
self.shutdown_pool()
return
if not self.pool:
logger.info("Received %s PoW(s) to generate, starting worker pool",
len(self.work_to_generate))
self.create_pool()
active_worker_count = len(self.get_all_active_workers())
available_worker_count = self.process_count - active_worker_count
work_iter = cycle(self.work_to_generate.values())
for _ in range(0, available_worker_count):
work_unit = next(work_iter)
self.start_worker(work_unit=work_unit)
def main():
parser = argparse.ArgumentParser(
description="Cosmosvanity - Create custom cosmos addresses.")
parser.add_argument(
"--startswith",
type=str,
help="Find an address ending with the provided argument.",
)
parser.add_argument(
"--endswith",
type=str,
help="Find an address starting with the provided argument.",
)
parser.add_argument(
"--contains",
type=str,
help="Find an address containing the provided argument.")
parser.add_argument(
"--letters",
type=int,
help="Find an address containing the provided number of letters.",
)
parser.add_argument(
"--digits",
type=int,
help="Find an address containing the provided number of digits.",
)
parser.add_argument("-n",
type=int,
help="Number of addresses to search for.",
default=1)
args = parser.parse_args()
vanity_args = {}
if args.startswith and _is_valid_bech32(args.startswith):
vanity_args[starts_with] = str(args.startswith)
if args.endswith and _is_valid_bech32(args.endswith):
vanity_args[ends_with] = str(args.endswith)
if args.contains and _is_valid_bech32(args.contains):
vanity_args[contains] = args.contains
if args.letters and args.letters:
vanity_args[letters] = args.letters
if args.digits and args.digits:
vanity_args[digits] = args.digits
with Pool(processes=mp.cpu_count()) as pool:
manager = Manager()
event = manager.Event()
queue = manager.Queue()
for i in range(args.n):
for _ in range(mp.cpu_count()):
pool.apply_async(find_vanity_addr,
args=(vanity_args, event, queue))
vanity_addresses = []
address = None
while not address:
address = queue.get()
vanity_addresses.append(address)
address = None
if len(vanity_addresses) >= args.n:
event.set()
break
for address in vanity_addresses:
print(address)
class MultiProcessBatcher(Iterator):
def __init__(
self,
base_iterable: Iterable,
batch_size: int,
stack_fn: Callable,
num_workers: int,
max_queue_size: Optional[int] = None,
decode_fn: Callable = lambda x: x,
):
assert num_workers >= 1
assert max_queue_size is None or max_queue_size >= num_workers
self.base_iterable = base_iterable
self.batch_size = batch_size
self.stack_fn = stack_fn
self.decode_fn = decode_fn
self.num_workers = num_workers
self.max_queue_size = (max_queue_size if max_queue_size is not None
else 5 * num_workers)
self.manager = Manager()
self.batch_queue = self.manager.Queue(maxsize=self.max_queue_size)
self.terminate_event = self.manager.Event()
self.exhausted_events = [
self.manager.Event() for _ in range(self.num_workers)
]
self.processes = []
for worker_id, event in enumerate(self.exhausted_events):
p = Process(
target=self.worker_fn,
args=(
worker_id,
self.num_workers,
self.base_iterable,
self.batch_size,
self.stack_fn,
self.batch_queue,
self.terminate_event,
event,
),
)
p.start()
self.processes.append(p)
self.count = 0
@staticmethod
def worker_fn(
worker_id: int,
num_workers: int,
iterable: Iterable,
batch_size: int,
stack_fn: Callable,
batch_queue: Queue,
terminate_event,
exhausted_event,
):
MPWorkerInfo.worker_process = True
MPWorkerInfo.worker_id = worker_id
MPWorkerInfo.num_workers = num_workers
for batch in batcher(iterable, batch_size):
stacked_batch = stack_fn(batch)
try:
if terminate_event.is_set():
return
buf = io.BytesIO()
ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(
(worker_id, stacked_batch))
batch_queue.put(buf.getvalue())
except (EOFError, BrokenPipeError):
return
exhausted_event.set()
def __iter__(self):
return self
def __next__(self):
if (all(event.is_set() for event in self.exhausted_events)
and self.batch_queue.empty()):
self._halt_processes()
raise StopIteration
try:
# TODO make timeout configurable
got = self.batch_queue.get(timeout=120)
worker_id, batch = pickle.loads(got)
batch = self.decode_fn(batch)
except Empty:
raise StopIteration()
return batch
def _empty_queue(self):
try:
batch = self.batch_queue.get(block=False)
while batch:
self.batch_queue.get(block=False)
except (Empty, FileNotFoundError):
pass
def _halt_processes(self):
# Send termination message to workers
self.terminate_event.set()
# Empty queue to make sure workers get the message
self._empty_queue()
for p in self.processes:
p.join()
if self.waitingEvent.is_set():
self.waitingEvent.clear()
self.waitingEvent.wait()
def f(queueLock, idleSemaphore, waitingEvent, queue, time, i):
s = ShiftQueue(queueLock, idleSemaphore, waitingEvent, queue, time, i)
s.setShift(3)
s.setIdle()
assert(s.getTime() == 3)
s.setShift(9)
s.setShift(4)
s.setIdle()
assert(s.getTime() == 7)
s.setIdle()
assert(s.getTime() == 12)
return
if __name__ == '__main__':
manager = Manager()
queueLock = manager.Lock()
idleSemaphore = manager.Semaphore(NUM_MASTER - 1)
waitingEvent = manager.Event()
queue = manager.list()
time = manager.list([0])
processes = [Process(target=f, args=(queueLock, idleSemaphore, waitingEvent, queue, time, i)) for i in range(0, NUM_MASTER)]
for p in processes:
p.start()
for p in processes:
p.join()
class BrokerSimulator(Process):
def __init__(self, grid_info, timeout=5, debug_tags="log|debug|verbose", enable_caching=True):
Process.__init__(self)
self.debug_tags = debug_tags
self.id = "B-{}".format(ShortId().generate())
# Store the grid info for this broker
self.grid = grid_info
# Instantiate MQTT client
self.verbose("Instantiating MQTT client")
self.client = mqtt.Client()
self.client.on_connect = self.onConnectHandler
self.client.on_message = self.onMessageHandler
# Initialize MQTT variables
self.verbose("Initializing MQTT variables")
self.broker_sub = "broker_{}".format(self.id)
self.cache_sub = "cache_{}".format(self.id)
self.subs_list = [ "all", "broker_general", "cache_general", "cache_index",
self.broker_sub, self.cache_sub ]
# Instantiate a Manager object for variables that need to be multiprocessed
self.verbose("Instantiating Manager object")
self.broker_manager = Manager()
#self.broker_manager.start()
#self.broker_manager.start()
self.active_queries = self.broker_manager.dict()
self.finished_queries = self.broker_manager.dict()
# Instantiate Task Queue objects
self.verbose("Instantiating Task Queue objects")
self.tq_info = AttrDict()
self.tq_info.request = AttrDict()
self.tq_info.response = AttrDict()
self.tq_info.request.queue = self.broker_manager.Queue()
self.tq_info.request.event = self.broker_manager.Event()
self.tq_info.response.queue = self.broker_manager.Queue()
self.tq_info.response.event = self.broker_manager.Event()
self.cache = Cache()
self.proc_nodes = []
self.agg_nodes = []
self.timeout = timeout
self.status = "INITIALIZED"
self.caching_enabled = enable_caching
self.verbose("Started.")
return
def run(self):
self.verbose("Connecting...")
self.client.connect("localhost", 1883, 60)
self.client.loop_forever()
# self.broker_manager.shutdown()
self.verbose("Shutdown.")
return
def onConnectHandler(self, client, userdata, flags, rc):
self.verbose("Connected.")
def launchNodes():
self.verbose("Starting processing node/s")
# Start the task processing nodes
self.tq_info.request.event.clear()
for i in range(0, self.grid.node_count):
self.proc_nodes.append(ProcNodeSimulator(self.tq_info, self.broker_sub))
self.proc_nodes[i].start()
self.verbose("Processing nodes started.")
# Start the aggregation node/s
self.tq_info.response.event.clear()
agg_info = AttrDict()
agg_info.request = AttrDict()
agg_info.request.queue = self.tq_info.response.queue
agg_info.request.event = self.tq_info.response.event
self.agg_nodes.append( AggNodeSimulator( agg_info,
self.active_queries,
self.broker_sub) )
self.verbose("Starting aggregation node/s")
self.agg_nodes[0].start()
self.verbose("Aggregation nodes started.")
return
self.verbose("Launching Nodes in a separate process")
Process(target=launchNodes).start()
# Subscribe to general MQTT topics and own topics
client.subscribe([ (sub, 0) for sub in self.subs_list ])
self.verbose("Subscribed to: {}".format(self.subs_list))
# Announce connection to channel
payload_intro = {
'type' : 'announce',
'class': 'broker',
'id' : self.id, # TODO include other capabilities here too
'grid' : { 'x' : self.grid.x, 'y' : self.grid.y, 'cache_limit' : self.cache.cache_limit, 'latency_map' : self.grid.latency_map.latency_map }
}
self.verbose("Announcement payload: {}".format(json.dumps(payload_intro)))
client.publish("all", json.dumps(payload_intro))
self.verbose("Announcement made.")
return
def onMessageHandler(self, client, userdata, msg):
topic = str(msg.topic)
request = json.loads(msg.payload)
#self.verbose("Received from {}: {}".format(topic, request))
if (request['type'] == 'shutdown') and (topic in ['all', 'broker_general', self.broker_sub]):
self.verbose("Shutting down...")
shutdown_request = { 'type' : 'shutdown' }
self.verbose("Shutting down processing nodes...")
for i in range(0, len(self.proc_nodes)):
self.tq_info.request.queue.put(shutdown_request)
self.tq_info.request.event.set()
self.verbose("Waiting for processing nodes to shutdown...")
for pn in self.proc_nodes:
pn.join()
self.verbose("Shutting down aggregation nodes...")
self.tq_info.response.queue.put(shutdown_request)
self.tq_info.response.event.set()
self.verbose("Waiting for aggregation nodes to shutdown...")
for an in self.agg_nodes:
an.join()
client.disconnect()
elif (request['type'] == 'status') and (topic in ['all', 'broker_general', self.broker_sub]):
rc = self.handleStatusRequest(request, client, topic)
if rc != True:
self.debug("Error Occurred during handling of status request!")
client.disconnect()
elif (request['type'] == 'query') and (topic == self.broker_sub):
rc = self.handleQueryRequest(request, client)
if rc != True:
self.debug("Error Occurred during handling of query request!")
client.disconnect()
elif (request['type'] == 'get_cached_item') and (topic == self.cache_sub):
rc = self.handleGetCachedItemRequest(request, client)
if rc != True:
self.debug("Error Occurred during handling of query request!")
client.disconnect()
elif (request['type'] == 'cache_index_response') and (topic == self.broker_sub):
rc = self.handleQueryCacheIndexResponse(request, client)
if rc != True:
self.debug("Error Occurred during handling of cache index response!")
client.disconnect()
elif (request['type'] == 'get_cached_item_response') and (topic == self.broker_sub):
rc = self.handleQueryCachedItemResponse(request, client)
if rc != True:
self.debug("Error Occurred during handling of cache response!")
client.disconnect()
elif (request['type'] == 'aggregation_result') and (topic == self.broker_sub):
rc = self.handleAggregationResult(request, client)
if rc != True:
self.debug("Error Occurred during handling of aggregation result!")
client.disconnect()
elif (request['type'] == 'cache_reassign') and (topic in ['all', 'broker_general', self.broker_sub]):
rc = self.handleCacheReassign(request, client)
if rc != True:
self.debug("Error Occurred during handling of cache reassign request!")
client.disconnect()
return
def handleQueryRequest(self, query, client):
self.verbose("Handling Query Request")
# Save information about the pending query
self.active_queries[query['id']] = self.broker_manager.dict({
'inputs' : query['inputs'],
'grid' : query['grid'],
'tasks' : query['tasks'],
'load' : query['load'],
'count' : query['tasks']['processing'],
'started' : time.time(),
'ended' : None,
})
if not self.caching_enabled:
self.verbose("Caching disabled. Switching over to full processing...")
query['query_id'] = query['id']
return self.handleQueryProcessing(query, client)
# Check if this query's result can be retrieved from some other cache
request = {
'query_id' : query['id'],
'type' : 'get_cache_list',
'input_key' : query['inputs'],
'broker_sub_id' : self.broker_sub,
}
self.verbose("Sending Cache Index Request: {}".format(request))
client.publish("cache_index_requests", json.dumps(request))
return True
def handleQueryCacheIndexResponse(self, query, client):
self.verbose("Handling Query Cache Index Response")
# Check if this does not have an active query -- if so, disregard it
if not query['query_id'] in self.active_queries.keys():
return False
# [ Expected CacheIndex Response ]
#
# response = {
# 'query_id' : query['id'],
# 'type' : 'cache_index_response',
# 'target_id' : broker id of requestor,
# 'input_key' : key requested,
# 'cache_list' : list of known holders of this info.
# }
#
# If the returned list is empty, then we have to process this on our own
if not query['cache_list']:
self.verbose("Not in cache. Switching over to full processing...")
return self.handleQueryProcessing(query, client)
# Otherwise, attempt to retrieve the result from another cache
target_id = random.choice(query['cache_list'])
target_sub = "cache_{}".format(target_id)
# Reload the input key from the list of known active queries
input_key = self.active_queries[query['query_id']]['inputs']
# Request the result from another cache
request = {
'query_id' : query['query_id'],
'type' : 'get_cached_item',
'resp_sub' : self.broker_sub,
'input_key' : input_key,
}
self.verbose("Requesting cached item from {}...".format(target_sub))
client.publish(target_sub, json.dumps(request))
return True
def handleGetCachedItemRequest(self, query, client):
self.verbose("Handling Get Cached Item Request")
# Context: This is a standalone request for the value of an item
# that is currently cache in this broker. Usually, this is done
# after getting confirmation from the cache index of the cached
# result's location.
# [ Expected Broker Request ]
#
# request = {
# 'query_id' : query['id'],
# 'type' : 'get_cached_item',
# 'resp_sub' : subscription topic of the response,
# 'input_key' : key of result to be retrieved from the cache,
# }
#
# Load the result from the cache
cached_result = self.cache.getItem(query['input_key'])
# Send the result
target_sub = query['resp_sub']
response = {
'query_id' : query['query_id'],
'type' : 'get_cached_item_response',
'cache_sub' : self.cache_sub,
'grid' : {'x' : self.grid.x, 'y' : self.grid.y},
'input_key' : query['input_key'],
'result' : cached_result,
}
client.publish(target_sub, json.dumps(response))
return True
def handleQueryCachedItemResponse(self, query, client):
self.verbose("Handling Get Cached Item Response")
# Context: A cached item was previously requested from a target broker.
# In this function, we process the response of that broker and try
# to retrieve the value of the cached item from it
# Check if this does not have an active query -- if so, disregard it
q_id = query['query_id']
if not q_id in self.active_queries.keys():
return self.handleQueryProcessing(query, client)
# [ Expected Broker Response ]
#
# response = {
# 'query_id' : query['id'],
# 'type' : 'get_cached_item_response',
# 'cache_sub' : self.cache_sub,
# 'grid' : {'x' : self.grid.x, 'y' : self.grid.y},
# 'input_key' : input_key,
# 'result' : cached_result,
# }
#
# If nothing was found, then we handover to normal processing
if query['result'] == None:
# TODO Should this incur any cache retrieval penalties?
return False
# Otherwise, get the latency between the current grid and the cached item source
targ_x = query['grid']['x']
targ_y = query['grid']['y']
cache_retrieval_delay = self.grid.getLatency(targ_x, targ_y)
def delayed_finish_query():
# Simulate the delay with sleep
if cache_retrieval_delay > 0.0:
time.sleep(cache_retrieval_delay)
# Reload the ongoing query's information
query_info = self.active_queries[q_id]
# Add the aggregation result to the finished results
self.verbose("Publishing finished result")
finished_result = {
'type' : 'completion_result',
'assigned': { 'x' : targ_x, 'y' : targ_y },
'origin' : query_info['grid'],
'query_id': q_id,
'inputs' : query_info['inputs'],
'tasks' : query_info['tasks'],
'load' : query_info['load'],
'count' : query_info['count'],
'started' : query_info['started'],
'ended' : time.time(),
'result' : query['result'],
'from_cache' : True,
'cache_retrieval_delay' : cache_retrieval_delay,
}
query_client = mqtt.Client()
query_client.connect("localhost", 1883, 60)
query_client.publish(QUERY_COMPLETION_SUB, json.dumps(finished_result))
# Remove query id from active queries
del self.active_queries[q_id]
self.verbose("Result found after {} secs: {}".format(cache_retrieval_delay, query['result']))
query_client.disconnect()
return
Process(target=delayed_finish_query).start()
return True
# If not, then process it as normal
# Make a note of the active tasks
# Push the tasks to the task queue and set the event flag for it
def handleQueryProcessing(self, query, client):
self.verbose("Handling Query Processing")
# Check if this does not have an active query -- if so, disregard it
if not query['query_id'] in self.active_queries.keys():
return False
# Reload the ongoing query's information
query_info = self.active_queries[query['query_id']]
# Start a number of tasks by putting them on the task queue
self.verbose("Starting tasks for query processing")
self.verbose(" Query: {}".format(query))
self.verbose(" Query Info: {}".format(query_info))
task_count = query_info['count']
for i in range(0, task_count):
task_inputs = {
'type' : 'task',
'query_id' : query['query_id'],
'exec_time' : query_info['load'],
'inputs' : query_info['inputs'],
}
self.tq_info.request.queue.put(task_inputs)
# Tell the processing tasks to start
self.tq_info.request.event.set()
self.verbose("Tasks started")
return True
def handleAggregationResult(self, query, client):
self.verbose("Handling Aggregation Result")
# Check if this does not have an active query -- if so, disregard it
q_id = query['query_id']
if not q_id in self.active_queries.keys():
return False
# Add the aggregation result to the finished results
self.verbose("Publishing finished result")
finished_result = {
'type' : 'completion_result',
'assigned': { 'x' : self.grid.x, 'y' : self.grid.y },
'origin' : { 'x' : self.grid.x, 'y' : self.grid.y },
'query_id': q_id,
'inputs' : query['inputs'],
'grid' : query['grid'],
'tasks' : query['tasks'],
'load' : query['load'],
'count' : query['tasks']['processing'],
'started' : query['started'],
'ended' : query['ended'],
'result' : query['result'],
'from_cache' : False,
'cache_retrieval_delay' : 0.0,
}
client.publish(QUERY_COMPLETION_SUB, json.dumps(finished_result))
# Remove query id from active queries
del self.active_queries[q_id]
# Add the *result* to the cache
self.cache.add( query['inputs'], query['result'])
# Tell the cache index to add a new entry as well
request = {
'type' : 'add',
'input_key' : query['inputs'],
'broker_id' : self.id,
}
self.verbose("Sending Cache Index Request: {}".format(request))
client.publish("cache_index_requests", json.dumps(request))
return True
def handleCacheReassign(self, request, client, topic):
# TODO
nqa_map = request['map']
# Cycle through each input key in the reassigned query map
# for k in self.cache.getKeys():
# # Check if the input key is in our cache, if so
# cached_item = self.cache.getItem(k)
return True
def handleStatusRequest(self, request, client, topic):
response = {'id' : self.id,
'status' : self.status}
# Send the result
client.publish(topic, json.dumps(response))
return True
def log(self, message):
print("[{}] {}".format(self.id, message))
return
def verbose(self, message):
if "verbose" in self.debug_tags:
self.log(message)
return
def debug(self, message):
if "debug" in self.debug_tags:
self.log(message)
return
if __name__ == '__main__':
import sys, os
from time import sleep
from multiprocessing import Manager
from DegenPrimer.Primer import Primer
from DegenPrimer.SeqUtils import load_sequence
from DegenPrimer.WorkCounter import WorkCounterManager
from DegenPrimer.WaitingThread import WaitingThread
from DegenPrimer.TD_Functions import PCR_P
from DegenPrimer.BlastPrimers import BlastPrimers
from threading import Lock
os.chdir('../')
mgr = Manager()
abort_event = mgr.Event()
PCR_P.PCR_T = 53
PCR_P.Mg = 3e-3
PCR_P.dNTP = 300e-6
PCR_P.DNA = 1e-10
fwd_primer = Primer(
load_sequence('ATATTCTACRACGGCTATCC', 'F-TGAM_0057-268_d1',
'F-TGAM_0057-268_d1'), 0.43e-6, True)
rev_primer = Primer(
load_sequence('GAASGCRAAKATYGGGAAC', 'R-TGAM_0055-624-d4',
'R-TGAM_0055-624-d4'), 0.43e-6, True)
blastp = BlastPrimers(abort_event,
job_id='F-TGAM_0057-268_d1-R-TGAM_0055-624-d4',
class TestOwnerPrintingInterProcessFileLock(unittest.TestCase):
def setUp(self):
self.lock_dir = tempfile.mkdtemp()
self.lock_path = os.path.join(self.lock_dir, 'lock')
self.lock = OwnerPrintingInterProcessFileLock(self.lock_path)
self.manager = Manager()
self.lock_held = self.manager.Event()
self.terminate = self.manager.Event()
self.lock_process = Process(
target=hold_lock_until_terminate,
args=(self.lock_path, self.lock_held, self.terminate),
)
def tearDown(self):
self.terminate.set()
try:
shutil.rmtree(self.lock_dir)
except OSError:
pass
def test_non_blocking_attempt(self):
self.lock_process.start()
self.lock_held.wait()
self.assertFalse(self.lock.acquire(blocking=False))
def test_message(self):
self.lock_process.start()
self.lock_held.wait()
self.assertTrue(os.path.exists(self.lock.message_path))
with open(self.lock.message_path, 'rb') as f:
message_content = f.read()
self.assertIn(str(self.lock_process.pid), message_content)
os.unlink(self.lock.message_path)
def message_fn(message):
self.assertIn(self.lock.missing_message_output, message)
self.lock.acquire(blocking=False, message_fn=message_fn)
def test_blocking(self):
self.lock_process.start()
self.lock_held.wait()
self.assertFalse(self.lock.acquire(timeout=.1))
acquire_is_blocking = self.manager.Event()
def terminate_subproc(terminate, acquire_is_blocking):
acquire_is_blocking.wait()
terminate.set()
Thread(target=terminate_subproc,
args=(self.terminate, acquire_is_blocking)).start()
def message_fn(message):
self.assertIn(str(self.lock_process.pid), message)
acquire_is_blocking.set()
# NOTE: We shouldn't ever wait this long (locally this runs in ~milliseconds)
# but sometimes CI containers are extremely slow, so we choose a very large
# value just in case.
self.assertTrue(self.lock.acquire(timeout=30, message_fn=message_fn))
def test_reentrant(self):
self.assertTrue(self.lock.acquire())
self.assertTrue(self.lock.acquire())
def test_release(self):
self.assertTrue(self.lock.acquire())
self.assertTrue(self.lock.acquired)
self.lock.release()
self.assertFalse(self.lock.acquired)
class Control(inLib.Device):
'''
The device control of pco.edge sCMOS camera.
'''
def __init__(self, settings):
inLib.Device.__init__(self, 'cooke.pcoedge.camera_api', settings)
#Camera properties
self._props = {}
#Loaded yaml settings
self.loadedSettings = settings
self.saved_settings = []
self.newSettings('no_file', settings_dict=settings)
#Sensor settings
self._bpp = 14
status, (x0, y0, x1, y1) = self._api.getROI()
self._props['dimensions'] = x1 - x0 + 1, y1 - y0 + 1
self._props['x_start'] = x0
self._props['y_start'] = y0
self._initializedBuffers = 0
self._lastBuffer = 0
self.buffer0_allocd = False
self.buffer1_allocd = False
self._buffersAllocated = np.ones((16), dtype=bool) * False
self._buffersQueued = np.ones((16), dtype=bool) * False
self._buffers = []
self._bufferEvents = []
self._images = np.array([])
self._fillBottom = True
self._alternatingBuffer = 0
self.daxfile = 0
self._api.setRecorderSubmode(1)
self.fastImages = None
#Multiprocessing tests
self.grabFrames = 100
self.emitSignalFrame = 4000
self.initFramesMP = 100
self.subProcessStart = False
self.subProcess2Start = False
self.parent_conn, self.child_conn = Pipe()
self.mgr = Manager()
self.q = self.mgr.Queue()
self.qIms = self.mgr.Queue()
self.resultsQueue = self.mgr.Queue()
self.resultsEvent = self.mgr.Event()
self.p = Process(target=imageAnalysisTest3,
args=(
self.grabFrames,
self.q,
self.resultsQueue,
self.qIms,
))
self.furtherAnalysis = Process(target=useImageAnalysisResults,
args=(
self.emitSignalFrame,
self.resultsQueue,
self.resultsEvent,
))
self.image1 = 0
self.image2 = 100
self.point1 = 0
self.point2 = 0
def newSettings(self, filename, settings_dict=None):
if settings_dict is None:
settings_dict = inLib.load_settings(filename)
if settings_dict.has_key('devices'):
settings_dict = settings_dict['devices']['camera']
settings_dict['settings_filename'] = filename
self.saved_settings = [settings_dict] + self.saved_settings
def updateSettings(self, settings_dict):
try:
roi = settings_dict['roi']
self._setROI(roi)
except:
print "Unable to set new ROI"
try:
exposure_time = settings_dict['exposure_time']
delay_time = settings_dict['delay_time']
self._setDelayExposureTime(delay_time, exposure_time, 'ms', 'ms')
except:
print "Unable to set new timings"
def unSetEvent(self):
self.resultsEvent.clear()
def whenToEmitAndCapture(self):
return self.emitSignalFrame, self.grabFrames
def changeWhenToEmitAndCapture(self, sig, capt):
self.emitSignalFrame = sig
self.grabFrames = capt
def getRecordingStruct(self):
prec = self._api.getRecordingStruct()
return prec
def _setROI(self, ROI):
x0, x1, y0, y1 = ROI
status = self._api.setROI(x0, y0, x1, y1)
def _getROI(self):
status, (x0, y0, x1, y1) = self._api.getROI()
return x0, x1, y0, y1
def _getResolution(self):
status, (x0, y0, x1, y1) = self._api.getROI()
return x1 - x0 + 1, y1 - y0 + 1
def _getDelayExposureTime(self):
err, times = self._api.getDelayExposureTime()
self._props['exposure_time'] = times[1]
return times[0], times[1]
def _setDelayExposureTime(self, delay, exposure, delay_units, exp_units):
if delay_units == 'ns':
delay_base = 0
elif delay_units == 'us':
delay_base = 1
elif delay_units == 'ms':
delay_base = 2
else:
delay_base = 2
if exp_units == 'ns':
exp_base = 0
elif exp_units == 'us':
exp_base = 1
elif exp_units == 'ms':
exp_base = 2
else:
exp_base = 2
status = self._api.setDelayExposureTime(delay, exposure, delay_base,
exp_base)
def _initBuffers(self, num_to_init=16):
xres, yres = self._props['dimensions']
self._buffers = []
for i in range(0, num_to_init):
self._buffers.append(np.zeros((xres * yres), dtype=np.uint16))
for i in range(0, num_to_init):
if self._buffersAllocated[i]:
status, eHandler, self._buffers[i] = self._api.allocateBuffer(
xres, yres, i)
#self._bufferEvents[i] = eHandler
print "Buffer size: ", (i, self._buffers[i].shape)
else:
status, eHandler, self._buffers[i] = self._api.allocateBuffer(
xres, yres, -1)
self._bufferEvents.append(eHandler)
print "Buffer size: ", (i, self._buffers[i].shape)
print "Init buff status: ", (i, status)
print "Event: ", self._bufferEvents[i].value
if status == 0:
self._buffersAllocated[i] = True
def _initImages(self):
self._fillBottom = True
self._images = np.array([])
def _addBuffers(self, start=0, end=16):
xres, yres = self._props['dimensions']
for i in range(start, end):
if self._buffersAllocated[i]:
self._api.addBuffer(0, 0, i, xres, yres, self._bpp)
self._buffersQueued[i] = True
def addManyBuffers(self, total):
self._initBuffers()
i = 0
j = 0
xres, yres = self._props['dimensions']
while i < total:
err = self._api.addBuffer(0, 0, j, xres, yres, self._bpp)
if err != 0:
print "error on ", i
print "j=", j
print "addBuffer error: ", err
i += 1
j += 1
if j == 16:
j = 0
print "Pending buffers: ", self._buffersPending()
def beginDAXRecording(self, filename):
self.daxfile = writer.DaxFile(filename, self._props)
self.recordedFrames = 0
def endDAX(self):
if self.daxfile != 0:
self.daxfile.closeFile([0, 0, 0], 0)
print "Closing DAX file " + str(self.daxfile.filename)
def writeMemoryToDAX(self, filename):
self.beginDAXRecording(filename)
print "Length of self._images: ", len(self._images)
for i in range(0, len(self._images)):
temp = self._images[i].astype(np.dtype('>H'))
self.daxfile.saveFrames(temp.tostring(), 1)
self.recordedFrames += 1
self.endDAX()
def captureFast(self, numImages):
#Stop camera if it is recording
if self._api.getRecordingState() > 0:
self.stopRecording()
else:
self._api.removeBuffer()
xres, yres = self._getResolution()
#First need to create valid events
err, event1, buf1 = self._api.allocateBuffer(xres, yres, -1)
err, event2, buf2 = self._api.allocateBuffer(xres, yres, -1)
#Add buffers to queue
self._api.addBuffer(0, 0, 0, xres, yres, 14)
self._api.addBuffer(0, 0, 1, xres, yres, 14)
#Set recording to on to fill those two buffers:
self._api.camlinkSetParams(xres, yres)
self._api.armCamera()
self._api.setRecordingState(0x0001)
#Wait for 2nd buffer to fill, then close
win32event.WaitForSingleObject(
event1.value, 5000) #Waits for event1 or for 5 seconds
#Stop recording
self.stopRecording()
#Fill with numImages:
self._api.camlinkSetParams(xres, yres)
self._api.armCamera()
self._api.setRecordingState(0x0001)
self.fastImages = self._api.largeBufferFill(xres,
yres,
numImages,
events=[event1, event2])
'''
self.fastImages will be an array of np.uint16
Dimensions of xres,yres,15,variable (variable = totalnumberofframes/15)
'''
#Wait for event2:
win32event.WaitForSingleObject(event2.value, 5000)
self.stopRecording()
def getFastImageFrame(self, frameNumber):
yres = self.fastImages[0][0][0]._length_
xres = self.fastImages[0][0]._length_
print "Dimensions: ", [xres, yres]
num1 = frameNumber / 15
num2 = frameNumber % 15
image = np.frombuffer(self.fastImages[num1][num2],
dtype=np.uint16).reshape(yres, xres)
print "max: ", image.max()
print "min: ", image.min()
return image
def setInitFramesMP(self, frames):
self.initFramesMP = frames
def _transferOutOfBuffer(self, start=0, end=8, toDAX=False, verbose=False):
global mpArray
if verbose:
print "Size of image buffer: ", len(self._images)
#self._images = copy.deepcopy(self._buffers[start:end])
if toDAX:
for i in range(start, end):
temp = self._buffers[i].astype(np.dtype('>H')).copy()
self.daxfile.saveFrames(temp.tostring(), 1)
self.recordedFrames += 1
self._images = np.array(self._buffers[start:end]).copy()
#print "Shape of _images: ", self._images.shape
return self.recordedFrames
numImages = len(self._images)
'''
if self.image1 == 0:
self.image2 = self.initFramesMP
if numImages == 0:
self._images = copy.deepcopy(self._buffers[start:end])
globalImageBuffer = copy.deepcopy(self._buffers[start:end])
self.p.start()
self.image1 = 0
self.image2 = self.initFramesMP
else:
self._images.extend(copy.deepcopy(self._buffers[start:end]))
globalImageBuffer = copy.deepcopy(self._images)
print "Length of globImageBuffer from inside: ", len(globalImageBuffer)
if numImages > self.initFramesMP:
if self.p.is_alive():
self.q.put([self.image1,self.image2])
self.image1 = self.image2
self.image2 = numImages
'''
if len(self._images) == 0:
self._images = copy.deepcopy(self._buffers[start:end])
else:
self._images.extend(copy.deepcopy(self._buffers[start:end]))
#arrData = self.mgr.Array('H',list(chain.from_iterable(self._buffers[start:end])),lock=False)
if not self.subProcessStart and not self.p.is_alive():
self.p.start()
self.subProcessStart = True
self.q.put(self._buffers[start:end])
if not self.subProcess2Start:
self.furtherAnalysis.start()
self.subProcess2Start = True
return numImages
def flushImQ(self):
while not self.qIms.empty():
self.qIms.get()
def endMP(self):
print "in EndMP: ", mpArray
self.q.put(None)
#if not self.subProcess2Start:
# self.furtherAnalysis.start()
self.subProcess2Start = True
print "Ending q...\n"
self.q.put(None)
#self.q.close()
time.sleep(3)
#self.p.terminate()
if self.furtherAnalysis.is_alive():
self.furtherAnalysis.join()
if self.p.is_alive():
self.p.join()
self.subProcessStart = False
self.subProcess2Start = False
#self.furtherAnalysis.terminate()
#self.q.close()
del self.mgr
self.mgr = Manager()
del self.q
del self.resultsQueue
#del self.resultsEvent
self.q = self.mgr.Queue()
self.resultsQueue = self.mgr.Queue()
#self.resultsEvent = self.mgr.Event()
del self.p
self.p = Process(target=imageAnalysisTest3,
args=(
self.grabFrames,
self.q,
self.resultsQueue,
self.qIms,
))
del self.furtherAnalysis
self.furtherAnalysis = Process(target=useImageAnalysisResults,
args=(
self.emitSignalFrame,
self.resultsQueue,
self.resultsEvent,
))
#self.furtherAnalysis.start()
#self.furtherAnalysis.join()
def stateOfProcesses(self):
print "imageAnalysisTesting process: ", self.p.is_alive()
print "Further analysis proces: ", self.furtherAnalysis.is_alive()
def imageAnalysisTest(self, conn):
print "New process starting..."
#images = conn.recv()
#print "Got images. Length = ", len(images)
def getImageBuffer(self):
return self._images
def _fillImages(self, toDAX=False):
num_pending = self._buffersPending()
nIms = -1
if num_pending <= 8:
if self._fillBottom:
nIms = self._transferOutOfBuffer(0, 8, toDAX=toDAX)
self._addBuffers(0, 8)
else:
nIms = self._transferOutOfBuffer(8, 16, toDAX=toDAX)
self._addBuffers(8, 16)
self._fillBottom = not self._fillBottom
return nIms, num_pending
'''
#print self._bufferEvents
numIms = 0
while numIms<400:
if self._fillBottom:
win32event.WaitForSingleObject(self._bufferEvents[8].value, 5000)
nIms = self._transferOutOfBuffer(0,8,toDAX=toDAX)
#self._addBuffers(0,8)
else:
win32event.WaitForSingleObject(self._bufferEvents[0].value, 5000)
nIms = self._transferOutOfBuffer(8,16,toDAX=toDAX)
#self._addBuffers(8,16)
self._fillBottom = not self._fillBottom
numIms = self._images.shape[0]
return self._images.shape[0]
'''
def _saveImages(self):
if self._images.shape[0] > 0:
np.save("D:\\Data\\test_images_out.npy", self._images)
def getNumberInImages(self):
return self._images.shape[0]
def _buffersPending(self):
return self._api.getPendingBuffer()[1]
def getImageForPreview(self):
xres, yres = self._props['dimensions']
pending = self._buffersPending()
#print "Pending buffers: ", pending
if pending < 2:
im = self._buffers[self._alternatingBuffer].reshape(xres, yres)
self._addBuffers(self._alternatingBuffer,
self._alternatingBuffer + 1)
self._alternatingBuffer = 1 * (not self._alternatingBuffer)
#print "Image returned for preview."
return im
else:
return None
def _freeBuffers(self):
for i in range(0, 16):
if self._buffersAllocated[i]:
self._api.freeBuffer(i)
def beginPreview(self):
if self._api.getRecordingState() > 0:
self.stopRecording()
xres, yres = self._getResolution()
self._props['dimensions'] = xres, yres
print "Resolution ", (xres, yres)
self._api.camlinkSetParams(xres, yres)
self._api.armCamera()
self._initBuffers(2)
self._addBuffers(0, 2)
self._api.setRecordingState(0x0001)
print "Preview started..."
def stopPreview(self):
if self._api.getRecordingState() > 0:
self.stopRecording()
self._api.freeBuffer(0)
self._api.freeBuffer(1)
self._buffersAllocated[0] = False
self._buffersAllocated[1] = False
def beginBufferFill(self):
self.stopRecording()
self._api.armCamera()
self._initBuffers(16)
self._addBuffers(0, 16)
#self.addManyBuffers(20)
self._api.setRecordingState(0x0001)
print "Recording started..."
def stopRecording(self):
self._api.setRecordingState(0x0000)
self._api.removeBuffer()
def shutDown(self):
print 'Shutting down pco.edge camera...'
self.stopRecording()
self._freeBuffers()
self._api.closeCamera()
lock = RLock()
#Data can be stored in a shared memory map using Value or Array
num = Value(
'd', 0.0) #typecode from array module, eg int(i), double(d), long(l),
arr = Array('i', range(10))
p1 = Process(target=sharedf, args=(num, arr, event))
p1.start()
event.wait() #or call event.is_set()
with lock:
print(num.value)
print(arr[:])
#Manager
#controls a server process which holds Python objects
#and allows other processes to manipulate them using proxies
#supports list, dict, Namespace,
#Lock, RLock, Semaphore, BoundedSemaphore, Condition, Event, Queue,
#Value and Array
manager = Manager()
event2 = manager.Event()
d = manager.dict()
l = manager.list(range(10))
p2 = Process(target=managerf, args=(d, l, event2))
p2.start()
event2.wait() #or call event.is_set()
with lock:
print(d)
print(l)
p1.join()
p2.join()
class DAI(Process):
daemon = True
def __init__(self,
api_url,
device_model,
device_addr=None,
device_name=None,
persistent_binding=False,
username=None,
extra_setup_webpage='',
device_webpage='',
register_callback=None,
on_register=None,
on_deregister=None,
on_connect=None,
on_disconnect=None,
push_interval=1,
interval=None,
device_features=None):
super(DAI, self).__init__()
self._manager = Manager()
self._event = self._manager.Event(
) # create Event proxy object at main process
self.api_url = api_url
self.device_model = device_model
self.device_addr = device_addr
self.device_name = device_name
self.persistent_binding = persistent_binding
self.username = username
self.extra_setup_webpage = extra_setup_webpage
self.device_webpage = device_webpage
self.register_callback = register_callback
self.on_register = on_register
self.on_deregister = on_deregister
self.on_connect = on_connect
self.on_disconnect = on_disconnect
self.push_interval = push_interval
self.interval = interval if interval else {}
self.device_features = device_features if device_features else {}
self.flags = self._manager.dict()
self.queues = self._manager.dict()
self.globals = self._manager.dict()
if not self.globals.get('index'):
self.globals['index'] = 1
def push_data(self, df_name):
if not self.device_features[df_name].push_data:
return
log.debug('%s:%s', df_name, self.flags[df_name])
while self.flags[df_name]:
_data = self.device_features[df_name].push_data(
self.queues[df_name])
if not isinstance(_data, NoData) and _data is not NoData:
self.dan.push(df_name, _data)
time.sleep(self.interval.get(df_name, self.push_interval))
def on_signal(self, signal, df_list):
log.info('Receive signal: \033[1;33m%s\033[0m, %s', signal, df_list)
if 'CONNECT' == signal:
for df_name in df_list:
# race condition
if not self.flags.get(df_name):
self.flags[df_name] = True
# https://stackoverflow.com/questions/56716470/
self.queues[df_name] = self._manager.Queue()
t = Thread(target=self.push_data, args=(df_name, ))
t.daemon = True
t.start()
elif 'DISCONNECT' == signal:
for df_name in df_list:
self.flags[df_name] = False
elif 'SUSPEND' == signal:
# Not use
pass
elif 'RESUME' == signal:
# Not use
pass
return True
def on_data(self, df_name, data):
self.device_features[df_name].on_data(data, self.queues, self.globals)
return True
@staticmethod
def df_func_name(df_name):
return re.sub(r'-(I|O)$', r'_\1', df_name)
def _check_parameter(self):
if self.api_url is None:
raise RegistrationError('api_url is required')
if self.device_model is None:
raise RegistrationError('device_model not given.')
if isinstance(self.device_addr, UUID):
self.device_addr = str(self.device_addr)
elif self.device_addr:
try:
UUID(self.device_addr)
except ValueError:
try:
self.device_addr = str(UUID(int=int(self.device_addr, 16)))
except ValueError:
log.warning(
'Invalid device_addr. Change device_addr to None.')
self.device_addr = None
if self.persistent_binding and self.device_addr is None:
msg = ('In case of `persistent_binding` set to `True`, '
'the `device_addr` should be set and fixed.')
raise ValueError(msg)
if not self.device_features.keys():
raise RegistrationError('Neither idf_list nor odf_list is empty.')
return True
def finalizer(self):
try:
if not self.persistent_binding:
self.dan.deregister()
except Exception as e:
log.warning('dai process cleanup exception: %s', e)
def start(self, *args, **kwargs):
ret = super(DAI, self).start(*args, **kwargs)
# conduct deregistration properly,
# if one doesn't stop process before main process ends
atexit.register(self.terminate)
return ret
def run(self): # this function will be executed in child process
self._check_parameter()
self.dan = Client()
idf_list = []
odf_list = []
for df in self.device_features.values():
if df.df_type == 'idf':
idf_list.append(df.profile())
else:
odf_list.append(df.profile())
def f():
for key in self.flags:
self.flags[key] = False
log.debug('on_disconnect: _flag = %s', str(self.flags))
if self.on_disconnect:
return self.on_disconnect()
self.dan.register(self.api_url,
on_signal=self.on_signal,
on_data=self.on_data,
accept_protos=['mqtt'],
id_=self.device_addr,
idf_list=idf_list,
odf_list=odf_list,
name=self.device_name,
profile={
'model': self.device_model,
'u_name': self.username,
'extra_setup_webpage': self.extra_setup_webpage,
'device_webpage': self.device_webpage,
},
register_callback=self.register_callback,
on_register=self.on_register,
on_deregister=self.on_deregister,
on_connect=self.on_connect,
on_disconnect=f)
log.info('Press Ctrl+C to exit DAI.')
try:
self._event.wait()
except KeyboardInterrupt:
pass
finally:
self.finalizer()
def wait(self):
try:
if platform.system() == 'Windows' or sys.version_info.major == 2:
# workaround for https://bugs.python.org/issue35935
while True:
time.sleep(86400)
else:
Event().wait()
except KeyboardInterrupt:
self.join() # wait for deregistration
def terminate(self, *args, **kwargs):
'''
Terminate DAI.
This is a blocking call.
'''
try:
self._event.set()
except Exception:
# this is triggered if the ``run`` function ended already.
pass
self.join()
return super(DAI, self).terminate(*args, **kwargs)
