def __init__(self,
client,
group,
topic,
auto_commit=True,
auto_commit_every_n=AUTO_COMMIT_MSG_COUNT,
auto_commit_every_t=AUTO_COMMIT_INTERVAL,
num_procs=1,
partitions_per_proc=0):
# Initiate the base consumer class
super(MultiProcessConsumer,
self).__init__(client,
group,
topic,
partitions=None,
auto_commit=auto_commit,
auto_commit_every_n=auto_commit_every_n,
auto_commit_every_t=auto_commit_every_t)
# Variables for managing and controlling the data flow from
# consumer child process to master
self.queue = MPQueue(1024) # Child consumers dump messages into this
self.start = Event() # Indicates the consumers to start fetch
self.exit = Event() # Requests the consumers to shutdown
self.pause = Event() # Requests the consumers to pause fetch
self.size = Value('i', 0) # Indicator of number of messages to fetch
partitions = self.offsets.keys()
# If unspecified, start one consumer per partition
# The logic below ensures that
# * we do not cross the num_procs limit
# * we have an even distribution of partitions among processes
if not partitions_per_proc:
partitions_per_proc = round(len(partitions) * 1.0 / num_procs)
if partitions_per_proc < num_procs * 0.5:
partitions_per_proc += 1
# The final set of chunks
chunker = lambda *x: [] + list(x)
chunks = map(chunker, *[iter(partitions)] * int(partitions_per_proc))
self.procs = []
for chunk in chunks:
chunk = filter(lambda x: x is not None, chunk)
args = (client.copy(), group, topic, chunk, self.queue, self.start,
self.exit, self.pause, self.size)
proc = Process(target=_mp_consume, args=args)
proc.daemon = True
proc.start()
self.procs.append(proc)
def _start(self):
if len(self._threads) == 0:
logging.debug("starting workers")
self._queue_loop = 0
self._end_ctr = 0
for i in range(self.num_processes):
self._queues.append(MPQueue(self.num_cached_per_queue))
self._threads.append(Process(target=producer, args=(self._queues[i], self.generator, self.transform, i, self.seeds[i])))
self._threads[-1].daemon = True
self._threads[-1].start()
else:
logging.debug("MultiThreadedGenerator Warning: start() has been called but workers are already running")
def run_multiple_ant_colony_system(self, file_to_write_path=None):
"""
开启另外的线程来跑multiple_ant_colony_system， 使用主线程来绘图
:return:
"""
path_queue_for_figure = MPQueue()
multiple_ant_colony_system_thread = Process(target=self._multiple_ant_colony_system, args=(path_queue_for_figure, file_to_write_path, ))
multiple_ant_colony_system_thread.start()
# 是否要展示figure
if self.whether_or_not_to_show_figure:
figure = VrptwAcoFigure(self.graph.nodes, path_queue_for_figure)
figure.run()
multiple_ant_colony_system_thread.join()
def test_read_data_multiprocessing(self):
set_start_method("spawn")
workers = 1
output_queue = MPQueue()
processes = [
Process(target=run_read, args=[output_queue])
for i in range(workers)
]
[process.start() for process in processes]
[process.join() for process in processes]
for i in range(workers):
data = output_queue.get(timeout=0.1)
assert_that(data["frame"][0][0][0], is_(equal_to(97)))
assert_that(data["frame"][0][0][1], is_(equal_to(98)))
assert_that(data["frame"][0][0][2], is_(equal_to(99)))
assert_that(data["test1"], is_(equal_to(1)))
assert_that(data["test2"], is_(equal_to(2)))
def _start(self):
self._queue = MPQueue(self.num_cached)
def producer(queue, generator):
try:
for item in generator:
queue.put(item)
except:
print "oops..."
finally:
queue.put("end")
for _ in xrange(self.num_processes):
np.random.seed()
self._threads.append(Process(target=producer, args=(self._queue, self.generator)))
self._threads[-1].daemon = True
self._threads[-1].start()
def __init__(self, buddy_configuration, logger):
"""
Initialize the object.
"""
self.__configuration__ = buddy_configuration
self.__logger__ = logger
self.__lcd__ = None
self.__lcd_status_id__ = 0
self.__initialize_lcd__()
self.__is_gas_detected__ = False
self.__system_start_time__ = datetime.datetime.now()
self.__sensors__ = Sensors(buddy_configuration)
serial_connection = self.__initialize_modem__()
if serial_connection is None and not local_debug.is_debug():
self.__logger__.log_warning_message(
"Unable to initialize serial connection, quiting.")
sys.exit()
self.__fona_manager__ = FonaManager(
self.__logger__, serial_connection,
self.__configuration__.cell_power_status_pin,
self.__configuration__.cell_ring_indicator_pin,
self.__configuration__.utc_offset)
# create heater relay instance
self.__relay_controller__ = RelayManager(
buddy_configuration, logger, self.__heater_turned_on_callback__,
self.__heater_turned_off_callback__,
self.__heater_max_time_off_callback__)
self.__gas_sensor_queue__ = MPQueue()
self.__logger__.log_info_message(
"Starting SMS monitoring and heater service")
self.__clear_existing_messages__()
self.__logger__.log_info_message("Begin monitoring for SMS messages")
self.__queue_message_to_all_numbers__(
"HangarBuddy monitoring started." + "\n" +
self.__get_help_status__())
self.__queue_message_to_all_numbers__(self.__get_full_status__())
self.__lcd__.clear()
self.__lcd__.write(0, 0, "Ready")
def crawl(self,
proc=None,
domain_depth=0,
crawl_depth=0,
page_limit=None,
wait_courtesy=0,
html2txt=False,
metas=None):
"""
:param proc: amount of processes to spawn, 0 or None can be used to exploit the current process
:param domain_depth: crawling depth for each seed element (inside original domain)
:param crawl_depth: crawling depth for each seed element (outside original domain)
:param page_limit: max amount of page to crawl
:param wait_courtesy: time in second between each fetch
:param html2txt: resulting pages must be raw html (default), or cleant txt
:param metas: metas we want to extract during crawling
"""
self.domain_depth = domain_depth
self.crawl_depth = crawl_depth
self.page_limit = page_limit
# lazy loading, to know if we need to implement seeds with multiproc or not
if self.seed is None:
if self.filename is not None:
self.seed = Seed(f=self.filename,
multiproc=not (proc is None or proc == 0))
elif self.seedlist is not None:
self.seed = Seed(s=self.seedlist,
multiproc=not (proc is None or proc == 0))
if proc is None or proc == 0:
self.storage = Queue() # Will contain shared crawl results
self._sub_crawl(self.seed.q, self.storage, Event(), wait_courtesy,
html2txt, metas, None)
while True:
try:
el = self.storage.get(block=False)
yield el
except Empty:
break
else:
self.storage = MPQueue() # Will contain shared crawl results
yield from self.spawn_crawl_processes(html2txt, metas, proc,
wait_courtesy)
def __init__(self, multiprocess=False, is_deque=True, max_len=200):
if multiprocess:
is_deque = False
self._queue = None
self._size = 0 # qsize() might not be implemented for MP queue
self._use_deque = is_deque
self._use_mpqueue = multiprocess
if self._use_deque:
if max_len == 0:
max_len = None
self._queue = deque(maxlen=max_len)
else:
if max_len is None:
max_len = 0
if self._use_mpqueue:
self._queue = MPQueue(maxsize=max_len)
else:
self._queue = Queue(maxsize=max_len)
def __init__(self, configuration, logger, heater_on_callback,
heater_off_callback, heater_max_time_callback):
""" Initialize the object. """
self.__configuration__ = configuration
self.__logger__ = logger
self.__on_callback__ = heater_on_callback
self.__off_callback__ = heater_off_callback
self.__max_time_callback__ = heater_max_time_callback
# create heater relay instance
self.__heater_relay__ = PowerRelay("heater_relay",
configuration.heater_pin)
self.__heater_queue__ = MPQueue()
# create queue to hold heater timer.
self.__heater_shutoff_timer__ = None
# make sure and turn heater off
self.__heater_relay__.switch_low()
def __init__(self):
super().__init__()
self.daemon = True
self.name = 'Collector'
self.inputQueue = Queue()
self.outputQueue = MPQueue()
self.receiver: Dict[str, Receiver] = {}
self.f450 = Format450()
for a in range(1, 17):
name = 'CH%02d' % a
ip = '239.192.0.%d' % a
port = 60000 + a
t = Receiver(name=name,
params=Antenna(ip=ip,
port=port,
streamType=StreamType.Type450),
qp=self.inputQueue)
self.receiver[name] = t
def __init__(self,
vname_to_line_config_dict=None,
ylim=(0, 70),
window_area=50,
update_interval=0.01):
'''
:param vname_to_line_config_dict:
以变量名作为key，config数组作为value，
:param ylim:
:param xlim:
:param window_area: 显示历史多少个数据，避免x无限大
:param update_interval:
'''
self.y_buff = 10
self.update_interval = update_interval
self.ylim = ylim
self.window_area = window_area
self.queue = MPQueue()
self.vname_to_lines = {}
self.vname_to_line_config_dict = vname_to_line_config_dict
self.vname_to_data = []
def test_tracer_usage_multiprocess():
q = MPQueue()
# Similar to test_multiprocess(), ensures that no collisions are
# generated between parent and child processes while using
# multiprocessing.
# Note that we have to be wary of the size of the underlying
# pipe in the queue: https://bugs.python.org/msg143081
def target(q):
ids_list = list(
chain.from_iterable(
(s.span_id, s.trace_id)
for s in [tracer.start_span("s") for _ in range(10)]))
q.put(ids_list)
ps = [mp.Process(target=target, args=(q, )) for _ in range(30)]
for p in ps:
p.start()
for p in ps:
p.join()
ids_list = list(
chain.from_iterable(
(s.span_id, s.trace_id)
for s in [tracer.start_span("s") for _ in range(100)]))
ids = set(ids_list)
assert len(ids) == len(ids_list), "Collisions found in ids"
while not q.empty():
child_ids_list = q.get()
child_ids = set(child_ids_list)
assert len(child_ids) == len(
child_ids_list), "Collisions found in subprocess ids"
assert ids & child_ids == set()
ids = ids | child_ids # accumulate the ids
def __init__(self, *, path: str, timeout: int = 5, buffersize: int = 32):
super().__init__()
self.daemon = False
self.name = 'SQLite'
self.live: bool = True
self.qp: MPQueue = MPQueue()
self.path = pathlib.Path(path) # path for *.db
self.nameformat: str = '%04d-%02d-%02d.db'
self.dateformat: str = '%Y-%m-%d %H:%M:%S.%f'
self.locker = Lock()
self.counter: int = 0
now = dt.now()
self.lastat: dt = now
self.lastsave: dt = now
self.timeout = timeout
# self.buffer: List[Record] = []
self.buffersize = buffersize
self.fifo = Queue()
self.schema = 'CREATE TABLE "sentence" ( \
def __init__(self, logger, serial_connection, power_status_pin,
ring_indicator_pin):
self.__logger__ = logger
self.__modem_access_lock__ = threading.Lock()
self.serial_connection = serial_connection
self.power_status_pin = power_status_pin
self.ring_indicator_pin = ring_indicator_pin
if self.serial_connection is not None:
self.serial_connection.flushInput()
self.serial_connection.flushOutput()
self.__send_command__("AT")
# self.send_command("AE0")
self.__disable_verbose_errors__()
self.__set_sms_mode__()
self.__read_from_fona__(10)
self.__message_waiting_queue__ = MPQueue()
self.__initialize_gpio_pins__()
self.__poll_for_messages__()
def __init__(self,
*,
serialPort: str = '',
baudrate: int = 0,
mcip: str = '',
mcport: int = 0,
outQueue: queue.Queue,
infoQueue: queue.Queue):
self.useVDO = False
self.fragment = []
self.seq = 0
self.engine = Dispatcher()
self.quePoint = MPQueue()
self.counter = 0
self.outQueue = outQueue
self.infoQueue = infoQueue
self.w = Thread(target=self.welcome, daemon=True)
self.w.start()
# if serialPort:
# logger.debug('+++ use Serial')
# self.serialPort = serialPort
# self.baudrate = baudrate
# self.s = Thread(target=self.fromSerial, daemon=True)
# self.s.start()
if mcip:
logger.debug('+++ use UDP(multicast)')
self.u = fromUDP(mcip=mcip, mcport=mcport, quePoint=self.quePoint)
self.u.start()
def __init__(
self,
roidb,
num_loaders=4,
minibatch_queue_size=64,
blobs_queue_capacity=8,
num_augmentation_processes=8,
):
self._roidb = roidb
for roi in roidb:
roi.pop('dataset', None) # pop the reference to prevent duplication
self._lock = threading.Lock()
self._perm = deque(range(len(self._roidb)))
self._cur = 0 # _perm cursor
self._counter = 0
# The minibatch queue holds prepared training data in host (CPU) memory
# When training with N > 1 GPUs, each element in the minibatch queue
# is actually a partial minibatch which contributes 1 / N of the
# examples to the overall minibatch
self._minibatch_queue = Queue.Queue(maxsize=minibatch_queue_size)
self._minibatch_queue_mp = MPQueue(minibatch_queue_size)
self._blobs_queue_capacity = blobs_queue_capacity
# Random queue name in case one instantiates multple RoIDataLoaders
self._loader_id = uuid.uuid4()
self._blobs_queue_name = 'roi_blobs_queue_{}'.format(self._loader_id)
# Loader threads construct (partial) minibatches and put them on the
# minibatch queue
self._num_loaders = num_loaders
self._num_gpus = cfg.NUM_GPUS
self.coordinator = Coordinator()
self._output_names = get_minibatch_blob_names()
self._shuffle_roidb_inds()
self.create_threads()
self.num_augmentation_processes = num_augmentation_processes
def test_tracer_usage_fork():
q = MPQueue()
pid = os.fork()
# Similar test to test_fork() above except we use the tracer API.
# In this case we expect to never have collisions.
if pid > 0:
# parent
parent_ids_list = list(
chain.from_iterable(
(s.span_id, s.trace_id)
for s in [tracer.start_span("s") for _ in range(100)]))
parent_ids = set(parent_ids_list)
assert len(parent_ids) == len(
parent_ids_list), "Collisions found in parent process ids"
child_ids_list = q.get()
child_ids = set(child_ids_list)
assert len(child_ids) == len(
child_ids_list), "Collisions found in child process ids"
assert parent_ids & child_ids == set()
else:
# child
try:
child_ids = list(
chain.from_iterable(
(s.span_id, s.trace_id)
for s in [tracer.start_span("s") for _ in range(100)]))
q.put(child_ids)
finally:
# Kill the process so it doesn't continue running the rest of the
# test suite in a separate process. Note we can't use sys.exit()
# as it raises an exception that pytest will detect as an error.
os._exit(0)
def test_fork_pid_check():
q = MPQueue()
pid = os.fork()
# Generate random numbers in the parent and child processes after forking.
# The child sends back their numbers to the parent where we check to see
# if we get collisions or not.
if pid > 0:
# parent
rns = {_rand.rand64bits() for _ in range(100)}
child_rns = q.get()
assert rns & child_rns == set()
else:
# child
try:
rngs = {_rand.rand64bits() for _ in range(100)}
q.put(rngs)
finally:
# Kill the process so it doesn't continue running the rest of the
# test suite in a separate process. Note we can't use sys.exit()
# as it raises an exception that pytest will detect as an error.
os._exit(0)
def run_real(self):
self.shared_memory = SharedMemory(create=True,
size=self.max_shared_memory)
self.log.debug(
f'Created shared memory of size: {self.shared_memory.size / 1024 / 1024:.02f} MiB'
)
# create the shared memory segments and add them to their respective pools
for i in range(
int(self.shared_memory.size / self.analysis.biggest_chunk)):
_sms = SharedMemorySegment(offset=i * self.analysis.biggest_chunk,
end=i * self.analysis.biggest_chunk +
self.analysis.biggest_chunk)
self.sms.append(_sms)
self.log.debug(f'Created {len(self.sms)} shared memory segments.')
# Create queues
self.dl_worker_queue = MPQueue(-1)
self.writer_queue = MPQueue(-1)
self.dl_result_q = MPQueue(-1)
self.writer_result_q = MPQueue(-1)
self.log.info(f'Starting download workers...')
for i in range(self.max_workers):
w = DLWorker(f'DLWorker {i + 1}',
self.dl_worker_queue,
self.dl_result_q,
self.shared_memory.name,
logging_queue=self.logging_queue,
dl_timeout=self.dl_timeout)
self.children.append(w)
w.start()
self.log.info('Starting file writing worker...')
writer_p = FileWorker(self.writer_queue, self.writer_result_q,
self.dl_dir, self.shared_memory.name,
self.cache_dir, self.logging_queue)
self.children.append(writer_p)
writer_p.start()
num_chunk_tasks = sum(isinstance(t, ChunkTask) for t in self.tasks)
num_dl_tasks = len(self.chunks_to_dl)
num_tasks = len(self.tasks)
num_shared_memory_segments = len(self.sms)
self.log.debug(
f'Chunks to download: {num_dl_tasks}, File tasks: {num_tasks}, Chunk tasks: {num_chunk_tasks}'
)
# active downloader tasks
self.active_tasks = 0
processed_chunks = 0
processed_tasks = 0
total_dl = 0
total_write = 0
# synchronization conditions
shm_cond = Condition()
task_cond = Condition()
self.conditions = [shm_cond, task_cond]
# start threads
s_time = time.time()
self.threads.append(
Thread(target=self.download_job_manager,
args=(task_cond, shm_cond)))
self.threads.append(
Thread(target=self.dl_results_handler, args=(task_cond, )))
self.threads.append(
Thread(target=self.fw_results_handler, args=(shm_cond, )))
for t in self.threads:
t.start()
last_update = time.time()
while processed_tasks < num_tasks:
delta = time.time() - last_update
if not delta:
time.sleep(self.update_interval)
continue
# update all the things
processed_chunks += self.num_processed_since_last
processed_tasks += self.num_tasks_processed_since_last
total_dl += self.bytes_downloaded_since_last
total_write += self.bytes_written_since_last
dl_speed = self.bytes_downloaded_since_last / delta
dl_unc_speed = self.bytes_decompressed_since_last / delta
w_speed = self.bytes_written_since_last / delta
r_speed = self.bytes_read_since_last / delta
# c_speed = self.num_processed_since_last / delta
# set temporary counters to 0
self.bytes_read_since_last = self.bytes_written_since_last = 0
self.bytes_downloaded_since_last = self.num_processed_since_last = 0
self.bytes_decompressed_since_last = self.num_tasks_processed_since_last = 0
last_update = time.time()
perc = (processed_chunks / num_chunk_tasks) * 100
runtime = time.time() - s_time
total_avail = len(self.sms)
total_used = (num_shared_memory_segments - total_avail) * (
self.analysis.biggest_chunk / 1024 / 1024)
if runtime and processed_chunks:
rt_hours, runtime = int(runtime // 3600), runtime % 3600
rt_minutes, rt_seconds = int(runtime // 60), int(runtime % 60)
average_speed = processed_chunks / runtime
estimate = (num_chunk_tasks - processed_chunks) / average_speed
hours, estimate = int(estimate // 3600), estimate % 3600
minutes, seconds = int(estimate // 60), int(estimate % 60)
else:
hours = minutes = seconds = 0
rt_hours = rt_minutes = rt_seconds = 0
# rounds the numbers to the base
def round_spec(x, base):
return round(x * (base / 100))
# returns the progress bar
def pounds(perc, rest):
num_pounds = round_spec(perc, rest)
return ('#' * num_pounds) + (' ' * (rest - num_pounds))
term_length = shutil.get_terminal_size().columns
# base message
message = f"{perc:.02f}%"
stats = f"{hours:02d}:{minutes:02d}:{seconds:02d} {dl_speed / 1024 / 1024:.02f} Mib/s {total_dl / 1024 / 1024:.02f} MiB "
# some old math i used to used, leaving it here for reference
# rest = term_length - len(message) - ((round(shutil.get_terminal_size().columns / 2) + 2) - len(stats)) # Gets how many whitespaces we have to leave
# I have no idea how this math works...
rest = round((term_length - len(message) - len(stats)) /
3) - 1 # gets how many whitespaces we have to leave
# adds as many spaces to the message as we need...
message += ' ' * rest
message += stats
message += f"[{pounds(perc, (rest * 2))}]"
# clears the line, and prints the message
sys.stdout.write("\r" + message)
# flushes stdout
sys.stdout.flush()
# send status update to back to instantiator (if queue exists)
if self.status_queue:
try:
self.status_queue.put(UIUpdate(progress=perc,
download_speed=dl_unc_speed,
write_speed=w_speed,
read_speed=r_speed,
memory_usage=total_used *
1024 * 1024),
timeout=1.0)
except Exception as e:
self.log.warning(
f'Failed to send status update to queue: {e!r}')
time.sleep(self.update_interval)
for i in range(self.max_workers):
self.dl_worker_queue.put_nowait(DownloaderTask(kill=True))
self.log.info('Waiting for installation to finish...')
self.writer_queue.put_nowait(WriterTask('', kill=True))
writer_p.join(timeout=10.0)
if writer_p.exitcode is None:
self.log.warning(f'Terminating writer process, no exit code!')
writer_p.terminate()
# forcibly kill DL workers that are not actually dead yet
for child in self.children:
if child.exitcode is None:
child.terminate()
# make sure all the threads are dead.
for t in self.threads:
t.join(timeout=5.0)
if t.is_alive():
self.log.warning(f'Thread did not terminate! {repr(t)}')
# clean up resume file
if self.resume_file:
try:
os.remove(self.resume_file)
except OSError as e:
self.log.warning(f'Failed to remove resume file: {e!r}')
# close up shared memory
self.shared_memory.close()
self.shared_memory.unlink()
self.shared_memory = None
self.log.info('All done! Download manager quitting...')
# finally, exit the process.
exit(0)
def run_real(self):
self.shared_memory = SharedMemory(create=True, size=self.max_shared_memory)
self.log.debug(f'Created shared memory of size: {self.shared_memory.size / 1024 / 1024:.02f} MiB')
# create the shared memory segments and add them to their respective pools
for i in range(int(self.shared_memory.size / self.analysis.biggest_chunk)):
_sms = SharedMemorySegment(offset=i * self.analysis.biggest_chunk,
end=i * self.analysis.biggest_chunk + self.analysis.biggest_chunk)
self.sms.append(_sms)
self.log.debug(f'Created {len(self.sms)} shared memory segments.')
# Create queues
self.dl_worker_queue = MPQueue(-1)
self.writer_queue = MPQueue(-1)
self.dl_result_q = MPQueue(-1)
self.writer_result_q = MPQueue(-1)
self.log.info(f'Starting download workers...')
for i in range(self.max_workers):
w = DLWorker(f'DLWorker {i + 1}', self.dl_worker_queue, self.dl_result_q,
self.shared_memory.name, logging_queue=self.logging_queue,
dl_timeout=self.dl_timeout)
self.children.append(w)
w.start()
self.log.info('Starting file writing worker...')
writer_p = FileWorker(self.writer_queue, self.writer_result_q, self.dl_dir,
self.shared_memory.name, self.cache_dir, self.logging_queue)
self.children.append(writer_p)
writer_p.start()
num_chunk_tasks = sum(isinstance(t, ChunkTask) for t in self.tasks)
num_dl_tasks = len(self.chunks_to_dl)
num_tasks = len(self.tasks)
num_shared_memory_segments = len(self.sms)
self.log.debug(f'Chunks to download: {num_dl_tasks}, File tasks: {num_tasks}, Chunk tasks: {num_chunk_tasks}')
# active downloader tasks
self.active_tasks = 0
processed_chunks = 0
processed_tasks = 0
total_dl = 0
total_write = 0
# synchronization conditions
shm_cond = Condition()
task_cond = Condition()
self.conditions = [shm_cond, task_cond]
# start threads
s_time = time.time()
self.threads.append(Thread(target=self.download_job_manager, args=(task_cond, shm_cond)))
self.threads.append(Thread(target=self.dl_results_handler, args=(task_cond,)))
self.threads.append(Thread(target=self.fw_results_handler, args=(shm_cond,)))
for t in self.threads:
t.start()
last_update = time.time()
while processed_tasks < num_tasks:
delta = time.time() - last_update
if not delta:
time.sleep(self.update_interval)
continue
# update all the things
processed_chunks += self.num_processed_since_last
processed_tasks += self.num_tasks_processed_since_last
total_dl += self.bytes_downloaded_since_last
total_write += self.bytes_written_since_last
dl_speed = self.bytes_downloaded_since_last / delta
dl_unc_speed = self.bytes_decompressed_since_last / delta
w_speed = self.bytes_written_since_last / delta
r_speed = self.bytes_read_since_last / delta
# c_speed = self.num_processed_since_last / delta
# set temporary counters to 0
self.bytes_read_since_last = self.bytes_written_since_last = 0
self.bytes_downloaded_since_last = self.num_processed_since_last = 0
self.bytes_decompressed_since_last = self.num_tasks_processed_since_last = 0
last_update = time.time()
perc = (processed_chunks / num_chunk_tasks) * 100
runtime = time.time() - s_time
total_avail = len(self.sms)
total_used = (num_shared_memory_segments - total_avail) * (self.analysis.biggest_chunk / 1024 / 1024)
if runtime and processed_chunks:
rt_hours, runtime = int(runtime // 3600), runtime % 3600
rt_minutes, rt_seconds = int(runtime // 60), int(runtime % 60)
average_speed = processed_chunks / runtime
estimate = (num_chunk_tasks - processed_chunks) / average_speed
hours, estimate = int(estimate // 3600), estimate % 3600
minutes, seconds = int(estimate // 60), int(estimate % 60)
else:
hours = minutes = seconds = 0
rt_hours = rt_minutes = rt_seconds = 0
bar.set_fraction(perc)
bar.set_text( f'{perc:.02f}% ({processed_chunks}/{num_chunk_tasks}), '
f'Elapsed: {rt_hours:02d}:{rt_minutes:02d}:{rt_seconds:02d}, '
f'ETA: {hours:02d}:{minutes:02d}:{seconds:02d}'
f'{dl_speed / 1024 / 1024:.02f} MiB/s'
)
#self.log.info(f'= Progress: {perc:.02f}% ({processed_chunks}/{num_chunk_tasks}), '
# f'Running for {rt_hours:02d}:{rt_minutes:02d}:{rt_seconds:02d}, '
# f'ETA: {hours:02d}:{minutes:02d}:{seconds:02d}')
#self.log.info(f' - Downloaded: {total_dl / 1024 / 1024:.02f} MiB, '
# f'Written: {total_write / 1024 / 1024:.02f} MiB')
#self.log.info(f' - Cache usage: {total_used} MiB, active tasks: {self.active_tasks}')
#self.log.info(f' + Download\t- {dl_speed / 1024 / 1024:.02f} MiB/s (raw) '
# f'/ {dl_unc_speed / 1024 / 1024:.02f} MiB/s (decompressed)')
#self.log.info(f' + Disk\t- {w_speed / 1024 / 1024:.02f} MiB/s (write) / '
# f'{r_speed / 1024 / 1024:.02f} MiB/s (read)')
# send status update to back to instantiator (if queue exists)
if self.status_queue:
try:
self.status_queue.put(UIUpdate(
progress=perc, download_speed=dl_unc_speed, write_speed=w_speed, read_speed=r_speed,
memory_usage=total_used * 1024 * 1024
), timeout=1.0)
except Exception as e:
self.log.warning(f'Failed to send status update to queue: {e!r}')
time.sleep(self.update_interval)
for i in range(self.max_workers):
self.dl_worker_queue.put_nowait(DownloaderTask(kill=True))
self.log.info('Waiting for installation to finish...')
self.writer_queue.put_nowait(WriterTask('', kill=True))
writer_p.join(timeout=10.0)
if writer_p.exitcode is None:
self.log.warning(f'Terminating writer process, no exit code!')
writer_p.terminate()
# forcibly kill DL workers that are not actually dead yet
for child in self.children:
if child.exitcode is None:
child.terminate()
# make sure all the threads are dead.
for t in self.threads:
t.join(timeout=5.0)
if t.is_alive():
self.log.warning(f'Thread did not terminate! {repr(t)}')
# clean up resume file
if self.resume_file:
try:
os.remove(self.resume_file)
except OSError as e:
self.log.warning(f'Failed to remove resume file: {e!r}')
# close up shared memory
self.shared_memory.close()
self.shared_memory.unlink()
self.shared_memory = None
self.log.info('All done! Download manager quitting...')
# finally, exit the process.
exit(0)
def start(self):
print("started")
def producer(target_queues, data_loader):
num_queues = len(target_queues)
ctr = 0
for item in data_loader:
q = ctr % num_queues
target_queues[q].put(item)
ctr += 1
[target_queue.put("end") for target_queue in target_queues]
def transformer(target_queue, source_queue, transform, seed):
np.random.seed(seed)
item = source_queue.get()
while item != "end":
target_queue.put(transform(**item))
item = source_queue.get()
target_queue.put("end")
def joiner(transformed_queues, ready_queues, join_method, batch_size):
# collects and joins samples to batches
stop = False
num_queues = len(transformed_queues)
ctr = 0
num_rdy = len(ready_queues)
rdy_ctr = 0
while not stop:
items = []
for _ in range(batch_size):
q = ctr % num_queues
item = transformed_queues[q].get()
ctr += 1
if item == "end":
stop = True
break
items.append(item)
if stop:
break
else:
joined = join_method(items)
rdy_q = rdy_ctr % num_rdy
ready_queues[rdy_q].put(joined)
rdy_ctr += 1
[ready_queue.put("end") for ready_queue in ready_queues]
self.sample_queues = [
MPQueue(self.num_raw_cached) for i in range(self.num_processes)
]
self.transformed_queues = [
MPQueue(self.num_transformed_cached)
for i in range(self.num_processes)
]
self.ready_queues = [MPQueue(2) for i in range(2)]
self.sample_generating_process = ProcessTerminateOnJoin(
target=producer, args=(self.sample_queues, self.dataloader))
self.sample_generating_process.daemon = True
self.sample_generating_process.start()
self.joining_process = ProcessTerminateOnJoin(
target=joiner,
args=(self.transformed_queues, self.ready_queues,
self.batch_joiner, self.batch_size))
self.joining_process.daemon = True
self.joining_process.start()
self.q_ctr = 0
self.transformers = []
for i in range(self.num_processes):
p = ProcessTerminateOnJoin(target=transformer,
args=(self.transformed_queues[i],
self.sample_queues[i],
self.transform, self.seeds[i]))
p.daemon = True
p.start()
self.transformers.append(p)
self.was_started = True
def __init__(self):
# 这个是进程间通信用的Queue
self.queue = MPQueue()
def start(self):
self.mpQueue = MPQueue()
self.process = Process(target=self.processMain)
self.process.start()
from udplistner import Receiver as UDPListner
from archive import Archive
if __name__ == '__main__':
logger = logging.getLogger('Log')
logger.setLevel(logging.DEBUG)
formatter = '%(asctime)s %(module)s(%(lineno)s):%(funcName)s [%(levelname)s]: %(message)s'
streamhandler = logging.StreamHandler()
streamhandler.setFormatter(logging.Formatter(formatter, datefmt='%H:%M:%S'))
streamhandler.setLevel(logging.DEBUG)
logger.addHandler(streamhandler)
logger.debug('Session start')
mpqueue = MPQueue() # queue for multiprocessing
threadqueue = ThreadQueue() # queue for threading
aqueue = MPQueue() # for archiver
ws = WSServer(port=Constants.wsport)
ws.start()
wd = WebServer(name='Flask')
wd.start()
serialReceiver: Dict[str, dict] = {
'sGPS': {
'port': '/dev/ttyACM0',
'baud': 9600,
},
'sAIS': {
def _create_queues(self):
return {
CoreID(core): MPQueue()
for core in range(self.config.workers_count + 1)
}
def __init__(self):
logger.info('CUEMS ENGINE INITIALIZATION')
# Main thread ids
logger.info(f'Main thread PID: {os.getpid()}')
try:
logger.info(f'Hardware discovery launched...')
hw_discovery()
except Exception as e:
logger.exception(f'Exception: {e}')
exit(-1)
# Running flag
self.stop_requested = False
self._editor_request_uuid = None
#########################################################
# System signals handlers
signal.signal(signal.SIGINT, self.sigIntHandler)
signal.signal(signal.SIGTERM, self.sigTermHandler)
signal.signal(signal.SIGUSR1, self.sigUsr1Handler)
signal.signal(signal.SIGUSR2, self.sigUsr2Handler)
signal.signal(signal.SIGCHLD, self.sigChldHandler)
# Conf load manager
try:
self.cm = ConfigManager(path=CUEMS_CONF_PATH)
except FileNotFoundError:
logger.critical(
'Node config file could not be found. Exiting !!!!!')
exit(-1)
# Our empty script object
self.script = None
'''
CUE "POINTERS":
here we use the "standard" point of view that there is an
ongoing cue already running (one or many, at least the last to be gone)
and a pointer indicating which is the next to be gone when go is pressed
'''
self.ongoing_cue = None
self.next_cue_pointer = None
self.armedcues = list()
# MTC master object creation through bound library and open port
self.mtcmaster = libmtcmaster.MTCSender_create()
self.go_offset = 0
# MTC listener (could be usefull)
try:
self.mtclistener = MtcListener(
port=self.cm.node_conf['mtc_port'],
step_callback=partial(CuemsEngine.mtc_step_callback, self),
reset_callback=partial(CuemsEngine.mtc_step_callback, self,
CTimecode('0:0:0:0')))
except KeyError:
logger.error(
'mtc_port config could bot be properly loaded. Exiting.')
exit(-1)
# WebSocket server
settings_dict = {}
settings_dict['session_uuid'] = str(uuid1())
settings_dict['library_path'] = self.cm.library_path
settings_dict['tmp_upload_path'] = self.cm.tmp_upload_path
settings_dict['database_name'] = self.cm.database_name
settings_dict['load_timeout'] = self.cm.node_conf['load_timeout']
settings_dict['discovery_timeout'] = self.cm.node_conf[
'discovery_timeout']
self.engine_queue = MPQueue()
self.editor_queue = MPQueue()
self.ws_server = CuemsWsServer(self.engine_queue, self.editor_queue,
settings_dict)
try:
self.ws_server.start(self.cm.node_conf['websocket_port'])
except KeyError:
self.stop_all_threads()
logger.exception(
'Config error, websocket_port key not found in settings. Exiting.'
)
exit(-1)
except Exception as e:
self.stop_all_threads()
logger.error('Exception when starting websocket server. Exiting.')
logger.exception(e)
exit(-1)
else:
# Threaded own queue consumer loop
self.engine_queue_loop = threading.Thread(
target=self.engine_queue_consumer, name='engineq_consumer')
self.engine_queue_loop.start()
# OSSIA OSCQuery server
self.ossia_queue = queue.Queue()
self.ossia_server = OssiaServer(self.cm.node_conf['id'],
self.cm.node_conf['oscquery_port'],
self.cm.node_conf['oscquery_out_port'],
self.ossia_queue)
# Initial OSC nodes to tell ossia to configure
OSC_ENGINE_CONF = {
'/engine': [ossia.ValueType.Impulse, None],
'/engine/command': [ossia.ValueType.Impulse, None],
'/engine/command/load':
[ossia.ValueType.String, self.load_project_callback],
'/engine/command/loadcue':
[ossia.ValueType.String, self.load_cue_callback],
'/engine/command/go': [ossia.ValueType.Impulse, self.go_callback],
'/engine/command/gocue':
[ossia.ValueType.String, self.go_cue_callback],
'/engine/command/pause':
[ossia.ValueType.Impulse, self.pause_callback],
'/engine/command/stop':
[ossia.ValueType.Impulse, self.stop_callback],
'/engine/command/resetall':
[ossia.ValueType.Impulse, self.reset_all_callback],
'/engine/command/preload':
[ossia.ValueType.String, self.load_cue_callback],
'/engine/command/unload':
[ossia.ValueType.String, self.unload_cue_callback],
'/engine/status/timecode': [ossia.ValueType.Int, None],
'/engine/status/currentcue': [ossia.ValueType.String, None],
'/engine/status/nextcue': [ossia.ValueType.String, None],
'/engine/status/running': [ossia.ValueType.Int, None]
}
self.ossia_queue.put(QueueData('add', OSC_ENGINE_CONF))
# Check, start and OSC register video devices/players
self._video_players = {}
try:
self.check_video_devs()
except Exception as e:
logger.error(f'Error checking & starting video devices...')
logger.exception(e)
logger.error(f'Exiting...')
exit(-1)
# Everything is ready now and should be working, let's run!
while not self.stop_requested:
time.sleep(0.005)
self.stop_all_threads()
def __init__(self, logger=None):
in_queue = MPQueue()
out_queue = MPQueue()
super(ProcessCallRelay, self).__init__(logger, in_queue, out_queue)
empty = pool.join(timeout=10, raise_error=False)
if not empty:
logger.warning('{} join timeout. kill it'.format(worker_name))
pool.kill()
logger.warning('{} QUIT !!!'.format(worker_name))
# return state
return '{}: {}'.format(worker_name, str(state))
def call(q: MPQueue, func, *args):
result = func(*args)
q.put(result)
q.close()
result_queue = MPQueue()
ps = []
for bind in binds:
for w in range(workers):
print('bind = {}, w = {}'.format(bind, w))
p = Process(target=call, args=(result_queue, run, bind, w))
p.start()
ps.append(p)
states = []
while len(states) != len(ps):
state = result_queue.get()
states.append(state)
for state in states:
# Copyright (C) 2016 Red Hat, Inc
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
All global queues.
"""
from multiprocessing import Queue as MPQueue
#: Input queue for the investigator thread(s)
INVESTIGATE_QUEUE = MPQueue()
# ROUTER_QUEUE = Queue()
# QUEUES = {
# "ALL": [Queue(), Queue()],
# "10.2.0.2": [Queue()],
# }
def __init__(self, maxsize=64, multiprocess=False):
self.q = (MPQueue(maxsize=maxsize) if multiprocess else Queue(
maxsize=maxsize))
