class MyStore(Store):
def on_init(self, **kwargs):
self.values = Manager().dict()
def _set(self, key, value):
self.values[key] = value
def _get(self, key):
return self.values.get(key, None)
def _clear(self, key):
if not key:
self.values.clear()
else:
self.values[key] = None
class QdiscRedStatCollector:
def __init__(self, get_stats_func, parent):
self._get_stats_func = get_stats_func
self._stop_flag = Value("b", True)
self._stats = Manager().dict()
self._p = None
self._old_stats = {}
self._backlogs = None
self._parent = parent
def _collect_qdisc_red_stats(self, stop_flag, stats, backlogs):
while not stop_flag.value:
new_stats = self._get_stats_func(self._parent)
if new_stats == {}:
continue
if new_stats["tx_packets"] > stats["tx_packets"]:
stats.update(new_stats)
backlogs.append(stats["backlog"])
sleep(0.25)
def start(self):
if not self._stop_flag.value:
return
self._stats.clear()
self._backlogs = Manager().list()
self._old_stats = self._get_stats_func(self._parent)
self._stats.update(self._old_stats)
self._stop_flag.value = False
self._p = Process(target=self._collect_qdisc_red_stats,
args=(self._stop_flag, self._stats, self._backlogs))
self._p.start()
def stop(self):
if self._stop_flag.value:
return
self._stop_flag.value = True
self._p.join(0)
backlogs = list(self._backlogs)
stats = self._stats.copy()
for key in self._old_stats.keys():
if key not in stats:
continue
if type(stats[key]) in (int, long):
stats[key] -= self._old_stats[key]
return (backlogs, stats)
# 标签分页链接地址
data_tgpl = Manager().dict()
tag_pool = Pool(processes=50)
for key in tag_url:
tag_pool.apply_async(get_all_tag_page_links, args=(key, tag_url[key], data_tgpl))
tag_url.clear()
tag_url.clear()
tag_pool.close()
tag_pool.join()
# 获取分类分页中的cartoon首页地址
data_ctindex = Manager().dict()
cindex_pool = Pool(processes=50)
for key in dict(data_tgpl):
cindex_pool.apply_async(get_all_cartoon_index, args=(key, data_tgpl[key], data_ctindex))
data_tgpl.clear()
cindex_pool.close()
cindex_pool.join()
# 获取章节地址
data_cplink = Manager().dict()
cplink_pool = Pool(processes=50)
for key in dict(data_ctindex):
cplink_pool.apply_async(get_all_cartoon_cpages, args=(key, data_ctindex[key], data_cplink))
data_ctindex.clear()
cplink_pool.close()
cplink_pool.join()
# 获取章节下页面详情(图片地址)
data_cpldetial = Manager().dict()
cpldetial_pool = Pool(processes=50)
class FrameArrivalHandler(object):
"""
execute some tasks when fixed frames arrive
"""
def __init__(self,
cfg: VideoConfig,
scfg: ServerConfig,
detect_index,
future_frames,
msg_queue: Queue,
rect_stream_path,
original_stream_path,
render_rect_cache,
original_frame_cache,
notify_queue,
region_path,
preview_path=None,
detect_params=None) -> None:
super().__init__()
self.cfg = cfg
self.scfg = scfg
self.detect_index = detect_index
self.rect_stream_path = rect_stream_path
self.original_stream_path = original_stream_path
self.preview_path = preview_path
self.task_cnt = 0
self.index = cfg.index
self.cache_size = cfg.cache_size
self.future_frames = future_frames
self.sample_rate = cfg.sample_rate
# self.render_frame_cache = render_frame_cache
self.render_rect_cache = render_rect_cache
self.original_frame_cache: SharedMemoryFrameCache = original_frame_cache
# shared event lock, will be released until all future frames has come
self.lock_window = Manager().Event()
self.lock_window.set()
self.msg_queue = msg_queue
self.fourcc = cv2.VideoWriter_fourcc(*'MP4V')
self.quit = Manager().Event()
self.quit.clear()
self.status = Manager().Value('i', SystemStatus.RUNNING)
self.notify_queue = notify_queue
self.LOG_PREFIX = f'Frame Arrival Handler [{self.cfg.index}]: '
self.post_filter = Filter(self.cfg, region_path, detect_params)
self.last_detection = time.time(
) # record the last task triggered time.
self.pre_candidate_rect = [
] # record the last rects seed for detection or tracking
self.task_msg_queue = Manager().Queue()
self.dis_thresh = max(self.cfg.shape[0], self.cfg.shape[1]) * 1 / 4
def is_window_reach(self, detect_index):
return detect_index - self.detect_index > self.future_frames
def reset(self, msg: ArrivalMessage):
"""
release window lock
all frames in a fixed slide window will be skipped until the final frame arrival
:param msg:
:return:
"""
if self.is_window_reach(msg.current_index):
self.detect_index = msg.current_index
# self.lock_window = False
if not self.lock_window.is_set():
self.lock_window.set()
# some messages may be cached when new candidates appear, will be post to task handler when
# current window released(all future frames have arrive.)
while not self.task_msg_queue.empty():
msg = self.task_msg_queue.get()
self.task(msg)
# logger.info(self.LOG_PREFIX + 'Release window lock')
def next_st(self, detect_index):
if detect_index - self.detect_index > self.future_frames:
return detect_index
else:
return self.detect_index
def notify(self, msg: ArrivalMessage):
"""
execute some analysis tasks asynchronously before the the future frame comes
:param msg:
:return:
"""
# if not self.lock_window:
# continuous arrival signal in current window will be ignored
if self.lock_window.is_set() or msg.no_wait:
self.lock_window.clear()
# skipped this detection if it is closed enough to the previous one
# avoid generating videos frequently
# It's common that the river situation becomes terrible if frequent
# detections were triggered.
if self.cfg.limit_freq and time.time(
) - self.last_detection < self.cfg.freq_thresh:
logger.info(
self.LOG_PREFIX +
f'Detection frequency: {round(time.time() - self.last_detection, 2)} is lower than the thresh,ignored.'
)
self.detect_index = msg.current_index
self.last_detection = time.time()
return
# occupy the whole window until a sequent task is done
self.update_record(msg)
self.task(msg)
# new candidate may appear during a window
else:
new_rects = self.cal_potential_new_candidate(msg)
if len(new_rects):
self.update_record(msg)
msg.rects = new_rects
# self.task(msg)
# put into msg queue, waiting processed until window released.
self.task_msg_queue.put(msg)
logger.info(
self.LOG_PREFIX +
f'Appear new candidate during a window, frame index: {msg.current_index}, {new_rects}.'
)
def cal_potential_new_candidate(self, msg):
if msg.rects is None:
return []
else:
new_rects = []
for old in self.pre_candidate_rect:
for new in msg.rects:
if Obj.cal_dst(old, new) > self.dis_thresh:
logger.info(
f'Distance betweent new rect and old rect: [{Obj.cal_dst(old, new)}]'
)
new_rects.append(new)
return new_rects
def update_record(self, msg):
self.detect_index = msg.current_index
self.last_detection = time.time()
self.pre_candidate_rect = msg.rects
def task(self, msg: ArrivalMessage):
"""
override by subclass,do everything what you want in a single frame window
task must be executed asynchronously in case blocking caller
:param msg: arrival msg
:return: return immediately
"""
pass
def wait(self, task_cnt, task_type, msg: ArrivalMessage):
"""
wait frames arrival,support time out
:param msg:
:param task_cnt:
:param task_type:
:return:
"""
if not self.lock_window.is_set():
logger.debug(
f'{self.LOG_PREFIX} {task_type} [{task_cnt}] wait frames arrival....'
)
start = time.time()
# wait the future frames prepared,if trigger time out, give up waits
if not msg.no_wait:
self.lock_window.wait(30)
logger.debug(
f"{self.LOG_PREFIX} {task_type} " +
f"[{task_cnt}] wait [{round(time.time() - start, 2)}] seconds")
logger.debug(
f'{self.LOG_PREFIX} Rect Render Task [{task_cnt}] frames accessible...'
)
def listen(self):
if self.quit.wait():
self.lock_window.set()
self.status.set(SystemStatus.SHUT_DOWN)
def loop(self):
"""
loop message inside a sub-process
:return:
"""
logger.info(
f'*******************************{self.LOG_PREFIX}: Init Frame Arrival Handle Service********************************'
)
threading.Thread(target=self.listen, daemon=True).start()
while self.status.get() == SystemStatus.RUNNING:
try:
# index, type = self.notify_queue.get()
msg: ArrivalMessage = self.notify_queue.get()
if msg.type == ArrivalMsgType.DETECTION:
self.notify(msg)
if msg.type == ArrivalMsgType.UPDATE:
self.reset(msg)
except Empty as e:
# task service will timeout if track request is empty in pipe
# ignore
pass
except Exception as e:
logger.error(e)
logger.info(
f'*******************************{self.LOG_PREFIX}: Exit Frame Arrival Handle Service********************************'
)
class ParallelBatchIterator(BatchIterator):
def __init__(
self,
shuffle=False,
iteration_transforms=[],
batch_size=32,
iteration_keys=None,
number_threads=16,
max_batch_queue_size=128,
):
super(ParallelBatchIterator,
self).__init__(epoch_transforms=[],
iteration_transforms=iteration_transforms,
batch_size=batch_size,
iteration_keys=iteration_keys)
self.shuffle = shuffle
self.number_threads = number_threads
self.max_batch_queue_size = max_batch_queue_size
self.batch_start_stop = Manager().Queue(maxsize=1000)
self.batch_queue = Manager().Queue(maxsize=self.max_batch_queue_size)
self.should_run = True
self.p_list = []
def __call__(self, data):
super(ParallelBatchIterator, self).__call__(data)
data_length = self.data[self.iteration_keys[0]].shape[0]
num_batches = np.floor(data_length / self.batch_size)
if (data_length % self.batch_size) != 0:
num_batches += 1
def create_batches():
# done by just one process. (sequential)
while self.should_run:
if self.shuffle:
new_indices = np.random.permutation(data_length)
else:
new_indices = range(data_length)
for i in range(int(num_batches)):
batch_start = i * self.batch_size
batch_stop = np.min(
[data_length + 1, batch_start + self.batch_size])
indices = new_indices[batch_start:batch_stop]
self.batch_start_stop.put(indices)
def load_batches():
np.random.seed()
while self.should_run:
batch = {}
indices = self.batch_start_stop.get(block=True)
for key in self.iteration_keys:
batch[key] = self.data[key][indices]
for transform in self.iteration_transforms:
batch = transform(batch)
self.batch_queue.put(batch)
for i in range(self.number_threads):
p = Process(target=load_batches, args=())
p.start()
self.p_list.append(p)
pb = Process(target=create_batches, args=())
pb.start()
self.p_list.append(pb)
def __iter__(self):
data_length = self.data[self.iteration_keys[0]].shape[0]
num_batches = np.floor(data_length / self.batch_size)
if (data_length % self.batch_size) != 0:
num_batches += 1
for i in range(int(num_batches)):
batch = self.batch_queue.get(block=True)
yield batch
def __del__(self):
self.should_run = False
self.batch_queue.clear(
) # in case process is waiting to put something in a full queue
for p in self.p_list[:-1]:
p.join()
self.batch_start_stop.clear(
) # in case process is waiting to put something in a full queue
self.p_list[-1].join()
class AudioPlayer(object):
""" Play audio files.
"""
def __init__(self, filename='', show_position=False, **kwargs):
""" AudioPlayer(filename='', show_position=False, **kwargs) -> Open
filename and an appropriate audio io for it.
"""
self._filename = filename
self._show_position = show_position
# Setup the msg_dict for sending messages to the child process.
self._msg_dict = Manager().dict()
self._control_dict = {}
# Create a pipe for sending and receiving messages.
self._control_conn, self._player_conn = Pipe()
# Open the file.
if filename:
self.open(filename, **kwargs)
def __str__(self):
""" The information about the open file.
"""
# Return nothing if no file is open.
if not self._filename: return ''
# Wait for the stream to open.
while 'info' not in self._msg_dict:
pass
# Return the info string.
return self._msg_dict.get('info', '')
def __repr__(self):
""" __repr__ -> Returns a python expression to recreate this instance.
"""
repr_str = "filename='%(_filename)s'" % self.__dict__
return '%s(%s)' % (self.__class__.__name__, repr_str)
def __enter__(self):
""" Provides the ability to use pythons with statement.
"""
try:
return self
except Exception as err:
print(err)
return None
def __exit__(self, exc_type, exc_value, traceback):
""" Stop playback when finished.
"""
try:
self.stop()
self._control_conn.close()
self._player_conn.close()
return not bool(exc_type)
except Exception as err:
print(err)
return False
def __del__(self):
""" Stop playback before deleting.
"""
if self._control_dict.get('playing', False):
try:
self.stop()
except IOError:
pass
def __len__(self):
""" The length of the file if it has one.
"""
return self.length if self.length >= 0 else 0
def playing_wrapper(func):
""" Wrap methods and only call them if the stream is playing
"""
@functools_wraps(func)
def wrapper(self, *args, **kwargs):
""" Check if stream is playing and if it is then call func
otherwise print a message and exit.
"""
if not self.playing:
print("%(filename)s is not playing." % self._msg_dict)
return None
return func(self, *args, **kwargs)
return wrapper
def _play_proc(self, msg_dict, pipe):
""" Player process
"""
# Open the file to play.
try:
with open_file(cached=True, **msg_dict) as fileobj:
# Put the file info in msg_dict.
msg_dict['info'] = str(fileobj)
msg_dict['length'] = fileobj.length
if fileobj._rate < 44100:
# if py_imp == 'PyPy':
# blacklist = msg_dict.get('blacklist', [])
# blacklist.append('portaudio')
# msg_dict['blacklist'] = blacklist
# else:
import audioop
# msg_dict['rate'] = 44100
state = None
# Open an audio output device that can handle the data
# from fileobj.
device = open_device(fileobj, 'w', cached=True, **msg_dict)
try:
# Set the default number of loops to infinite.
fileobj.loops = msg_dict.get('loops', -1)
# Initialize variable.
buf = '\x00' * device.buffer_size
written = 0
# Loop until stopped.
while msg_dict.get('playing', True):
# Stop if the read buffer is empty or player is
# not paused.
if not (buf or msg_dict.get('paused', False)):
break
# Print the stream position.
if msg_dict.get('show_position', False):
# Only print the position if the stream has a
# length.
if fileobj.length > 0:
# Calculate the percentage played.
pos = (fileobj.position * 100) / float(
fileobj.length)
# Make the string.
pos_str = 'Position: %.2f%%' % pos
# Find the length of the string.
format_len = len(pos_str) + 2
# Print the string and after erasing the old
# one using ansi escapes.
print('\033[%dD\033[K%s' %
(format_len, pos_str),
end='')
sys_stdout.flush()
# Keep playing if not paused.
if not msg_dict.get('paused', False):
# Re-open the device after comming out of
# paused state.
if device.closed:
device = open_device(fileobj,
'w',
cached=True,
**msg_dict)
# Read the next buffer full of data.
try:
buf = fileobj.readline()
except KeyboardInterrupt:
break
# if device._rate != fileobj._rate \
# and py_imp != 'PyPy' and fileobj._rate != 0:
if device._rate != fileobj._rate \
and fileobj._rate != 0:
# Convert the input sample rate to that of
# the output device.
buf, state = audioop.ratecv(
buf, fileobj._width, fileobj._channels,
fileobj._rate, int(device._rate), state)
# Filler for end of partial buffer to elminiate
# end of audio noise.
if type(buf) == bytes:
filler = '\x00' * (device.buffer_size -
len(buf))
else:
filler = ''
# Write buf.
try:
written = device.write(buf + filler)
except KeyboardInterrupt:
break
else:
# Close the device when paused and sleep to
# open the audio for another process and
# save cpu cycles.
if not device.closed:
device.close()
time_sleep(0.05)
# Write a buffer of null bytes so the audio
# system can keep its buffer full.
# device.write(b'\x00' * device.buffer_size)
# Get and process any commands from the parent process.
if pipe.poll():
# Get the data into temp.
command = pipe.recv()
if 'getposition' in command:
pipe.send(fileobj.position)
elif 'setposition' in command:
fileobj.position = command['setposition']
elif 'getloops' in command:
pipe.send(fileobj.loops)
elif 'setloops' in command:
fileobj.loops = command['setloops']
elif 'getloopcount' in command:
pipe.send(fileobj.loop_count)
except Exception as err:
print(err)
finally:
if not device.closed:
device.close()
except IOError as err:
from time import sleep
msg_dict['error'] = err
msg_dict['info'] = ''
msg_dict['length'] = 0
print(err)
finally:
try:
# Set playing to False for the parent.
msg_dict['playing'] = False
except BrokenPipeError:
pass
def open(self, filename, **kwargs):
""" open(filename) -> Open an audio file to play.
"""
# Stop the current file from playing.
self.stop()
# Set the new filename.
self._filename = filename
# Reset the message dictionary so none of the old info is
# re-used.
self._msg_dict.clear()
# Fill the message dictionary with the new info.
self._msg_dict['show_position'] = self._show_position
self._msg_dict['filename'] = filename
self._msg_dict.update(kwargs)
self._control_dict.update(self._msg_dict)
# Pause it so when we call play later it will start the player
# but not the audio playback. Call play again to start audio
# playback.
self.pause()
# Start the playback process in a paused state. Requires a
# second call to play to un-pause.
self.play()
def play(self):
""" play() -> Start playback.
"""
if not self._msg_dict.get('playing', False):
# Set playing to True for the child process.
self._msg_dict['playing'] = True
# Open a new process to play a file in the background.
self._play_p = Process(target=self._play_proc,
args=(self._msg_dict, self._player_conn))
# Start the process.
self._play_p.start()
elif self._msg_dict.get('paused', True):
# Un-pause if paused.
self._msg_dict['paused'] = False
self._control_dict.update(self._msg_dict)
def stop(self):
""" stop() -> Stop playback.
"""
if self._msg_dict.get('playing', False):
# Stop playback.
self._msg_dict['playing'] = False
# Wait for the player process to stop.
self._play_p.join()
# Un-Pause.
self._msg_dict['paused'] = False
self._control_dict.update(self._msg_dict)
def pause(self):
""" pause() -> Pause playback.
"""
# Pause playback.
self._msg_dict['paused'] = True
self._control_dict.update(self._msg_dict)
@property
def error(self):
""" True if playing.
"""
return self._msg_dict.get('error', False)
@property
def paused(self):
""" True if playback is paused.
"""
return self._msg_dict.get('paused', False)
@property
def playing(self):
""" True if playing.
"""
return self._msg_dict.get('playing', False)
@property
def length(self):
""" Length of audio.
"""
return self._msg_dict.get('length', 0)
@property
@playing_wrapper
def position(self):
""" Current position.
"""
self._control_conn.send('getposition')
return self._control_conn.recv()
@position.setter
@playing_wrapper
def position(self, value):
""" Set the current position.
"""
self._control_conn.send({'setposition': int(value)})
@property
@playing_wrapper
def loops(self):
""" Number of times to loop (playback time + 1).
"""
self._control_conn.send('getloops')
return self._control_conn.recv()
@loops.setter
@playing_wrapper
def loops(self, value):
""" Number of times to loop (playback time + 1).
"""
self._control_conn.send({'setloops': int(value)})
@property
@playing_wrapper
def loop_count(self):
""" Number of times the player has looped.
"""
self._control_conn.send('getloopcount')
return self._control_conn.recv()
@playing_wrapper
def seek(self, offset, whence=SEEK_SET):
""" seek(position) -> Seek to position in mod.
"""
if whence == SEEK_CUR:
self.position += offset
elif whence == SEEK_END:
self.position = self.length - offset
else:
self.position = offset
return self.position
@playing_wrapper
def tell(self):
""" tell -> Returns the current position.
"""
return self.position
class MultiThreader:
def __init__(self):
self.max_threads = 100
self.queue = Manager().Queue(maxsize=0)
self.active_threads = []
self.inactive_threads = []
self.monitor = None
def run_queue_monitor(self, total):
monitor = QueueMonitor.QueueMonitor(self.queue, total)
self.monitor = Thread(target=monitor.run)
self.monitor.start()
@thread_error()
def add_thread(self, *args, **kwargs):
thread = Thread(target=self.wrap_thread, args=args, kwargs=kwargs)
self.inactive_threads.append(thread)
def wrap_thread(self, func, *args, **kwargs):
result = func(*args, **kwargs)
self.queue.put(result)
def schedule_threads(self):
thread_chunks = self.chunk_threads()
for chunk in thread_chunks:
self.run_threads(chunk)
self.join_monitor()
return self.queue
def chunk_threads(self):
chunk_size = self.max_threads
chunks = [
self.inactive_threads[i:i + chunk_size]
for i in range(0, len(self.inactive_threads), chunk_size)
]
return chunks
def run_threads(self, chunk, block_until_complete=True):
for thread in chunk:
self.run_thread(thread)
if block_until_complete:
self.join_threads(chunk)
def run_thread(self, thread):
self.inactive_threads.remove(thread)
self.active_threads.append(thread)
thread.start()
@staticmethod
def join_threads(chunk):
for thread in chunk:
thread.join()
def join_monitor(self):
if self.monitor:
self.monitor.join()
def kill_active_threads(self):
for thread in self.active_threads:
self.kill_thread(thread)
@thread_error()
def kill_thread(self, thread):
thread.stop()
self.active_threads.remove(thread)
def suspend_threads(self):
for thread in self.active_threads:
self.suspend_thread(thread)
@thread_error()
def suspend_thread(self, thread):
thread.stop()
self.inactive_threads.append(thread)
self.active_threads.remove(thread)
def clear_queue(self):
self.queue.clear()
def clear_inactive_threads(self):
self.inactive_threads = []
for jr in ir:
id_set = find_zid(jr)
if not id_set:
pass
else:
tset = tset | id_set
pool = multiprocessing.Pool(Ncpu - 2)
func = partial(searching_part1, mlist=mlist)
pool.map(func, scf_list)
pool.close()
pool.join()
with open('tmp_%s.txt' % str(a), 'w') as W:
for line in mlist.keys():
W.write(line + '\n')
mlist.clear()
tset.clear()
txts = sorted(glob.glob('tmp_*.txt'))
dfs = []
for i in txts:
df = pd.read_csv(i, header=None)
dfs.append(df)
tdf = pd.concat(dfs)
global tdf
lili = []
for i in sorted(df[0].drop_duplicates()):
df1 = dropop(i)
lili.append(df1)
df_fin = pd.concat(lili)
class Context(object):
"""
Stores settings across multiple independent processing units
This is a key-value store, that supports concurrency
across multiple processes.
"""
def __init__(self, settings=None, filter=None):
"""
Stores settings across multiple independent processing units
:param settings: the set of variables managed in this context
:type settings: dict
:param filter: a function to interpret values on check()
:type filter: callable
"""
# prevent Manager() process to be interrupted
handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
self.lock = Lock()
self.values = Manager().dict()
# restore current handler for the rest of the program
signal.signal(signal.SIGINT, handler)
self.filter = filter if filter else self._filter
if settings:
self.apply(settings)
def apply(self, settings={}):
"""
Applies multiple settings at once
:param settings: variables to be added to this context
:type settings: dict
"""
with self.lock:
for key in settings.keys():
if isinstance(settings[key], dict):
for label in settings[key].keys():
self.values[key + '.' +
label] = settings[key].get(label)
else:
self.values[key] = settings[key]
def clear(self):
"""
Clears content of a context
"""
with self.lock:
self.values.clear()
@property
def is_empty(self):
"""
Does the context store something?
:return: True if there at least one value, False otherwise
"""
with self.lock:
return len(self.values.keys()) < 1
def check(self,
key,
default=None,
is_mandatory=False,
validate=None,
filter=False):
"""
Checks some settings
:param key: the key that has to be checked
:type primary: str
:param default: the default value if no statement can be found
:type default: str
:param is_mandatory: raise an exception if keys are not found
:type is_mandatory: bool
:param validate: a function called to validate values before the import
:type validate: callable
:param filter: look at the content, and change it eventually
:type filter: bool
Example::
context = Context({
'spark': {
'room': 'My preferred room',
'participants':
['*****@*****.**', '*****@*****.**'],
'team': 'Anchor team',
'token': 'hkNWEtMJNkODk3ZDZLOGQ0OVGlZWU1NmYtyY>',
'webhook': "http://73a1e282.ngrok.io",
'weird_token', '$WEIRD_TOKEN',
}
})
context.check('spark.room', is_mandatory=True)
context.check('spark.team')
context.check('spark.weird_token', filter=True)
When a default value is provided, it is used to initialize
properly a missing key::
context.check('general.switch', 'on')
Another usage is to ensure that a key has been set::
context.check('spark.room', is_mandatory=True)
Additional control can be added with the validation function::
context.check('general.switch',
validate=lambda x: x in ('on', 'off'))
When filter is True, if the value is a string starting with '$',
then a variable with the same name is loaded from the environment::
>>>token=context.check('spark.weird_token', filter=True)
>>>assert token == os.environ.get('WEIRD_TOKEN')
True
The default filter can be changed at the creation of a context::
>>>context=Context(filter=lambda x : x + '...')
This function raises ``KeyError`` if a mandatory key is absent.
If a validation function is provided, then a ``ValueError`` can be
raised as well in some situations.
"""
with self.lock:
if default is not None:
value = self.values.get(key, None)
if value is None:
self.values[key] = default
value = default
elif (is_mandatory or validate):
try:
value = self.values[key]
except KeyError:
raise KeyError(u"Missing '{}' in context".format(key))
else:
try:
value = self.values[key]
except KeyError:
value = None
if validate and validate(value) is False:
raise ValueError(
u"Invalid value for '{}' in context".format(key))
if filter:
if value == default:
default = None # else kills filtering of empty variables
self.values[key] = self.filter(value, default)
@classmethod
def _filter(self, value, default=None):
"""
Loads a value from the environment
:param value: if it starts with '$',
then it names an environment variable
:type value: str
:param default: the default value if no variable can be found
:type default: str
:return: the same or a different text string
:rtype: str
If the string provided starts with the char '$', then the function
looks for an environment variable of this name and returns its value::
>>>print(context._filter('$HOME'))
/Users/bernard
This is useful if you want to secure your configuration files.
Instead of putting secrets in these files, you can store them
in the environment, and only make a reference.
Example::
context = Context({
'spark': {
'token': '$MY_BOT_TOKEN',
'weird_token', '$WEIRD_TOKEN',
}
})
context.check('spark.token', filter=True)
context.check('spark.weird_token', filter=True)
"""
if value is None or len(value) < 1 or value[0] != '$':
return value
imported = os.environ.get(value[1:], default)
if imported is None:
logging.warning(u"Missing {} in environment".format(value))
return imported
def has(self, prefix):
"""
Checks the presence of some prefix
:param prefix: key prefix to be checked
:type prefix: str
:return: True if one or more key start with the prefix, else False
This function looks at keys actually used in this context,
and return True if prefix is found. Else it returns False.
Example::
context = Context(settings={'space': {'title', 'a title'}})
>>>context.has('space')
True
>>>context.has('space.title')
True
>>>context.has('spark')
False
"""
with self.lock:
for key in self.values.keys():
if key.startswith(prefix):
return True
return False
def get(self, key, default=None):
"""
Retrieves the value of one configurationkey
:param key: name of the value
:type key: str
:param default: default value
:type default: any serializable type is accepted
:return: the actual value, or the default value, or None
Example::
message = context.get('bot.on_start')
This function is safe on multiprocessing and multithreading.
"""
with self.lock:
value = self.values.get(key)
if value is not None:
return value
values = {}
for label in self.values.keys():
if label.startswith(key + '.'):
values[label[len(key) + 1:]] = self.values[label]
if values.keys():
return values
return default
def set(self, key, value):
"""
Changes the value of one configuration key
:param key: name of the value
:type key: str
:param value: new value
:type value: any serializable type is accepted
Example::
context.set('bot.on_start', 'hello world')
This function is safe on multiprocessing and multithreading.
"""
with self.lock:
self.values[key] = value
def increment(self, key, delta=1):
"""
Increments a value
"""
with self.lock:
value = self.values.get(key, 0)
if not isinstance(value, int):
value = 0
value += delta
self.values[key] = value
return value
def decrement(self, key, delta=1):
"""
Decrements a value
"""
with self.lock:
value = self.values.get(key, 0)
if not isinstance(value, int):
value = 0
value -= delta
self.values[key] = value
return value
@classmethod
def set_logger(cls, level=logging.DEBUG):
"""
Configure logging
:param level: expected level of verbosity
This utility function should probably be put elsewhere
"""
handler = colorlog.StreamHandler()
formatter = colorlog.ColoredFormatter(
"%(asctime)-2s %(log_color)s%(message)s",
datefmt='%H:%M:%S',
reset=True,
log_colors={
'DEBUG': 'cyan',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red,bg_white',
},
secondary_log_colors={},
style='%')
handler.setFormatter(formatter)
logging.getLogger('').handlers = []
logging.getLogger('').addHandler(handler)
logging.getLogger('').setLevel(level=level)
class AudioPlayer(object):
""" Play audio files.
"""
def __init__(self, filename: str='', show_position: bool=False, **kwargs):
""" AudioPlayer(filename='', show_position=False, **kwargs) -> Open
filename and an appropriate audio io for it.
"""
self._filename = filename
self._show_position = show_position
# Setup the msg_dict for sending messages to the child process.
self._msg_dict = Manager().dict()
self._control_dict = {}
# Create a pipe for sending and receiving messages.
self._control_conn, self._player_conn = Pipe()
# Open the file.
if filename:
self.open(filename, **kwargs)
def __str__(self) -> str:
""" The information about the open file.
"""
# Wait for the stream to open.
while 'info' not in self._msg_dict: pass
# Return the info string.
return self._msg_dict.get('info', '')
def __repr__(self) -> str:
""" __repr__ -> Returns a python expression to recreate this instance.
"""
repr_str = "filename='%(_filename)s'" % self.__dict__
return '%s(%s)' % (self.__class__.__name__, repr_str)
def __enter__(self):
""" Provides the ability to use pythons with statement.
"""
try:
return self
except Exception as err:
print(err)
return None
def __exit__(self, exc_type, exc_value, traceback):
""" Stop playback when finished.
"""
try:
self.stop()
self._control_conn.close()
self._player_conn.close()
return not bool(exc_type)
except Exception as err:
print(err)
return False
def __del__(self):
""" Stop playback before deleting.
"""
if self._control_dict.get('playing', False):
try:
self.stop()
except IOError:
pass
def __len__(self):
""" The length of the file if it has one.
"""
return self.length if self.length >= 0 else 0
def playing_wrapper(func):
""" Wrap methods and only call them if the stream is playing
"""
@functools_wraps(func)
def wrapper(self, *args, **kwargs):
""" Check if stream is playing and if it is then call func
otherwise print a message and exit.
"""
if not self.playing:
print("%(filename)s is not playing." % self._msg_dict)
return None
return func(self, *args, **kwargs)
return wrapper
def _play_proc(self, msg_dict: dict, pipe: Pipe):
""" Player process
"""
# Open the file to play.
try:
with open_file(cached=True, **msg_dict) as fileobj:
# Put the file info in msg_dict.
msg_dict['info'] = str(fileobj)
msg_dict['length'] = fileobj.length
if fileobj._rate < 44100:
# if py_imp == 'PyPy':
# blacklist = msg_dict.get('blacklist', [])
# blacklist.append('portaudio')
# msg_dict['blacklist'] = blacklist
# else:
import audioop
# msg_dict['rate'] = 44100
state = None
# Open an audio output device that can handle the data
# from fileobj.
device = open_device(fileobj, 'w', cached=True, **msg_dict)
try:
# Set the default number of loops to infinite.
fileobj.loops = msg_dict.get('loops', -1)
# Initialize variable.
buf = b'\x00' * device.buffer_size
written = 0
# Loop until stopped.
while msg_dict.get('playing', True):
# Stop if the read buffer is empty or player is
# not paused.
if not (buf or msg_dict.get('paused', False)):
break
# Print the stream position.
if msg_dict.get('show_position', False):
# Only print the position if the stream has a
# length.
if fileobj.length > 0:
# Calculate the percentage played.
pos = (fileobj.position * 100) / fileobj.length
# Make the string.
pos_str = 'Position: %.2f%%' % pos
# Find the length of the string.
format_len = len(pos_str) + 2
# Print the string and after erasing the old
# one using ansi escapes.
if py_imp == 'PyPy':
# Running in pypy which doesn't have the
# flush parameter in the print function.
print('\033[%dD\033[K%s' % (format_len,
pos_str), end='')
sys_stdout.flush()
else:
print('\033[%dD\033[K%s' % (format_len,
pos_str), end='', flush=True)
# Keep playing if not paused.
if not msg_dict.get('paused', False):
# Re-open the device after comming out of
# paused state.
if device.closed:
device = open_device(fileobj, 'w', cached=True,
**msg_dict)
# Read the next buffer full of data.
try:
buf = fileobj.readline()
except KeyboardInterrupt:
break
# if device._rate != fileobj._rate \
# and py_imp != 'PyPy' and fileobj._rate != 0:
if device._rate != fileobj._rate \
and fileobj._rate != 0:
# Convert the input sample rate to that of
# the output device.
buf, state = audioop.ratecv(buf,
fileobj._width,
fileobj._channels,
fileobj._rate,
int(device._rate),
state)
# Filler for end of partial buffer to elminiate
# end of audio noise.
if type(buf) == bytes:
filler = b'\x00' * (device.buffer_size - len(buf))
else:
filler = ''
# Write buf.
try:
written = device.write(buf + filler)
except KeyboardInterrupt:
break
else:
# Close the device when paused and sleep to
# open the audio for another process and
# save cpu cycles.
if not device.closed:
device.close()
time_sleep(0.05)
# Write a buffer of null bytes so the audio
# system can keep its buffer full.
# device.write(b'\x00' * device.buffer_size)
# Get and process any commands from the parent process.
if pipe.poll():
# Get the data into temp.
command = pipe.recv()
if 'getposition' in command:
pipe.send(fileobj.position)
elif 'setposition' in command:
fileobj.position = command['setposition']
elif 'getloops' in command:
pipe.send(fileobj.loops)
elif 'setloops' in command:
fileobj.loops = command['setloops']
elif 'getloopcount' in command:
pipe.send(fileobj.loop_count)
except Exception as err:
print(err)
finally:
if not device.closed:
device.close()
except IOError as err:
from time import sleep
msg_dict['error'] = err
msg_dict['info'] = ''
msg_dict['length'] = 0
print(err)
finally:
try:
# Set playing to False for the parent.
msg_dict['playing'] = False
except BrokenPipeError:
pass
def open(self, filename: str, **kwargs):
""" open(filename) -> Open an audio file to play.
"""
# Stop the current file from playing.
self.stop()
# Set the new filename.
self._filename = filename
# Reset the message dictionary so none of the old info is
# re-used.
self._msg_dict.clear()
# Fill the message dictionary with the new info.
self._msg_dict['show_position'] = self._show_position
self._msg_dict['filename'] = filename
self._msg_dict.update(kwargs)
self._control_dict.update(self._msg_dict)
# Pause it so when we call play later it will start the player
# but not the audio playback. Call play again to start audio
# playback.
self.pause()
# Start the playback process in a paused state. Requires a
# second call to play to un-pause.
self.play()
def play(self):
""" play() -> Start playback.
"""
if not self._msg_dict.get('playing', False):
# Set playing to True for the child process.
self._msg_dict['playing'] = True
# Open a new process to play a file in the background.
self._play_p = Process(target=self._play_proc,
args=(self._msg_dict, self._player_conn))
# Start the process.
self._play_p.start()
elif self._msg_dict.get('paused', True):
# Un-pause if paused.
self._msg_dict['paused'] = False
self._control_dict.update(self._msg_dict)
def stop(self):
""" stop() -> Stop playback.
"""
if self._msg_dict.get('playing', False):
# Stop playback.
self._msg_dict['playing'] = False
# Wait for the player process to stop.
self._play_p.join()
# Un-Pause.
self._msg_dict['paused'] = False
self._control_dict.update(self._msg_dict)
def pause(self):
""" pause() -> Pause playback.
"""
# Pause playback.
self._msg_dict['paused'] = True
self._control_dict.update(self._msg_dict)
@property
def error(self) -> bool:
""" True if playing.
"""
return self._msg_dict.get('error', False)
@property
def paused(self) -> bool:
""" True if playback is paused.
"""
return self._msg_dict.get('paused', False)
@property
def playing(self) -> bool:
""" True if playing.
"""
return self._msg_dict.get('playing', False)
@property
def length(self) -> int:
""" Length of audio.
"""
return self._msg_dict.get('length', 0)
@property
@playing_wrapper
def position(self) -> int:
""" Current position.
"""
self._control_conn.send('getposition')
return self._control_conn.recv()
@position.setter
@playing_wrapper
def position(self, value: int):
""" Set the current position.
"""
self._control_conn.send({'setposition': int(value)})
@property
@playing_wrapper
def loops(self) -> int:
""" Number of times to loop (playback time + 1).
"""
self._control_conn.send('getloops')
return self._control_conn.recv()
@loops.setter
@playing_wrapper
def loops(self, value: int):
""" Number of times to loop (playback time + 1).
"""
self._control_conn.send({'setloops': int(value)})
@property
@playing_wrapper
def loop_count(self) -> int:
""" Number of times the player has looped.
"""
self._control_conn.send('getloopcount')
return self._control_conn.recv()
@playing_wrapper
def seek(self, offset: int, whence=SEEK_SET) -> int:
""" seek(position) -> Seek to position in mod.
"""
if whence == SEEK_CUR:
self.position += offset
elif whence == SEEK_END:
self.position = self.length - offset
else:
self.position = offset
return self.position
@playing_wrapper
def tell(self) -> int:
""" tell -> Returns the current position.
"""
return self.position
class MemoryStore(Store):
"""
Stores data for one space
This is a key-value store, that supports concurrency
across multiple processes.
Example::
store = MemoryStore()
"""
def on_init(self, **kwargs):
"""
Adds processing to initialization
"""
# prevent Manager() process to be interrupted
handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
self.values = Manager().dict()
# restore current handler for the rest of the program
signal.signal(signal.SIGINT, handler)
def _set(self, key, value):
"""
Sets a permanent value
:param key: name of the value
:type key: str
:param value: actual value
:type value: any serializable type is accepted
This functions stores or updates a value in the back-end storage
system.
Example::
store._set('parameter_123', 'George')
"""
self.values[key] = value
def _get(self, key):
"""
Gets a permanent value
:param key: name of the value
:type key: str
:return: the actual value, or None
Example::
value = store._get('parameter_123')
"""
return self.values.get(key)
def _clear(self, key=None):
"""
Forgets a value or all values
:param key: name of the value to forget, or None
:type key: str
To clear only one value, provide the name of it.
For example::
store._clear('parameter_123')
To clear all values in the store, just call the function
without a value.
For example::
store._clear()
"""
if not key:
self.values.clear()
else:
self.values[key] = None
class VideoCaptureThreading:
"""
read video frames based cv2.VideoCapture
"""
def __init__(self, video_path: Path, sample_path: Path, index_pool: Queue, frame_queue: Queue, cfg: VideoConfig,
idx,
sample_rate=5, width=640, height=480, delete_post=True):
self.cfg = cfg
self.video_path = video_path
self.sample_path = sample_path
self.index_pool = index_pool
self.idx = idx
# self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
# self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
# self.grabbed, self.frame = self.cap.read()
self.status = Manager().Value('i', SystemStatus.SHUT_DOWN)
self.src = -1
self.cap = None
self.sample_rate = sample_rate
self.frame_queue = frame_queue
self.delete_post = delete_post
self.runtime = 0
self.posix = None
self.quit = Manager().Event()
self.quit.clear()
def set(self, var1, var2):
self.cap.set(var1, var2)
def __start__(self, *args):
"""
capture initialization
:param args:
:return:
"""
if self.status.get() == SystemStatus.RUNNING:
print('[!] Threaded video capturing has already been started.')
return None
self.update_capture(0)
self.status.set(SystemStatus.RUNNING)
threading.Thread(target=self.listen, args=(), daemon=True).start()
time.sleep(5) # wait detection service init done
self.update(*args)
return self
def listen(self):
"""
controller capture process to shutdown
:return:
"""
logger.info('Video Capture [{}]: Start listen event'.format(self.cfg.index))
if self.quit.wait():
logger.info('Video Capture [{}]: Receive quit signal'.format(self.cfg.index))
self.cancel()
def cancel(self):
self.status.set(SystemStatus.SHUT_DOWN)
def load_next_src(self):
"""
load next video source
:return:
"""
logger.debug('Loading video stream from video index pool....')
self.posix = self.get_posix()
self.src = str(self.posix)
if self.posix == -1:
return self.src
basename = os.path.basename(self.src)
filename, extention = os.path.splitext(basename)
if extention == '.mp4' or extention == '.mov':
return self.src
else:
return 0
def get_posix(self):
"""
a base pointer, can be overrived by sub-class
:return:
"""
# DEPRECATED
return self.video_path / self.index_pool.get()
def update(self, *args):
"""
video capture service, loop video frames from a rtsp stream
:param args:
:return:
"""
cnt = 0
start = time.time()
logger.info('*******************************Init video capture [{}]********************************'.format(
self.cfg.index))
ssd_detector = None
classifier = None
server_cfg = args[0]
# if server_cfg.detect_mode == ModelType.SSD:
# ssd_detector = SSDDetector(model_path=server_cfg.detect_model_path, device_id=server_cfg.cd_id)
# ssd_detector.run()
# logger.info(
# f'*******************************Capture [{self.cfg.index}]: Running SSD Model********************************')
# elif server_cfg.detect_mode == ModelType.CLASSIFY:
# classifier = DolphinClassifier(model_path=server_cfg.classify_model_path, device_id=server_cfg.dt_id)
# classifier.run()
# logger.info(
# f'*******************************Capture [{self.cfg.index}]: Running Classifier Model********************************')
while self.status.get() == SystemStatus.RUNNING:
# with self.read_lock:
s = time.time()
grabbed, frame = self.read_frame()
e = 1 / (time.time() - s)
logger.debug(f'Video capture [{self.cfg.index}]: Receive Speed Rate [{round(e, 2)}]/FPS')
# if e > 25:
# sleep_time = 1 / (e / 25) / 2
# time.sleep(sleep_time)
# logger.info(
# f'Video capture [{self.cfg.index}]: too quick receive speed rate [{e}/FPS],sleep [{sleep_time}] seconds.')
s = time.time()
if not grabbed:
# if current video source is end,load the next video sources
self.update_capture(cnt)
end = time.time()
logger.info('Current src consumes time: [{}] seconds'.format(end - start))
start = time.time()
cnt = 0
continue
# if cnt % self.sample_rate == 0:
self.pass_frame(frame, args[0], ssd_detector, classifier)
e = 1 / (time.time() - s)
logger.debug(f'Video capture [{self.cfg.index}]: Operation Speed Rate [{round(e, 2)}]/FPS')
cnt += 1
self.post_frame_process(frame)
self.runtime = time.time() - start
logger.info(
'*******************************Video capture [{}] exit********************************'.format(
self.cfg.index))
# logger.info('Video Capture [{}]: cancel..'.format(self.cfg.index))
def read_frame(self):
return self.cap.read()
def pass_frame(self, *args):
self.frame_queue.put(args[0], block=True)
# logger.info('Passed frame...')
def update_capture(self, cnt):
"""
release old video capture instance and init a new video capture when update video source
:param cnt:
:return:
"""
logger.debug('Read frame done from [{}].Has loaded [{}] frames'.format(self.src, cnt))
logger.debug('Read next frame from video ....')
self.handle_history()
while True:
src = self.load_next_src()
if src == str(-1):
self.cancel()
return False
elif src == 0:
continue
else:
break
self.reload_cap(src)
return True
def reload_cap(self, src):
if self.cap is not None:
self.cap.release()
self.cap = cv2.VideoCapture(src)
def handle_history(self):
if self.posix.exists() and self.delete_post:
self.posix.unlink()
def post_frame_process(self, frame):
pass
def read(self, *args):
"""
:param args:
:return:
"""
try:
if self.status.get() == SystemStatus.SHUT_DOWN:
self.__start__(*args)
return True
except Exception as e:
logger.error(e)
traceback.print_exc()
# with self.read_lock:
# frame = self.frame.copy()
# grabbed = self.grabbed
# return self.grabbed, self.frame
def stop(self):
self.status.set(SystemStatus.SHUT_DOWN)
self.cap.release()
self.thread.join()
def __exit__(self, exec_type, exc_value, traceback):
self.cap.release()
class DetectionMonitor(object):
"""
A global manager that creates camera workdirs based on the camera's configuration file,
initializes many services, and defines basic state control functions.
"""
def __init__(self,
cfgs: List[VideoConfig],
scfg: ServerConfig,
stream_path: Path,
sample_path: Path,
frame_path: Path,
region_path: Path,
offline_path: Path = None,
build_pool=True) -> None:
super().__init__()
# self.cfgs = I.load_video_config(cfgs)[-1:]
# self.cfgs = I.load_video_config(cfgs)
# self.cfgs = [c for c in self.cfgs if c.enable]
# self.cfgs = [c for c in cfgs if enable_options[c.index]]
self.scfg = scfg
self.cfgs = cfgs
self.quit = False
# Communication Pipe between detector and stream receiver
self.pipes = [Manager().Queue(c.max_streams_cache) for c in self.cfgs]
self.time_stamp = generate_time_stamp('%m%d')
self.stream_path = stream_path / self.time_stamp
self.sample_path = sample_path / self.time_stamp
self.frame_path = frame_path / self.time_stamp
self.region_path = region_path / self.time_stamp
self.offline_path = offline_path
self.process_pool = None
self.thread_pool = None
self.shut_down_event = Manager().Event()
self.shut_down_event.clear()
self.scheduler = BackgroundScheduler()
if build_pool:
# build service
# pool_size = min(len(cfgs) * 2, cpu_count() - 1)
# self.process_pool = Pool(processes=pool_size)
self.process_pool = Pool(processes=len(self.cfgs) * 5)
self.thread_pool = ThreadPoolExecutor()
# self.clean()
self.stream_receivers = [
stream.StreamReceiver(self.stream_path / str(c.index),
offline_path, c, self.pipes[idx])
for idx, c in enumerate(self.cfgs)
]
self.scheduler.add_job(self.notify_shut_down,
'cron',
month=self.scfg.cron['end']['month'],
day=self.scfg.cron['end']['day'],
hour=self.scfg.cron['end']['hour'],
minute=self.scfg.cron['end']['minute'])
self.scheduler.start()
self.frame_cache_manager = SharedMemoryManager()
self.frame_cache_manager.start()
def monitor(self):
self.call()
self.wait()
# def set_runtime(self, runtime):
# self.runtime = runtime
def shut_down_from_keyboard(self):
logger.info('Click Double Enter to shut down system.')
while True and not self.shut_down_event.is_set():
c = sys.stdin.read(1)
logger.info(c)
if c == '\n':
self.notify_shut_down()
break
# if keycode == Key.enter:
# self.shut_down_event.set()
def shut_down_after(self, runtime=-1):
if runtime == -1:
return
logger.info('System will exit after [{}] seconds'.format(runtime))
threading.Timer(runtime, self.notify_shut_down).start()
def notify_shut_down(self):
if not self.shut_down_event.is_set():
self.shut_down_event.set()
def listen(self):
# Listener(on_press=self.shut_down_from_keyboard).start()
threading.Thread(target=self.shut_down_from_keyboard,
daemon=True).start()
logger.info(
'*******************************Monitor: Listening exit event********************************'
)
# if self.runtime != -1:
# time.sleep(self.runtime)
# else:
# input('')
self.shut_down_event.wait()
logger.info(
'*******************************Monitor: preparing exit system********************************'
)
self.cancel()
def cancel(self):
pass
def call(self):
for i, cfg in enumerate(self.cfgs):
# clean all legacy streams and candidates files before initialization
self.init_stream_receiver(i)
self.init_detection(cfg, i)
def wait(self):
logger.info('Wait processes done.')
if self.process_pool is not None:
self.process_pool.close()
self.process_pool.join()
logger.info('Closed Pool')
def init_detection(self, cfg, i):
if self.process_pool is not None:
self.process_pool.apply_async(detect, (
self.stream_path / str(cfg.index),
self.region_path / str(cfg.index),
self.pipes[i],
cfg,
))
def init_stream_receiver(self, i):
if self.process_pool is not None:
return self.process_pool.apply_async(
self.stream_receivers[i].receive_online)
def clean(self):
clean_dir(self.sample_path)
clean_dir(self.stream_path)
clean_dir(self.region_path)
class AudioPlayer:
"""AudioPlayer.
An audio player object.
"""
def __init__(self, **kwargs):
"""Play audio in the background using a subprocess."""
self._kwargs = kwargs
self._filename = kwargs.pop("filename", "")
self._show_position = kwargs.pop("show_position", False)
# Setup the msg_dict for sending messages to the child process.
self._msg_dict = Manager().dict()
self._control_dict = {}
# Create a pipe for sending and receiving messages.
self._control_conn, self._player_conn = Pipe()
# Open the file.
if self._filename:
self.open(self._filename, **kwargs)
def __str__(self) -> str:
"""Get the information about the open file."""
# Return nothing if no file is open.
if not self._filename:
return ''
# Wait for the stream to open.
while 'info' not in self._msg_dict:
pass
# Return the info string.
return self._msg_dict.get('info', '')
def __repr__(self) -> str:
"""Return a python expression to recreate this instance."""
kwargs_lst = []
for key, value in self._kwargs.items():
if type(value) == str:
value = f'"{value}"'
kwargs_lst.append(f"{key}={value}")
return f"{self.__class__.__name__}({', '.join(kwargs_lst)})"
def __enter__(self) -> 'AudioPlayer':
"""Provide the ability to use pythons with statement."""
return self
def __exit__(self, exc_type, exc_value, traceback) -> bool:
"""Stop playback when finished."""
try:
self.stop()
self._control_conn.close()
self._player_conn.close()
return not bool(exc_type)
except Exception as err:
print(err)
return False
def __del__(self):
"""Stop playback before deleting."""
self.close()
def __len__(self) -> int:
"""Get the length of the file if it has one."""
return self.length if self.length >= 0 else 0
def playing_wrapper(func: Callable) -> Callable:
"""Wrap methods and only call them if the stream is playing."""
@functools_wraps(func)
def wrapper(self, *args, **kwargs) -> Callable:
"""Check if stream is playing.
Check if stream is playing and if it is then call func otherwise
print a message and exit.
"""
if not self.playing:
print(f"{self._msg_dict['filename']} is not playing.")
return lambda **a: print(a)
return func(self, *args, **kwargs)
return wrapper
def _play_proc(self, msg_dict: dict, pipe: Connection):
"""Player process."""
# Open the file to play.
try:
with open_file(**msg_dict) as fileobj:
# Put the file info in msg_dict.
msg_dict['info'] = str(fileobj)
msg_dict['length'] = fileobj.length
state = None
# Open an audio output device that can handle the data
# from fileobj.
device = open_device(fileobj, 'w', **msg_dict)
msg_out(f"\nFile: {repr(fileobj)}\n")
msg_out(f"\nDevice: {repr(device)}\n")
try:
# Set the default number of loops to infinite.
fileobj.loops = msg_dict.get('loops', -1)
# Initialize variable.
buf = b'\x00' * device.buffer_size
# Loop until stopped.
while msg_dict.get('playing', True):
# Stop if the read buffer is empty or player is
# not paused.
if not (buf or msg_dict.get('paused', False)):
break
# Print the stream position.
if msg_dict.get('show_position', False):
print_position(fileobj)
# Keep playing if not paused.
if not msg_dict.get('paused', False):
# Re-open the device after comming out of
# paused state.
if device.closed:
device = open_device(
fileobj,
'w',
cached=True,
**msg_dict
)
# Read the next buffer full of data.
try:
buf = fileobj.readline()
except KeyboardInterrupt:
break
# Sometimes a read needs to be made before the
# length is available.
if not msg_dict['length']:
msg_dict['length'] = fileobj.length
if device._rate != fileobj._rate \
and fileobj._rate != 0:
# Convert the input sample rate to that of
# the output device.
buf, state = audioop.ratecv(
buf,
fileobj._width,
fileobj._channels,
fileobj._rate,
int(device._rate),
state
)
# Filler for end of partial buffer to elminiate
# end of audio noise.
if type(buf) == bytes:
filler = b'\x00' * \
(device.buffer_size - len(buf))
else:
filler = ''
# Write buf.
try:
_ = device.write(buf + filler)
except KeyboardInterrupt:
break
else:
# Close the device when paused and sleep to
# open the audio for another process and
# save cpu cycles.
if not device.closed:
device.close()
sleep(0.05)
# Write a buffer of null bytes so the audio
# system can keep its buffer full.
# device.write(b'\x00' * device.buffer_size)
# Get and process any commands from the parent process.
if pipe.poll():
# Get the data into temp.
command = pipe.recv()
if 'getposition' in command:
pipe.send(fileobj.position)
elif 'setposition' in command:
fileobj.position = command['setposition']
elif 'getloops' in command:
pipe.send(fileobj.loops)
elif 'setloops' in command:
fileobj.loops = command['setloops']
elif 'getloopcount' in command:
pipe.send(fileobj.loop_count)
elif 'getlength' in command:
pipe.send(fileobj.length)
except Exception as err:
print(err)
finally:
if not device.closed:
device.close()
except IOError as err:
msg_dict['error'] = err
msg_dict['info'] = ''
msg_dict['length'] = 0
print(err)
finally:
try:
# Set playing to False for the parent.
msg_dict['playing'] = False
except BrokenPipeError:
pass
def open(self, filename: str, **kwargs):
"""Open an audio file to play."""
# Stop the current file from playing.
self.stop()
# Skip non-files.
if not Path(filename).is_file():
raise(IOError(f"{filename} is not a file."))
# Skip unsupported files.
if get_codec(
filename,
blacklist=[*kwargs.get('blacklist', []), 'all']
) == DummyFile:
raise(IOError(f"File {filename} not supported."))
# Set the new filename.
self._filename = filename
# Reset the message dictionary so none of the old info is
# re-used.
self._msg_dict.clear()
# Fill the message dictionary with the new info.
self._msg_dict['show_position'] = self._show_position
self._msg_dict['filename'] = filename
self._msg_dict.update(kwargs)
self._control_dict.update(self._msg_dict)
# Pause it so when we call play later it will start the player
# but not the audio playback. Call play again to start audio
# playback.
self.pause()
# Start the playback process in a paused state. Requires a
# second call to play to un-pause.
self.play()
def play(self):
"""Start playback."""
if not self._msg_dict.get('playing', False):
# Set playing to True for the child process.
self._msg_dict['playing'] = True
# Open a new process to play a file in the background.
self._play_p = Process(
target=self._play_proc,
args=(self._msg_dict, self._player_conn)
)
# Start the process.
self._play_p.start()
elif self._msg_dict.get('paused', True):
# Un-pause if paused.
self._msg_dict['paused'] = False
self._control_dict.update(self._msg_dict)
def stop(self):
"""Stop playback."""
if self._msg_dict.get('playing', False):
# Stop playback.
self._msg_dict['playing'] = False
# Wait for the player process to stop.
self._play_p.join()
# Un-Pause.
self._msg_dict['paused'] = False
self._control_dict.update(self._msg_dict)
def pause(self):
"""Pause playback."""
# Pause playback.
self._msg_dict['paused'] = True
self._control_dict.update(self._msg_dict)
@property
def error(self) -> bool:
"""Return True if playing."""
return self._msg_dict.get('error', False)
@property
def paused(self) -> bool:
"""Return True if playback is paused."""
return self._msg_dict.get('paused', False)
@property
def playing(self) -> bool:
"""Return True if playing."""
return self._msg_dict.get('playing', False)
@property
def length(self) -> int:
"""Get the length of audio."""
# self._control_conn.send('getlength')
# return self._control_conn.recv()
return self._msg_dict.get('length', 0)
@property
@playing_wrapper
def position(self) -> int:
"""Get the current position."""
self._control_conn.send('getposition')
return self._control_conn.recv()
@position.setter
@playing_wrapper
def position(self, value: int):
"""Set the current position."""
self._control_conn.send({'setposition': value})
@property
@playing_wrapper
def loops(self) -> int:
"""Get the number of times to loop (playback time + 1)."""
self._control_conn.send('getloops')
return self._control_conn.recv()
@loops.setter
@playing_wrapper
def loops(self, value: int):
"""Set the number of times to loop (playback time + 1)."""
self._control_conn.send({'setloops': int(value)})
@property
@playing_wrapper
def loop_count(self) -> int:
"""Get the number of times the player has looped."""
self._control_conn.send('getloopcount')
return self._control_conn.recv()
@playing_wrapper
def seek(self, offset: int, whence: int = SEEK_SET) -> int:
"""Seek to position in mod."""
if whence == SEEK_CUR:
self.position += offset
elif whence == SEEK_END:
self.position = self.length - offset
else:
self.position = offset
return self.position
@playing_wrapper
def tell(self) -> int:
"""Return the current position."""
return self.position
def close(self):
"""Close the audio file and device."""
if self._control_dict.get('playing', False):
try:
self.stop()
except IOError:
pass
class PushStreamer(object):
def __init__(self, cfg: VideoConfig, stream_stack: List) -> None:
super().__init__()
self.cfg = cfg
self.stream_stack = stream_stack
self.LOG_PREFIX = f'Push Streamer [{self.cfg.index}]: '
self.quit = Manager().Event()
self.quit.clear()
self.status = Manager().Value('i', SystemStatus.RUNNING)
def listen(self):
if self.quit.wait():
self.status.set(SystemStatus.SHUT_DOWN)
# self.stream_render.quit.set()
def push_stream(self):
logger.info(
f'*******************************Controller [{self.cfg.index}]: Init push stream service********************************')
draw_cnt = 0
tmp_results = []
video_streamer = FFMPEG_VideoStreamer(self.cfg.push_to, size=(self.cfg.shape[1], self.cfg.shape[0]), fps=25,
codec='h264', )
video_streamer.write_frame(np.zeros((self.cfg.shape[1], self.cfg.shape[0], 3), dtype=np.uint8))
# time.sleep(6)
pre_index = 0
threading.Thread(target=self.listen, daemon=True).start()
while self.status.get() == SystemStatus.RUNNING:
try:
ps = time.time()
# if self.status.get() == SystemStatus.SHUT_DOWN:
# video_streamer.close()
# break
# se = 1 / (time.time() - ps)
# logger.debug(self.LOG_PREFIX + f'Get Signal Speed Rate: [{round(se, 2)}]/FPS')
# gs = time.time()
if self.cfg.use_sm:
# 4K frame has large size, get it from shared memory instead pipe serialization between
# multi-processes
frame = self.stream_stack[0][0]
# current index and other info are smaller than frame buffer,so we can use Manager().list()
if not len(self.stream_stack[1]):
continue
proc_res, frame_index = self.stream_stack[1].pop()
else:
# if frame shape is 4K, obtain FPS from Manager().list is around 10,which is over slow than video fps(25)
# it causes much latent when pushing stream
if not len(self.stream_stack):
continue
frame, proc_res, frame_index = self.stream_stack.pop()
logger.debug(f'Push Streamer [{self.cfg.index}]: Cache queue size: [{len(self.stream_stack)}]')
if not self.cfg.use_sm and len(self.stream_stack) > 1000:
self.stream_stack[:] = []
logger.info(self.LOG_PREFIX + 'Too much frames blocked in stream queue.Cleared')
continue
if pre_index < frame_index:
pre_index = frame_index
else:
continue
# end = 1 / (time.time() - gs)
# logger.debug(self.LOG_PREFIX + f'Get Frame Speed Rate: [{round(end, 2)}]/FPS')
detect_flag = (proc_res is not None and proc_res.detect_flag)
# logger.info(f'Draw cnt: [{draw_cnt}]')
# if proc_res is not None:
# logger.info(f'Detect flag: [{proc_res.detect_flag}]')
# ds = time.time()
if detect_flag:
# logger.info('Detect flag~~~~~~~~~~')
draw_cnt = 0
tmp_results = proc_res.results
is_draw_over = draw_cnt <= 36
if is_draw_over:
# logger.info('Draw next frames~~~~~~~~~~~~~~~~~~~~~~~~~~~')
for r in tmp_results:
for rect in r.rects:
color = np.random.randint(0, 255, size=(3,))
color = [int(c) for c in color]
p1, p2 = bbox_points(self.cfg, rect, frame.shape)
# p1 = (int(rect[0]), int(rect[1]))
# p2 = (int(rect[2]), int(rect[3]))
cv2.putText(frame, 'Asaeorientalis', p1,
cv2.FONT_HERSHEY_COMPLEX, 2, color, 2, cv2.LINE_AA)
cv2.rectangle(frame, p1, p2, color, 2)
# if self.server_cfg.detect_mode == ModelType.SSD:
# cv2.putText(frame, str(round(r[4], 2)), (p2[0], p2[1]),
# cv2.FONT_HERSHEY_COMPLEX, 2, color, 2, cv2.LINE_AA)
draw_cnt += 1
if self.cfg.write_timestamp:
time_stamp = generate_time_stamp("%Y-%m-%d %H:%M:%S")
cv2.putText(frame, time_stamp, (100, 100),
cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 2, cv2.LINE_AA)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# de = 1 / (time.time() - ds)
# logger.debug(self.LOG_PREFIX + f'Draw Speed Rate: [{round(de, 2)}]/FPS')
# logger.info(f'Frame index [{frame_index}]')
# if frame_index % self.cfg.sample_rate == 0:
# for _ in range(2):
# video_streamer.write_frame(frame)
# else:
# video_streamer.write_frame(frame)
# end = 1 / (time.time() - ps)
# ws = time.time()
video_streamer.write_frame(frame)
# w_end = 1 / (time.time() - ws)
end = 1 / (time.time() - ps)
# logger.debug(self.LOG_PREFIX + f'Writing Speed Rate: [{round(w_end, 2)}]/FPS')
logger.info(f'Streamer [{self.cfg.index}]: Streaming Speed Rate: [{round(end, 2)}]/FPS')
except Exception as e:
logger.error(e)
traceback.print_stack()
# logger.warning(e)
logger.info(
'*******************************Controller [{}]: Push stream service exit********************************'.format(
self.cfg.index))
