def test_multi_processing_handler(self):
from multiprocessing import Process, Queue
from logbook.queues import MultiProcessingHandler, \
MultiProcessingSubscriber
queue = Queue(-1)
test_handler = logbook.TestHandler()
subscriber = MultiProcessingSubscriber(queue)
def send_back():
handler = MultiProcessingHandler(queue)
handler.push_thread()
try:
logbook.warn('Hello World')
finally:
handler.pop_thread()
p = Process(target=send_back)
p.start()
p.join()
test_handler.push_thread()
try:
subscriber.dispatch_once()
self.assert_(test_handler.has_warning('Hello World'))
finally:
test_handler.pop_thread()
def enable_multiprocess_log_handing(args):
"""
Set up logging when using multiprocessing
"""
normal_handler = log_handler(args, thread_wrapping=False)
manager = multiprocessing.Manager()
queue = manager.Queue(-1)
mp_handler = MultiProcessingHandler(queue)
mp_handler.push_application()
mp_sub = MultiProcessingSubscriber(queue)
mp_sub.dispatch_in_background(normal_handler)
return queue
def test_multi_processing_handler():
from multiprocessing import Process, Queue
from logbook.queues import MultiProcessingSubscriber
queue = Queue(-1)
test_handler = logbook.TestHandler()
subscriber = MultiProcessingSubscriber(queue)
p = Process(target=MultiProcessingHandlerSendBack(queue))
p.start()
p.join()
with test_handler:
subscriber.dispatch_once()
assert test_handler.has_warning('Hello World')
def test_multi_processing_handler():
from multiprocessing import Process, Queue
from logbook.queues import MultiProcessingSubscriber
queue = Queue(-1)
test_handler = logbook.TestHandler()
subscriber = MultiProcessingSubscriber(queue)
p = Process(target=MultiProcessingHandlerSendBack(queue))
p.start()
p.join()
with test_handler:
subscriber.dispatch_once()
assert test_handler.has_warning('Hello World')
def test_multi_processing_handler():
if os.getenv('APPVEYOR') == 'True':
pytest.skip('Test hangs on AppVeyor CI')
from multiprocessing import Process, Queue
from logbook.queues import MultiProcessingSubscriber
queue = Queue(-1)
test_handler = logbook.TestHandler()
subscriber = MultiProcessingSubscriber(queue)
p = Process(target=MultiProcessingHandlerSendBack(queue))
p.start()
p.join()
with test_handler:
subscriber.dispatch_once()
assert test_handler.has_warning('Hello World')
def test_multi_processing_handler():
if os.getenv('APPVEYOR') == 'True':
pytest.skip('Test hangs on AppVeyor CI')
from multiprocessing import Process, Queue
from logbook.queues import MultiProcessingSubscriber
queue = Queue(-1)
test_handler = logbook.TestHandler()
subscriber = MultiProcessingSubscriber(queue)
p = Process(target=MultiProcessingHandlerSendBack(queue))
p.start()
p.join()
with test_handler:
subscriber.dispatch_once()
assert test_handler.has_warning('Hello World')
def test_subscriber_group():
from multiprocessing import Process, Queue
from logbook.queues import MultiProcessingSubscriber, SubscriberGroup
a_queue = Queue(-1)
b_queue = Queue(-1)
subscriber = SubscriberGroup([
MultiProcessingSubscriber(a_queue),
MultiProcessingSubscriber(b_queue)
])
for _ in range(10):
p1 = Process(target=SubscriberGroupSendBack('foo', a_queue))
p2 = Process(target=SubscriberGroupSendBack('bar', b_queue))
p1.start()
p2.start()
p1.join()
p2.join()
messages = [subscriber.recv().message for i in (1, 2)]
assert sorted(messages) == ['bar', 'foo']
def test_subscriber_group():
if os.getenv('APPVEYOR') == 'True':
pytest.skip('Test hangs on AppVeyor CI')
from multiprocessing import Process, Queue
from logbook.queues import MultiProcessingSubscriber, SubscriberGroup
a_queue = Queue(-1)
b_queue = Queue(-1)
subscriber = SubscriberGroup([
MultiProcessingSubscriber(a_queue),
MultiProcessingSubscriber(b_queue)
])
for _ in range(10):
p1 = Process(target=SubscriberGroupSendBack('foo', a_queue))
p2 = Process(target=SubscriberGroupSendBack('bar', b_queue))
p1.start()
p2.start()
p1.join()
p2.join()
messages = [subscriber.recv().message for i in (1, 2)]
assert sorted(messages) == ['bar', 'foo']
def main():
start = time.time()
args = parse_args()
wwise_dir = Path(args.wwise_dir)
out_dir = Path(args.out_dir)
out_dir.mkdir(exist_ok=True)
id_to_filename_path = out_dir / ID_TO_FILENAME
manager = mp.Manager()
queue = manager.Queue()
id_queue = manager.Queue()
setup_logging(queue)
target_handlers = logbook.NestedSetup([
logbook.NullHandler(),
logbook.StreamHandler(sys.stdout, level=logbook.INFO, bubble=True),
logbook.FileHandler("extract.log",
mode="w",
level=logbook.INFO,
bubble=True),
])
sub = MultiProcessingSubscriber(queue)
controller = sub.dispatch_in_background(target_handlers)
quickbms_log_lock = manager.Lock()
quickbms_log = out_dir / "quickbms.log"
try:
id_to_filename_path.unlink()
logbook.info("removed old {id_file}",
id_file=id_to_filename_path.absolute())
except FileNotFoundError:
pass
logbook.info("QuickBMS log: '{qlog}'", qlog=quickbms_log.absolute())
try:
quickbms_log.unlink()
logbook.info("removed old {f}", f=quickbms_log.absolute())
except FileNotFoundError:
pass
id_to_filename_path.touch()
logbook.info("writing old ID -> new filename info in '{id_file}'",
id_file=id_to_filename_path.absolute())
id_to_filename_p = mp.Process(target=id_to_filename_worker,
args=(id_queue, id_to_filename_path, queue))
id_to_filename_p.start()
logbook.info("processing audio files in '{wd}'", wd=wwise_dir.absolute())
fut2func = {}
# Parse .bnk files and metadata.
with ProcessPoolExecutor(max_workers=MAX_WORKERS) as executor:
fut2func[executor.submit(parse_banks_metadata, wwise_dir,
queue)] = parse_banks_metadata
fut2func[executor.submit(decode_banks, wwise_dir, out_dir,
quickbms_log, quickbms_log_lock,
queue)] = decode_banks
memory_bnk_meta_file2metadata = {}
streamed_bnk_meta_file2metadata = {}
orig_bnk2decode_info = {}
for completed_fut in futures.as_completed(fut2func):
if fut2func[completed_fut] == parse_banks_metadata:
result = completed_fut.result()
memory_bnk_meta_file2metadata = result[0]
streamed_bnk_meta_file2metadata = result[1]
elif fut2func[completed_fut] == decode_banks:
orig_bnk2decode_info = completed_fut.result()
if len(memory_bnk_meta_file2metadata) != len(orig_bnk2decode_info):
logbook.warning(
"Amount of Bank and metadata files "
"do not match ({first}) != {second})",
first=len(orig_bnk2decode_info),
second=len(memory_bnk_meta_file2metadata))
s1 = memory_bnk_meta_file2metadata.keys()
s2 = set([key.stem for key in orig_bnk2decode_info])
to_del = []
diff = s2.difference(s1)
for d in diff:
# TODO: expensive!
for key in orig_bnk2decode_info:
if key.stem == d:
logbook.warn("ignoring {f}", f=str(key))
to_del.append(key)
for td in to_del:
del orig_bnk2decode_info[td]
wem_src2wem_dst = {}
# Move .wem files to out_dir in correct places.
with ProcessPoolExecutor(max_workers=MAX_WORKERS) as executor:
for bnk_meta_file, meta in streamed_bnk_meta_file2metadata.items():
for m in meta:
src_dir = bnk_meta_file.parent
src = src_dir / Path(m.generated_audio_file)
if src.exists():
wwise_path = Path(m.wwise_object_path)
dst = out_dir / wwise_path.relative_to(
wwise_path.anchor).with_suffix(".wem")
executor.submit(copy, src, dst, queue, id_queue)
wem_src2wem_dst[src] = dst
else:
logbook.warning(
"found references to {src} in metadata, but "
"the file cannot be found in wwise_dir",
src=src)
decoded_file2metas = {}
for orig_bnk_file, decode_info in orig_bnk2decode_info.items():
orig_bnk_file = orig_bnk_file.stem
meta = memory_bnk_meta_file2metadata[orig_bnk_file]
if len(decode_info) != len(meta):
logbook.error(
"decode_info and meta length mismatch: "
"{len1} != {len2} for bnk: '{bnk}'",
len1=len(decode_info),
len2=len(meta),
bnk=orig_bnk_file)
# print(decode_info)
# print(meta)
continue
# raise ValueError(f"decode_info and meta length mismatch "
# f"{len(decode_info)} != {len(meta)}")
for m, (decoded_stem, decoded_size) in zip(meta, decode_info.items()):
if m.data_size != decoded_size:
# raise ValueError(f"{m.data_size} != {decoded_size}")
logbook.error(
"metadata size and decoded data size length mismatch: "
"{len1} != {len2}",
len1=m.data_size,
len2=decoded_size)
continue
decoded_file2metas[decoded_stem] = m
fs = []
# Move output from decoding .bnk files to correct places in out_dir.
executor = ProcessPoolExecutor(max_workers=MAX_WORKERS)
for decoded_file, meta in decoded_file2metas.items():
src = out_dir / f"{decoded_file}.bin"
wwise_path = Path(meta.wwise_object_path)
dst = out_dir / wwise_path.relative_to(
wwise_path.anchor).with_suffix(".bin")
fs.append(executor.submit(move, src, dst, queue, id_queue))
futures.wait(fs, return_when=futures.ALL_COMPLETED)
fs = []
# Convert all .wem and .bin files to .ogg.
executor = ProcessPoolExecutor(max_workers=MAX_WORKERS)
for bin_file in out_dir.rglob("*.bin"):
fs.append(executor.submit(ww2ogg, bin_file, queue))
for wem_file in out_dir.rglob("*.wem"):
fs.append(executor.submit(ww2ogg, wem_file, queue))
futures.wait(fs, return_when=futures.ALL_COMPLETED)
done_wems_stems = set([ws.stem for ws in wem_src2wem_dst.keys()])
source_wems = [w for w in wwise_dir.rglob("*.wem")]
source_wems_stems = set([w.stem for w in source_wems])
wem_diff = source_wems_stems.difference(done_wems_stems)
if wem_diff:
logbook.warn("failed to determine filename for "
"{num} files",
num=len(wem_diff))
for ws in source_wems:
if str(ws.stem) in wem_diff:
logbook.info("couldn't determine filename for: {ws}", ws=ws)
copy_seq(ws, out_dir, id_to_filename_queue=None)
# Convert leftovers.
leftovers_fs = []
with ProcessPoolExecutor(max_workers=MAX_WORKERS) as executor:
for wem_file in out_dir.rglob("*.wem"):
leftovers_fs.append(executor.submit(ww2ogg, wem_file, queue))
futures.wait(leftovers_fs, return_when=futures.ALL_COMPLETED)
id_queue.put(SENTINEL)
id_to_filename_p.join()
secs = time.time() - start
logbook.info("finished successfully in {secs:.2f} seconds", secs=secs)
controller.stop()
from logbook import TimedRotatingFileHandler
from multiprocessing import Queue
from logbook.queues import MultiProcessingSubscriber
from test import queue
#queue = Queue(-1)
subscriber = MultiProcessingSubscriber(queue.queue)
controller = subscriber.dispatch_in_background(TimedRotatingFileHandler('multiprocessfoo.log'))
subscriber.dispatch_in_background(TimedRotatingFileHandler('foo.log'))
a = 1
while a > 0:
record = subscriber.recv()
print "**** Log Record **** \n" , record
controller.stop()
