def create_logger():
multiprocessing.log_to_stderr()
logger = multiprocessing.get_logger()
logger.setLevel(logging.INFO)
fh = logging.FileHandler("process.log")
fmt = '%(asctime)s - %(processName)s - %(levelname)s - %(funcName)s - %(message)s'
formatter = logging.Formatter(fmt)
fh.setFormatter(formatter)
logger.addHandler(fh)
return logger
def info(cls, message: str) -> None:
# pylint: disable=no-self-use
if cls._logger is None:
cls._logger = log_to_stderr()
cls._logger.setLevel(logging.INFO)
cls._logger.info(message)
def info(cls, message: str) -> None:
# pylint: disable=no-self-use
if cls._logger is None:
cls._logger = log_to_stderr()
cls._logger.setLevel(logging.INFO)
cls._logger.info(message)
def info(cls, message: str) -> None:
if cls._logger is None:
cls._logger = log_to_stderr()
cls._logger.setLevel(logging.INFO)
cls._logger.info(message)
from typing import List, Iterable, Iterator
import glob
import logging
import random
from torch.multiprocessing import Manager, Process, Queue, log_to_stderr
from allennlp.data.dataset_readers.dataset_reader import DatasetReader
from allennlp.data.instance import Instance
logger = log_to_stderr() # pylint: disable=invalid-name
logger.setLevel(logging.INFO)
def _worker(reader: DatasetReader,
input_queue: Queue,
output_queue: Queue,
index: int) -> None:
"""
A worker that pulls filenames off the input queue, uses the dataset reader
to read them, and places the generated instances on the output queue.
When there are no filenames left on the input queue, it puts its ``index``
on the output queue and doesn't do anything else.
"""
# Keep going until you get a file_path that's None.
while True:
file_path = input_queue.get()
if file_path is None:
# Put my index on the queue to signify that I'm finished
output_queue.put(index)
break
# @Author: richman
# @Date: 2017-10-23 20:47:21
# @Last Modified by: Jiaqi guo
# @Last Modified time: 2018-01-18 13:44:21
import os
import numpy as np
import torch
from ..utils import kaldi_io
import itertools
from torch.utils.data import TensorDataset
import torch.multiprocessing as multiprocessing
from torch.utils.data import DataLoader
import logging
from datetime import datetime
logger=multiprocessing.log_to_stderr()
logger.setLevel(logging.INFO)
# Fetches a cache ( e.g. some frames from a stream ). Used as a background
# process
def _fetch_cache(datastream, index_queue, data_queue, labels, nFinished, nj, batchsize, shuffle=False):
while True:
# Use None as breaking indicator for both queues
cachesize = index_queue.get()
#logger.debug(cachesize)
if cachesize is None:
data_queue.put(None)
logger.error('get None cachesize')
break
st = datetime.now()
try:
def _fetch_cache(datastream,
index_queue,
data_queue,
labels,
nFinished,
nj,
batchsize,
shuffle=False):
filehandler = logging.FileHandler(
'/mnt/lustre/sjtu/users/jqg01/asr/aurora4/readprocessing.log',
mode='w')
logger = multiprocessing.log_to_stderr()
logger.addHandler(filehandler)
logger.setLevel(logging.DEBUG)
while True:
# Use None as breaking indicator for both queues
cachesize = index_queue.get()
logger.info(cachesize)
flog.write(str(cachesize) + '\n')
flog.flush()
'''if cachesize is None:
data_queue.put(None)
break'''
try:
if labels is None:
feats, targets = zip(
*[(v, np.array([1] * len(v)))
for k, v in (next(datastream)
for _ in range(cachesize))])
else:
feats, targets = zip(*[(v, labels[k])
for k, v in (next(datastream)
for _ in range(cachesize))
if k in labels])
# Finished iterating over the dataset, two cases:
# 1. Last element has been reached and data is nonempty
# 2. Last element has been reached and data is empty
# The second case will be catched in ValueError
except StopIteration as e:
# Just return the data
logger.error(e)
flog.write(e)
flog.flush()
raise (e)
nFinished.value = nFinished.value + 1
logger.error('current nFinished: ' + str(nFinished.value))
break
# Returns Value error, if zip fails (list is empty, thus no data)
except ValueError as e:
logger.error(e)
flog.write(e)
flog.flush()
raise (e)
nFinished.value = nFinished.value + 1
logger.error('current nFinished:' + str(nFinished.value))
if nFinished.value == nj:
data_queue.put(None)
break
assert feats is not None, "Check the labels!"
# No features to return, just tell the iterator its finished
feats = np.concatenate(feats)
# Assuming multiple labels for each feature, targets has size 2xDATA ->
# DATAx2
targets = np.concatenate(targets)
tnetdataset = TensorDataset(torch.from_numpy(feats),
torch.from_numpy(targets).long())
dataloader = DataLoader(tnetdataset,
batch_size=batchsize,
shuffle=shuffle,
drop_last=False)
data_queue.put(dataloader)
def log_start(level=logging.INFO):
logger = mp.log_to_stderr()
logger.setLevel(level) #DEBUG, INFO, WARNING, ERROR, CRITICAL
logger.propagate = False