def __init__(self, data_dir, dir_name="train", ext="wav"):
com.logger.info("target_dir : %s", data_dir)
file_list_path = os.path.abspath(
"{data_dir}/{dir_name}/*.{ext}".format(data_dir=data_dir,
dir_name=dir_name,
ext=ext))
files = sorted(glob.glob(file_list_path))
if not files:
com.logger.exception("no_wav_file!!")
com.logger.info("train_file num : %s", len(files))
for file_id, file_name in enumerate(files):
vector_array = com.file_to_vector_array(
file_name,
n_mels=PARAM["feature"]["n_mels"],
frames=PARAM["feature"]["frames"],
n_fft=PARAM["feature"]["n_fft"],
hop_length=PARAM["feature"]["hop_length"],
power=PARAM["feature"]["power"])
if file_id == 0:
dataset_array = numpy.zeros(
(vector_array.shape[0] * len(files),
PARAM["feature"]["n_mels"] * PARAM["feature"]["frames"]),
numpy.float32)
dataset_array[vector_array.shape[0] *
file_id:vector_array.shape[0] *
(file_id + 1), :] = vector_array
self.feat_data = dataset_array
def list_to_vector_array(file_list, feat_path, feature_range, scaler,
msg="calc...",
n_mels=64,
frames=5,
n_fft=1024,
hop_length=512,
power=2.0):
"""
convert the file_list to features and save features.
file_to_vector_array() is iterated, and the output vector array is saved.
file_list : list [ str ]
.wav filename list of dataset
msg : str ( default = "calc..." )
description for tqdm.
this parameter will be input into "desc" param at tqdm.
"""
###### uncomment to compute scaler
#scaler = preprocessing.StandardScaler()
# iterate file_to_vector_array()
for idx in tqdm(range(len(file_list)), desc=msg):
vector_array = com.file_to_vector_array(file_list[idx], feature_range, scaler,
n_mels=n_mels,
frames=frames,
n_fft=n_fft,
hop_length=hop_length,
power=power)
###### uncomment to compute scaler
#scaler.partial_fit(X=vector_array)
if idx == 0:
X = np.empty((len(file_list), vector_array.shape[0], vector_array.shape[1]))
X[idx,] = vector_array
#save features
numpy.save(feat_path+"\\data.npy", X)
###### uncomment to compute scaler
'''
def list_to_vector_array(file_list,
msg="calc...",
n_mels=64,
frames=5,
n_fft=1024,
hop_length=512,
power=2.0,
method='normal'
):
"""
convert the file_list to a vector array.
file_to_vector_array() is iterated, and the output vector array is concatenated.
file_list : list [ str ]
.wav filename list of dataset
msg : str ( default = "calc..." )
description for tqdm.
this parameter will be input into "desc" param at tqdm.
return : numpy.array( numpy.array( float ) )
vector array for training (this function is not used for test.)
* dataset.shape = (number of feature vectors, dimensions of feature vectors)
"""
# calculate the number of dimensions
dims = n_mels * frames
# iterate file_to_vector_array()
for idx in tqdm(range(len(file_list)), desc=msg):
vector_array = com.file_to_vector_array(file_list[idx],
n_mels=n_mels,
frames=frames,
n_fft=n_fft,
hop_length=hop_length,
power=power,
method=method)
if idx == 0:
dataset = vector_array
continue
dataset = numpy.vstack((dataset, vector_array))
# print(dataset.shape)
return dataset
def list_to_vector_array(file_list,
noise,
msg="calc...",
n_mels=64,
frames=5,
n_fft=1024,
hop_length=512,
power=2.0):
"""
convert the file_list to a vector array.
file_to_vector_array() is iterated, and the output vector array is concatenated.
file_list : list [ str ]
.wav filename list of dataset
msg : str ( default = "calc..." )
description for tqdm.
this parameter will be input into "desc" param at tqdm.
return : numpy.array( numpy.array( float ) )
vector array for training (this function is not used for test.)
* dataset.shape = (number of feature vectors, dimensions of feature vectors)
"""
# calculate the number of dimensions
# iterate file_to_vector_array()
dataset = []
for idx in tqdm(range(len(file_list)), desc=msg):
vector_array = com.file_to_vector_array(file_list[idx],
noise,
n_mels=n_mels,
frames=frames,
n_fft=n_fft,
hop_length=hop_length,
power=power)
# if idx == 0:
# dataset = numpy.zeros((vector_array.shape[0] * len(file_list), dims), float)
# dataset[vector_array.shape[0] * idx: vector_array.shape[0] * (idx + 1), :] = vector_array
dataset.append(vector_array)
dataset = numpy.array(dataset)
return dataset
def __init__(self,
target_dir,
dir_name="train",
ext="wav",
n_mels=64,
frames=5,
n_fft=1024,
hop_length=512,
power=2.0,
transform=None):
self.transform = transform
com.logger.info("target_dir : {}".format(target_dir))
file_list_path = os.path.abspath("{dir}/{dir_name}/*.{ext}".format(
dir=target_dir, dir_name=dir_name, ext=ext))
files = sorted(glob.glob(file_list_path))
if len(files) == 0:
com.logger.exception("no_wav_file!!")
com.logger.info("train_file num : {num}".format(num=len(files)))
dims = n_mels * frames
for idx in range(len(files)):
vector_array = com.file_to_vector_array(files[idx],
n_mels=n_mels,
frames=frames,
n_fft=n_fft,
hop_length=hop_length,
power=power)
if idx == 0:
dataset = numpy.zeros(
(vector_array.shape[0] * len(files), dims), float)
dataset[vector_array.shape[0] * idx:vector_array.shape[0] *
(idx + 1), :] = vector_array
self.feat_data = dataset
# setup anomaly score file path
anomaly_score_csv = "{result}/anomaly_score_{machine_type}_{id_str}.csv".format(
result=param["result_directory"],
machine_type=machine_type,
id_str=id_str)
anomaly_score_list = []
print(
"\n============== BEGIN TEST FOR A MACHINE ID ==============")
y_pred = [0. for k in test_files]
for file_idx, file_path in tqdm(enumerate(test_files),
total=len(test_files)):
data = com.file_to_vector_array(
file_path,
False,
n_mels=param["feature"]["n_mels"],
frames=param["feature"]["frames"],
n_fft=param["feature"]["n_fft"],
hop_length=param["feature"]["hop_length"],
power=param["feature"]["power"])
data = np.array([data])
errors = numpy.square(data - model.predict(data))
print(errors)
y_pred[file_idx] = numpy.mean(errors)
anomaly_score_list.append(
[os.path.basename(file_path), y_pred[file_idx]])
# save anomaly score
save_csv(save_file_path=anomaly_score_csv,
save_data=anomaly_score_list)
com.logger.info(
"anomaly score result -> {}".format(anomaly_score_csv))
.format(result=param["result_directory"],
machine_type=machine_type,
id_str=id_str)
anomaly_score_list = []
com.logger.info(
"============== BEGIN TEST FOR A MACHINE ID ==============")
y_pred = [0. for k in test_files]
for file_idx, file_path in enumerate(test_files):
try:
data = com.file_to_vector_array(
file_path,
n_mels=config["mel_spectrogram_param"]["n_mels"],
frames=config["mel_spectrogram_param"]["frames"],
n_fft=config["mel_spectrogram_param"]["n_fft"],
hop_length=config["mel_spectrogram_param"]
["hop_length"],
power=config["mel_spectrogram_param"]["power"])
# reconstruction through auto encoder in pytorch
feed_data = torch.as_tensor(data,
device=device,
dtype=torch.float32)
with torch.no_grad():
pred = model(feed_data,
device).to('cpu').detach().numpy().copy()
errors = numpy.mean(numpy.square(data - pred), axis=1)
y_pred[file_idx] = numpy.mean(errors)
anomaly_score_list.append(
id_str=id_str)
anomaly_score_list = []
print(
"\n============== BEGIN TEST FOR A MACHINE ID ==============")
y_pred = [0. for k in test_files]
for file_idx, file_path in tqdm(enumerate(test_files),
total=len(test_files)):
try:
# get audio features
vector_array = com.file_to_vector_array(
file_path,
param["train_data"][machine_type],
scaler,
n_mels=param["feature"]["n_mels"],
frames=param["feature"]["frames"],
n_fft=param["feature"]["n_fft"],
hop_length=param["feature"]["hop_length"],
power=param["feature"]["power"])
length, _ = vector_array.shape
dim = param["autoencoder"]["shape0"]
step = param["step"]
idex = numpy.arange(length - dim + step, step=step)
for idx in range(len(idex)):
start = min(idex[idx], length - dim)
machine_type=machine_type,
id_str=id_str)
anomaly_score_list = []
com.logger.info(
"============== BEGIN TEST FOR A MACHINE ID ==============")
y_pred = [0. for k in test_files]
for file_idx, file_path in enumerate(test_files):
try:
# load mean and stddev
data = com.file_to_vector_array(
file_path,
n_mels=PARAM["feature"]["n_mels"],
frames=PARAM["feature"]["frames"],
n_fft=PARAM["feature"]["n_fft"],
hop_length=PARAM["feature"]["hop_length"],
power=PARAM["feature"]["power"])
# data = (data - mean) / stddev
# reconstruction through auto encoder in pytorch
feed_data = torch.from_numpy(data).clone()
feed_data.to(device)
feed_data = feed_data.float()
with torch.no_grad():
pred, _, _ = model(feed_data)
pred = pred.to('cpu').detach().numpy().copy()
# 次元方向に平均
errors = numpy.mean(numpy.square(data - pred), axis=1)
anomaly_score_csv = "{result}/anomaly_score_{machine_type}_{id_str}.csv".format(
result=params.result_directory,
machine_type=machine_type,
id_str=id_str)
anomaly_score_list = []
print(
"\n============== BEGIN TEST FOR A MACHINE ID ==============")
y_pred = [0. for k in test_files]
for file_idx, file_path in tqdm(enumerate(test_files),
total=len(test_files)):
try:
data = com.file_to_vector_array(
file_path,
n_mels=params.feature.n_mels,
frames=params.feature.frames,
n_fft=params.feature.n_fft,
hop_length=params.feature.hop_length,
power=params.feature.power)
data = pytorch_common.normalize_0to1(data)
with torch.no_grad():
yhat = model(
to_tensor(data)).cpu().detach().numpy().reshape(
data.shape)
errors = numpy.mean(numpy.square(data - yhat), axis=1)
y_pred[file_idx] = numpy.mean(errors)
anomaly_score_list.append(
[os.path.basename(file_path), y_pred[file_idx]])
except:
com.logger.error("file broken!!: {}".format(file_path))
sys.exit(-1)
