def __init__(self, data_dir, dir_name="train", ext="wav"): com.logger.info("target_dir : %s", data_dir) file_list_path = os.path.abspath( "{data_dir}/{dir_name}/*.{ext}".format(data_dir=data_dir, dir_name=dir_name, ext=ext)) files = sorted(glob.glob(file_list_path)) if not files: com.logger.exception("no_wav_file!!") com.logger.info("train_file num : %s", len(files)) for file_id, file_name in enumerate(files): vector_array = com.file_to_vector_array( file_name, n_mels=PARAM["feature"]["n_mels"], frames=PARAM["feature"]["frames"], n_fft=PARAM["feature"]["n_fft"], hop_length=PARAM["feature"]["hop_length"], power=PARAM["feature"]["power"]) if file_id == 0: dataset_array = numpy.zeros( (vector_array.shape[0] * len(files), PARAM["feature"]["n_mels"] * PARAM["feature"]["frames"]), numpy.float32) dataset_array[vector_array.shape[0] * file_id:vector_array.shape[0] * (file_id + 1), :] = vector_array self.feat_data = dataset_array
def list_to_vector_array(file_list, feat_path, feature_range, scaler, msg="calc...", n_mels=64, frames=5, n_fft=1024, hop_length=512, power=2.0): """ convert the file_list to features and save features. file_to_vector_array() is iterated, and the output vector array is saved. file_list : list [ str ] .wav filename list of dataset msg : str ( default = "calc..." ) description for tqdm. this parameter will be input into "desc" param at tqdm. """ ###### uncomment to compute scaler #scaler = preprocessing.StandardScaler() # iterate file_to_vector_array() for idx in tqdm(range(len(file_list)), desc=msg): vector_array = com.file_to_vector_array(file_list[idx], feature_range, scaler, n_mels=n_mels, frames=frames, n_fft=n_fft, hop_length=hop_length, power=power) ###### uncomment to compute scaler #scaler.partial_fit(X=vector_array) if idx == 0: X = np.empty((len(file_list), vector_array.shape[0], vector_array.shape[1])) X[idx,] = vector_array #save features numpy.save(feat_path+"\\data.npy", X) ###### uncomment to compute scaler '''
def list_to_vector_array(file_list, msg="calc...", n_mels=64, frames=5, n_fft=1024, hop_length=512, power=2.0, method='normal' ): """ convert the file_list to a vector array. file_to_vector_array() is iterated, and the output vector array is concatenated. file_list : list [ str ] .wav filename list of dataset msg : str ( default = "calc..." ) description for tqdm. this parameter will be input into "desc" param at tqdm. return : numpy.array( numpy.array( float ) ) vector array for training (this function is not used for test.) * dataset.shape = (number of feature vectors, dimensions of feature vectors) """ # calculate the number of dimensions dims = n_mels * frames # iterate file_to_vector_array() for idx in tqdm(range(len(file_list)), desc=msg): vector_array = com.file_to_vector_array(file_list[idx], n_mels=n_mels, frames=frames, n_fft=n_fft, hop_length=hop_length, power=power, method=method) if idx == 0: dataset = vector_array continue dataset = numpy.vstack((dataset, vector_array)) # print(dataset.shape) return dataset
def list_to_vector_array(file_list, noise, msg="calc...", n_mels=64, frames=5, n_fft=1024, hop_length=512, power=2.0): """ convert the file_list to a vector array. file_to_vector_array() is iterated, and the output vector array is concatenated. file_list : list [ str ] .wav filename list of dataset msg : str ( default = "calc..." ) description for tqdm. this parameter will be input into "desc" param at tqdm. return : numpy.array( numpy.array( float ) ) vector array for training (this function is not used for test.) * dataset.shape = (number of feature vectors, dimensions of feature vectors) """ # calculate the number of dimensions # iterate file_to_vector_array() dataset = [] for idx in tqdm(range(len(file_list)), desc=msg): vector_array = com.file_to_vector_array(file_list[idx], noise, n_mels=n_mels, frames=frames, n_fft=n_fft, hop_length=hop_length, power=power) # if idx == 0: # dataset = numpy.zeros((vector_array.shape[0] * len(file_list), dims), float) # dataset[vector_array.shape[0] * idx: vector_array.shape[0] * (idx + 1), :] = vector_array dataset.append(vector_array) dataset = numpy.array(dataset) return dataset
def __init__(self, target_dir, dir_name="train", ext="wav", n_mels=64, frames=5, n_fft=1024, hop_length=512, power=2.0, transform=None): self.transform = transform com.logger.info("target_dir : {}".format(target_dir)) file_list_path = os.path.abspath("{dir}/{dir_name}/*.{ext}".format( dir=target_dir, dir_name=dir_name, ext=ext)) files = sorted(glob.glob(file_list_path)) if len(files) == 0: com.logger.exception("no_wav_file!!") com.logger.info("train_file num : {num}".format(num=len(files))) dims = n_mels * frames for idx in range(len(files)): vector_array = com.file_to_vector_array(files[idx], n_mels=n_mels, frames=frames, n_fft=n_fft, hop_length=hop_length, power=power) if idx == 0: dataset = numpy.zeros( (vector_array.shape[0] * len(files), dims), float) dataset[vector_array.shape[0] * idx:vector_array.shape[0] * (idx + 1), :] = vector_array self.feat_data = dataset
# setup anomaly score file path anomaly_score_csv = "{result}/anomaly_score_{machine_type}_{id_str}.csv".format( result=param["result_directory"], machine_type=machine_type, id_str=id_str) anomaly_score_list = [] print( "\n============== BEGIN TEST FOR A MACHINE ID ==============") y_pred = [0. for k in test_files] for file_idx, file_path in tqdm(enumerate(test_files), total=len(test_files)): data = com.file_to_vector_array( file_path, False, n_mels=param["feature"]["n_mels"], frames=param["feature"]["frames"], n_fft=param["feature"]["n_fft"], hop_length=param["feature"]["hop_length"], power=param["feature"]["power"]) data = np.array([data]) errors = numpy.square(data - model.predict(data)) print(errors) y_pred[file_idx] = numpy.mean(errors) anomaly_score_list.append( [os.path.basename(file_path), y_pred[file_idx]]) # save anomaly score save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list) com.logger.info( "anomaly score result -> {}".format(anomaly_score_csv))
.format(result=param["result_directory"], machine_type=machine_type, id_str=id_str) anomaly_score_list = [] com.logger.info( "============== BEGIN TEST FOR A MACHINE ID ==============") y_pred = [0. for k in test_files] for file_idx, file_path in enumerate(test_files): try: data = com.file_to_vector_array( file_path, n_mels=config["mel_spectrogram_param"]["n_mels"], frames=config["mel_spectrogram_param"]["frames"], n_fft=config["mel_spectrogram_param"]["n_fft"], hop_length=config["mel_spectrogram_param"] ["hop_length"], power=config["mel_spectrogram_param"]["power"]) # reconstruction through auto encoder in pytorch feed_data = torch.as_tensor(data, device=device, dtype=torch.float32) with torch.no_grad(): pred = model(feed_data, device).to('cpu').detach().numpy().copy() errors = numpy.mean(numpy.square(data - pred), axis=1) y_pred[file_idx] = numpy.mean(errors) anomaly_score_list.append(
id_str=id_str) anomaly_score_list = [] print( "\n============== BEGIN TEST FOR A MACHINE ID ==============") y_pred = [0. for k in test_files] for file_idx, file_path in tqdm(enumerate(test_files), total=len(test_files)): try: # get audio features vector_array = com.file_to_vector_array( file_path, param["train_data"][machine_type], scaler, n_mels=param["feature"]["n_mels"], frames=param["feature"]["frames"], n_fft=param["feature"]["n_fft"], hop_length=param["feature"]["hop_length"], power=param["feature"]["power"]) length, _ = vector_array.shape dim = param["autoencoder"]["shape0"] step = param["step"] idex = numpy.arange(length - dim + step, step=step) for idx in range(len(idex)): start = min(idex[idx], length - dim)
machine_type=machine_type, id_str=id_str) anomaly_score_list = [] com.logger.info( "============== BEGIN TEST FOR A MACHINE ID ==============") y_pred = [0. for k in test_files] for file_idx, file_path in enumerate(test_files): try: # load mean and stddev data = com.file_to_vector_array( file_path, n_mels=PARAM["feature"]["n_mels"], frames=PARAM["feature"]["frames"], n_fft=PARAM["feature"]["n_fft"], hop_length=PARAM["feature"]["hop_length"], power=PARAM["feature"]["power"]) # data = (data - mean) / stddev # reconstruction through auto encoder in pytorch feed_data = torch.from_numpy(data).clone() feed_data.to(device) feed_data = feed_data.float() with torch.no_grad(): pred, _, _ = model(feed_data) pred = pred.to('cpu').detach().numpy().copy() # 次元方向に平均 errors = numpy.mean(numpy.square(data - pred), axis=1)
anomaly_score_csv = "{result}/anomaly_score_{machine_type}_{id_str}.csv".format( result=params.result_directory, machine_type=machine_type, id_str=id_str) anomaly_score_list = [] print( "\n============== BEGIN TEST FOR A MACHINE ID ==============") y_pred = [0. for k in test_files] for file_idx, file_path in tqdm(enumerate(test_files), total=len(test_files)): try: data = com.file_to_vector_array( file_path, n_mels=params.feature.n_mels, frames=params.feature.frames, n_fft=params.feature.n_fft, hop_length=params.feature.hop_length, power=params.feature.power) data = pytorch_common.normalize_0to1(data) with torch.no_grad(): yhat = model( to_tensor(data)).cpu().detach().numpy().reshape( data.shape) errors = numpy.mean(numpy.square(data - yhat), axis=1) y_pred[file_idx] = numpy.mean(errors) anomaly_score_list.append( [os.path.basename(file_path), y_pred[file_idx]]) except: com.logger.error("file broken!!: {}".format(file_path)) sys.exit(-1)