def f_save_trained_name(args): """ f_save_trained_name(args) Args: args, argument object by arg_parse Return: name of trained network file, e.g., trained_network.pt """ return nii_str_tk.f_realpath(args.save_model_dir, \ args.save_trained_name, \ args.save_model_ext)
def f_save_epoch_name(args, epoch_idx): """ f_save_epoch_name(args, epoch_idx) Args: args, argument object by arg_parse epoch_idx, int, epoch index Return: name of epoch state file, str, e.g. epoch_001.pt """ tmp_name = "{}_{:03d}".format(args.save_epoch_name, epoch_idx) return nii_str_tk.f_realpath(args.save_model_dir, tmp_name, \ args.save_model_ext)
def f_save_trained_name(args, suffix=''): """ str = f_save_trained_name(args) Return the name of the best trained model file Args: args: argument object by arg_parse args.save_trained_name, args.save_model_dir, args.save_model_ext suffix: a suffix added to the name (default '') Return: str: name of trained network file, e.g., trained_network.pt """ return nii_str_tk.f_realpath( args.save_model_dir, args.save_trained_name + suffix, args.save_model_ext)
def f_save_epoch_name(args, epoch_idx, suffix=''): """ str = f_save_epoch_name(args, epoch_idx) Return the name of the model file saved during training Args: args: argument object by arg_parse, we will use args.save_epoch_name, args.save_model_dir, args.save_model_ext epoch_idx:, int, epoch index suffix: a suffix to the name (default '') Return: str: name of epoch state file, str, e.g. epoch_001.pt """ tmp_name = "{}_{:03d}".format(args.save_epoch_name, epoch_idx) + suffix return nii_str_tk.f_realpath(args.save_model_dir, tmp_name, \ args.save_model_ext)
def f_putitem(self, output_data, save_dir, data_infor_str): """ """ # Change the dimension to (length, dim) if output_data.ndim == 3 and output_data.shape[0] == 1: # When input data is (batchsize=1, length, dim) output_data = output_data[0] elif output_data.ndim == 2 and output_data.shape[0] == 1: # When input data is (batchsize=1, length) output_data = np.expand_dims(output_data[0], -1) else: nii_warn.f_print("Output data format not supported.", "error") nii_warn.f_print("Format is not (batch, len, dim)", "error") nii_warn.f_die("Please use batch_size = 1 in generation") # Save output if output_data.shape[1] != self.m_output_all_dim: nii_warn.f_print("Output data dim != expected dim", "error") nii_warn.f_print("Output:%d" % (output_data.shape[1]), \ "error") nii_warn.f_print("Expected:%d" % (self.m_output_all_dim), \ "error") nii_warn.f_die("Please check configuration") if not os.path.isdir(save_dir): try: os.mkdir(save_dir) except OSError: nii_warn.f_die("Cannot carete {}".format(save_dir)) # read the sentence information tmp_seq_info = nii_seqinfo.SeqInfo() tmp_seq_info.parse_from_str(data_infor_str) # write the data file_name = tmp_seq_info.seq_tag() s_dim = 0 e_dim = 0 for t_ext, t_dim in zip(self.m_output_exts, self.m_output_dims): e_dim = s_dim + t_dim file_path = nii_str_tk.f_realpath(save_dir, file_name, t_ext) self.f_write_data(output_data[:, s_dim:e_dim], file_path) return
def f_check_specific_data(self, file_name): """ check the data length of a specific file """ tmp_dirs = self.m_input_dirs.copy() tmp_exts = self.m_input_exts.copy() tmp_dims = self.m_input_dims.copy() tmp_reso = self.m_input_reso.copy() tmp_dirs.extend(self.m_output_dirs) tmp_exts.extend(self.m_output_exts) tmp_dims.extend(self.m_output_dims) tmp_reso.extend(self.m_output_reso) # loop over each input/output feature type for t_dir, t_ext, t_dim, t_res in \ zip(tmp_dirs, tmp_exts, tmp_dims, tmp_reso): file_path = nii_str_tk.f_realpath(t_dir, file_name, t_ext) if not nii_io_tk.file_exist(file_path): nii_warn.f_die("%s not found" % (file_path)) else: t_len = self.f_length_data(file_path) // t_dim print("%s, length %d, dim %d, reso: %d" % \ (file_path, t_len, t_dim, t_res)) return
def f_calculate_stats(self, flag_cal_data_len, flag_cal_mean_std): """ f_calculate_stats Log down the number of time steps for each file Calculate the mean/std """ # check #if not self.m_output_dirs: # nii_warn.f_print("Calculating mean/std", 'error') # nii_warn.f_die("But output_dirs is not provided") # prepare the directory, extension, and dimensions tmp_dirs = self.m_input_dirs.copy() tmp_exts = self.m_input_exts.copy() tmp_dims = self.m_input_dims.copy() tmp_reso = self.m_input_reso.copy() tmp_norm = self.m_input_norm.copy() tmp_dirs.extend(self.m_output_dirs) tmp_exts.extend(self.m_output_exts) tmp_dims.extend(self.m_output_dims) tmp_reso.extend(self.m_output_reso) tmp_norm.extend(self.m_output_norm) # starting dimension of one type of feature s_dim = 0 # ending dimension of one type of feature e_dim = 0 # loop over each input/output feature type for t_dir, t_ext, t_dim, t_reso, t_norm in \ zip(tmp_dirs, tmp_exts, tmp_dims, tmp_reso, tmp_norm): s_dim = e_dim e_dim = s_dim + t_dim t_cnt = 0 mean_i, var_i = np.zeros([t_dim]), np.zeros([t_dim]) # loop over all the data for file_name in self.m_file_list: # get file path file_path = nii_str_tk.f_realpath(t_dir, file_name, t_ext) if not nii_io_tk.file_exist(file_path): nii_warn.f_die("%s not found" % (file_path)) # read the length of the data if flag_cal_data_len: t_len = self.f_length_data(file_path) // t_dim self.f_log_data_len(file_name, t_len, t_reso) # accumulate the mean/std recursively if flag_cal_mean_std: t_data = self.f_load_data(file_path, t_dim) # if the is F0 data, only consider voiced data if t_ext in nii_dconf.f0_unvoiced_dic: unvoiced_value = nii_dconf.f0_unvoiced_dic[t_ext] t_data = t_data[t_data > unvoiced_value] # mean_i, var_i, t_cnt will be updated using online # accumulation method mean_i, var_i, t_cnt = nii_stats.f_online_mean_std( t_data, mean_i, var_i, t_cnt) # save mean and std for one feature type if flag_cal_mean_std: # if not normalize this dimension, set mean=0, std=1 if not t_norm: mean_i[:] = 0 var_i[:] = 1 if s_dim < self.m_input_all_dim: self.m_input_mean[s_dim:e_dim] = mean_i std_i = nii_stats.f_var2std(var_i) self.m_input_std[s_dim:e_dim] = std_i else: tmp_s = s_dim - self.m_input_all_dim tmp_e = e_dim - self.m_input_all_dim self.m_output_mean[tmp_s:tmp_e] = mean_i std_i = nii_stats.f_var2std(var_i) self.m_output_std[tmp_s:tmp_e] = std_i if flag_cal_data_len: # create seq_info self.f_log_seq_info() self.f_save_data_len(self.m_data_len_path) if flag_cal_mean_std: self.f_save_mean_std(self.m_ms_input_path, self.m_ms_output_path) # done return
def __getitem__(self, idx): """ __getitem__(self, idx): Return input, output For test set data, output can be None """ try: tmp_seq_info = self.m_seq_info[idx] except IndexError: nii_warn.f_die("Sample %d is not in seq_info" % (idx)) # file_name file_name = tmp_seq_info.seq_tag() # For input data input_reso = self.m_input_reso[0] seq_len = int(tmp_seq_info.seq_length() // input_reso) s_idx = (tmp_seq_info.seq_start_pos() // input_reso) e_idx = s_idx + seq_len input_dim = self.m_input_all_dim in_data = np.zeros([seq_len, input_dim], dtype=nii_dconf.h_dtype) s_dim = 0 e_dim = 0 # loop over each feature type for t_dir, t_ext, t_dim, t_res in \ zip(self.m_input_dirs, self.m_input_exts, \ self.m_input_dims, self.m_input_reso): e_dim = s_dim + t_dim # get file path and load data file_path = nii_str_tk.f_realpath(t_dir, file_name, t_ext) try: tmp_d = self.f_load_data(file_path, t_dim) except IOError: nii_warn.f_die("Cannot find %s" % (file_path)) # write data if tmp_d.shape[0] == 1: # input data has only one frame, duplicate if tmp_d.ndim > 1: in_data[:,s_dim:e_dim] = tmp_d[0,:] elif t_dim == 1: in_data[:,s_dim] = tmp_d else: nii_warn.f_die("Dimension wrong %s" % (file_path)) else: # normal case if tmp_d.ndim > 1: # write multi-dimension data in_data[:,s_dim:e_dim] = tmp_d[s_idx:e_idx,:] elif t_dim == 1: # write one-dimension data in_data[:,s_dim] = tmp_d[s_idx:e_idx] else: nii_warn.f_die("Dimension wrong %s" % (file_path)) s_dim = e_dim # load output data if self.m_output_dirs: seq_len = tmp_seq_info.seq_length() s_idx = tmp_seq_info.seq_start_pos() e_idx = s_idx + seq_len out_dim = self.m_output_all_dim out_data = np.zeros([seq_len, out_dim], \ dtype = nii_dconf.h_dtype) s_dim = 0 e_dim = 0 for t_dir, t_ext, t_dim in zip(self.m_output_dirs, \ self.m_output_exts, \ self.m_output_dims): e_dim = s_dim + t_dim # get file path and load data file_path = nii_str_tk.f_realpath(t_dir, file_name, t_ext) try: tmp_d = self.f_load_data(file_path, t_dim) except IOError: nii_warn.f_die("Cannot find %s" % (file_path)) if tmp_d.shape[0] == 1: if tmp_d.ndim > 1: out_data[:,s_dim:e_dim] = tmp_d[0,:] elif t_dim == 1: out_data[:,s_dim]=tmp_d else: nii_warn.f_die("Dimension wrong %s" % (file_path)) else: if tmp_d.ndim > 1: out_data[:,s_dim:e_dim] = tmp_d[s_idx:e_idx,:] elif t_dim == 1: out_data[:,s_dim]=tmp_d[s_idx:e_idx] else: nii_warn.f_die("Dimension wrong %s" % (file_path)) s_dim = s_dim + t_dim else: out_data = [] return in_data, out_data, tmp_seq_info.print_to_str(), idx