def load_data(): args = parse_arguments() window_nuc = 2001 half_wx = window_nuc // 2 window_rna = 10 half_wx_rna = window_rna // 2 args = parse_arguments() path_to_directory = os.path.dirname(os.path.dirname(args.directory)) # we get the path conducting to seq_chr_sacCer3 path_to_file = os.path.join(path_to_directory, 'seq_chr_sacCer3', args.directory, 'chr16.hdf5') f = h5py.File(path_to_file, 'r') nucleotid = np.array(f['data']) f.close() X_one_hot = (np.arange(nucleotid.max()) == nucleotid[..., None] - 1).astype(int) X_ = X_one_hot.reshape(X_one_hot.shape[0], X_one_hot.shape[1] * X_one_hot.shape[2]) nuc_directory = os.path.dirname(args.nuc) nuc_file = os.path.join(nuc_directory, 'Start_data', args.nuc) nuc_density = pd.read_csv(nuc_file) y_true = nuc_density[nuc_density.chr == 'chr16'].value.values X_slide = rolling_window(X_, window=(window_nuc, 4)) X_ = X_slide.reshape(X_slide.shape[0], X_slide.shape[2], X_slide.shape[3], 1) threshold = nuc_occupancy(nuc_file, return_threshold=True) rna_directory = os.path.dirname(args.rna_seq) rna_file = os.path.join(rna_directory, 'Start_data', args.rna_seq) rna_density = pd.read_csv(rna_file) rna_density = rna_density[rna_density.chr == 'chr16'].value.values if not args.predicted_rnaseq: rna_density[rna_density > 0] = np.log(rna_density[rna_density > 0]) rna_density[rna_density < 0] = -np.log(-rna_density[rna_density < 0]) rna_inputs = rolling_window(rna_density, window=(window_rna, )) rna_inputs = rna_inputs[half_wx - half_wx_rna:-half_wx + half_wx_rna - 1] rna_inputs = rna_inputs.reshape(rna_inputs.shape[0], window_rna, 1) y_true = y_true[half_wx:-half_wx] y_true /= float(threshold) return X_, rna_inputs, y_true
def load_data(seq2seq=False, args=None): window = 2001 half_wx = window // 2 args = parse_arguments(args) path_to_directory = os.path.dirname(os.path.dirname(args.directory)) # we get the path conducting to seq_chr_sacCer3 path_to_file = os.path.join( path_to_directory, "seq_chr_sacCer3", args.directory, "chr" + args.test + ".hdf5", ) f = h5py.File(path_to_file, "r") nucleotid = np.array(f["data"]) f.close() if args.reversed_seq: nucleotid[nucleotid == 1] = 5 nucleotid[nucleotid == 2] = 6 nucleotid[nucleotid == 3] = 7 nucleotid[nucleotid == 4] = 8 nucleotid[nucleotid == 5] = 2 nucleotid[nucleotid == 6] = 1 nucleotid[nucleotid == 7] = 4 nucleotid[nucleotid == 8] = 3 nucleotid = nucleotid[::-1] n = int(args.k) X_ = nmer_patch(nucleotid, n) if seq2seq: _, output_len = model_dictionary()[args.model] X_slide = rolling_window(X_, window=(window, pow(4, n)), asteps=(output_len, pow(4, n))) X_ = X_slide.reshape(X_slide.shape[0], X_slide.shape[2], 1, X_slide.shape[3]) windows_num = len(X_slide) else: X_slide = rolling_window(X_, window=(window, pow(4, n))) X_ = X_slide.reshape(X_slide.shape[0], X_slide.shape[2], 1, X_slide.shape[3]) output_len = 1 windows_num = 1 return X_
def _calculate_rolling_mean(x, batch_size, sample_len, output_len, num_classes): x = rolling_window(x, window=(x.shape[0], sample_len, num_classes), asteps=(x.shape[0], sample_len, num_classes)) x = x.reshape((output_len, batch_size, sample_len, num_classes)) x = np.mean(x, axis=2) x = np.swapaxes(x, 0, 1) return x
def nmer_patch(nucleotid_, window, n): nucleotid_ = rolling_window(nucleotid_ - 1, window=n) weights = [pow(4, i) for i in range(n)] nucleotid_ = np.average(nucleotid_, weights=weights, axis=1) * sum(weights) X_one_hot = (np.arange(pow(4, n)) == nucleotid_[..., None]).astype(int) _X_ = np.zeros((window, 1, pow(4, n))) _X_[(n - 1) // 2:-(n // 2), 0] = X_one_hot return _X_
def nmer_patch(nucleotid, n): nucleotid = nucleotid[:, 0] nucleotid_ = rolling_window(nucleotid - 1, window=n) weights = [pow(4, i) for i in range(n)] nucleotid_ = np.average(nucleotid_, weights=weights, axis=1) * sum(weights) X_one_hot = (np.arange(pow(4, n)) == nucleotid_[..., None]).astype(int) if n == 1: return X_one_hot else: _X_ = np.zeros((len(nucleotid), pow(4, n))) _X_[(n - 1) // 2:-(n // 2)] = X_one_hot return _X_
def process(self): """ Load the data from a .fa file and process it with a rolling window so that the model can handle it. """ WX = 299 path_to_directory = os.path.dirname(os.path.dirname(self.directory)) path_to_directory = os.path.join(path_to_directory, 'seq_chr', self.directory + '/') f = h5py.File(path_to_directory + 'chr' + str(self.num_chr) + '.hdf5') seq = np.array(f['data']) f.close() reverse = self._reverse_dna(seq) seq = self._one_hot_encoder(seq) reverse = self._one_hot_encoder(reverse) seq_slide = rolling_window(seq, window=(WX, 4), asteps=None, wsteps=None, axes=None, toend=True) seq = seq_slide.reshape(seq_slide.shape[0], WX, 4, 1) reverse_slide = rolling_window(reverse, window=(WX, 4), asteps=None, wsteps=None, axes=None, toend=True) reverse = reverse_slide.reshape(reverse_slide.shape[0], WX, 4, 1) return seq, reverse
def process(nucleotid): """ Take a numpy array corresponding to a DNA sequence and transform it so that the model is able to make prediction on it. Args: nucleotid: array corresponding to the DNA sequence shape = (len, 1) return: x_seq: array ready to be passed as input of a model to make prediction. The shape is (len, 2001, 4, 1) """ WX = 2001 x = one_hot_encoder(nucleotid) x_slide = rolling_window(x, window=(WX, 4)) x_seq = x_slide.reshape(x_slide.shape[0], WX, 4, 1) return x_seq
def load_data(): args = parse_arguments() window = 2001 half_wx = window // 2 args = parse_arguments() path_to_directory = os.path.dirname(os.path.dirname( args.directory)) # we get the path conducting to seq_chr_sacCer3 path_to_file = os.path.join(path_to_directory, 'seq_chr_sacCer3', args.directory, 'chr16.hdf5') f = h5py.File(path_to_file, 'r') nucleotid = np.array(f[f.keys()[0]]) f.close() X_one_hot = (np.arange(nucleotid.max()) == nucleotid[..., None] - 1).astype(int) X_ = X_one_hot.reshape(X_one_hot.shape[0], X_one_hot.shape[1] * X_one_hot.shape[2]) proba_directory = os.path.dirname(args.file) proba_file = os.path.join(proba_directory, 'Start_data', args.file) proba = pd.read_csv(proba_file) y_true = proba[proba.chr == 'chr16'].value.values _, output_len = model_dictionary()[args.model] if output_len % 2 == 0: half_len = output_len // 2 else: half_len = output_len // 2 + 1 X_slide = rolling_window(X_, window=(window, 4), asteps=(half_len, 4)) X_ = X_slide.reshape(X_slide.shape[0], X_slide.shape[2], X_slide.shape[3], 1) X_1 = X_[::2] X_2 = X_[1::2] y_true = y_true[half_wx:X_1.shape[0] * output_len + half_wx - half_len] return X_1, X_2, y_true, half_len
def _convert_array_to_multi(myArray, number_of_lines, number_of_column): """ Convert a numpy array to multi array and if the shape is not correct then reshape it (removing starting elements in most of case). """ if len(myArray) > number_of_lines * number_of_column: # if the array has not the right shape, # then reshape it by removing x starting elements resized_array = np.delete( myArray, range(0, len(myArray) - (number_of_lines * number_of_column)), 0) res = np.reshape(resized_array, (number_of_lines, number_of_column)) elif len(myArray) < number_of_lines * number_of_column: myArray = rolling_window(myArray, window=number_of_column) np.random.shuffle(myArray) res = myArray[:number_of_lines] else: res = np.reshape(myArray, (number_of_lines, number_of_column)) return res
def _process(self, nucleotid): x = self._rescale(nucleotid) x = self._one_hot_encoder(x) x_slide = rolling_window(x, window=(Sequence.WX, 4)) x_seq = x_slide.reshape(x_slide.shape[0], Sequence.WX, 4, 1) return x_seq
def _max_norm(y, wx=3001): y_roll = rolling_window(y, window=wx) max_roll = np.max(y_roll, axis=1).astype(float) half = wx // 2 y = np.concatenate((y[:half], y[half:-half] / max_roll, y[-half:]), axis=0) return y