def read_and_extract_features(reader, period, features): print("number of get_number_of_examples", reader.get_number_of_examples()) ret = common_utils.read_chunk(reader, reader.get_number_of_examples()) # ret = common_utils.read_chunk(reader, 100) X = common_utils.extract_features_from_rawdata(ret['X'], ret['header'], period, features) return (X, ret['y'], ret['name'])
def _generator(self): B = self.batch_size while True: if self.shuffle: self.reader.random_shuffle() remaining = self.n_examples while remaining > 0: current_size = min(self.chunk_size, remaining) remaining -= current_size ret = common_utils.read_chunk(self.reader, current_size) Xs = ret["X"] ts = ret["t"] ys = ret["y"] names = ret["name"] Xs = preprocess_chunk(Xs, ts, self.discretizer, self.normalizer) (Xs, ys, ts, names) = common_utils.sort_and_shuffle([Xs, ys, ts, names], B) for i in range(0, current_size, B): X = common_utils.pad_zeros(Xs[i:i + B]) y = np.array(ys[i:i + B]) batch_names = names[i:i+B] batch_ts = ts[i:i+B] batch_data = (X, y) if not self.return_names: yield batch_data else: yield {"data": batch_data, "names": batch_names, "ts": batch_ts}
def __init__(self, reader, discretizer, normalizer, ihm_pos, partition, target_repl, batch_size, small_part, shuffle, return_names=False): self.discretizer = discretizer self.normalizer = normalizer self.ihm_pos = ihm_pos self.partition = partition self.target_repl = target_repl self.batch_size = batch_size self.shuffle = shuffle self.return_names = return_names N = reader.get_number_of_examples() if small_part: N = 1000 self.steps = (N + batch_size - 1) // batch_size self.lock = threading.Lock() ret = common_utils.read_chunk(reader, N) Xs = ret['X'] ts = ret['t'] ihms = ret['ihm'] loss = ret['los'] phenos = ret['pheno'] decomps = ret['decomp'] self.data = dict() self.data['pheno_ts'] = ts self.data['names'] = ret['name'] self.data['decomp_ts'] = [] self.data['los_ts'] = [] for i in range(N): self.data['decomp_ts'].append( [pos for pos, m in enumerate(decomps[i][0]) if m == 1]) self.data['los_ts'].append( [pos for pos, m in enumerate(loss[i][0]) if m == 1]) (Xs[i], ihms[i], decomps[i], loss[i], phenos[i]) = \ self._preprocess_single(Xs[i], ts[i], ihms[i], decomps[i], loss[i], phenos[i]) self.data['X'] = Xs self.data['ihm_M'] = [x[0] for x in ihms] self.data['ihm_y'] = [x[1] for x in ihms] self.data['decomp_M'] = [x[0] for x in decomps] self.data['decomp_y'] = [x[1] for x in decomps] self.data['los_M'] = [x[0] for x in loss] self.data['los_y'] = [x[1] for x in loss] self.data['pheno_y'] = phenos self.generator = self._generator()
def load_data(reader, discretizer, normalizer, small_part=False, return_names=False): N = reader.get_number_of_examples() if small_part: N = 1000 ret = common_utils.read_chunk(reader, N) data = ret["X"] ts = ret["t"] labels = ret["y"] names = ret["name"] data = [discretizer.transform(X, end=t)[0] for (X, t) in zip(data, ts)] if normalizer is not None: data = [normalizer.transform(X) for X in data] whole_data = (np.array(data), labels) if not return_names: return whole_data return {"data": whole_data, "names": names}