def train_test_split(unrolled, test_size): cutPoint = unrolled.count() - test_size train = unrolled.filter(lambda x: x.index < cutPoint) \ .map(lambda x: Sample.from_ndarray(np.array(x.feature), np.array(x.label))) test = unrolled.filter(lambda x: x.index >= cutPoint) \ .map(lambda x: Sample.from_ndarray(np.array(x.feature), np.array(x.label))) return [train, test]
def gen_rand_user_item_feature(user_num, item_num, class_num): user_id = random.randint(1, user_num) item_id = random.randint(1, item_num) rating = random.randint(1, class_num) sample = Sample.from_ndarray(np.array([user_id, item_id]), np.array([rating])) return UserItemFeature(user_id, item_id, sample)
def to_sample(vectors, label, embedding_dim): # flatten nested list flatten_features = list(itertools.chain(*vectors)) features = np.array(flatten_features, dtype='float').reshape( [sequence_len, embedding_dim]) return Sample.from_ndarray(features, np.array(label))
def to_sample(vectors, label, embedding_dim): # flatten nested list flatten_features = list(itertools.chain(*vectors)) features = np.array(flatten_features, dtype='float').reshape( [sequence_len, embedding_dim]) return Sample.from_ndarray(features, np.array(label))
def get_featureset(x, y, shuffle=True): x = np.split(x.data.numpy(), x.shape[0]) y = np.split(y.data.numpy(), y.shape[0]) print(x[0].shape) print(y[0].shape) samples = [Sample.from_ndarray(np.squeeze(x[i]), np.squeeze(y[i])) for i in range(len(x))] sample_rdd = sc.parallelize(samples) return FeatureSet.sample_rdd(sample_rdd, shuffle=shuffle)
def to_sample(vectors, label, embedding_dim): # flatten nested list flatten_features = list(itertools.chain(*vectors)) features = np.array(flatten_features, dtype='float').reshape([sequence_len, embedding_dim]) if model_type.lower() == "cnn": features = features.transpose(1, 0) return Sample.from_ndarray(features, np.array(label))
def to_sample(data): from bigdl.util.common import Sample data = check_type_and_convert(data, allow_list=True, allow_tuple=False) features = data["x"] labels = data["y"] length = features[0].shape[0] for i in range(length): fs = [feat[i] for feat in features] ls = [l[i] for l in labels] yield Sample.from_ndarray(np.array(fs), np.array(ls))
def to_sample_rdd(x, y, numSlices=None): """ Conver x and y into RDD[Sample] :param x: ndarray and the first dimension should be batch :param y: ndarray and the first dimension should be batch :param numSlices: :return: """ from bigdl.util.common import Sample x_rdd = sc.parallelize(x, numSlices) y_rdd = sc.parallelize(y, numSlices) return x_rdd.zip(y_rdd).map(lambda item: Sample.from_ndarray(item[0], item[1]))
def prepare_data(sc, folder, vocabsize, training_split): if not folder.startswith('hdfs://'): file = download_data(folder) else: file = folder sentences_rdd = sc.textFile(file) \ .map(lambda line: sentence.sentences_split(line)) pad_sent = sentences_rdd.flatMap(lambda x: x). \ map(lambda sent: sentence.sentences_bipadding(sent)) tokens = pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) train_tokens, val_tokens = tokens.randomSplit( [training_split, 1 - training_split]) train_tokens.cache() val_tokens.cache() train_max_len = train_tokens.map(lambda x: len(x)).max() print("max length %s" % train_max_len) words = train_tokens.flatMap(lambda x: x) print("%s words and %s sentences processed in train data" % (words.count(), train_tokens.count())) val_max_len = val_tokens.map(lambda x: len(x)).max() print("val max length %s" % val_max_len) val_words = val_tokens.flatMap(lambda x: x) print("%s words and %s sentences processed in validation data" % (val_words.count(), val_tokens.count())) sort_words = words.map(lambda w: (w, 1)) \ .reduceByKey(lambda a, b: a + b) \ .sortBy(lambda w_c: w_c[1]) vocabulary = np.array(sort_words.map(lambda w: w[0]).collect()) fre_len = vocabulary.size if vocabsize > fre_len: length = fre_len else: length = vocabsize discard_vocab = vocabulary[:fre_len - length] used_vocab = vocabulary[fre_len - length:fre_len] used_vocab_size = used_vocab.size index = np.arange(used_vocab_size) index2word = dict(enumerate(used_vocab)) word2index = dict(zip(used_vocab, index)) total_vocab_len = used_vocab_size + 1 startIdx = word2index.get("SENTENCESTART") endIdx = word2index.get("SENTENCEEND") def text2labeled(sent): indexes = [word2index.get(x, used_vocab_size) for x in sent] data = indexes[0:-1] label = indexes[1:len(indexes)] return data, label def labeled2onehotformat(labeled_sent): label = [x + 1 for x in labeled_sent[1]] size = len(labeled_sent[0]) feature_onehot = np.zeros(size * total_vocab_len, dtype='int').reshape([size, total_vocab_len]) for i, el in enumerate(labeled_sent[0]): feature_onehot[i, el] = 1 return feature_onehot, label def padding(features, label, length): pad_len = length - len(label) padded_label = (label + [startIdx] * length)[:length] feature_padding = np.zeros((pad_len, total_vocab_len), dtype=np.int) feature_padding[:, endIdx + 1] = np.ones(pad_len) padded_feautres = np.concatenate((features, feature_padding), axis=0) return padded_feautres, padded_label sample_rdd = train_tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ .map(lambda x: padding(x[0], x[1], train_max_len)) \ .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))).cache() val_sample_rdd = val_tokens.map(lambda sentence_t: text2labeled(sentence_t)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ .map(lambda x: padding(x[0], x[1], val_max_len)) \ .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))).cache() return sample_rdd, val_sample_rdd, total_vocab_len
def get_rdd(x, y, shuffle=False): x = np.split(x.data.numpy(), x.shape[0]) y = np.split(x.data.numpy(), y.shape[0]) samples = [Sample.from_ndarray(np.squeeze(x[i]), np.squeeze(y[i])) for i in range(len(x))] sample_rdd = sc.parallelize(samples) return sample_rdd
def prepare_data(sc, folder, vocabsize, training_split): if not folder.startswith( 'hdfs://' ): file = download_data(folder) else: file = folder sentences_rdd = sc.textFile(file) \ .map(lambda line: sentence.sentences_split(line)) pad_sent = sentences_rdd.flatMap(lambda x: x). \ map(lambda sent: sentence.sentences_bipadding(sent)) tokens = pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) train_tokens, val_tokens = tokens.randomSplit([training_split, 1 - training_split]) train_tokens.cache() val_tokens.cache() train_max_len = train_tokens.map(lambda x: len(x)).max() print("max length %s" % train_max_len) words = train_tokens.flatMap(lambda x: x) print("%s words and %s sentences processed in train data" % (words.count(), train_tokens.count())) val_max_len = val_tokens.map(lambda x: len(x)).max() print("val max length %s" % val_max_len) val_words = val_tokens.flatMap(lambda x: x) print("%s words and %s sentences processed in validation data" % (val_words.count(), val_tokens.count())) sort_words = words.map(lambda w: (w, 1)) \ .reduceByKey(lambda a, b: a + b) \ .sortBy(lambda w_c: w_c[1]) vocabulary = np.array(sort_words.map(lambda w: w[0]).collect()) fre_len = vocabulary.size if vocabsize > fre_len: length = fre_len else: length = vocabsize discard_vocab = vocabulary[: fre_len-length] used_vocab = vocabulary[fre_len-length: fre_len] used_vocab_size = used_vocab.size index = np.arange(used_vocab_size) index2word = dict(enumerate(used_vocab)) word2index = dict(zip(used_vocab, index)) total_vocab_len = used_vocab_size + 1 startIdx = word2index.get("SENTENCESTART") endIdx = word2index.get("SENTENCEEND") def text2labeled(sent): indexes = [word2index.get(x, used_vocab_size) for x in sent] data = indexes[0: -1] label = indexes[1: len(indexes)] return data, label def labeled2onehotformat(labeled_sent): label = [x+1 for x in labeled_sent[1]] size = len(labeled_sent[0]) feature_onehot = np.zeros(size * total_vocab_len, dtype='int').reshape( [size, total_vocab_len]) for i, el in enumerate(labeled_sent[0]): feature_onehot[i, el] = 1 return feature_onehot, label def padding(features, label, length): pad_len = length - len(label) padded_label = (label + [startIdx] * length)[:length] feature_padding = np.zeros((pad_len, total_vocab_len), dtype=np.int) feature_padding[:, endIdx + 1] = np.ones(pad_len) padded_feautres = np.concatenate((features, feature_padding), axis=0) return padded_feautres, padded_label sample_rdd = train_tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ .map(lambda x: padding(x[0], x[1], train_max_len)) \ .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))).cache() val_sample_rdd = val_tokens.map(lambda sentence_t: text2labeled(sentence_t)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ .map(lambda x: padding(x[0], x[1], val_max_len)) \ .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))).cache() return sample_rdd, val_sample_rdd, total_vocab_len
def gen_rand_user_item_feature(user_num, item_num, class_num): user_id = random.randint(1, user_num) item_id = random.randint(1, item_num) rating = random.randint(1, class_num) sample = Sample.from_ndarray(np.array([user_id, item_id]), np.array([rating])) return UserItemFeature(user_id, item_id, sample)
def normalizer(mean, std): """ Normalize features by standard deviation """ return lambda sample: Sample.from_ndarray((sample.features - mean) / std, sample.label, sample.bigdl_type)
import matplotlib.pyplot as plt from pyspark import SparkContext from bigdl.util.common import init_engine from bigdl.util.common import create_spark_conf from bigdl.util.common import JavaCreator from bigdl.util.common import Sample from vision.image3d.transformation import * import h5py from math import pi img_path = os.path.abspath(__file__ + "/../../resources/image_3d/a.mat") sample = h5py.File(img_path)['meniscus_im'] sample = np.array(sample) sample = Sample.from_ndarray(features=sample, label=np.array(-1)) # sample = np.expand_dims(sample,0) print(sample.features[0].shape) sc = SparkContext(appName="test", conf=create_spark_conf()) JavaCreator.set_creator_class( "com.intel.analytics.zoo.transform.vision.image3d.python.api.VisionPythonBigDL" ) init_engine() data_rdd = sc.parallelize([sample]) start_loc = [13, 80, 125] patch = [5, 40, 40] # end_loc = [17,119,164]