def test_load_openvino(self): local_path = self.create_temp_dir() model = InferenceModel(1) model_url = data_url + "/analytics-zoo-models/openvino/2018_R5/resnet_v1_50.xml" weight_url = data_url + "/analytics-zoo-models/openvino/2018_R5/resnet_v1_50.bin" model_path = maybe_download("resnet_v1_50.xml", local_path, model_url) weight_path = maybe_download("resnet_v1_50.bin", local_path, weight_url) model.load_openvino(model_path, weight_path) input_data = np.random.random([4, 1, 224, 224, 3]) model.predict(input_data)
def read_data_sets(train_dir, data_type="train"): """ Parse or download mnist data if train_dir is empty. :param: train_dir: The directory storing the mnist data :param: data_type: Reading training set or testing set.It can be either "train" or "test" :return: ``` (ndarray, ndarray) representing (features, labels) features is a 4D unit8 numpy array [index, y, x, depth] representing each pixel valued from 0 to 255. labels is 1D unit8 nunpy array representing the label valued from 0 to 9. ``` """ TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' TEST_IMAGES = 't10k-images-idx3-ubyte.gz' TEST_LABELS = 't10k-labels-idx1-ubyte.gz' if data_type == "train": local_file = base.maybe_download(TRAIN_IMAGES, train_dir, SOURCE_URL + TRAIN_IMAGES) with open(local_file, 'rb') as f: train_images = extract_images(f) local_file = base.maybe_download(TRAIN_LABELS, train_dir, SOURCE_URL + TRAIN_LABELS) with open(local_file, 'rb') as f: train_labels = extract_labels(f) return train_images, train_labels else: local_file = base.maybe_download(TEST_IMAGES, train_dir, SOURCE_URL + TEST_IMAGES) with open(local_file, 'rb') as f: test_images = extract_images(f) local_file = base.maybe_download(TEST_LABELS, train_dir, SOURCE_URL + TEST_LABELS) with open(local_file, 'rb') as f: test_labels = extract_labels(f) return test_images, test_labels
def download_news20(dest_dir): file_name = "20news-18828.tar.gz" file_abs_path = base.maybe_download(file_name, dest_dir, NEWS20_URL) tar = tarfile.open(file_abs_path, "r:gz") extracted_to = os.path.join(dest_dir, "20news-18828") if not os.path.exists(extracted_to): print("Extracting %s to %s" % (file_abs_path, extracted_to)) tar.extractall(dest_dir) tar.close() return extracted_to
def load_roberta(self): os.makedirs(local_path, exist_ok=True) model_url = data_url + "/analytics-zoo-data/roberta/roberta.tar" model_path = maybe_download("roberta.tar", local_path, model_url) tar = tarfile.open(model_path) tar.extractall(path=local_path) tar.close() model_path = os.path.join(local_path, "roberta/model.xml") self.est = Estimator.from_openvino(model_path=model_path)
def download_glove_w2v(dest_dir): file_name = "glove.6B.zip" file_abs_path = base.maybe_download(file_name, dest_dir, GLOVE_URL) import zipfile zip_ref = zipfile.ZipFile(file_abs_path, 'r') extracted_to = os.path.join(dest_dir, "glove.6B") if not os.path.exists(extracted_to): print("Extracting %s to %s" % (file_abs_path, extracted_to)) zip_ref.extractall(extracted_to) zip_ref.close() return extracted_to
def download_reuters(dest_dir): """Download pre-processed reuters newswire data :argument dest_dir: destination directory to store the data :return The absolute path of the stored data """ file_name = 'reuters.pkl' file_abs_path = base.maybe_download( file_name, dest_dir, 'https://s3.amazonaws.com/text-datasets/reuters.pkl') return file_abs_path
def download_imdb(dest_dir): """Download pre-processed IMDB movie review data :argument dest_dir: destination directory to store the data :return The absolute path of the stored data """ file_name = "imdb_full.pkl" file_abs_path = base.maybe_download( file_name, dest_dir, 'https://s3.amazonaws.com/text-datasets/imdb_full.pkl') return file_abs_path
def load_resnet(self): input_file_path = os.path.join(resource_path, "orca/learn/resnet_input") output_file_path = os.path.join(resource_path, "orca/learn/resnet_output") self.input = read_file_and_cast(input_file_path) self.output = read_file_and_cast(output_file_path) self.input = np.array(self.input).reshape([3, 224, 224]) self.output = np.array(self.output).reshape([4, 1000])[:1] os.makedirs(local_path, exist_ok=True) model_url = data_url + "/analytics-zoo-data/openvino2020_resnet50.tar" model_path = maybe_download("openvino2020_resnet50.tar", local_path, model_url) tar = tarfile.open(model_path) tar.extractall(path=local_path) tar.close() model_path = os.path.join(local_path, "openvino2020_resnet50/resnet_v1_50.xml") self.est = Estimator.from_openvino(model_path=model_path)
def get_word_index(dest_dir='/tmp/.zoo/dataset', filename='reuters_word_index.pkl'): """Retrieves the dictionary mapping word indices back to words. # Arguments dest_dir: where to cache the data (relative to `~/.zoo/dataset`). filename: dataset file name # Returns The word index dictionary. """ path = base.maybe_download( filename, dest_dir, 'https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl') f = open(path, 'rb') data = cPickle.load(f, encoding='latin1') f.close() return data
def read_data_sets(data_dir): """ Parse or download movielens 1m data if train_dir is empty. :param data_dir: The directory storing the movielens data :return: a 2D numpy array with user index and item index in each row """ WHOLE_DATA = 'ml-1m.zip' local_file = base.maybe_download(WHOLE_DATA, data_dir, SOURCE_URL + WHOLE_DATA) zip_ref = zipfile.ZipFile(local_file, 'r') extracted_to = os.path.join(data_dir, "ml-1m") if not os.path.exists(extracted_to): print("Extracting %s to %s" % (local_file, data_dir)) zip_ref.extractall(data_dir) zip_ref.close() rating_files = os.path.join(extracted_to, "ratings.dat") rating_list = [ i.strip().split("::") for i in open(rating_files, "r").readlines() ] movielens_data = np.array(rating_list).astype(int) return movielens_data
def download_data(dest_dir): TINYSHAKESPEARE_URL = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt' # noqa file_name = "input.txt" file_abs_path = base.maybe_download(file_name, dest_dir, TINYSHAKESPEARE_URL) return file_abs_path