def load_images(self, img_size): if os.path.exists(self.data_path + f"/image_data_{img_size}.plk"): with open(self.data_path + f"/image_data_{img_size}.plk", "rb") as fp: data = pickle.load(fp) else: file_name = "CUB_200_2011.tgz" url = "http://www.vision.caltech.edu.s3-us-west-2.amazonaws.com/visipedia-data/CUB-200-2011/CUB_200_2011.tgz" if not os.path.exists(self.data_path + f"/{file_name}"): downloder(url, self.data_path + f"/{file_name}") if len(glob.glob(self.data_path + "/images/*.jpg")) != IMAGE_SIZE: print("Info:Extracting image data from tar file") import tarfile, shutil with tarfile.open(self.data_path + f"/{file_name}", 'r') as tar_fp: tar_fp.extractall(self.data_path) shutil.move(self.data_path + "/CUB_200_2011/images", self.data_path) data = {} files = glob.glob(self.data_path + "/images/*/*.jpg") print(f"Info:load {img_size}x{img_size} image data") for i, _path in enumerate(files): arr_id = int(_path.split("_")[-1].split(".")[0]) arr = self.path2array(_path, img_size) data[arr_id] = arr progress(i + 1, IMAGE_SIZE) print("") with open(self.data_path + f"/image_data_{img_size}.plk", "wb") as fp: pickle.dump(data, fp) return data
def load_english_captions(self, sent_len): if os.path.exists(self.data_path + "/english_captions.pkl"): with open(self.data_path + "/english_captions.pkl", "rb") as fp: data = pickle.load(fp) return data["train_data"], data["val_data"], data["index2tok"] else: url = "http://images.cocodataset.org/annotations/annotations_trainval2014.zip" file_name = "annotations_trainval2014.zip" if not os.path.exists(self.data_path + f"/{file_name}"): downloder(url, self.data_path + f"/{file_name}") annotations_dir = self.data_path + "/annotations" if not os.path.exists(annotations_dir): os.mkdir(annotations_dir) if len(os.listdir(annotations_dir)) != 6: print("Info:extracting zip file") import zipfile with zipfile.ZipFile(self.data_path + f"/{file_name}", 'r') as zip_fp: zip_fp.extractall(self.data_path) print("Info:loading english captin data") def load_caption(data_path): def preprocess_caption(line): line = line.replace("\ufffd\ufffd", " ") tokenizer = RegexpTokenizer(r"\w+") tokens = tokenizer.tokenize(line.lower()) return tokens with open(data_path, "r", encoding="utf_8") as fp: data = json.load(fp) anns = data["annotations"] data_dict = defaultdict(list) for ann in anns: data_dict[ann["image_id"]].append( preprocess_caption(ann["caption"])) return data_dict files = glob.glob(annotations_dir + "/captions_*2014.json") train = None val = None for data_path in files: captions = load_caption(data_path) if "train" in data_path: train = captions elif "val" in data_path: val = captions train_data, val_data, index2tok, tok2index = self.make_vocaburaly( train, val, sent_len) with open(self.data_path + "/english_captions.pkl", "wb") as fp: data = { "train_data": train_data, "val_data": val_data, "index2tok": index2tok, "tok2index": tok2index } pickle.dump(data, fp) return train_data, val_data, index2tok
def load_japanese_captions(sent_len, data_path, threshpld): if os.path.exists(data_path + "/japanese_caption_data.pkl"): with open(data_path + "/japanese_caption_data.pkl", "rb") as fp: data = pickle.load(fp) return data["train_data"], data["val_data"], data["index2tok"] else: url = "https://github.com/STAIR-Lab-CIT/STAIR-captions/raw/master/stair_captions_v1.2.tar.gz" file_name = "stair_captions_v1.2.tar.gz" if not os.path.exists(data_path + f"/{file_name}"): downloder(url, data_path + f"/{file_name}") path = data_path + "/stair_captions_v1.2" if not os.path.exists(path): os.mkdir(path) print("Info:extracting caption data") import tarfile with tarfile.open(data_path + f"/{file_name}", "r") as tar_fp: tar_fp.extractall(path) files = glob.glob(path + "/*_tokenized.json") print(f"Info:loading japanese caption data") def load_caption(data_path): def preprocess_caption(line): prep_line = re.sub("[%s]" % re.escape(string.punctuation), " ", line.rstrip()) prep_line = prep_line.replace("-", " ").replace("\n", "") return prep_line.lower().split(" ") with open(data_path, "r", encoding="utf_8") as fp: data = json.load(fp) anns = data["annotations"] data_dict = defaultdict(list) for ann in anns: data_dict[ann["image_id"]].append( preprocess_caption(ann["tokenized_caption"])) return data_dict train = None val = None for _path in files: captions = load_caption(_path) if "train" in _path: train = captions elif "val" in _path: val = captions train_data, val_data, index2tok, tok2index = make_vocaburaly( train, val, sent_len, threshpld) with open(data_path + "/japanese_caption_data.pkl", "wb") as fp: data = { "train_data": train_data, "val_data": val_data, "index2tok": index2tok, "tok2index": tok2index } pickle.dump(data, fp) return train_data, val_data, index2tok
def load_images(self, is_train=True): if is_train: data_type = "train" data_len = train_image_size else: data_type = "val" data_len = val_image_size path = self.data_path + "/image" if not os.path.exists(path): os.mkdir(path) if os.path.exists(path + f"/{data_type}2014_array/{self.image_size[0]}"): files = glob.glob( path + f"/{data_type}2014_array/{self.image_size[0]}/*.npy") id_list = [ int(path.split("_")[-1].split(".")[0]) for path in files ] data = {id: path for id, path in zip(id_list, files)} return data else: url = f"http://images.cocodataset.org/zips/{data_type}2014.zip" file_name = url.split("/")[-1] if not os.path.exists(path + f"/{file_name}"): downloder(url, path + f"/{file_name}") if len(glob.glob(path + f"/{data_type}2014/*.jpg")) != data_len: print(f"Info:Extracting {data_type} image data from zip file") import zipfile with zipfile.ZipFile(path + f"/{file_name}") as zip_fp: zip_fp.extractall(path) files = glob.glob(path + f"/{data_type}2014/*.jpg") path = path + f"/{data_type}2014_array" if not os.path.exists(path): os.mkdir(path) path = path + f"/{self.image_size[0]}" if not os.path.exists(path): os.mkdir(path) print(f"Info:Conveting {data_type} image data path to ndarray") for i, _path in enumerate(files): file_name = _path.split("/")[-1].split(".")[0] arr = self.path2array(_path) np.save(path + f"/{file_name}.npy", arr) progress(i + 1, data_len) print("") files = glob.glob(path + "/*.npy") id_list = [ int(path.split("_")[-1].split(".")[0]) for path in files ] data = {id: path for id, path in zip(id_list, files)} return data
def load_images(self, is_train=True): if is_train: data_type = "train" data_len = train_image_size else: data_type = "val" data_len = val_image_size if len(glob.glob(self.data_path + f"/{data_type}2014_array/*.npy")) == data_len: files = glob.glob(self.data_path + f"/{data_type}2014_array/*.npy") id_list = [ int(path.split("_")[-1].split(".")[0]) for path in files ] data = {id: path for id, path in zip(id_list, files)} return data else: url = f"http://images.cocodataset.org/zips/{data_type}2014.zip" file_name = url.split("/")[-1] if not os.path.exists(self.data_path + f"/{file_name}"): downloder(url, self.data_path + f"/{file_name}") path = self.data_path + f"/{data_type}2014_array" if len(glob.glob(self.data_path + f"/{data_type}2014/*.jpg")) != data_len: print( f"Info:Extract {data_type} images from zip file and convert images to ndarray" ) if not os.path.exists(path): os.mkdir(path) import zipfile with zipfile.ZipFile(self.data_path + f"/{file_name}") as zip_fp: file_count = len(zip_fp.filelist) for i, item in enumerate(zip_fp.filelist): file_name = item.filename.split("/")[-1].split(".")[0] zip_fp.extract(item, self.data_path) if file_name != "": arr = self.path2array( self.data_path + f"/{data_type}2014/{file_name}.jpg") np.save(path + f"/{file_name}", arr) progress(i + 1, file_count) print("") files = glob.glob(path + "/*.npy") id_list = [ int(path.split("_")[-1].split(".")[0]) for path in files ] data = {id: path for id, path in zip(id_list, files)} return data
def load_captions(self): path = self.data_path + "/annotations" if not os.path.exists(path): os.mkdir(path) if os.path.exists(path + "/caption_data.pkl"): with open(path + "/caption_data.pkl", "rb") as fp: data = pickle.load(fp) return data["train_data"], data["val_data"], data[ "embed_mat"], data["index2tok"], data["tok2index"] else: url = "http://images.cocodataset.org/annotations/annotations_trainval2014.zip" file_name = url.split("/")[-1] if not os.path.exists(path + f"/{file_name}"): downloder(url, path + f"/{file_name}") import zipfile with zipfile.ZipFile(path + f"/{file_name}") as zip_fp: zip_fp.extractall(os.path.abspath(path + "/..")) print("Info:loading caption data") files = glob.glob(path + "/captions*.json") def load_caption(data_path): with open(data_path, "r", encoding="utf_8") as fp: data = json.load(fp) anns = data["annotations"] data_dict = defaultdict(list) for ann in anns: data_dict[ann["image_id"]].append(ann["caption"]) return data_dict train = None val = None for data_path in files: captions = load_caption(data_path) if "train" in data_path: train = captions elif "val" in data_path: val = captions train_data, val_data, embed_mat, index2tok, tok2index = self.make_vocaburaly( train, val, path) return train_data, val_data, embed_mat, index2tok, tok2index
def image2depth(self, img): try: sess = getattr(self, "sess") depth_net = getattr(self, "depth_net") input_node = getattr(self, "input_node") except: import sys import tensorflow as tf os.environ[ 'TF_CPP_MIN_LOG_LEVEL'] = '0' sys.path.append(os.path.abspath(__file__ + "/../models")) from models.fcrn import ResNet50UpProj model_params_path = os.path.abspath(__file__ + "/../data/NYU_ResNet-UpProj.npy") if not os.path.exists(model_params_path): url = "http://campar.in.tum.de/files/rupprecht/depthpred/NYU_ResNet-UpProj.npy" downloder(url, model_params_path) self.input_node = tf.placeholder(tf.float32, shape=(None, 256, 256, 3)) self.depth_net = ResNet50UpProj({'data': self.input_node}, 1, 1, False) self.sess = tf.Session() print('Loading the model') self.depth_net.load(model_params_path, self.sess) sess = getattr(self, "sess") depth_net = getattr(self, "depth_net") input_node = getattr(self, "input_node") def normalization(arr): _min, _max = np.min(arr), np.max(arr) arr = (arr - _min) / (_max - _min) return arr img = img.transpose((1,2,0)) img = np.expand_dims(np.asarray(img), axis = 0) pred = np.asarray(sess.run(depth_net.get_output(), feed_dict={input_node: img}))[0,:,:,0] return normalization(pred)
def load_images(self): files_cont = len(glob.glob(self.data_path + f"/array/{self.image_size[0]}/*.npy")) if files_cont == image_size: files = glob.glob(self.data_path + f"/array/{self.image_size[0]}/*.npy") id_list = [int(path.split("_")[-1].split(".")[0]) for path in files] data = {id:path for id, path in zip(id_list,files)} else: url = "http://www.vision.caltech.edu.s3-us-west-2.amazonaws.com/visipedia-data/CUB-200-2011/CUB_200_2011.tgz" title = "CUB_200_2011.tgz" if not os.path.exists(self.data_path + f"/{title}"):downloder(url, self.data_path + f"/{title}") if len(glob.glob(self.data_path + "/image/*/*.jpg")) != image_size: print("Info:Extracting image data from tar file") import tarfile, shutil with tarfile.open(self.data_path + f"/{title}", 'r') as tar_fp: tar_fp.extractall(self.data_path) shutil.move(self.data_path + "/CUB_200_2011/images", self.data_path) os.rename(self.data_path + "/images", self.data_path + "/image") files = glob.glob(self.data_path + "/image/*/*.jpg") path = self.data_path + "/array" if not os.path.exists(path):os.mkdir(path) path = path + f"/{self.image_size[0]}" if not os.path.exists(path): os.mkdir(path) print("Info:Conveting image data path to ndarray") for i, _path in enumerate(files): file_name = _path.split("/")[-1].split(".")[0] arr = self.path2array(_path) np.save(path + f"/{file_name}.npy", arr) progress(i+1, image_size) print("") files = glob.glob(path + "/*.npy") id_list = [int(path.split("_")[-1].split(".")[0]) for path in files] data = {id:path for id, path in zip(id_list,files)} return data
def load_images(self): if os.path.exists(self.data_path + "/image_data.plk"): with open(self.data_path + "/image_data.plk", "rb") as fp: data = pickle.load(fp) else: file_name = "102flowers.tgz" url = "http://www.robots.ox.ac.uk/~vgg/data/flowers/102/102flowers.tgz" if not os.path.exists(self.data_path + f"/{file_name}"): downloder(url, self.data_path + f"/{file_name}") if len(glob.glob(self.data_path + "/images/*.jpg")) != IMAGE_SIZE: print("Info:Extracting image data from tar file") import tarfile with tarfile.open(self.data_path + f"/{file_name}", 'r') as tar_fp: tar_fp.extractall(self.data_path) os.rename(self.data_path + "/jpg", self.data_path + "/images") data = {} files = glob.glob(self.data_path + "/images/*.jpg") print("Info:load image data") for i, _path in enumerate(files): arr_id = int(_path.split("_")[-1].split(".")[0]) arr_256 = self.path2array(_path, 256) arr_128 = self.path2array(_path, 128) arr_64 = self.path2array(_path, 64) data[arr_id] = { "x_256": arr_256, "x_128": arr_128, "x_64": arr_64 } progress(i + 1, IMAGE_SIZE) print("") with open(self.data_path + "/image_data.plk", "wb") as fp: pickle.dump(data, fp) return data
def load_images(self): files_cont = len(glob.glob(self.data_path + f"/array/{self.image_size[0]}/*.npy")) if files_cont == image_size: files = glob.glob(self.data_path + f"/array/{self.image_size[0]}/*.npy") id_list = [int(path.split("_")[-1].split(".")[0]) for path in files] data = {id:path for id, path in zip(id_list,files)} else: url = "http://www.robots.ox.ac.uk/~vgg/data/flowers/102/102flowers.tgz" title = "102flowers.tgz" if not os.path.exists(self.data_path + f"/{title}"):downloder(url, self.data_path + f"/{title}") if len(glob.glob(self.data_path + "/image/*.jpg")) != image_size: print("Info:Extracting image data from tar file") import tarfile with tarfile.open(self.data_path + f"/{title}", 'r') as tar_fp: tar_fp.extractall(self.data_path) os.rename(self.data_path + "/jpg", self.data_path + "/image") files = glob.glob(self.data_path + "/image/*.jpg") path = self.data_path + "/array" if not os.path.exists(path):os.mkdir(path) path = path + f"/{self.image_size[0]}" if not os.path.exists(path): os.mkdir(path) print("Info:Conveting image data path to ndarray") for i, _path in enumerate(files): file_name = _path.split("/")[-1].split(".")[0] arr = self.path2array(_path) np.save(path + f"/{file_name}.npy", arr) progress(i+1, image_size) print("") files = glob.glob(path + "/*.npy") id_list = [int(path.split("_")[-1].split(".")[0]) for path in files] data = {id:path for id, path in zip(id_list,files)} return data
def make_embed_mat(self, vocaburaly): embed_mat_path = os.path.abspath(self.data_path + "/../crawl-300d-2M.vec") if not os.path.exists(embed_mat_path): url = "https://dl.fbaipublicfiles.com/fasttext/vectors-english/crawl-300d-2M.vec.zip" if not os.path.exists(embed_mat_path + ".zip"): downloder(url, embed_mat_path + ".zip") import zipfile with zipfile.ZipFile(embed_mat_path + ".zip") as zip_fp: zip_fp.extractall(os.path.abspath(self.data_path + "/../")) with open(embed_mat_path, "r", encoding="utf_8") as fp: n, d = map(int, fp.readline().split()) embed_data = {} for line in fp: tokens = line.rstrip().split(' ') if tokens[0] in vocaburaly: embed_data[tokens[0]] = [float(v) for v in tokens[1:]] del vocaburaly[vocaburaly.index(tokens[0])] index2tok = {i + 3: key for i, key in enumerate(embed_data.keys())} tok2index = {key: i + 3 for i, key in enumerate(embed_data.keys())} embed_mat = np.array([ np.random.normal(size=d), np.random.normal(size=d), np.random.normal(size=d) ] + [ embed_data[index2tok[i + 3]] for i in range(len(index2tok.keys())) ]) index2tok[0] = "<S>" index2tok[1] = "</S>" index2tok[2] = "<UNK>" tok2index["<S>"] = 0 tok2index["</S>"] = 1 tok2index["<UNK>"] = 2 return embed_mat, index2tok, tok2index