def load_dataset(path):
image_info_list = []
with open(os.path.join(path, "attributes", "images-dirs.txt"), "r") as fr:
images = [filename.split()[1] for filename in fr.readlines()]
for image in images:
image_info = {}
anno = scipy.io.loadmat(os.path.join(path, "annotations-mat", image[:-4]))
image_info["anno"] = anno
image_info["image_path"] = os.path.join(path, "images", image)
image_info["label"] = image.split("/")[0]
if os.path.exists(os.path.join(path, "images", image)):
image_info_list.append(image_info)
ds = dataset.generate(CubGenerator(), image_info_list)
return ds
def load_dataset(path):
df = pd.read_csv(
os.path.join(path, "processed", "image_download_actual.csv"))
country_dirs = os.listdir(os.path.join(path, "countries"))
image_info_list = []
for index, row in df.iterrows():
image_info = {}
image_info["image_path"] = get_image_path(row['country'],
row['image_name'], path)
image_info["nightlights_bin"] = row["nightlights_bin"]
try:
Image.open(image_info["image_path"])
image_info_list.append(image_info)
except Exception as identifier:
print("Image not found")
# the generator iterates through the argument given, one by one and applies forward. This is done lazily.
ds = dataset.generate(OmdenaGenerator(), image_info_list)
return ds
def load_dataset(base_path):
labels_list = os.listdir(base_path)
labels_dict = map_labels(labels_list)
image_info_list = []
for label in labels_list:
for label_num in range(1, NUM_FEATURES + 1):
curr_path = base_path + "/" + label + "/" + label + "_" + str(
label_num)
images_list = os.listdir(curr_path)
for image in images_list:
image_info = {}
if image.lower().startswith(
label
): # all images' name starts with the feature name (observation)
image_info["image_path"] = curr_path + "/" + image
image_info["image_label"] = labels_dict[label]
image_info["named_image_label"] = label
image_info_list.append(image_info)
# the generator iterates through the argument given, one by one and applies forward. This is done lazily.
ds = dataset.generate(DatasetGenerator(), image_info_list)
return ds
def load_dataset(path):
df = pd.read_csv(os.path.join(path, "processed",
"image_download_locs.csv"))
country_dirs = os.listdir(os.path.join(path, "countries"))
image_info_list = []
for dir in country_dirs:
images_path = os.path.join(path, "countries", dir, "images")
image_list = os.listdir(images_path)
for image_name in image_list:
image_info = {}
image_info["image_name"] = image_name
res_df = df.loc[df["image_name"] == image_name]
image_info["image_lat"] = res_df.iloc[0]["image_lat"]
image_info["image_lon"] = res_df.iloc[0]["image_lon"]
image_info["cluster_lat"] = res_df.iloc[0]["cluster_lat"]
image_info["cluster_lon"] = res_df.iloc[0]["cluster_lon"]
image_info["cons_pc"] = res_df.iloc[0]["cons_pc"]
image_info["nightlights"] = res_df.iloc[0]["nightlights"]
image_info["country"] = res_df.iloc[0]["country"]
image_info["nightlights_bin"] = res_df.iloc[0]["nightlights_bin"]
image_info_list.append(image_info)
ds = dataset.generate(OmdenaGenerator(path), image_info_list)
return ds
import numpy as np
from hub import Transform, dataset
class Crop(Transform):
def forward(self, input):
return {"image": input[0:1, :256, :256]}
def meta(self):
return {"image": {"shape": (1, 256, 256), "dtype": "uint8"}}
class Flip(Transform):
def forward(self, input):
img = np.expand_dims(input["image"], axis=0)
img = np.flip(img, axis=(1, 2))
return {"image": img}
def meta(self):
return {"image": {"shape": (1, 256, 256), "dtype": "uint8"}}
images = [np.ones((1, 512, 512), dtype="uint8") for i in range(20)]
ds = dataset.generate(Crop(), images)
ds = dataset.generate(Flip(), ds)
ds.store("/tmp/cropflip")