def __init__(self, args):
"""args type is EasyDict class
"""
labels = list('ABCDEFGHIJKLMNOPQRS')
self.label2clsval = {l:i for i,l in enumerate(labels)}
self.args = args
self.gray = args.converse_gray
self.image_normalizer = ImageNormalizer()
self.pairs = self.read_paths()
self.counter = 0
self.image_size_in_batch = [None, None] # height, width
def create_normalized_dataset(self,
target_width,
target_height,
root_path,
max_db_gb=2,
image_limit=None):
if not os.path.exists(root_path):
os.makedirs(root_path)
normalizer = ImageNormalizer(target_width, target_height)
mono_table_name = "mono"
colour_table_name = "colour"
index_table_name = "index"
meatadata = {
mono_table_name: {
"width": target_width,
"height": target_height,
"channels": 1,
"dtype": "float32"
},
colour_table_name: {
"width": target_width,
"height": target_height,
"channels": 3,
"dtype": "float32"
},
index_table_name: {
"encoding": "ascii",
"dtype": "string"
}
}
with open(os.path.join(root_path, "metadata.json"),
"w") as metadata_file:
json.dump(meatadata, metadata_file)
env = lmdb.open(root_path,
map_size=(max_db_gb * one_gb),
create=True,
max_dbs=4)
mono_db = env.open_db(mono_table_name.encode(encoding="ascii"))
colour_db = env.open_db(colour_table_name.encode(encoding="ascii"))
index_db = env.open_db(index_table_name.encode(encoding="ascii"))
i = 0
blob_service = self._get_or_create_blob_service()
blobs_to_be_normalized = blob_service.list_blob_names(
self.original_image_blob_container, num_results=image_limit)
source_count = len(list(blobs_to_be_normalized))
source_index = 0
for original_blob_name in blobs_to_be_normalized:
source_index += 1
print("Normalizing {} of {} : {}".format(source_index,
source_count,
original_blob_name))
self.blob_service.get_blob_to_path(
self.original_image_blob_container, original_blob_name,
original_blob_name)
im = Image.open(original_blob_name)
image_id = original_blob_name.split(".")[0]
normalized = normalizer.normalize_image(im)
for normal_image in normalized:
i_key = i.to_bytes(4, byteorder="big")
i += 1
new_image_id = "{0}.{1}.png".format(image_id,
str(uuid.uuid4()))
mono_image_array = ToTensor()(
normal_image["mono"]).flatten().numpy()
colour_image_array = ToTensor()(
normal_image["colour"]).flatten().numpy()
# insert in db
with env.begin(db=index_db, write=True, buffers=True) as txn:
txn.put(i_key, new_image_id.encode(encoding="ascii"))
with env.begin(db=mono_db, write=True, buffers=True) as txn:
txn.put(i_key, mono_image_array.data)
with env.begin(db=colour_db, write=True, buffers=True) as txn:
txn.put(i_key, colour_image_array.data)
print("{} normalized images for {}".format(len(normalized),
original_blob_name))
os.remove(original_blob_name)
env.close()
# open and close it again as readonly to compress the file
env2 = lmdb.open(root_path, max_dbs=1)
env2.close()
def __init__(self, args):
self.args = args
self.image_normalizer = ImageNormalizer()
self.pairs = self.read_paths()
self.counter = 0
self.image_size_in_batch = [None, None] # height, width