def read_flag():
parser = argparse.ArgumentParser()
parser.add_argument('--batch-size', default=BATCH_SIZE, type=int, help='batch size (10)')
parser.add_argument('--ds-name', default=DS_NAME, type=str, help='dataset name')
parser.add_argument('--tile-size', default=TILE_SIZE, type=tuple, help='tile size 5000')
parser.add_argument('--patch-size', default=PATCH_SIZE, type=tuple, help='patch size 572')
parser.add_argument('--epochs', default=EPOCHS, type=int, help='# epochs (1)')
parser.add_argument('--num-classes', type=int, default=NUM_CLASS, help='# classes (including background)')
parser.add_argument('--par-dir', type=str, default=PAR_DIR, help='parent directory name to save the model')
parser.add_argument('--n-train', type=int, default=N_TRAIN, help='# samples per epoch')
parser.add_argument('--n-valid', type=int, default=N_VALID, help='# patches to valid')
parser.add_argument('--val-mult', type=int, default=VAL_MULT, help='validation_bs=val_mult*train_bs')
parser.add_argument('--GPU', type=str, default=GPU, help="GPU used for computation.")
parser.add_argument('--decay-step', type=float, default=DECAY_STEP, help='Learning rate decay step in number of epochs.')
parser.add_argument('--decay-rate', type=float, default=DECAY_RATE, help='Learning rate decay rate')
parser.add_argument('--verb-step', type=int, default=VERB_STEP, help='#steps between two verbose prints')
parser.add_argument('--save-epoch', type=int, default=SAVE_EPOCH, help='#epochs between two model save')
parser.add_argument('--data-dir', type=str, default=DATA_DIR, help='root directory of cityscapes')
parser.add_argument('--from-scratch', type=bool, default=FROM_SCRATCH, help='from scratch or not')
parser.add_argument('--start-layer', type=str, default=START_LAYER, help='start layer to unfreeze')
parser.add_argument('--model-dir', type=str, default=MODEL_DIR, help='pretrained model directory')
parser.add_argument('--learn-rate', type=str, default=LEARN_RATE, help='learning rate')
flags = parser.parse_args()
flags.par_dir = 'aemo/' + flags.par_dir
flags.learn_rate = ersa_utils.str2list(flags.learn_rate, d_type=float)
flags.start_layer = ersa_utils.str2list(flags.start_layer, d_type=int)
return flags
def load_files(self, field_name=None, field_id=None, field_ext=None):
"""
Load all files meet the given filters, each one above can be left blank
:param field_name: name of the field
:param field_id: name of the id
:param field_ext: name of the field extension
:return:
"""
if field_name is None:
field_name = self.field_name
if field_id is None:
field_id = self.field_id
field_ext = ersa_utils.str2list(field_ext, d_type=str)
files = []
for fe in field_ext:
if fe in self.rgb_ext:
select_file_ext = self.file_ext[self.rgb_ext.index(fe)]
else:
select_file_ext = self.file_ext[-1]
if type(field_id) is not str:
field_id = str(field_id)
file = ersa_utils.rotate_list(
self.get_file_selection(field_name, field_id, fe,
select_file_ext))[0]
files.append(file)
files = ersa_utils.rotate_list(files)
if len(files) == 1:
if len(files[0]) == 1:
# only one file been requested
return files[0][0]
return files
def get_file_selection(self, field_name, field_id, field_ext, file_ext):
"""
Get list of lists of files selected by given field names, field ids, field extensions and file extensions
:param field_name: name of the fields (e.g., city names)
:param field_id: id of the fields (e.g., tile numbers)
:param field_ext: extension of the fields (e.g., RGB)
:param file_ext: file extension (e.g., tif)
:return: list of lists, where each row is file names of same place with different files
"""
field_name = ersa_utils.str2list(field_name, d_type=str)
field_id = ersa_utils.str2list(field_id, d_type=str)
field_ext = ersa_utils.str2list(field_ext, d_type=str)
file_ext = ersa_utils.str2list(file_ext, d_type=str)
if len(file_ext) == 1:
file_ext = [file_ext[0] for _ in range(len(field_ext))]
file_selection = []
for field, file in zip(field_ext, file_ext):
regexp = self.make_regexp(field_name, field_id, field, file)
file_selection.append(self.get_files(regexp, full_path=True))
file_selection = ersa_utils.rotate_list(file_selection)
return file_selection
def __init__(self,
raw_data_path,
field_name,
field_id,
rgb_ext,
gt_ext,
file_ext,
files=None,
clc_name=None,
force_run=False):
"""
Create a collection
:param raw_data_path: path to where the data are stored
:param field_name: could be name of the cities, or other prefix of the images
:param field_id: could be id of the tiles, or other suffix of the images
:param rgb_ext: name extensions that indicates the images are not ground truth, use ',' to separate if you have
multiple extensions
:param gt_ext: name extensions that indicates the images are ground truth, you can only have at most one ground
truth extension
:param file_ext: extension of the files, use ',' to separate if you have multiple extensions, if all the files
have the same extension, you only need to specify one
:param files: files in the raw_data_path, can be specified by user to exclude some of the raw files, if it is
None, all files will be found automatically
:param clc_name: name of the collection, if set to None, it will be the name of the raw_data_path folder
:param force_run: force run the collection maker even if it already exists
"""
self.raw_data_path = raw_data_path
self.field_name = ersa_utils.str2list(
field_name,
d_type=str) # name of the 'cities' to include in the collection
self.field_id = ersa_utils.str2list(
field_id,
d_type=str) # id of the 'cities' to include in the collection
self.rgb_ext = ersa_utils.str2list(rgb_ext, d_type=str)
self.gt_ext = gt_ext
if len(gt_ext) == 0:
has_gt_ext = 0
else:
has_gt_ext = 1
self.file_ext = ersa_utils.str2list(file_ext, d_type=str)
assert len(self.file_ext) == 1 or len(
self.file_ext) == len(self.rgb_ext) + has_gt_ext
if len(self.file_ext) == 1:
self.file_ext = [
self.file_ext[0]
for _ in range(len(self.rgb_ext) + has_gt_ext)
]
if clc_name is None:
clc_name = os.path.basename(raw_data_path)
self.clc_name = clc_name # name of the collection
self.clc_dir = self.get_dir() # directory to store the collection
self.force_run = force_run
# make collection
if files is None:
self.files = sorted(glob(os.path.join(self.raw_data_path, '*.*')))
else:
self.files = files
self.clc_pb = processBlock.BasicProcess('collection_maker',
self.clc_dir,
self.make_collection)
self.clc_pb.run(self.force_run)
self.meta_data = self.read_meta_data()
def main(flags):
run_id_list = ersa_utils.str2list(flags.run_id, d_type=int)
for flags.run_id in run_id_list:
tf.reset_default_graph()
np.random.seed(int(flags.run_id))
tf.set_random_seed(int(flags.run_id))
if flags.start_layer >= 10:
flags.model_name = flags.model_name[:-1] + str(flags.run_id)
else:
if int(flags.run_id) == 0:
flags.model_name = flags.model_name[:-1] + str(flags.run_id)
else:
flags.model_name = flags.model_name[:-5] + str(flags.run_id)
flags.model_name += '_up{}'.format(flags.start_layer)
# make network
# define place holder
X = tf.placeholder(tf.float32, shape=[None, flags.input_size[0], flags.input_size[1], 3], name='X')
y = tf.placeholder(tf.int32, shape=[None, flags.input_size[0], flags.input_size[1], 1], name='y')
mode = tf.placeholder(tf.bool, name='mode')
model = uabMakeNetwork_UNet.UnetModelCrop({'X': X, 'Y': y},
trainable=mode,
model_name=flags.model_name,
input_size=flags.input_size,
batch_size=flags.batch_size,
learn_rate=flags.learning_rate,
decay_step=flags.decay_step,
decay_rate=flags.decay_rate,
epochs=flags.epochs,
start_filter_num=flags.sfn)
model.create_graph('X', class_num=flags.num_classes)
# create collection
# the original file is in /ei-edl01/data/uab_datasets/inria
blCol = uab_collectionFunctions.uabCollection(flags.ds_name)
blCol.readMetadata()
img_mean = blCol.getChannelMeans([1, 2, 3]) # get mean of rgb info
# extract patches
extrObj = uab_DataHandlerFunctions.uabPatchExtr([0, 1, 2, 3],
cSize=flags.input_size,
numPixOverlap=int(model.get_overlap()),
extSave=['png', 'jpg', 'jpg', 'jpg'],
isTrain=True,
gtInd=0,
pad=int(model.get_overlap()//2))
patchDir = extrObj.run(blCol)
# make data reader
# use first 5 tiles for validation
idx, file_list = uabCrossValMaker.uabUtilGetFolds(patchDir, 'fileList.txt', 'tile')
file_list_train = uabCrossValMaker.make_file_list_by_key(idx, file_list, [0, 1, 2, 3])
file_list_valid = uabCrossValMaker.make_file_list_by_key(idx, file_list, [4, 5])
with tf.name_scope('image_loader'):
# GT has no mean to subtract, append a 0 for block mean
dataReader_train = uabDataReader.ImageLabelReader([0], [1, 2, 3], patchDir, file_list_train, flags.input_size,
flags.tile_size,
flags.batch_size, dataAug='flip,rotate',
block_mean=np.append([0], img_mean))
# no augmentation needed for validation
dataReader_valid = uabDataReader.ImageLabelReader([0], [1, 2, 3], patchDir, file_list_valid, flags.input_size,
flags.tile_size,
flags.batch_size, dataAug=' ', block_mean=np.append([0], img_mean))
# train
start_time = time.time()
if flags.start_layer >= 10:
model.train_config('X', 'Y', flags.n_train, flags.n_valid, flags.input_size, uabRepoPaths.modelPath,
loss_type='xent', par_dir='aemo/{}'.format(flags.ds_name))
else:
model.train_config('X', 'Y', flags.n_train, flags.n_valid, flags.input_size, uabRepoPaths.modelPath,
loss_type='xent', par_dir='aemo/{}'.format(flags.ds_name),
train_var_filter=['layerup{}'.format(i) for i in range(flags.start_layer, 10)])
model.run(train_reader=dataReader_train,
valid_reader=dataReader_valid,
pretrained_model_dir=flags.model_dir, # train from scratch, no need to load pre-trained model
isTrain=True,
img_mean=img_mean,
verb_step=100, # print a message every 100 step(sample)
save_epoch=20, # save the model every 5 epochs
gpu=GPU,
tile_size=flags.tile_size,
patch_size=flags.input_size)
duration = time.time() - start_time
print('duration {:.2f} hours'.format(duration/60/60))
parser = argparse.ArgumentParser()
parser.add_argument('--test-list', default=TEST_LIST, type=str, help='test list')
parser.add_argument('--min-size', default=MIN_SIZE, type=int, help='minimum size to remove')
flags = parser.parse_args()
return flags
if __name__ == '__main__':
flags = read_flag()
model_dirs = [
r'/hdd/Results/domain_selection/UnetCrop_inria_aug_grid_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60_DR0.1_SFN32',
r'/hdd/Results/domain_selection/DeeplabV3_inria_aug_grid_0_PS(321, 321)_BS5_EP100_LR1e-05_DS40_DR0.1_SFN32',
r'/hdd/Results/ugan/UnetGAN_V3_inria_gan_xregion_0_PS(572, 572)_BS20_EP30_LR0.0001_1e-06_1e-06_DS30.0_30.0_30.0_DR0.1_0.1_0.1',
]
model_list = ersa_utils.str2list(flags.test_list)
model_dirs = [model_dirs[x] for x in model_list]
img_dir, task_dir = sis_utils.get_task_img_folder()
for cnt, model_dir in enumerate(model_dirs):
pred_files = sorted(glob(os.path.join(model_dir, 'inria', 'pred', '*.png')))
save_file = os.path.join(task_dir, '{}_xregion_wolverine.txt'.format(model_dir))
with open(save_file, 'w'):
pass
for pred_file in pred_files:
city_name = pred_file.split('/')[-1].split('.')[0]
pred = ersa_utils.load_file(pred_file)
pred = remove_small_objects(pred, flags.min_size)
lbl, idx = get_object_id(pred)