def main(flags, weight_dict):
path_to_save = os.path.join(flags.weight_dir, 'shift_dict.pkl')
shift_dict = ersa_utils.load_file(path_to_save)
# make network
# define place holder
X = tf.placeholder(tf.float32, shape=[None, flags.input_size[0], flags.input_size[1], 3], name='X')
Z = tf.placeholder(tf.float32, shape=[None, flags.input_size[0], flags.input_size[1], 3], name='Z')
y = tf.placeholder(tf.int32, shape=[None, flags.input_size[0], flags.input_size[1], 1], name='y')
mode = tf.placeholder(tf.bool, name='mode')
model = uabMakeNetwork_UNet.UnetModelDTDA({'X': X, 'Z': Z, 'Y': y},
trainable=mode,
model_name=flags.model_name,
input_size=flags.input_size,
batch_size=flags.batch_size,
learn_rate=flags.learning_rate,
decay_step=flags.decay_step,
decay_rate=flags.decay_rate,
epochs=flags.epochs,
start_filter_num=flags.sfn)
model.create_graph('X', 'Z', class_num=flags.num_classes)
# create collection
# the original file is in /ei-edl01/data/uab_datasets/inria
blCol = uab_collectionFunctions.uabCollection('inria')
opDetObj = bPreproc.uabOperTileDivide(255) # inria GT has value 0 and 255, we map it back to 0 and 1
# [3] is the channel id of GT
rescObj = uabPreprocClasses.uabPreprocMultChanOp([], 'GT_Divide.tif', 'Map GT to (0, 1)', [3], opDetObj)
rescObj.run(blCol)
img_mean = blCol.getChannelMeans([0, 1, 2]) # get mean of rgb info
# extract patches
extrObj = uab_DataHandlerFunctions.uabPatchExtr([0, 1, 2, 4],
cSize=flags.input_size,
numPixOverlap=int(model.get_overlap()),
extSave=['jpg', 'jpg', 'jpg', 'png'],
isTrain=True,
gtInd=3,
pad=model.get_overlap() // 2)
patchDir = extrObj.run(blCol)
# make data reader
# use uabCrossValMaker to get fileLists for training and validation
idx, file_list = uabCrossValMaker.uabUtilGetFolds(patchDir, 'fileList.txt', 'force_tile')
# use first 5 tiles for validation
file_list_source = uabCrossValMaker.make_file_list_by_key(idx, file_list, [i for i in range(6, 37)])
file_list_valid = uabCrossValMaker.make_file_list_by_key(idx, file_list, [i for i in range(0, 6)])
# AIOI dataset
blCol = uab_collectionFunctions.uabCollection(CITY_LIST[flags.leave_city])
# extract patches
extrObj = uab_DataHandlerFunctions.uabPatchExtr([0, 1, 2, 3],
cSize=flags.input_size,
numPixOverlap=int(model.get_overlap()),
extSave=['jpg', 'jpg', 'jpg', 'png'],
isTrain=True,
gtInd=3,
pad=model.get_overlap() // 2)
patchDir_target = extrObj.run(blCol)
idx, file_list = uabCrossValMaker.uabUtilGetFolds(patchDir_target, 'fileList.txt', 'force_tile')
file_list_target = uabCrossValMaker.make_file_list_by_key(idx, file_list, [i for i in range(5)])
with tf.name_scope('image_loader'):
# GT has no mean to subtract, append a 0 for block mean
dataReader_source = uabDataReader.ImageLabelReader([3], [0, 1, 2], patchDir, file_list_source, flags.input_size,
flags.tile_size,
flags.batch_size, dataAug='flip,rotate',
block_mean=np.append([0], img_mean))
# no augmentation needed for validation
dataReader_target = uabDataReader.ImageLabelReader([3], [0, 1, 2], patchDir_target, file_list_target, flags.input_size,
flags.tile_size,
flags.batch_size, dataAug='flip,rotate',
block_mean=np.append([0], img_mean))
dataReader_valid = uabDataReader.ImageLabelReader([3], [0, 1, 2], patchDir, file_list_valid, flags.input_size,
flags.tile_size,
flags.batch_size, dataAug='flip,rotate',
block_mean=np.append([0], img_mean))
# train
start_time = time.time()
model.train_config('X', 'Y', 'Z', flags.n_train, flags.n_valid, flags.input_size, uabRepoPaths.modelPath,
loss_type='xent', par_dir='domain_baseline/contorl_valid', lam=flags.lam)
model.load_source_weights(flags.model_dir, shift_dict, gpu=flags.GPU)
model.run(train_reader_source=dataReader_source,
train_reader_target=dataReader_target,
valid_reader=dataReader_valid,
pretrained_model_dir=None, # train from scratch, no need to load pre-trained model
isTrain=True,
img_mean=img_mean,
verb_step=100, # print a message every 100 step(sample)
save_epoch=5, # save the model every 5 epochs
gpu=flags.GPU,
tile_size=flags.tile_size,
patch_size=flags.input_size)
duration = time.time() - start_time
print('duration {:.2f} hours'.format(duration/60/60))
blCol.readMetadata()
file_list, parent_dir = blCol.getAllTileByDirAndExt([0, 1, 2])
file_list_truth, parent_dir_truth = blCol.getAllTileByDirAndExt(4)
idx, file_list = uabCrossValMaker.uabUtilGetFolds(None, file_list, 'force_tile')
idx_truth, file_list_truth = uabCrossValMaker.uabUtilGetFolds(None, file_list_truth, 'force_tile')
# use first 5 tiles for validation
exclude_cities = [city_list[a] for a in range(5) if a != city_id]
file_list_valid = uabCrossValMaker.make_file_list_by_key(
idx, file_list, [i for i in range(0, 6)],
filter_list=['bellingham', 'bloomington', 'sfo', 'tyrol-e', 'innsbruck'] + exclude_cities)
file_list_valid_truth = uabCrossValMaker.make_file_list_by_key(
idx_truth, file_list_truth, [i for i in range(0, 6)],
filter_list=['bellingham', 'bloomington', 'sfo', 'tyrol-e', 'innsbruck']+ exclude_cities)
img_mean = blCol.getChannelMeans([0, 1, 2])
# make the model
# define place holder
X = tf.placeholder(tf.float32, shape=[None, input_size[0], input_size[1], 3], name='X')
Z = tf.placeholder(tf.float32, shape=[None, input_size[0], input_size[1], 3], name='Z')
y = tf.placeholder(tf.int32, shape=[None, input_size[0], input_size[1], 1], name='y')
mode = tf.placeholder(tf.bool, name='mode')
model = uabMakeNetwork_UNet.UnetModelDTDA({'X': X, 'Z': Z, 'Y': y}, trainable=mode, input_size=input_size, batch_size=5)
# create graph
model.create_graph('X', 'Z', class_num=2)
# evaluate on tiles
model.evaluate(file_list_valid, file_list_valid_truth, parent_dir, parent_dir_truth,
input_size, tile_size, batch_size, img_mean, model_dir, gpu,
save_result_parent_dir='domain_baseline2', ds_name='inria', best_model=False,
load_epoch_num=55, show_figure=False)