def main():
# Parse the commandline
parser = argparse.ArgumentParser(description='Process a dataset for SSD')
parser.add_argument('--data-source',
default='pascal_voc',
help='data source')
parser.add_argument('--data-dir',
default='pascal-voc',
help='data directory')
parser.add_argument('--expand-probability',
type=float,
default=0.5,
help='probability of running sample expander')
parser.add_argument('--sampler-trials',
type=int,
default=50,
help='number of time a sampler tries to find a sample')
parser.add_argument('--preset',
default='vgg300',
choices=['vgg300', 'vgg512'])
args = parser.parse_args()
print('[i] Data source: ', args.data_source)
print('[i] Data directory: ', args.data_dir)
print('[i] Expand probability: ', args.expand_probability)
print('[i] Sampler trials: ', args.sampler_trials)
print('[i] Preset: ', args.preset)
# Load the data source
print('[i] Configuring the data source...')
try:
source = load_data_source(args.data_source)
source.load_trainval_data(args.data_dir)
print('[i] # training samples: ', source.num_train)
print('[i] # validation samples: ', source.num_valid)
print('[i] # classes: ', source.num_classes)
except (ImportError, AttributeError, RuntimeError) as e:
print('[!] Unable to load data source:', str(e))
return 1
# Compute the training data
preset = get_preset_by_name(args.preset)
with open(args.data_dir + '/train-samples.pkl', 'wb') as f:
pickle.dump(source.train_samples, f)
with open(args.data_dir + '/valid-samples.pkl', 'wb') as f:
pickle.dump(source.valid_samples, f)
with open(args.data_dir + '/training-data.pkl', 'wb') as f:
data = {
'preset':
preset,
'num-classes':
source.num_classes,
'colors':
source.colors,
'lid2name':
source.lid2name,
'lname2id':
source.lname2id,
'train-transforms':
build_train_transforms(preset, source.num_classes,
args.sampler_trials,
args.expand_probability),
'valid-transforms':
build_valid_transforms(preset, source.num_classes)
}
pickle.dump(data, f)
return 0
def main():
#---------------------------------------------------------------------------
# Parse the commandline
#---------------------------------------------------------------------------
parser = argparse.ArgumentParser(description='Process a dataset for SSD')
parser.add_argument('--data-source', default='brats18', help='data source')
parser.add_argument('--data-dir',
default='/data/tng016/brats18',
help='data directory')
parser.add_argument('--validation-fraction',
type=float,
default=0.025,
help='fraction of the data to be used for validation')
parser.add_argument('--expand-probability',
type=float,
default=0.5,
help='probability of running sample expander')
parser.add_argument('--sampler-trials',
type=int,
default=50,
help='number of time a sampler tries to find a sample')
parser.add_argument('--annotate',
type=str2bool,
default='False',
help="Annotate the data samples")
parser.add_argument('--compute-td',
type=str2bool,
default='True',
help="Compute training data")
parser.add_argument('--preset',
default='vgg300',
choices=['vgg300', 'vgg512'],
help="The neural network preset")
parser.add_argument('--process-test',
type=str2bool,
default='False',
help="process the test dataset")
args = parser.parse_args()
print('[i] Data source: ', args.data_source)
print('[i] Data directory: ', args.data_dir)
print('[i] Validation fraction: ', args.validation_fraction)
print('[i] Expand probability: ', args.expand_probability)
print('[i] Sampler trials: ', args.sampler_trials)
print('[i] Annotate: ', args.annotate)
print('[i] Compute training data:', args.compute_td)
print('[i] Preset: ', args.preset)
print('[i] Process test dataset: ', args.process_test)
#---------------------------------------------------------------------------
# Load the data source
#---------------------------------------------------------------------------
print('[i] Configuring the data source...')
#try:
source = load_data_source(args.data_source)
source.load_trainval_data(args.data_dir, args.validation_fraction)
if args.process_test:
source.load_test_data(args.data_dir)
print('[i] # training samples: ', source.num_train)
print('[i] # validation samples: ', source.num_valid)
print('[i] # testing samples: ', source.num_test)
print('[i] # classes: ', source.num_classes)
#except (ImportError, AttributeError, RuntimeError) as e:
# print('[!] Unable to load data source:', str(e))
# return 1
#---------------------------------------------------------------------------
# Annotate samples
#---------------------------------------------------------------------------
if args.annotate:
print('[i] Annotating samples...')
annotate(args.data_dir, source.train_samples, source.colors, 'train')
annotate(args.data_dir, source.valid_samples, source.colors, 'valid')
if args.process_test:
annotate(args.data_dir, source.test_samples, source.colors,
'test ')
#---------------------------------------------------------------------------
# Compute the training data
#---------------------------------------------------------------------------
if args.compute_td:
preset = get_preset_by_name(args.preset)
with open(args.data_dir + '/train-samples.pkl', 'wb') as f:
pickle.dump(source.train_samples, f)
with open(args.data_dir + '/valid-samples.pkl', 'wb') as f:
pickle.dump(source.valid_samples, f)
with open(args.data_dir + '/training-data.pkl', 'wb') as f:
data = {
'preset':
preset,
'num-classes':
source.num_classes,
'colors':
source.colors,
'lid2name':
source.lid2name,
'lname2id':
source.lname2id,
'train-transforms':
build_train_transforms(preset, source.num_classes,
args.sampler_trials,
args.expand_probability),
'valid-transforms':
build_valid_transforms(preset, source.num_classes)
}
pickle.dump(data, f)
return 0
def main():
#---------------------------------------------------------------------------
# Parse commandline
#---------------------------------------------------------------------------
parser = argparse.ArgumentParser(description='SSD inference')
parser.add_argument("files", nargs="*")
parser.add_argument('--name', default='test', help='project name')
parser.add_argument('--checkpoint',
type=int,
default=-1,
help='checkpoint to restore; -1 is the most recent')
parser.add_argument('--training-data',
default='pascal-voc/training-data.pkl',
help='Information about parameters used for training')
parser.add_argument('--output-dir',
default='test-output',
help='directory for the resulting images')
parser.add_argument('--annotate',
type=str2bool,
default='False',
help="Annotate the data samples")
parser.add_argument('--dump-predictions',
type=str2bool,
default='False',
help="Dump raw predictions")
parser.add_argument('--compute-stats',
type=str2bool,
default='True',
help="Compute the mAP stats")
parser.add_argument('--data-source',
default=None,
help='Use test files from the data source')
parser.add_argument('--data-dir',
default='pascal-voc',
help='Use test files from the data source')
parser.add_argument('--batch-size',
type=int,
default=32,
help='batch size')
parser.add_argument('--sample',
default='test',
choices=['test', 'trainval'],
help='sample to run on')
parser.add_argument('--threshold',
type=float,
default=0.5,
help='confidence threshold')
parser.add_argument('--pascal-summary',
type=str2bool,
default='False',
help='dump the detections in Pascal VOC format')
args = parser.parse_args()
#---------------------------------------------------------------------------
# Print parameters
#---------------------------------------------------------------------------
print('[i] Project name: ', args.name)
print('[i] Training data: ', args.training_data)
print('[i] Batch size: ', args.batch_size)
print('[i] Data source: ', args.data_source)
print('[i] Data directory: ', args.data_dir)
print('[i] Output directory: ', args.output_dir)
print('[i] Annotate: ', args.annotate)
print('[i] Dump predictions: ', args.dump_predictions)
print('[i] Sample: ', args.sample)
print('[i] Threshold: ', args.threshold)
print('[i] Pascal summary: ', args.pascal_summary)
#---------------------------------------------------------------------------
# Check if we can get the checkpoint
#---------------------------------------------------------------------------
state = tf.train.get_checkpoint_state(args.name)
if state is None:
print('[!] No network state found in ' + args.name)
return 1
try:
checkpoint_file = state.all_model_checkpoint_paths[args.checkpoint]
except IndexError:
print('[!] Cannot find checkpoint ' + str(args.checkpoint_file))
return 1
metagraph_file = checkpoint_file + '.meta'
if not os.path.exists(metagraph_file):
print('[!] Cannot find metagraph ' + metagraph_file)
return 1
#---------------------------------------------------------------------------
# Load the training data
#---------------------------------------------------------------------------
try:
with open(args.training_data, 'rb') as f:
data = pickle.load(f)
preset = data['preset']
colors = data['colors']
lid2name = data['lid2name']
num_classes = data['num-classes']
image_size = preset.image_size
anchors = get_anchors_for_preset(preset)
except (FileNotFoundError, IOError, KeyError) as e:
print('[!] Unable to load training data:', str(e))
return 1
#---------------------------------------------------------------------------
# Load the data source if defined
#---------------------------------------------------------------------------
compute_stats = False
source = None
if args.data_source:
print('[i] Configuring the data source...')
try:
source = load_data_source(args.data_source)
if args.sample == 'test':
source.load_test_data(args.data_dir)
num_samples = source.num_test
samples = source.test_samples
else:
source.load_trainval_data(args.data_dir, 0)
num_samples = source.num_train
samples = source.train_samples
print('[i] # samples: ', num_samples)
print('[i] # classes: ', source.num_classes)
except (ImportError, AttributeError, RuntimeError) as e:
print('[!] Unable to load data source:', str(e))
return 1
if args.compute_stats:
compute_stats = True
#---------------------------------------------------------------------------
# Create a list of files to analyse and make sure that the output directory
# exists
#---------------------------------------------------------------------------
files = []
if source:
for sample in samples:
files.append(sample.filename)
if not source:
if args.files:
files = args.files
if not files:
print('[!] No files specified')
return 1
files = list(filter(lambda x: os.path.exists(x), files))
if files:
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
#---------------------------------------------------------------------------
# Print model and dataset stats
#---------------------------------------------------------------------------
print('[i] Compute stats: ', compute_stats)
print('[i] Network checkpoint:', checkpoint_file)
print('[i] Metagraph file: ', metagraph_file)
print('[i] Image size: ', image_size)
print('[i] Number of files: ', len(files))
#---------------------------------------------------------------------------
# Create the network
#---------------------------------------------------------------------------
if compute_stats:
ap_calc = APCalculator()
if args.pascal_summary:
pascal_summary = PascalSummary()
with tf.Session() as sess:
print('[i] Creating the model...')
net = SSDVGG(sess, preset)
net.build_from_metagraph(metagraph_file, checkpoint_file)
#-----------------------------------------------------------------------
# Process the images
#-----------------------------------------------------------------------
generator = sample_generator(files, image_size, args.batch_size)
n_sample_batches = int(math.ceil(len(files) / args.batch_size))
description = '[i] Processing samples'
for x, idxs in tqdm(generator,
total=n_sample_batches,
desc=description,
unit='batches'):
feed = {net.image_input: x, net.keep_prob: 1}
enc_boxes = sess.run(net.result, feed_dict=feed)
#-------------------------------------------------------------------
# Process the predictions
#-------------------------------------------------------------------
for i in range(enc_boxes.shape[0]):
boxes = decode_boxes(enc_boxes[i], anchors, args.threshold,
lid2name, None)
boxes = suppress_overlaps(boxes)[:200]
filename = files[idxs[i]]
basename = os.path.basename(filename)
#---------------------------------------------------------------
# Annotate samples
#---------------------------------------------------------------
if args.annotate:
img = cv2.imread(filename)
for box in boxes:
draw_box(img, box[1], colors[box[1].label])
fn = args.output_dir + '/' + basename
cv2.imwrite(fn, img)
#---------------------------------------------------------------
# Dump the predictions
#---------------------------------------------------------------
if args.dump_predictions:
raw_fn = args.output_dir + '/' + basename + '.npy'
np.save(raw_fn, enc_boxes[i])
#---------------------------------------------------------------
# Add predictions to the stats calculator and to the Pascal
# summary
#---------------------------------------------------------------
if compute_stats:
ap_calc.add_detections(samples[idxs[i]].boxes, boxes)
if args.pascal_summary:
pascal_summary.add_detections(filename, boxes)
#---------------------------------------------------------------------------
# Compute and print the stats
#---------------------------------------------------------------------------
if compute_stats:
aps = ap_calc.compute_aps()
for k, v in aps.items():
print('[i] AP [{0}]: {1:.3f}'.format(k, v))
print('[i] mAP: {0:.3f}'.format(APs2mAP(aps)))
#---------------------------------------------------------------------------
# Write the pascal summary files
#---------------------------------------------------------------------------
if args.pascal_summary:
pascal_summary.write_summary(args.output_dir)
print('[i] All done.')
return 0
