def next_minibatch(self,
num_samples,
number_of_workers=1,
worker_rank=0,
device=None):
features = []
labels = []
sweep_end = False
f_sample_count = 0
l_sample_count = 0
while max(f_sample_count, l_sample_count) < num_samples:
if self.next_seq_idx == len(self.sequences):
sweep_end = True
self.next_seq_idx = 0
seq_id = self.sequences[self.sequences[self.next_seq_idx]]
f_data = self.data[seq_id]['features']
l_data = self.data[seq_id]['labels']
if (features or labels) and max(
f_sample_count + len(f_data),
l_sample_count + len(l_data)) > num_samples:
break
f_sample_count += len(f_data)
features.append(f_data)
l_sample_count += len(l_data)
labels.append(l_data)
self.next_seq_idx += 1
num_seq = len(features)
f_data = Value.one_hot(batch=features, num_classes=self.f_dim)
l_data = Value(batch=np.asarray(labels, dtype=np.float32))
result = {
self.fsi: MinibatchData(f_data, num_seq, f_sample_count,
sweep_end),
self.lsi: MinibatchData(l_data, num_seq, l_sample_count, sweep_end)
}
return result
def next_minibatch(self, num_samples, number_of_workers=1, worker_rank=0, device=None):
samples = []
sweep_end = False
for i in range(num_samples):
feature_to_label = self.reader.next()
if feature_to_label is None:
samples = samples[: (len(samples) // self.seqlength) * self.seqlength]
self.reader.reset()
sweep_end = True
break
feature, label = feature_to_label
curr_word = self.parse_word(feature)
next_word = self.parse_word(label)
samples.append((curr_word, next_word))
batchsize = len(samples) / self.seqlength
# Divide batch into every gpu
batchrange = list(map(int, [
(batchsize // number_of_workers) * worker_rank,
min((batchsize // number_of_workers) * (worker_rank + 1), batchsize)
]))
samples = samples[batchrange[0] * self.seqlength: batchrange[1] * self.seqlength]
minibatch = self.make_minibatch(samples)
sample_count = len(samples)
num_seq = len(minibatch[0])
minibatch = {
self.input1: MinibatchData(minibatch[0], num_seq, sample_count, sweep_end),
self.input2: MinibatchData(minibatch[1], num_seq, sample_count, sweep_end),
self.label1: MinibatchData(minibatch[2], num_seq, sample_count, sweep_end),
self.label2: MinibatchData(minibatch[3], num_seq, sample_count, sweep_end),
self.word1: MinibatchData(minibatch[4], num_seq, sample_count, sweep_end),
self.word2: MinibatchData(minibatch[5], num_seq, sample_count, sweep_end)
}
return minibatch
def next_minibatch_with_proposals(self, num_samples, number_of_workers=1, worker_rank=1, device=None, input_map=None):
if num_samples > 1:
print("Only single item mini batches are supported currently by od_mb_source.py")
exit(1)
img_data, roi_data, img_dims, buffered_proposals = self.od_reader.get_next_input()
sweep_end = self.od_reader.sweep_end()
if input_map is None:
result = {
self.image_si: MinibatchData(Value(batch=img_data), 1, 1, sweep_end),
self.roi_si: MinibatchData(Value(batch=roi_data), 1, 1, sweep_end),
self.dims_si: MinibatchData(Value(batch=np.asarray(img_dims, dtype=np.float32)), 1, 1, sweep_end),
}
else:
result = {
input_map[self.image_si]: MinibatchData(Value(batch=np.asarray(img_data, dtype=np.float32)), 1, 1, sweep_end),
input_map[self.roi_si]: MinibatchData(Value(batch=np.asarray(roi_data, dtype=np.float32)), 1, 1, sweep_end),
input_map[self.dims_si]: MinibatchData(Value(batch=np.asarray(img_dims, dtype=np.float32)), 1, 1, sweep_end),
}
return result, buffered_proposals
def next_minibatch(self,
url,
num_samples,
number_of_workers=1,
worker_rank=1,
device=None,
input_map=None):
img_data, roi_data, img_dims = self.od_reader.get_next_input(url)
sweep_end = self.od_reader.sweep_end()
if input_map is None:
result = {
self.image_si:
MinibatchData(Value(batch=img_data), 1, 1, sweep_end),
self.roi_si:
MinibatchData(Value(batch=roi_data), 1, 1, sweep_end),
self.dims_si:
MinibatchData(
Value(batch=np.asarray(img_dims, dtype=np.float32)), 1, 1,
sweep_end),
}
else:
result = {
input_map[self.image_si]:
MinibatchData(
Value(batch=np.asarray(img_data, dtype=np.float32)), 1, 1,
sweep_end),
input_map[self.roi_si]:
MinibatchData(
Value(batch=np.asarray(roi_data, dtype=np.float32)), 1, 1,
sweep_end),
input_map[self.dims_si]:
MinibatchData(
Value(batch=np.asarray(img_dims, dtype=np.float32)), 1, 1,
sweep_end),
}
return result
def train_model(image_input,
roi_input,
dims_input,
loss,
pred_error,
lr_per_sample,
mm_schedule,
l2_reg_weight,
epochs_to_train,
rpn_rois_input=None,
buffered_rpn_proposals=None):
if isinstance(loss, cntk.Variable):
loss = combine([loss])
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
if cfg["CNTK"].DEBUG_OUTPUT:
print("biases")
for p in biases:
print(p)
print("others")
for p in others:
print(p)
print("bias_lr_mult: {}".format(bias_lr_mult))
# Instantiate the learners and the trainer object
lr_schedule = learning_rate_schedule(lr_per_sample, unit=UnitType.sample)
learner = momentum_sgd(others,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
bias_lr_per_sample = [v * bias_lr_mult for v in lr_per_sample]
bias_lr_schedule = learning_rate_schedule(bias_lr_per_sample,
unit=UnitType.sample)
bias_learner = momentum_sgd(
biases,
bias_lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
od_minibatch_source = ObjectDetectionMinibatchSource(
globalvars['train_map_file'],
globalvars['train_roi_file'],
max_annotations_per_image=cfg["CNTK"].INPUT_ROIS_PER_IMAGE,
pad_width=image_width,
pad_height=image_height,
pad_value=img_pad_value,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["CNTK"].NUM_TRAIN_IMAGES,
buffered_rpn_proposals=buffered_rpn_proposals)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.roi_si: roi_input,
od_minibatch_source.dims_si: dims_input
}
use_buffered_proposals = buffered_rpn_proposals is not None
progress_printer = ProgressPrinter(tag='Training',
num_epochs=epochs_to_train,
gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data, proposals = od_minibatch_source.next_minibatch_with_proposals(
min(mb_size, epoch_size - sample_count), input_map=input_map)
if use_buffered_proposals:
data[rpn_rois_input] = MinibatchData(
Value(batch=np.asarray(proposals, dtype=np.float32)), 1, 1,
False)
# remove dims input if no rpn is required to avoid warnings
del data[[k for k in data if '[6]' in str(k)][0]]
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
from utils.annotations.annotations_helper import parse_class_map_file
from config import cfg
from od_mb_source import ObjectDetectionMinibatchSource
from cntk_helpers import regress_rois
###############################################################
###############################################################
mb_size = cfg["CNTK"].MB_SIZE
image_width = cfg["CNTK"].IMAGE_WIDTH
image_height = cfg["CNTK"].IMAGE_HEIGHT
num_channels = cfg["CNTK"].NUM_CHANNELS
# dims_input -- (pad_width, pad_height, scaled_image_width, scaled_image_height, orig_img_width, orig_img_height)
dims_input_const = MinibatchData(
Value(batch=np.asarray([
image_width, image_height, image_width, image_height, image_width,
image_height
],
dtype=np.float32)), 1, 1, False)
# Color used for padding and normalization (Caffe model uses [102.98010, 115.94650, 122.77170])
img_pad_value = [103, 116, 123
] if cfg["CNTK"].BASE_MODEL == "VGG16" else [114, 114, 114]
normalization_const = Constant([[[103]], [[116]], [[
123
]]]) if cfg["CNTK"].BASE_MODEL == "VGG16" else Constant([[[114]], [[114]],
[[114]]])
globalvars = {}
globalvars['output_path'] = os.path.join(abs_path, "Output")
# dataset specific parameters
def next_minibatch(self, num_samples, number_of_workers=1, worker_rank=1, device=None, input_map=None):
if num_samples > 1:
print("Only single item mini batches are supported currently by od_mb_source.py")
exit(1)
img_data, roi_data, img_dims, proposals, label_targets, bbox_targets, bbox_inside_weights = self.od_reader.get_next_input()
sweep_end = self.od_reader.sweep_end()
if input_map is None:
result = {
self.image_si: MinibatchData(Value(batch=img_data), 1, 1, sweep_end),
self.roi_si: MinibatchData(Value(batch=roi_data), 1, 1, sweep_end),
self.dims_si: MinibatchData(Value(batch=np.asarray(img_dims, dtype=np.float32)), 1, 1, sweep_end),
self.proposals_si: MinibatchData(Value(batch=np.asarray(proposals, dtype=np.float32)), 1, 1, sweep_end),
self.label_targets_si: MinibatchData(Value(batch=np.asarray(label_targets, dtype=np.float32)), 1, 1, sweep_end),
self.bbox_targets_si: MinibatchData(Value(batch=np.asarray(bbox_targets, dtype=np.float32)), 1, 1, sweep_end),
self.bbiw_si: MinibatchData(Value(batch=np.asarray(bbox_inside_weights, dtype=np.float32)), 1, 1, sweep_end),
}
else:
result = {
input_map[self.image_si]: MinibatchData(Value(batch=np.asarray(img_data, dtype=np.float32)), 1, 1, sweep_end)
}
if self.roi_si in input_map:
result[input_map[self.roi_si]] = MinibatchData(Value(batch=np.asarray(roi_data, dtype=np.float32)), 1, 1, sweep_end)
if self.dims_si in input_map:
result[input_map[self.dims_si]] = MinibatchData(Value(batch=np.asarray(img_dims, dtype=np.float32)), 1, 1, sweep_end)
if self.proposals_si in input_map:
result[input_map[self.proposals_si]] = MinibatchData(Value(batch=np.asarray(proposals, dtype=np.float32)), 1, 1, sweep_end)
if self.label_targets_si in input_map:
result[input_map[self.label_targets_si]] = MinibatchData(Value(batch=np.asarray(label_targets, dtype=np.float32)), 1, 1, sweep_end)
if self.bbox_targets_si in input_map:
result[input_map[self.bbox_targets_si]] = MinibatchData(Value(batch=np.asarray(bbox_targets, dtype=np.float32)), 1, 1, sweep_end)
if self.bbiw_si in input_map:
result[input_map[self.bbiw_si]] = MinibatchData(Value(batch=np.asarray(bbox_inside_weights, dtype=np.float32)), 1, 1, sweep_end)
return result
def set_global_vars(use_arg_parser=True):
global globalvars
global image_width
global image_height
global dims_input_const
global img_pad_value
global normalization_const
global map_file_path
global epoch_size
global num_test_images
global model_folder
global base_model_file
global feature_node_name
global last_conv_node_name
global start_train_conv_node_name
global pool_node_name
global last_hidden_node_name
global roi_dim
global prediction
global prediction_in
global prediction_out
if use_arg_parser:
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--config',
help='Configuration file in YAML format',
required=False,
default=None)
parser.add_argument('-t',
'--device_type',
type=str,
help="The type of the device (cpu|gpu)",
required=False,
default="cpu")
parser.add_argument(
'-d',
'--device',
type=int,
help="Force to run the script on a specified device",
required=False,
default=None)
parser.add_argument('-l',
'--list_devices',
action='store_true',
help="Lists the available devices and exits",
required=False,
default=False)
parser.add_argument('--prediction',
action='store_true',
help="Switches to prediction mode",
required=False,
default=False)
parser.add_argument(
'--prediction_in',
action='append',
type=str,
help=
"The input directory for images in prediction mode. Can be supplied mulitple times.",
required=False,
default=list())
parser.add_argument(
'--prediction_out',
action='append',
type=str,
help=
"The output directory for processed images and predicitons in prediction mode. Can be supplied mulitple times.",
required=False,
default=list())
parser.add_argument(
'--no_headers',
action='store_true',
help="Whether to suppress the header row in the ROI CSV files",
required=False,
default=False)
parser.add_argument(
'--output_width_height',
action='store_true',
help=
"Whether to output width/height instead of second x/y in the ROI CSV files",
required=False,
default=False)
parser.add_argument(
'--suppressed_labels',
type=str,
help=
"Comma-separated list of labels to suppress from being output in ROI CSV files.",
required=False,
default="")
args = vars(parser.parse_args())
# prediction mode?
prediction = args['prediction']
if prediction:
prediction_in = args['prediction_in']
if len(prediction_in) == 0:
raise RuntimeError("No prediction input directory provided!")
for p in prediction_in:
if not os.path.exists(p):
raise RuntimeError(
"Prediction input directory '%s' does not exist" % p)
prediction_out = args['prediction_out']
if len(prediction_out) == 0:
raise RuntimeError("No prediction output directory provided!")
for p in prediction_out:
if not os.path.exists(p):
raise RuntimeError(
"Prediction output directory '%s' does not exist" % p)
if len(prediction_in) != len(prediction_out):
raise RuntimeError(
"Number of input and output directories don't match: %i != %i"
% (len(prediction_in), len(prediction_out)))
for i in range(len(prediction_in)):
if prediction_in[i] == prediction_out[i]:
raise RuntimeError(
"Input and output directories #%i for prediction are the same: %s"
% ((i + 1), prediction_in[i]))
if args['list_devices']:
print("Available devices (Type - ID - description)")
for d in cntk.device.all_devices():
if d.type() == 0:
type = "cpu"
elif d.type() == 1:
type = "gpu"
else:
type = "<unknown:" + str(d.type()) + ">"
print(type + " - " + str(d.id()) + " - " + str(d))
sys.exit(0)
if args['config'] is not None:
cfg_from_file(args['config'])
if args['device'] is not None:
if args['device_type'] == 'gpu':
cntk.device.try_set_default_device(
cntk.device.gpu(args['device']))
else:
cntk.device.try_set_default_device(cntk.device.cpu())
image_width = cfg["CNTK"].IMAGE_WIDTH
image_height = cfg["CNTK"].IMAGE_HEIGHT
# dims_input -- (pad_width, pad_height, scaled_image_width, scaled_image_height, orig_img_width, orig_img_height)
dims_input_const = MinibatchData(
Value(batch=np.asarray([
image_width, image_height, image_width, image_height, image_width,
image_height
],
dtype=np.float32)), 1, 1, False)
# Color used for padding and normalization (Caffe model uses [102.98010, 115.94650, 122.77170])
img_pad_value = [103, 116, 123] if cfg["CNTK"].BASE_MODEL == "VGG16" else [
114, 114, 114
]
normalization_const = Constant([[[103]], [[116]], [[
123
]]]) if cfg["CNTK"].BASE_MODEL == "VGG16" else Constant([[[114]], [[114]],
[[114]]])
# dataset specific parameters
map_file_path = os.path.join(abs_path, cfg["CNTK"].MAP_FILE_PATH)
globalvars['class_map_file'] = cfg["CNTK"].CLASS_MAP_FILE
globalvars['train_map_file'] = cfg["CNTK"].TRAIN_MAP_FILE
globalvars['test_map_file'] = cfg["CNTK"].TEST_MAP_FILE
globalvars['train_roi_file'] = cfg["CNTK"].TRAIN_ROI_FILE
globalvars['test_roi_file'] = cfg["CNTK"].TEST_ROI_FILE
globalvars['output_path'] = cfg["CNTK"].OUTPUT_PATH
epoch_size = cfg["CNTK"].NUM_TRAIN_IMAGES
num_test_images = cfg["CNTK"].NUM_TEST_IMAGES
# model specific parameters
if cfg["CNTK"].PRETRAINED_MODELS.startswith(".."):
model_folder = os.path.join(abs_path, cfg["CNTK"].PRETRAINED_MODELS)
else:
model_folder = cfg["CNTK"].PRETRAINED_MODELS
base_model_file = os.path.join(model_folder, cfg["CNTK"].BASE_MODEL_FILE)
feature_node_name = cfg["CNTK"].FEATURE_NODE_NAME
last_conv_node_name = cfg["CNTK"].LAST_CONV_NODE_NAME
start_train_conv_node_name = cfg["CNTK"].START_TRAIN_CONV_NODE_NAME
pool_node_name = cfg["CNTK"].POOL_NODE_NAME
last_hidden_node_name = cfg["CNTK"].LAST_HIDDEN_NODE_NAME
roi_dim = cfg["CNTK"].ROI_DIM
data_path = map_file_path
# set and overwrite learning parameters
globalvars['rpn_lr_factor'] = cfg["CNTK"].RPN_LR_FACTOR
globalvars['frcn_lr_factor'] = cfg["CNTK"].FRCN_LR_FACTOR
globalvars['e2e_lr_factor'] = cfg["CNTK"].E2E_LR_FACTOR
globalvars['momentum_per_mb'] = cfg["CNTK"].MOMENTUM_PER_MB
globalvars['e2e_epochs'] = 1 if cfg["CNTK"].FAST_MODE else cfg[
"CNTK"].E2E_MAX_EPOCHS
globalvars[
'rpn_epochs'] = 1 if cfg["CNTK"].FAST_MODE else cfg["CNTK"].RPN_EPOCHS
globalvars['frcn_epochs'] = 1 if cfg["CNTK"].FAST_MODE else cfg[
"CNTK"].FRCN_EPOCHS
globalvars['rnd_seed'] = cfg.RNG_SEED
globalvars['train_conv'] = cfg["CNTK"].TRAIN_CONV_LAYERS
globalvars['train_e2e'] = cfg["CNTK"].TRAIN_E2E
if not os.path.isdir(data_path):
raise RuntimeError("Directory %s does not exist" % data_path)
globalvars['class_map_file'] = os.path.join(data_path,
globalvars['class_map_file'])
globalvars['train_map_file'] = os.path.join(data_path,
globalvars['train_map_file'])
globalvars['test_map_file'] = os.path.join(data_path,
globalvars['test_map_file'])
globalvars['train_roi_file'] = os.path.join(data_path,
globalvars['train_roi_file'])
globalvars['test_roi_file'] = os.path.join(data_path,
globalvars['test_roi_file'])
globalvars['headers'] = not args['no_headers']
globalvars['output_width_height'] = args['output_width_height']
suppressed_labels = []
if len(args['suppressed_labels']) > 0:
suppressed_labels = args['suppressed_labels'].split(",")
globalvars['suppressed_labels'] = suppressed_labels
if cfg["CNTK"].FORCE_DETERMINISTIC:
force_deterministic_algorithms()
np.random.seed(seed=globalvars['rnd_seed'])
globalvars['classes'] = parse_class_map_file(globalvars['class_map_file'])
globalvars['num_classes'] = len(globalvars['classes'])
if cfg["CNTK"].DEBUG_OUTPUT:
# report args
print("Using the following parameters:")
print("Flip image : {}".format(cfg["TRAIN"].USE_FLIPPED))
print("Train conv layers: {}".format(globalvars['train_conv']))
print("Random seed : {}".format(globalvars['rnd_seed']))
print("Momentum per MB : {}".format(globalvars['momentum_per_mb']))
if globalvars['train_e2e']:
print("E2E epochs : {}".format(globalvars['e2e_epochs']))
else:
print("RPN lr factor : {}".format(globalvars['rpn_lr_factor']))
print("RPN epochs : {}".format(globalvars['rpn_epochs']))
print("FRCN lr factor : {}".format(globalvars['frcn_lr_factor']))
print("FRCN epochs : {}".format(globalvars['frcn_epochs']))