def checkArgs(Args):
# print('Using config:')
# pprint.pprint(cfg)
print yellow("Checking the args ...")
if Args.fineTune == 'True':
Args.fine_tune = True
else:
Args.fine_tune = False
if Args.useDemo == 'True':
Args.use_demo = True
else:
Args.use_demo = False
if Args.method == 'test':
if Args.weights is None:
print red(" Specify the testing network weights!")
sys.exit(3)
else:
print blue(" Test the weight: \n {}".format(Args.weights))
elif Args.fine_tune:
if Args.weights is None:
print red(" Specify the finetune network weights!")
sys.exit(4)
else:
print blue(" Finetune the weight: {}".format(Args.weights))
else:
print red(" The network will RE-TRAIN from empty ! ! ")
print ' Called with args:', args
def __init__(self):
self.data_path = cfg.DATA_DIR
self.folder = 'demo_p3_gaojiaqiao'
self.test_set = self.load_dataset()
self.testing_rois_length = len(self.test_set)
print blue(
'Dataset {} initialization has been done successfully.'.format(
self.testing_rois_length))
time.sleep(2)
def load_dataset(self):
Instruction_cache_file = path_add(self.data_path,
'Instruction_cache_data.pkl')
train_cache_file = path_add(self.data_path, 'train_cache_data.pkl')
valid_cache_file = path_add(self.data_path, 'valid_cache_data.pkl')
test_cache_file = path_add(self.data_path, 'test_cache_data.pkl')
if os.path.exists(train_cache_file) & os.path.exists(
valid_cache_file) & os.path.exists(
test_cache_file) & os.path.exists(Instruction_cache_file):
print blue('Loaded the STi dataset from pkl cache files ...')
with open(Instruction_cache_file, 'rb') as fid:
key_points = cPickle.load(fid)
print yellow(' NOTICE: the groundtruth range is [{}] meters, the label to keep is {},\n including folders:{},\n Please verify that meets requirement !' \
.format(key_points[0], key_points[1], key_points[2]))
with open(train_cache_file, 'rb') as fid:
train_set = cPickle.load(fid)
print ' train gt set(cnt:{}) loaded from {}'.format(
len(train_set), train_cache_file)
with open(valid_cache_file, 'rb') as fid:
valid_set = cPickle.load(fid)
print ' valid gt set(cnt:{}) loaded from {}'.format(
len(valid_set), valid_cache_file)
with open(test_cache_file, 'rb') as fid:
test_set = cPickle.load(fid)
print ' test gt set(cnt:{}) loaded from {}'.format(
len(test_set), test_cache_file)
return train_set, valid_set, test_set
print blue('Prepare the STi dataset for training, please wait ...')
self.total_roidb = self.load_sti_annotation()
self.filter_roidb = self.filter(self.total_roidb, self.type_to_keep)
train_set, valid_set, test_set = self.assign_dataset(
self.filter_roidb) # train,valid percent
with open(Instruction_cache_file, 'wb') as fid:
cPickle.dump(
[cfg.DETECTION_RANGE, self.type_to_keep, self.folder_list],
fid, cPickle.HIGHEST_PROTOCOL)
print yellow(' NOTICE: the groundtruth range is [{}] meters, the label to keep is {},\n use the dataset:{},\n Please verify that meets requirement !' \
.format(cfg.DETECTION_RANGE, self.type_to_keep, self.folder_list))
with open(train_cache_file, 'wb') as fid:
cPickle.dump(train_set, fid, cPickle.HIGHEST_PROTOCOL)
print ' Wrote and loaded train gt roidb(cnt:{}) to {}'.format(
len(train_set), train_cache_file)
with open(valid_cache_file, 'wb') as fid:
cPickle.dump(valid_set, fid, cPickle.HIGHEST_PROTOCOL)
print ' Wrote and loaded valid gt roidb(cnt:{}) to {}'.format(
len(valid_set), valid_cache_file)
with open(test_cache_file, 'wb') as fid:
cPickle.dump(test_set, fid, cPickle.HIGHEST_PROTOCOL)
print ' Wrote and loaded test gt roidb(cnt:{}) to {}'.format(
len(test_set), test_cache_file)
return train_set, valid_set, test_set
def __init__(self):
self.data_path = cfg.DATA_DIR
# self.folder_list = ['32_yuanqu_11804041320']
self.folder_list = [
'32_yuanqu_11804041320', 'p3_beihuan_B16_11803221546',
'32beams_dingdianlukou_2018-03-12-11-02-41',
'32_daxuecheng_01803191740', '32_gaosulu_test', 'xuanchuan'
]
self._classes = [
'unknown', 'smallMot', 'bigMot', 'nonMot', 'pedestrian'
] #TODO:declare: there is another label dont care! becareful
self.type_to_keep = ['smallMot', 'bigMot', 'nonMot', 'pedestrian']
self.num_classes = len(self._classes)
self.class_convert = dict(zip(self._classes, xrange(self.num_classes)))
self.total_roidb = []
self.filter_roidb = []
self.percent_train = 0.7
self.percent_valid = 0.3
self.train_set, self.valid_set, self.test_set = self.load_dataset()
self.validing_rois_length = len(self.valid_set)
self.training_rois_length = len(self.train_set)
print blue('Dataset initialization has been done successfully.')
time.sleep(2)
def eat_data_in_one_piece(self):
if not os.path.exists(path_add(self.path, 'data_in_one_piece')):
os.mkdir(path_add(self.path, 'data_in_one_piece'))
TrainSet_POS_file_name = path_add(self.path, 'data_in_one_piece',
'TrainSet_POS.npy')
TrainSet_NEG_file_name = path_add(self.path, 'data_in_one_piece',
'TrainSet_NEG.npy')
ValidSet_POS_file_name = path_add(self.path, 'data_in_one_piece',
'ValidSet_POS.npy')
ValidSet_NEG_file_name = path_add(self.path, 'data_in_one_piece',
'ValidSet_NEG.npy')
if not os.path.exists(path_add(self.path, 'filter_data_in_one_piece')):
os.mkdir(path_add(self.path, 'filter_data_in_one_piece'))
info_file_name = path_add(self.path, 'filter_data_in_one_piece',
'information_about_files.npy')
TrainSet_POS_filter_file_name = path_add(self.path,
'filter_data_in_one_piece',
'Filter_TrainSet_POS.npy')
ValidSet_POS_filter_file_name = path_add(self.path,
'filter_data_in_one_piece',
'Filter_ValidSet_POS.npy')
TrainSet_NEG_filter_file_name = path_add(self.path,
'filter_data_in_one_piece',
'Filter_TrainSet_NEG.npy')
ValidSet_NEG_filter_file_name = path_add(self.path,
'filter_data_in_one_piece',
'Filter_ValidSet_NEG.npy')
if os.path.exists(TrainSet_POS_filter_file_name) and os.path.exists(ValidSet_POS_filter_file_name) \
and os.path.exists(TrainSet_NEG_filter_file_name) and os.path.exists(ValidSet_NEG_filter_file_name) \
and os.path.exists(info_file_name):
print(
'Eating filtered data(Points more than {}) from npy zip file in folder:filter_data_in_one_piece ...'
.format(darkyellow('[' + str(np.load(info_file_name)) + ']')))
self.TrainSet_POS = np.load(TrainSet_POS_filter_file_name)
self.TrainSet_NEG = np.load(TrainSet_NEG_filter_file_name)
self.ValidSet_POS = np.load(ValidSet_POS_filter_file_name)
self.ValidSet_NEG = np.load(ValidSet_NEG_filter_file_name)
self.train_positive_cube_cnt = self.TrainSet_POS.shape[0]
self.train_negative_cube_cnt = self.TrainSet_NEG.shape[0]
self.valid_positive_cube_cnt = self.ValidSet_POS.shape[0]
self.valid_negative_cube_cnt = self.ValidSet_NEG.shape[0]
print(
' emmm,there are TP:{} TN:{} VP:{} VN:{} in my belly.'.format(
purple(str(self.TrainSet_POS.shape[0])),
purple(str(self.TrainSet_NEG.shape[0])),
purple(str(self.ValidSet_POS.shape[0])),
purple(str(self.ValidSet_NEG.shape[0])),
))
return None
if os.path.exists(TrainSet_POS_file_name) and os.path.exists(TrainSet_NEG_file_name) \
and os.path.exists(ValidSet_POS_file_name) and os.path.exists(ValidSet_NEG_file_name):
print(blue('Let`s eating exiting data(without filter) !'))
self.TrainSet_POS = np.load(TrainSet_POS_file_name)
self.TrainSet_NEG = np.load(TrainSet_NEG_file_name)
self.ValidSet_POS = np.load(ValidSet_POS_file_name)
self.ValidSet_NEG = np.load(ValidSet_NEG_file_name)
else:
print(darkyellow('Let`s eating raw data onr by one !'))
train_pos_name_list = sorted(
os.listdir(path_add(self.path, 'KITTI_TRAIN_BOX', 'POSITIVE')))
train_neg_name_list = sorted(
os.listdir(path_add(self.path, 'KITTI_TRAIN_BOX', 'NEGATIVE')))
valid_pos_name_list = sorted(
os.listdir(path_add(self.path, 'KITTI_VALID_BOX', 'POSITIVE')))
valid_neg_name_list = sorted(
os.listdir(path_add(self.path, 'KITTI_VALID_BOX', 'NEGATIVE')))
for name in train_pos_name_list:
data = np.load(
path_add(self.path, 'KITTI_TRAIN_BOX', 'POSITIVE') + '/' +
name)
self.TrainSet_POS.append(data)
self.TrainSet_POS = np.array(self.TrainSet_POS, dtype=np.uint8)
np.save(TrainSet_POS_file_name, self.TrainSet_POS)
print(' Yummy!')
for name in train_neg_name_list:
data = np.load(
path_add(self.path, 'KITTI_TRAIN_BOX', 'NEGATIVE') + '/' +
name)
self.TrainSet_NEG.append(data)
self.TrainSet_NEG = np.array(self.TrainSet_NEG, dtype=np.uint8)
np.save(TrainSet_NEG_file_name, self.TrainSet_NEG)
print(' Take another piece!')
for name in valid_pos_name_list:
data = np.load(
path_add(self.path, 'KITTI_VALID_BOX', 'POSITIVE') + '/' +
name)
self.ValidSet_POS.append(data)
self.ValidSet_POS = np.array(self.ValidSet_POS, dtype=np.uint8)
np.save(ValidSet_POS_file_name, self.ValidSet_POS)
print(' One more!')
for name in valid_neg_name_list:
data = np.load(
path_add(self.path, 'KITTI_VALID_BOX', 'NEGATIVE') + '/' +
name)
self.ValidSet_NEG.append(data)
self.ValidSet_NEG = np.array(self.ValidSet_NEG, dtype=np.uint8)
np.save(ValidSet_NEG_file_name, self.ValidSet_NEG)
print(' I`m full ...')
print('All data has been saved in zip npy file!')
print(
'There are TP:{} TN:{} VP:{} VN:{} and has been successfully eaten!'
.format(self.TrainSet_POS.shape[0], self.TrainSet_NEG.shape[0],
self.ValidSet_POS.shape[0], self.ValidSet_NEG.shape[0]))
print(
darkyellow(
'Filter the positive data which has less points({}) inside ... '
.format(self.arg.positive_points_needed)))
train_sum = np.array([
self.TrainSet_POS[i].sum()
for i in range(self.TrainSet_POS.shape[0])
])
keep_mask1 = np.where(train_sum > self.arg.positive_points_needed)
self.TrainSet_POS = self.TrainSet_POS[keep_mask1]
np.save(TrainSet_POS_filter_file_name, self.TrainSet_POS)
valid_sum = np.array([
self.ValidSet_POS[i].sum()
for i in range(self.ValidSet_POS.shape[0])
])
keep_mask2 = np.where(valid_sum > self.arg.positive_points_needed)
self.ValidSet_POS = self.ValidSet_POS[keep_mask2]
np.save(ValidSet_POS_filter_file_name, self.ValidSet_POS)
np.save(ValidSet_NEG_filter_file_name, self.ValidSet_NEG)
np.save(TrainSet_NEG_filter_file_name, self.TrainSet_NEG)
np.save(info_file_name, self.arg.positive_points_needed)
self.train_positive_cube_cnt = self.TrainSet_POS.shape[0]
self.train_negative_cube_cnt = self.TrainSet_NEG.shape[0]
self.valid_positive_cube_cnt = self.ValidSet_POS.shape[0]
self.valid_negative_cube_cnt = self.ValidSet_NEG.shape[0]
print(
green(
'Done! TrainPositive remain: {},ValidPositive remain: {} and has been saved'
).format(
self.TrainSet_POS.shape[0],
self.ValidSet_POS.shape[0],
))