def select_frames():
if start > 1.0 or stop > 1.0 or start < 0 or stop < 0 or start >= stop:
raise ValueError('Please change start & stop, they should form a '
'normalized interval with 0 <= start < stop <= 1.')
else:
base_folder = os.path.join(video_path, 'data-' + task + '/')
auxiliary_functions.attempt_to_make_folder(base_folder)
videos = auxiliary_functions.get_video_list(filename, video_path,
video_type)
for vindex, video in enumerate(videos):
print("Loading ", video, '# ', str(vindex + 1), ' of ',
str(len(videos)))
clip = VideoFileClip(os.path.join(video_path, video))
# print("Duration of video [s], ", clip.duration, "fps, ", clip.fps,
# "Cropped frame dimensions: ", clip.size)
# Create folder with experiment name and extract random frames
folder = 'selected'
v_name = video.split('.')[0]
auxiliary_functions.attempt_to_make_folder(
os.path.join(base_folder, folder))
index_length = int(np.ceil(np.log10(clip.duration * clip.fps)))
# extract first frame (uncropped) - useful for data augmentation
# index = 0
# image = img_as_ubyte(clip.get_frame(index * 1. / clip.fps))
# io.imsave(os.path.join(base_folder, folder, 'img' + v_name + '-'
# + str(index).zfill(index_length) + '.png'), image)
if cropping is True:
clip = clip.crop(y1=y1, y2=y2, x1=x1, x2=x2)
# print("Extracting frames")
if selection_algorithm == 'uniform':
frames_to_pick = frame_selection_tools.uniform_frames(
clip, num_frames_to_pick, start, stop)
elif selection_algorithm == 'kmeans':
frames_to_pick = frame_selection_tools.k_means_based_frame_selection(
clip, num_frames_to_pick, start, stop)
else:
print('not implemented')
frames_to_pick = []
index_length = int(np.ceil(np.log10(clip.duration * clip.fps)))
for index in frames_to_pick:
try:
image = img_as_ubyte(clip.get_frame(index * 1. / clip.fps))
io.imsave(
os.path.join(
base_folder, folder, 'img' + v_name + '-' +
str(index).zfill(index_length) + '.png'), image)
except FileExistsError:
print('Frame # ', index, ' does not exist.')
clip.close()
def evaluate_model():
for shuffle_index, shuffle in enumerate(shuffles):
for train_fraction_index, train_fraction in enumerate(training_fraction):
experiment_name = task + date + '-trainset' + str(
int(train_fraction * 100)) + 'shuffle' + str(shuffle)
cfg = load_config(os.path.join(base_folder, experiment_name, 'test/pose_cfg.yaml'))
# get available snapshots
snapshots = np.array([
fn.split('.')[0] for fn in os.listdir(
os.path.join(
base_folder, 'trained-results'
)
) if 'index' in fn
])
# just pick most trained snapshot
increasing_indices = np.argsort([int(m.split('-')[1]) for m in snapshots])
snapshots = snapshots[increasing_indices]
cfg['init_weights'] = os.path.join(base_folder, 'trained-results', snapshots[0])
training_iterations = (cfg['init_weights'].split('/')[-1]).split('-')[-1]
# load metadata for training / test files and labels
dataset_folder = 'unaugmented-data-set-' + task + date
doc_file = '/documentation_data-' + task + str(int(train_fraction * 100)) + \
'shuffle' + str(int(shuffle)) + '.pickle'
with open(os.path.join(base_folder, dataset_folder + doc_file), 'rb') as f:
data_doc, train_indices, test_indices, __ignore__ = pickle.load(f)
data = pd.read_hdf(os.path.join(base_folder, dataset_folder, 'data-' + task,
'collected-data-' + scorer + '.h5'), 'df_with_missing')
# Load and setup CNN part detector and configuration
sess, inputs, outputs = predict.setup_pose_prediction(cfg)
num_images = len(data.index)
predict_data = np.zeros((num_images, 3 * len(cfg['all_joints_names'])))
test_set = np.zeros(num_images)
# compute predictions over images
for image_index, image_name in tqdm(enumerate(data.index)):
# get each image
image = io.imread(os.path.join(base_folder, dataset_folder, image_name))
image = skimage.color.gray2rgb(image)
image_batch = data_to_input(image)
# run into CNN
outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})
scmap, locref = predict.extract_cnn_output(outputs_np, cfg)
# extract maximum scoring location from the heatmap
pose = predict.argmax_pose_predict(scmap, locref, cfg.stride)
predict_data[image_index, :] = pose.flatten()
dlc_scorer = 'deep-cut-' + str(cfg['net_type']) + '-' + \
str(int(train_fraction * 100)) + 'shuffle' + str(shuffle) + \
'-' + str(training_iterations) + '-for-task-' + task
index = pd.MultiIndex.from_product(
[[dlc_scorer], cfg['all_joints_names'], ['x', 'y', 'likelihood']],
names=['scorer', 'bodyparts', 'coords']
)
# save results
auxiliary_functions.attempt_to_make_folder(os.path.join(base_folder, 'trained-results/evaluation/'))
data_machine = pd.DataFrame(
predict_data,
columns=index,
index=data.index.values
)
data_machine.to_hdf(os.path.join(base_folder,
'trained-results/evaluation',
dlc_scorer + '.h5'),
'df_with_missing', format='table', mode='w')
# data_machine.to_csv(os.path.join(base_folder,
# 'trained-results/evaluation',
# dlc_scorer + '.csv'))
print(data_machine)
def generate_training_file():
generate_base_config(base_folder)
folder = os.path.join(base_folder, 'data-' + task)
data = pd.read_hdf(folder + '/collected-data-' + scorer + '.h5',
'df_with_missing')[scorer]
bf = os.path.join(base_folder, 'unaugmented-data-set-' + task + date + '/', 'data-' + task)
shutil.copytree(folder, bf)
for shuffle in shuffles:
for train_fraction in training_fraction:
train_indexes, test_indexes = split_trials(
range(len(data.index)), train_fraction
)
filename_matfile = task + '_' + scorer + str(int(train_fraction * 100)) \
+ 'shuffle' + str(shuffle)
fn = os.path.join(base_folder, 'unaugmented-data-set-' + task + date +
'/documentation_' + 'data-' + task + str(int(train_fraction * 100))
+ 'shuffle' + str(shuffle))
# make matlab train file
dat = []
for jj in train_indexes:
H = {}
# load image to get dimensions
filename = data.index[jj]
im = io.imread(os.path.join(base_folder, filename))
H['image'] = os.path.join('../../unaugmented-data-set-' + task + date + '/',
filename)
if np.ndim(im) > 2:
H['size'] = np.array(
[np.shape(im)[2],
np.shape(im)[0],
np.shape(im)[1]]
)
else:
H['size'] = np.array([1, np.shape(im)[0], np.shape(im)[1]])
index_joints = 0
joints = np.zeros((len(body_parts), 3)) * np.nan
for bp_index, body_part in enumerate(body_parts):
if data[body_part]['x'][jj] < np.shape(im)[1] and data[body_part]['y'][jj] < np.shape(im)[0]:
joints[index_joints, 0] = int(bp_index)
joints[index_joints, 1] = data[body_part]['x'][jj]
joints[index_joints, 2] = data[body_part]['y'][jj]
index_joints += 1
joints = joints[np.where(np.prod(np.isfinite(joints), 1))[0], :]
assert (np.prod(np.array(joints[:, 2]) < np.shape(im)[0]))
assert (np.prod(np.array(joints[:, 1]) < np.shape(im)[1]))
H['joints'] = np.array(joints, dtype=int)
if np.size(joints) > 0:
dat.append(H)
with open(fn + '.pickle', 'wb') as f:
pickle.dump([data, train_indexes, test_indexes, train_fraction], f,
pickle.HIGHEST_PROTOCOL)
DTYPE = [('image', 'O'), ('size', 'O'), ('joints', 'O')]
matlab_data = np.array(
[(np.array([dat[item]['image']], dtype='U'),
np.array([dat[item]['size']]),
box_it_into_a_cell(dat[item]['joints']))
for item in range(len(dat))],
dtype=DTYPE)
sio.savemat(base_folder + '/unaugmented-data-set-' + task + date + '/'
+ filename_matfile + '.mat', {'dataset': matlab_data})
# Creating file structure for training and test files + pose_yaml
experiment_name = os.path.join(base_folder, task + date + '-trainset' + str(
int(train_fraction * 100)) + 'shuffle' + str(shuffle))
auxiliary_functions.attempt_to_make_folder(experiment_name)
auxiliary_functions.attempt_to_make_folder(experiment_name + '/train')
auxiliary_functions.attempt_to_make_folder(experiment_name + '/test')
items_to_change = {
'dataset': '../../' + 'unaugmented-data-set-' + task + date + '/' + filename_matfile + '.mat',
'num_joints': len(body_parts),
'all_joints': [[i] for i in range(len(body_parts))],
'all_joints_names': body_parts
}
training_data = make_train_pose_yaml(
items_to_change,
experiment_name + '/train/' + 'pose_cfg.yaml',
filename=os.path.join(base_folder, 'pose_cfg.yaml')
)
keys_to_save = [
'dataset', 'num_joints', 'all_joints', 'all_joints_names',
'net_type', 'init_weights', 'global_scale', 'location_refinement',
'locref_stdev'
]
make_test_pose_yaml(training_data, keys_to_save,
experiment_name + '/test/' + 'pose_cfg.yaml')
def check_labels():
color_scheme = get_cmap(len(body_parts))
num_joints = len(body_parts)
all_joints = map(lambda j: [j], range(num_joints))
all_joints_names = body_parts
num_body_parts = len(body_parts)
data_combined = None
data_combined = pd.read_hdf(
os.path.join(base_folder, 'data-' + task,
'collected-data-' + scorer + '.h5'), 'df_with_missing')
folder = os.path.join(base_folder, 'data-' + task + '/', 'selected')
print(folder)
tmp_folder = folder + '-labeled'
auxiliary_functions.attempt_to_make_folder(tmp_folder)
files = [
fn for fn in os.listdir(folder)
if (image_type in fn and '_labeled' not in fn)
]
files.sort(key=lambda f: int(''.join(filter(str.isdigit, f))))
comparison_body_parts = body_parts
for index, image_name in enumerate(files):
image = io.imread(os.path.join(folder, image_name))
plt.axis('off')
if np.ndim(image) == 2:
h, w = np.shape(image)
else:
h, w, cn = np.shape(image)
plt.figure(frameon=False,
figsize=(w * 1. / 100 * scale, h * 1. / 100 * scale))
plt.subplots_adjust(left=0,
bottom=0,
right=1,
top=1,
wspace=0,
hspace=0)
im_index = np.where(
np.array(data_combined.index.values) == 'data-' + task +
'/selected/' + image_name)[0]
plt.imshow(image, 'bone')
if index == 0:
print('Creating images with labels by', scorer)
for c, bp in enumerate(comparison_body_parts):
plt.plot(data_combined[scorer][bp]['x'].values[im_index],
data_combined[scorer][bp]['y'].values[im_index],
Labels[0],
color=color_scheme(c),
alpha=alpha_value,
ms=m_size)
plt.xlim(0, w)
plt.ylim(0, h)
plt.axis('off')
plt.subplots_adjust(left=0,
bottom=0,
right=1,
top=1,
wspace=0,
hspace=0)
plt.gca().invert_yaxis()
plt.savefig(tmp_folder + '/' + image_name)
plt.close('all')
def analyse_results():
dataset_folder = 'unaugmented-data-set-' + task + date
data = pd.read_hdf(os.path.join(base_folder, dataset_folder, 'data-' + task,
'collected-data-' + scorer + '.h5'), 'df_with_missing')
comparison_body_parts = list(np.unique(data.columns.get_level_values(1)))
colors = au.get_cmap(len(comparison_body_parts))
for train_fraction in training_fraction:
for shuffle in shuffles:
fns = [file for file in os.listdir(os.path.join(base_folder, 'trained-results/evaluation'))
if 'for-task-' + task in file and 'shuffle' + str(shuffle) in file
and str(int(train_fraction * 100)) in file]
doc_file = 'documentation_data-' + task + str(int(train_fraction * 100)) + \
'shuffle' + str(int(shuffle)) + '.pickle'
doc_file = os.path.join(base_folder, dataset_folder, doc_file)
with open(doc_file, 'rb') as f:
[training_data_details,
train_indices,
test_indices,
test_fraction_data] = pickle.load(f)
training_iterations = [(int(fns[j].split('-for-task-')[0].split('-')[-1]), j) for j in range(len(fns))]
training_iterations.sort(key=lambda tup: tup[0])
training_iterations = training_iterations
for training_iteration, index in training_iterations:
data_machine = pd.read_hdf(os.path.join(base_folder, 'trained-results/evaluation',
fns[index]),
'df_with_missing')
data_combined = pd.concat([data.T, data_machine.T], axis=0).T
scorer_machine = data_machine.columns.get_level_values(0)[0]
mse, mse_pcutoff = au.pairwise_distances(data_combined, scorer, scorer_machine,
pcutoff, comparison_body_parts)
test_error = np.nanmean(mse.iloc[test_indices].values.flatten())
train_error = np.nanmean(mse.iloc[train_indices].values.flatten())
test_error_pcutoff = np.nanmean(mse_pcutoff.iloc[test_indices].values.flatten())
train_error_pcutoff = np.nanmean(mse_pcutoff.iloc[train_indices].values.flatten())
print('train error: ', np.round(train_error, 2), 'pixels --- test error: ', np.round(test_error, 2),
'pixels')
print('train error with cutoff: ', np.round(train_error_pcutoff, 2),
'pixels --- test error with cutoff: ', np.round(test_error_pcutoff, 2), 'pixels')
# plotting
folder_name = os.path.join(base_folder, 'trained-results/evaluation/labeled')
auxiliary_functions.attempt_to_make_folder(folder_name)
num_frames = np.size(data_combined.index)
for ind in np.arange(num_frames):
fn = data_combined.index[ind]
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
au.make_labeled_image(data_combined,
ind,
os.path.join(base_folder, dataset_folder),
[scorer, scorer_machine],
comparison_body_parts,
colors,
pcutoff=pcutoff)
if ind in train_indices:
plt.savefig(os.path.join(folder_name, 'train-image' + str(ind) + '-'
+ fn.split('/')[0] + '-' + fn.split('/')[1]))
else:
plt.savefig(os.path.join(folder_name, 'test-image' + str(ind) + '-'
+ fn.split('/')[0] + '-' + fn.split('/')[1]))
plt.close('all')