def PWC_full(B,
lvl_h,
lvl_w,
pwc_h,
pwc_w,
lvl,
frameNum=5,
blur=False):
ratio_h = float(lvl_h) / float(pwc_h)
ratio_w = float(lvl_w) / float(pwc_w)
nn = ModelPWCNet(mode='test', options=nn_opts)
nn.print_config()
for i in range(frameNum):
B[i] = tf.image.resize_bilinear(B[i], (pwc_h, pwc_w),
align_corners=True)
tmp_list = []
for i in range(frameNum):
for j in range(frameNum):
tmp_list.append(tf.stack([B[i], B[j]], 1))
PWC_input = tf.concat(tmp_list, 0) # [batch_size*20, 2, H, W, 3]
PWC_input = tf.reshape(
PWC_input,
[FLAGS.batch_size * (frameNum * frameNum), 2, pwc_h, pwc_w, 3])
pred_labels, _ = nn.nn(PWC_input, reuse=tf.AUTO_REUSE)
print(pred_labels)
pred_labels = tf.image.resize_bilinear(pred_labels, (lvl_h, lvl_w),
align_corners=True)
"""
0: W
1: H
"""
ratio_tensor = tf.expand_dims(
tf.expand_dims(
tf.expand_dims(
tf.convert_to_tensor(np.asarray([ratio_w, ratio_h]),
dtype=tf.float32), 0), 0), 0)
FB = []
counter = 0
for i in range(frameNum):
FB_tmp = []
for j in range(frameNum):
FB_tmp.append(
tf.stop_gradient(pred_labels[FLAGS.batch_size *
counter:FLAGS.batch_size *
(counter + 1)] *
ratio_tensor))
counter += 1
FB.append(FB_tmp)
return FB
def load_optical_flow_model():
# Here, we're using a GPU (use '/device:CPU:0' to run inference on the CPU)
gpu_devices = ['/device:GPU:0']
controller = '/device:GPU:0'
# Set the path to the trained model
ckpt_path = ROOT_DIR + '/models/pwcnet-lg-6-2-multisteps-chairsthingsmix/pwcnet.ckpt-595000'
# Configure the model for inference, starting with the default options
nn_opts = deepcopy(_DEFAULT_PWCNET_TEST_OPTIONS)
nn_opts['verbose'] = True
nn_opts['ckpt_path'] = ckpt_path
nn_opts['batch_size'] = 1
nn_opts['gpu_devices'] = gpu_devices
nn_opts['controller'] = controller
# We're running the PWC-Net-large model in quarter-resolution mode
# That is, with a 6 level pyramid, and upsampling of level 2 by 4 in each dimension as the final flow prediction
nn_opts['use_dense_cx'] = True
nn_opts['use_res_cx'] = True
nn_opts['pyr_lvls'] = 6
nn_opts['flow_pred_lvl'] = 2
# The size of the images in this dataset are not multiples of 64, while the model generates flows padded to multiples
# of 64. Hence, we need to crop the predicted flows to their original size
nn_opts['adapt_info'] = (1, 436, 1024, 2)
# Instantiate the model in inference mode and display the model configuration
nn = ModelPWCNet(mode='test', options=nn_opts)
#Return model
return nn
def build_pwc_net_model(self, H, W):
'''
Parameters
----------
H, W : int
image mering height/width.
Returns
-------
None.
'''
gpu_devices = ['/device:CPU:0']
controller = '/device:CPU:0'
os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tfoptflow/', 'tfoptflow/')
ckpt_path = self.pwc_net_weight_path
# Configure the model for inference, starting with the default options
nn_opts = deepcopy(_DEFAULT_PWCNET_TEST_OPTIONS)
nn_opts['verbose'] = True
nn_opts['ckpt_path'] = ckpt_path
nn_opts['batch_size'] = 1
nn_opts['gpu_devices'] = gpu_devices
nn_opts['controller'] = controller
# We're running the PWC-Net-large model in quarter-resolution mode
# That is, with a 6 level pyramid, and upsampling of level 2 by 4 in
# each dimension as the final flow prediction
nn_opts['use_dense_cx'] = True
nn_opts['use_res_cx'] = True
nn_opts['pyr_lvls'] = 6
nn_opts['flow_pred_lvl'] = 2
# The size of the images in this dataset are not multiples of 64, while the model generates flows padded to multiples
# of 64. Hence, we need to crop the predicted flows to their original
# size
nn_opts['adapt_info'] = (1, H, W, 2)
# Instantiate the model in inference mode and display the model
# configuration
nn = ModelPWCNet(mode='test', options=nn_opts)
nn.print_config()
self.pwc_net_model = nn
def _main():
# Set controller device and devices
# A one-gpu setup would be something like controller='/device:GPU:0' and gpu_devices=['/device:GPU:0']
gpu_devices = ['/device:GPU:0']
controller = '/device:CPU:0'
# Useful settings
batch_size = 4
img_size = (448, 256)
"""
# Dataset options
ds_opts = {
'batch_size': batch_size,
'verbose': True,
'in_memory': False, # True loads all samples upfront, False loads them on-demand
'crop_preproc': None, # None or (h, w), use (384, 768) for FlyingThings3D
'scale_preproc': img_size, # None or (h, w),
'type': 'noc', # ['clean' | 'final'] for MPISintel, ['noc' | 'occ'] for KITTI, 'into_future' for FlyingThings3D
'tb_test_imgs': False, # If True, make test images available to model in training mode
# Sampling and split options
'random_seed': 1337, # random seed used for sampling
'val_split': 0.01, # portion of data reserved for the validation split
# Augmentation options
'aug_type': 'heavy', # in [None, 'basic', 'heavy'] to add augmented data to training set
'aug_labels': True, # If True, augment both images and labels; otherwise, only augment images
'fliplr': 0.5, # Horizontally flip 50% of images
'flipud': 0.0, # Vertically flip 50% of images
'translate': (0., 0.), # Translate 50% of images by a value between -5 and +5 percent of original size on x- and y-axis independently
'scale': (0., 0.), # Scale 50% of images by a factor between 95 and 105 percent of original size
}
# Load train dataset
ds_1 = KITTIDataset(mode='train_with_val', ds_root=data_path + kitti12, options=ds_opts)
ds_2 = KITTIDataset(mode='train_with_val', ds_root=data_path + kitti15, options=ds_opts)
ds = MixedDataset(mode='train_with_val', datasets=[ds_1, ds_2], options=ds_opts)
"""
# Dataset options
ds_opts = {
'batch_size': batch_size,
'verbose': True,
'in_memory':
False, # True loads all samples upfront, False loads them on-demand
'crop_preproc':
None, # None or (h, w), use (384, 768) for FlyingThings3D
'scale_preproc': img_size, # None or (h, w),
# ['clean' | 'final'] for MPISintel, ['noc' | 'occ'] for KITTI, 'into_future' for FlyingThings3D
'type': 'into_future',
'tb_test_imgs':
False, # If True, make test images available to model in training mode
# Sampling and split options
'random_seed': 1337, # random seed used for sampling
'val_split': 0.01, # portion of data reserved for the validation split
# Augmentation options
'aug_type':
'heavy', # in [None, 'basic', 'heavy'] to add augmented data to training set
'aug_labels':
True, # If True, augment both images and labels; otherwise, only augment images
'fliplr': 0.5, # Horizontally flip 50% of images
'flipud': 0.0, # Vertically flip 50% of images
# Translate 50% of images by a value between -5 and +5 percent of original size on x- and y-axis independently
'translate': (0, 0),
'scale': (
0, 0
), # Scale 50% of images by a factor between 95 and 105 percent of original size
'train_imgs': _TRAIN_IMGS,
'train_flow': _TRAIN_FLOW,
'test_imgs': _TEST_IMGS,
}
ds = FlowDroNetDataset(mode='train_with_val',
ds_root=_DATASET_ROOT,
options=ds_opts)
# Display dataset configuration
ds.print_config()
"""
# Fine-Tuning options (Large)
nn_opts = {
'verbose': True,
'ckpt_path': ckpt_path, # original checkpoint to finetune
'ckpt_dir': save_path, # where finetuning checkpoints are stored
'max_to_keep': 10,
'x_dtype': tf.float32, # image pairs input type
'x_shape': [2, img_size[1], img_size[0], 3], # image pairs input shape [2, H, W, 3]
'y_dtype': tf.float32, # u,v flows output type
'y_shape': [img_size[1], img_size[0], 2], # u,v flows output shape [H, W, 2]
'train_mode': 'fine-tune', # in ['train', 'fine-tune']
'adapt_info': None, # if predicted flows are padded by the model, crop them back by to this size
'sparse_gt_flow': False, # if gt flows are sparse (KITTI), only compute average EPE where gt flows aren't (0., 0.)
# Logging/Snapshot params
'display_step': 10, # show progress every 100 training batches
'snapshot_step': 500, # save trained model every 1000 training batches
'val_step': 1000, # Test trained model on validation split every 1000 training batches
'val_batch_size': 100, # Use -1 to use entire validation split, or set number of val samples (0 disables it)
'tb_val_imgs': 'top_flow', # None, 'top_flow', or 'pyramid'; runs model on batch_size val images, log results
'tb_test_imgs': None, # None, 'top_flow', or 'pyramid'; runs trained model on batch_size test images, log results
# Multi-GPU config
# list devices on which to run the model's train ops (can be more than one GPU)
'gpu_devices': gpu_devices,
# controller device to put the model's variables on (usually, /cpu:0 or /gpu:0 -> try both!)
'controller': controller,
# Training config and hyper-params
'use_tf_data': False, # Set to True to get data from tf.data.Dataset; otherwise, use feed_dict with numpy
'use_mixed_precision': False, # Set to True to use mixed precision training (fp16 inputs)
'loss_scaler': 128., # Loss scaler (only used in mixed precision training)
'batch_size': batch_size * len(gpu_devices),
'lr_policy': 'multisteps', # choose between None, 'multisteps', and 'cyclic'; adjust the max_steps below too
# Multistep lr schedule
'init_lr': 1e-05, # initial learning rate
'max_steps': 25000, # max number of training iterations (i.e., batches to run)
'lr_boundaries': [5000, 10000, 15000, 20000], # step schedule boundaries
'lr_values': [1e-05, 5e-06, 2.5e-06, 1.25e-06, 6.25e-07], # step schedule values
# Cyclic lr schedule
'cyclic_lr_max': 2e-05, # maximum bound
'cyclic_lr_base': 1e-06, # min bound
'cyclic_lr_stepsize': 20000 * 8 / ds_opts['batch_size'], # step schedule values
# 'max_steps': 200000, # max number of training iterations
# Loss functions hyper-params
'loss_fn': 'loss_multiscale', # 'loss_robust' doesn't really work; the loss goes down but the EPE doesn't
'alphas': [0.32, 0.08, 0.02, 0.01, 0.005], # See 'Implementation details" on page 5 of ref PDF
'gamma': 0.0004, # See 'Implementation details" on page 5 of ref PDF
'q': 0.4, # See 'Implementation details" on page 5 of ref PDF
'epsilon': 0.01, # See 'Implementation details" on page 5 of ref PDF
# Model hyper-params
'pyr_lvls': 6, # number of feature levels in the flow pyramid
'flow_pred_lvl': 2, # which level to upsample to generate the final optical flow prediction
'search_range': 4, # cost volume search range
# if True, use model with dense connections (4705064 params w/o, 9374274 params with (no residual conn.))
'use_dense_cx': True,
# if True, use model with residual connections (4705064 params w/o, 6774064 params with (+2069000) (no dense conn.))
'use_res_cx': True,
}
"""
# Fine-Tuning options (Small)
nn_opts = {
'verbose': True,
'ckpt_path': ckpt_path, # original checkpoint to finetune
'ckpt_dir': save_path, # where finetuning checkpoints are stored
'max_to_keep': 10,
'x_dtype': tf.float32, # image pairs input type
'x_shape': [2, img_size[1], img_size[0],
3], # image pairs input shape [2, H, W, 3]
'y_dtype': tf.float32, # u,v flows output type
'y_shape': [img_size[1], img_size[0],
2], # u,v flows output shape [H, W, 2]
'train_mode': 'fine-tune', # in ['train', 'fine-tune']
'adapt_info':
None, # if predicted flows are padded by the model, crop them back by to this size
'sparse_gt_flow':
False, # if gt flows are sparse (KITTI), only compute average EPE where gt flows aren't (0., 0.)
# Logging/Snapshot params
'display_step': 100, # show progress every 100 training batches
'snapshot_step': 500, # save trained model every 1000 training batches
'val_step':
500, # Test trained model on validation split every 1000 training batches
'val_batch_size':
20, # Use -1 to use entire validation split, or set number of val samples (0 disables it)
'tb_val_imgs':
'top_flow', # None, 'top_flow', or 'pyramid'; runs model on batch_size val images, log results
'tb_test_imgs':
None, # None, 'top_flow', or 'pyramid'; runs trained model on batch_size test images, log results
# Multi-GPU config
# list devices on which to run the model's train ops (can be more than one GPU)
'gpu_devices': gpu_devices,
# controller device to put the model's variables on (usually, /cpu:0 or /gpu:0 -> try both!)
'controller': controller,
# Training config and hyper-params
'use_tf_data':
False, # Set to True to get data from tf.data.Dataset; otherwise, use feed_dict with numpy
'use_mixed_precision':
False, # Set to True to use mixed precision training (fp16 inputs)
'loss_scaler':
128., # Loss scaler (only used in mixed precision training)
'batch_size': batch_size * len(gpu_devices),
'lr_policy':
'multisteps', # choose between None, 'multisteps', and 'cyclic'; adjust the max_steps below too
# Multistep lr schedule
'init_lr': 1e-05, # initial learning rate
'max_steps':
25000, # max number of training iterations (i.e., batches to run)
'lr_boundaries': [5000, 10000, 15000,
20000], # step schedule boundaries
'lr_values': [1e-05, 5e-06, 2.5e-06, 1.25e-06,
6.25e-07], # step schedule values
# Cyclic lr schedule
'cyclic_lr_max': 2e-05, # maximum bound
'cyclic_lr_base': 1e-06, # min bound
'cyclic_lr_stepsize':
20000 * 8 / ds_opts['batch_size'], # step schedule values
# 'max_steps': 200000, # max number of training iterations
# Loss functions hyper-params
'loss_fn':
'loss_multiscale', # 'loss_robust' doesn't really work; the loss goes down but the EPE doesn't
'alphas': [0.32, 0.08, 0.02, 0.01,
0.005], # See 'Implementation details" on page 5 of ref PDF
'gamma': 0.0004, # See 'Implementation details" on page 5 of ref PDF
'q': 0.4, # See 'Implementation details" on page 5 of ref PDF
'epsilon': 0.01, # See 'Implementation details" on page 5 of ref PDF
# Model hyper-params
'pyr_lvls': 6, # number of feature levels in the flow pyramid
'flow_pred_lvl':
2, # which level to upsample to generate the final optical flow prediction
'search_range': 4, # cost volume search range
# if True, use model with dense connections (4705064 params w/o, 9374274 params with (no residual conn.))
'use_dense_cx': False,
# if True, use model with residual connections (4705064 params w/o, 6774064 params with (+2069000) (no dense conn.))
'use_res_cx': False,
}
# Instantiate the model and display the model configuration
nn = ModelPWCNet(mode='train_with_val', options=nn_opts, dataset=ds)
nn.print_config()
# Train the model
nn.train()
image_path2 = f'./samples/mpisintel_test_clean_ambush_1_frame_00{pair+1:02d}.png'
image1, image2 = imread(image_path1), imread(image_path2)
img_pairs.append((image1, image2))
# Configure the model for inference, starting with the default options
nn_opts = deepcopy(_DEFAULT_PWCNET_TEST_OPTIONS)
nn_opts['verbose'] = True
nn_opts['ckpt_path'] = ckpt_path
nn_opts['batch_size'] = 1
nn_opts['gpu_devices'] = gpu_devices
nn_opts['controller'] = controller
# We're running the PWC-Net-large model in quarter-resolution mode
# That is, with a 6 level pyramid, and upsampling of level 2 by 4 in each dimension as the final flow prediction
nn_opts['use_dense_cx'] = True
nn_opts['use_res_cx'] = True
nn_opts['pyr_lvls'] = 6
nn_opts['flow_pred_lvl'] = 2
# The size of the images in this dataset are not multiples of 64, while the model generates flows padded to multiples
# of 64. Hence, we need to crop the predicted flows to their original size
nn_opts['adapt_info'] = (1, 436, 1024, 2)
# Instantiate the model in inference mode and display the model configuration
nn = ModelPWCNet(mode='test', options=nn_opts)
nn.print_config()
# Generate the predictions and display them
pred_labels = nn.predict_from_img_pairs(img_pairs, batch_size=1, verbose=False)
display_img_pairs_w_flows(img_pairs, pred_labels)
nn_opts['cyclic_lr_base'] = 1e-05
nn_opts['cyclic_lr_stepsize'] = 20000
nn_opts['max_steps'] = 200000
# Below,we adjust the schedule to the size of the batch and our number of GPUs (2).
nn_opts['cyclic_lr_stepsize'] /= len(gpu_devices)
nn_opts['max_steps'] /= len(gpu_devices)
nn_opts['cyclic_lr_stepsize'] = int(nn_opts['cyclic_lr_stepsize'] /
(float(ds_opts['batch_size']) / 8))
nn_opts['max_steps'] = int(nn_opts['max_steps'] /
(float(ds_opts['batch_size']) / 8))
# In[7]:
# Instantiate the model and display the model configuration
nn = ModelPWCNet(mode='train_with_val', options=nn_opts, dataset=ds)
nn.print_config()
# ## Train the model
# In[8]:
# Train the model
nn.train()
# ## Training log
# Here are the training curves for the run above:
#
# 
# 
nn_opts['gpu_devices'] = gpu_devices
nn_opts['controller'] = controller
# We're running the PWC-Net-large model in quarter-resolution mode
# That is, with a 6 level pyramid, and upsampling of level 2 by 4 in each dimension as the final flow prediction
nn_opts['use_dense_cx'] = True
nn_opts['use_res_cx'] = True
nn_opts['pyr_lvls'] = 6
nn_opts['flow_pred_lvl'] = 2
# The size of the images in this dataset are not multiples of 64, while the model generates flows padded to multiples
# of 64. Hence, we need to crop the predicted flows to their original size
nn_opts['adapt_info'] = (1, 96*2, 96*2, 2) # check, for better prediction, we multiply x2 (in larger spatial resolution)
# Instantiate the model in inference mode and display the model configuration
nn = ModelPWCNet(mode='test', options=nn_opts)
# Read data from mat file
data_path = 'E:/FISR_Github/data/train/LR_LFR/LR_Surfing_SlamDunk_5seq.mat' # check, the path where your .mat file located.
data = read_mat_file(data_path, 'LR_data')
sz = data.shape # check, in our case [N, 5, 96, 96, 3]
img_pairs = []
scale = 2 # check, upscaling factor for spatial resolution
ss = 1 # check, temporal stride (1 or 2)
pred = np.zeros((sz[0], 8//ss, sz[2], sz[3], 2), dtype=np.float32) # check, in our case
for num in range(sz[0]):
for seq in range(sz[1]-(ss*2-1)):
rgb_1 = YUV2RGB_matlab(data[num, ss*seq, :, :, :]) # check, since PWC-Net works on RGB images, we have to convert our YUV dataset.
rgb_2 = YUV2RGB_matlab(data[num, ss*(seq+1), :, :, :])
nn_opts['ckpt_path'] = ckpt_path
nn_opts['batch_size'] = batch_size
nn_opts['use_tf_data'] = True
nn_opts['gpu_devices'] = gpu_devices
nn_opts['controller'] = controller
# We're running the PWC-Net-large model in quarter-resolution mode
# That is, with a 6 level pyramid, and upsampling of level 2 by 4 in each dimension as the final flow prediction
nn_opts['use_dense_cx'] = True
nn_opts['use_res_cx'] = True
nn_opts['pyr_lvls'] = 6
nn_opts['flow_pred_lvl'] = 2
# The size of the images in this dataset are not multiples of 64, while the model generates flows padded to multiples
# of 64. Hence, we need to crop the predicted flows to their original size
nn_opts['adapt_info'] = (1, 436, 1024, 2)
# Instantiate the model in inference mode and display the model configuration
nn = ModelPWCNet(mode='test', options=nn_opts, dataset=ds)
nn.print_config()
# Generate the predictions and save them directly to disk; no need to return them here
nn.predict(return_preds=False, save_preds=True)
# Display a few random samples and their predictions
img_pairs, pred_labels, ids = ds.get_samples(num_samples,
split='test_with_preds',
deterministic=False,
as_tuple=True)
display_img_pairs_w_flows(img_pairs, pred_labels, titles=ids)
def PWC_full(F0, F1, F2, F3, F4, B0, B1, B2, B3, B4, lvl_h, lvl_w, pwc_h, pwc_w):
ratio_h = float(lvl_h) / float(pwc_h)
ratio_w = float(lvl_w) / float(pwc_w)
nn = ModelPWCNet(mode='test', options=nn_opts)
nn.print_config()
F0 = tf.image.resize_bilinear(F0, (pwc_h, pwc_w))
F1 = tf.image.resize_bilinear(F1, (pwc_h, pwc_w))
F2 = tf.image.resize_bilinear(F2, (pwc_h, pwc_w))
F3 = tf.image.resize_bilinear(F3, (pwc_h, pwc_w))
F4 = tf.image.resize_bilinear(F4, (pwc_h, pwc_w))
B0 = tf.image.resize_bilinear(B0, (pwc_h, pwc_w))
B1 = tf.image.resize_bilinear(B1, (pwc_h, pwc_w))
B2 = tf.image.resize_bilinear(B2, (pwc_h, pwc_w))
B3 = tf.image.resize_bilinear(B3, (pwc_h, pwc_w))
B4 = tf.image.resize_bilinear(B4, (pwc_h, pwc_w))
tmp_list = []
tmp_list.append(tf.stack([F0, F1], 1))
tmp_list.append(tf.stack([F0, F2], 1))
tmp_list.append(tf.stack([F0, F3], 1))
tmp_list.append(tf.stack([F0, F4], 1))
tmp_list.append(tf.stack([F1, F0], 1))
tmp_list.append(tf.stack([F1, F2], 1))
tmp_list.append(tf.stack([F1, F3], 1))
tmp_list.append(tf.stack([F1, F4], 1))
tmp_list.append(tf.stack([F2, F0], 1))
tmp_list.append(tf.stack([F2, F1], 1))
tmp_list.append(tf.stack([F2, F3], 1))
tmp_list.append(tf.stack([F2, F4], 1))
tmp_list.append(tf.stack([F3, F0], 1))
tmp_list.append(tf.stack([F3, F1], 1))
tmp_list.append(tf.stack([F3, F2], 1))
tmp_list.append(tf.stack([F3, F4], 1))
tmp_list.append(tf.stack([F4, F0], 1))
tmp_list.append(tf.stack([F4, F1], 1))
tmp_list.append(tf.stack([F4, F2], 1))
tmp_list.append(tf.stack([F4, F3], 1))
tmp_list.append(tf.stack([B0, B1], 1))
tmp_list.append(tf.stack([B0, B2], 1))
tmp_list.append(tf.stack([B0, B3], 1))
tmp_list.append(tf.stack([B0, B4], 1))
tmp_list.append(tf.stack([B1, B0], 1))
tmp_list.append(tf.stack([B1, B2], 1))
tmp_list.append(tf.stack([B1, B3], 1))
tmp_list.append(tf.stack([B1, B4], 1))
tmp_list.append(tf.stack([B2, B0], 1))
tmp_list.append(tf.stack([B2, B1], 1))
tmp_list.append(tf.stack([B2, B3], 1))
tmp_list.append(tf.stack([B2, B4], 1))
tmp_list.append(tf.stack([B3, B0], 1))
tmp_list.append(tf.stack([B3, B1], 1))
tmp_list.append(tf.stack([B3, B2], 1))
tmp_list.append(tf.stack([B3, B4], 1))
tmp_list.append(tf.stack([B4, B0], 1))
tmp_list.append(tf.stack([B4, B1], 1))
tmp_list.append(tf.stack([B4, B2], 1))
tmp_list.append(tf.stack([B4, B3], 1))
PWC_input = tf.concat(tmp_list, 0) # [batch_size*20, 2, H, W, 3]
PWC_input = tf.reshape(PWC_input, [FLAGS.batch_size * 40, 2, pwc_h, pwc_w, 3])
pred_labels, _ = nn.nn(PWC_input, reuse=tf.AUTO_REUSE)
print(pred_labels)
pred_labels = tf.image.resize_bilinear(pred_labels, (lvl_h, lvl_w))
"""
0: W
1: H
"""
ratio_tensor = tf.expand_dims(tf.expand_dims(tf.expand_dims(tf.convert_to_tensor(np.asarray([ratio_w, ratio_h]), dtype=tf.float32), 0), 0), 0)
FF01 = pred_labels[FLAGS.batch_size * 0:FLAGS.batch_size * 1] * ratio_tensor
FF02 = pred_labels[FLAGS.batch_size * 1:FLAGS.batch_size * 2] * ratio_tensor
FF03 = pred_labels[FLAGS.batch_size * 2:FLAGS.batch_size * 3] * ratio_tensor
FF04 = pred_labels[FLAGS.batch_size * 3:FLAGS.batch_size * 4] * ratio_tensor
FF10 = pred_labels[FLAGS.batch_size * 4:FLAGS.batch_size * 5] * ratio_tensor
FF12 = pred_labels[FLAGS.batch_size * 5:FLAGS.batch_size * 6] * ratio_tensor
FF13 = pred_labels[FLAGS.batch_size * 6:FLAGS.batch_size * 7] * ratio_tensor
FF14 = pred_labels[FLAGS.batch_size * 7:FLAGS.batch_size * 8] * ratio_tensor
FF20 = pred_labels[FLAGS.batch_size * 8:FLAGS.batch_size * 9] * ratio_tensor
FF21 = pred_labels[FLAGS.batch_size * 9:FLAGS.batch_size * 10] * ratio_tensor
FF23 = pred_labels[FLAGS.batch_size * 10:FLAGS.batch_size * 11] * ratio_tensor
FF24 = pred_labels[FLAGS.batch_size * 11:FLAGS.batch_size * 12] * ratio_tensor
FF30 = pred_labels[FLAGS.batch_size * 12:FLAGS.batch_size * 13] * ratio_tensor
FF31 = pred_labels[FLAGS.batch_size * 13:FLAGS.batch_size * 14] * ratio_tensor
FF32 = pred_labels[FLAGS.batch_size * 14:FLAGS.batch_size * 15] * ratio_tensor
FF34 = pred_labels[FLAGS.batch_size * 15:FLAGS.batch_size * 16] * ratio_tensor
FF40 = pred_labels[FLAGS.batch_size * 16:FLAGS.batch_size * 17] * ratio_tensor
FF41 = pred_labels[FLAGS.batch_size * 17:FLAGS.batch_size * 18] * ratio_tensor
FF42 = pred_labels[FLAGS.batch_size * 18:FLAGS.batch_size * 19] * ratio_tensor
FF43 = pred_labels[FLAGS.batch_size * 19:FLAGS.batch_size * 20] * ratio_tensor
FB01 = pred_labels[FLAGS.batch_size * 20:FLAGS.batch_size * 21] * ratio_tensor
FB02 = pred_labels[FLAGS.batch_size * 21:FLAGS.batch_size * 22] * ratio_tensor
FB03 = pred_labels[FLAGS.batch_size * 22:FLAGS.batch_size * 23] * ratio_tensor
FB04 = pred_labels[FLAGS.batch_size * 23:FLAGS.batch_size * 24] * ratio_tensor
FB10 = pred_labels[FLAGS.batch_size * 24:FLAGS.batch_size * 25] * ratio_tensor
FB12 = pred_labels[FLAGS.batch_size * 25:FLAGS.batch_size * 26] * ratio_tensor
FB13 = pred_labels[FLAGS.batch_size * 26:FLAGS.batch_size * 27] * ratio_tensor
FB14 = pred_labels[FLAGS.batch_size * 27:FLAGS.batch_size * 28] * ratio_tensor
FB20 = pred_labels[FLAGS.batch_size * 28:FLAGS.batch_size * 29] * ratio_tensor
FB21 = pred_labels[FLAGS.batch_size * 29:FLAGS.batch_size * 30] * ratio_tensor
FB23 = pred_labels[FLAGS.batch_size * 30:FLAGS.batch_size * 31] * ratio_tensor
FB24 = pred_labels[FLAGS.batch_size * 31:FLAGS.batch_size * 32] * ratio_tensor
FB30 = pred_labels[FLAGS.batch_size * 32:FLAGS.batch_size * 33] * ratio_tensor
FB31 = pred_labels[FLAGS.batch_size * 33:FLAGS.batch_size * 34] * ratio_tensor
FB32 = pred_labels[FLAGS.batch_size * 34:FLAGS.batch_size * 35] * ratio_tensor
FB34 = pred_labels[FLAGS.batch_size * 35:FLAGS.batch_size * 36] * ratio_tensor
FB40 = pred_labels[FLAGS.batch_size * 36:FLAGS.batch_size * 37] * ratio_tensor
FB41 = pred_labels[FLAGS.batch_size * 37:FLAGS.batch_size * 38] * ratio_tensor
FB42 = pred_labels[FLAGS.batch_size * 38:FLAGS.batch_size * 39] * ratio_tensor
FB43 = pred_labels[FLAGS.batch_size * 39:FLAGS.batch_size * 40] * ratio_tensor
return FF01, FF02, FF03, FF04, \
FF10, FF12, FF13, FF14, \
FF20, FF21, FF23, FF24, \
FF30, FF31, FF32, FF34, \
FF40, FF41, FF42, FF43, \
FB01, FB02, FB03, FB04, \
FB10, FB12, FB13, FB14, \
FB20, FB21, FB23, FB24, \
FB30, FB31, FB32, FB34, \
FB40, FB41, FB42, FB43
output = tf.concat([x_0, x_1, x_2], -1)
return output[:, 40:-40, 40:-40]
down_level = 0
pwc_h = int(np.ceil(float(CROP_PATCH_H // (2 ** down_level)) / 64.0)) * 64
pwc_w = int(np.ceil(float(CROP_PATCH_W // (2 ** down_level)) / 64.0)) * 64
B0_pred_0_down = tf.image.resize_bilinear(apply_gaussian_blur_image(B0_pred_0), (pwc_h, pwc_w), align_corners=True)
B1_pred_0_down = tf.image.resize_bilinear(apply_gaussian_blur_image(B1_pred_0), (pwc_h, pwc_w), align_corners=True)
B2_pred_0_down = tf.image.resize_bilinear(apply_gaussian_blur_image(B2_pred_0), (pwc_h, pwc_w), align_corners=True)
B3_pred_0_down = tf.image.resize_bilinear(apply_gaussian_blur_image(B3_pred_0), (pwc_h, pwc_w), align_corners=True)
B4_pred_0_down = tf.image.resize_bilinear(apply_gaussian_blur_image(B4_pred_0), (pwc_h, pwc_w), align_corners=True)
ratio_h = float(CROP_PATCH_H // (2 ** down_level)) / float(pwc_h)
ratio_w = float(CROP_PATCH_W // (2 ** down_level)) / float(pwc_w)
nn = ModelPWCNet(mode='test', options=nn_opts)
nn.print_config()
tmp_list = []
tmp_list.append(tf.stack([B2_pred_0_down, B0_pred_0_down], 1))
tmp_list.append(tf.stack([B2_pred_0_down, B1_pred_0_down], 1))
tmp_list.append(tf.stack([B2_pred_0_down, B3_pred_0_down], 1))
tmp_list.append(tf.stack([B2_pred_0_down, B4_pred_0_down], 1))
PWC_input = tf.concat(tmp_list, 0) # [batch_size*20, 2, H, W, 3]
PWC_input = tf.reshape(PWC_input, [FLAGS.batch_size * 4, 2, pwc_h, pwc_w, 3])
pred_labels, _ = nn.nn(PWC_input, reuse=tf.AUTO_REUSE)
pred_labels = tf.image.resize_bilinear(pred_labels, (RESIZED_H // (2 ** down_level), RESIZED_W // (2 ** down_level)), align_corners=True)
"""
0: W
1: H
"""
nn_opts['batch_size'] = 1 nn_opts['gpu_devices'] = gpu_devices nn_opts['controller'] = controller # We're running the PWC-Net-large model in quarter-resolution mode # That is, with a 6 level pyramid, and upsampling of level 2 by 4 in each dimension as the final flow prediction nn_opts['use_dense_cx'] = True nn_opts['use_res_cx'] = True nn_opts['pyr_lvls'] = 6 nn_opts['flow_pred_lvl'] = 2 # The size of the images in this dataset are not multiples of 64, while the model generates flows padded to multiples # of 64. Hence, we need to crop the predicted flows to their original size nn_opts['adapt_info'] = (1, 436, 1024, 2) nn = ModelPWCNet(mode='test', options=nn_opts) # nn.print_config() pred_labels = nn.predict_from_img_pairs(img_pairs[0:50], batch_size=1, verbose=False) zmf_save_flows(names[0:50], pred_labels, save_addr) print(1) pred_labels = nn.predict_from_img_pairs(img_pairs[50:100], batch_size=1, verbose=False) zmf_save_flows(names[50:100], pred_labels, save_addr) print(2) pred_labels = nn.predict_from_img_pairs(img_pairs[100:150], batch_size=1, verbose=False) zmf_save_flows(names[100:150], pred_labels, save_addr) print(3) pred_labels = nn.predict_from_img_pairs(img_pairs[150:200], batch_size=1, verbose=False) zmf_save_flows(names[150:200], pred_labels, save_addr) print(4) pred_labels = nn.predict_from_img_pairs(img_pairs[200:250], batch_size=1, verbose=False)
nn_opts['gpu_devices'] = gpu_devices
nn_opts['controller'] = controller
# We're running the PWC-Net-large model in quarter-resolution mode
# That is, with a 6 level pyramid, and upsampling of level 2 by 4 in each dimension as the final flow prediction
nn_opts['use_dense_cx'] = True
nn_opts['use_res_cx'] = True
nn_opts['pyr_lvls'] = 6
nn_opts['flow_pred_lvl'] = 2
# The size of the images in this dataset are not multiples of 64, while the model generates flows padded to multiples
# of 64. Hence, we need to crop the predicted flows to their original size
nn_opts['adapt_info'] = (1, 436, 1024, 2)
# Instantiate the model in inference mode and display the model configuration
nn = ModelPWCNet(mode='test', options=nn_opts)
#MASK_RCNN SETTINGS
class InferenceConfig(coco.CocoConfig):
# Set batch size to 1 since we'll be running inference on
# one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
GPU_COUNT = 1
IMAGES_PER_GPU = 1
def main():
# Definition of the parameters of DEEP_SORT
max_cosine_distance = 0.3
nn_budget = None
nn_opts['verbose'] = True
nn_opts['ckpt_path'] = ckpt_path
nn_opts[
'batch_size'] = 1 # Setting this to 1 leads to more accurate evaluations of the processing time
nn_opts['use_tf_data'] = False # Don't use tf.data reader for this simple task
nn_opts['gpu_devices'] = gpu_devices
nn_opts['controller'] = controller # Evaluate on CPU or GPU?
# We're evaluating the PWC-Net-small model in quarter-resolution mode
# That is, with a 6 level pyramid, and upsampling of level 2 by 4 in each dimension as the final flow prediction
nn_opts['use_dense_cx'] = False
nn_opts['use_res_cx'] = False
nn_opts['pyr_lvls'] = 6
nn_opts['flow_pred_lvl'] = 2
nn = ModelPWCNet(mode='test', options=nn_opts)
nn.print_config()
import vision_params
import cv2
import os
import numpy as np
import kociemba
from draw import *
from rotate import *
from recover import *
from tools import *
from collections import Counter
def grab_colors():