def train(config):
cv2.setNumThreads(1)
session_config = create_session(config, 'train')
input_data = InputTrainData(batch_size=config.train.batch_size, input_shape=config.input_shape,
json_path=config.train.annotation_path,
fill_with_current_image_mean=config.train.fill_with_current_image_mean,
cache_type=config.train.cache_type,
classes=config.classes, num_parallel_calls=config.train.execution.transformer_parallel_calls,
prefetch_size=config.train.execution.transformer_prefetch_size)
steps_per_epoch = config.train.save_checkpoints_steps if config.train.save_checkpoints_steps \
else input_data.dataset_size // input_data.batch_size
run_config = tf.estimator.RunConfig(session_config=session_config,
keep_checkpoint_every_n_hours=config.train.keep_checkpoint_every_n_hours,
save_summary_steps=config.train.save_summary_steps,
save_checkpoints_steps=steps_per_epoch,
tf_random_seed=config.train.random_seed)
config.detector_params['steps_per_epoch'] = steps_per_epoch
config.detector_params['log_dir'] = config.model_dir
predictor = tf.estimator.Estimator(
model_fn=detection_model,
params=config.detector_params,
model_dir=config.model_dir,
config=run_config
)
predictor.train(input_fn=input_data.input_fn, steps=config.train.steps, max_steps=config.train.max_steps)
def selective_search(pil_image=None,quality='f',size=800):
# speed-up using multithreads
cv2.setUseOptimized(True);
cv2.setNumThreads(4);
# resize image to limit number of proposals and to bypass a bug in OpenCV with non-square images
w,h = pil_image.size
h_factor,w_factor=h/size,w/size
pil_image=pil_image.resize((size,size))
im = cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR)
# create Selective Search Segmentation Object using default parameters
ss = cv2.ximgproc.segmentation.createSelectiveSearchSegmentation()
# set input image on which we will run segmentation
ss.setBaseImage(im)
# Switch to fast but low recall Selective Search method
if (quality == 'f'):
ss.switchToSelectiveSearchFast()
# Switch to high recall but slow Selective Search method
elif (quality == 'q'):
ss.switchToSelectiveSearchQuality()
# run selective search segmentation on input image
rects = ss.process()
# rect is in x,y,w,h format
# convert to xmin,ymin,xmax,ymax format
rects = np.vstack((rects[:,0]*w_factor, rects[:,1]*h_factor, (rects[:,0]+rects[:,2])*w_factor, (rects[:,1]+rects[:,3])*h_factor)).transpose()
return rects
def tile_and_correct_wrapper(params):
from skimage.external.tifffile import imread
import numpy as np
import cv2
try:
cv2.setNumThreads(1)
except:
1 # 'Open CV is naturally single threaded'
from caiman.motion_correction import tile_and_correct
img_name, out_fname, idxs, shape_mov, template, strides, overlaps, max_shifts,\
add_to_movie, max_deviation_rigid, upsample_factor_grid, newoverlaps, newstrides = params
imgs = imread(img_name, key=idxs)
mc = np.zeros(imgs.shape, dtype=np.float32)
shift_info = []
for count, img in enumerate(imgs):
if count % 10 == 0:
print(count)
mc[count], total_shift, start_step, xy_grid = tile_and_correct(img, template, strides, overlaps, max_shifts, add_to_movie=add_to_movie, newoverlaps=newoverlaps, newstrides=newstrides,
upsample_factor_grid=upsample_factor_grid, upsample_factor_fft=10, show_movie=False, max_deviation_rigid=max_deviation_rigid)
shift_info.append([total_shift, start_step, xy_grid])
if out_fname is not None:
outv = np.memmap(out_fname, mode='r+', dtype=np.float32,
shape=shape_mov, order='F')
outv[:, idxs] = np.reshape(
mc.astype(np.float32), (len(imgs), -1), order='F').T
return shift_info, idxs, np.nanmean(mc, 0)
def compare_with_sample_image_multi_process(self, input_sample_dp):
manager = Manager()
result_list = manager.list()
sample_fn_list = os.listdir(input_sample_dp)
if len(sample_fn_list) != 2:
return map(dict, result_list)
sample_fn_list.sort()
sample_fp_list = [os.path.join(input_sample_dp, item) for item in sample_fn_list]
sample_dct_list = [self.convert_to_dct(item) for item in sample_fp_list]
logger.info("Comparing sample file start %s" % time.strftime("%c"))
start = time.time()
# Execution will be blocked while converting color base if without cv2's setNumThreads attribute
if hasattr(cv2, 'setNumThreads'):
logger.debug("Image comparison from multiprocessing")
cv2.setNumThreads(0)
p_list = []
for index in range(len(sample_dct_list)):
args = [self.image_list, not index, sample_dct_list[index], result_list]
p_list.append(Process(target=self.parallel_compare_image, args=args))
p_list[index].start()
for index in range(len(p_list)):
p_list[index].join()
else:
logger.debug("Image comparison from single process")
for img_index in range(self.search_range[1] - 1, self.search_range[0], -1):
image_data = self.image_list[img_index]
comparing_dct = self.convert_to_dct(image_data['image_fp'])
if self.compare_two_images(sample_dct_list[0], comparing_dct):
logger.info("Comparing sample file end %s" % time.strftime("%c"))
result_list.append(image_data)
break
for img_index in range(self.search_range[2] - 1, self.search_range[3]):
image_data = self.image_list[img_index]
comparing_dct = self.convert_to_dct(image_data['image_fp'])
if self.compare_two_images(sample_dct_list[1], comparing_dct):
logger.info("Comparing sample file end %s" % time.strftime("%c"))
result_list.append(image_data)
break
end = time.time()
elapsed = end - start
logger.debug("Elapsed Time: %s" % str(elapsed))
map_result_list = sorted(map(dict, result_list), key=lambda k: k['time_seq'])
logger.info(map_result_list)
return map_result_list
def __init__(
self,
config: ConfigWrapper,
klippy: Klippy,
light_device: PowerDevice,
logging_handler: logging.Handler = None,
):
self.enabled: bool = bool(config.camera.enabled and config.camera.host)
self._host = int(config.camera.host) if str.isdigit(
config.camera.host) else config.camera.host
self._threads: int = config.camera.threads
self._flip_vertically: bool = config.camera.flip_vertically
self._flip_horizontally: bool = config.camera.flip_horizontally
self._fourcc: str = config.camera.fourcc
self._video_duration: int = config.camera.video_duration
self._video_buffer_size: int = config.camera.video_buffer_size
self._stream_fps: int = config.camera.stream_fps
self._klippy: Klippy = klippy
# Todo: refactor into timelapse class
self._base_dir: str = config.timelapse.base_dir
self._ready_dir: str = config.timelapse.ready_dir
self._cleanup: bool = config.timelapse.cleanup
self._target_fps: int = 15
self._min_lapse_duration: int = 0
self._max_lapse_duration: int = 0
self._last_frame_duration: int = 5
self._light_need_off: bool = False
self._light_need_off_lock: threading.Lock = threading.Lock()
self.light_timeout: int = config.camera.light_timeout
self.light_device: PowerDevice = light_device
self._camera_lock: threading.Lock = threading.Lock()
self.light_lock = threading.Lock()
self.light_timer_event: threading.Event = threading.Event()
self.light_timer_event.set()
self._hw_accel: bool = False
self._img_extension: str
if config.camera.picture_quality == "low":
self._img_extension = "jpeg"
elif config.camera.picture_quality == "high":
self._img_extension = "webp"
else:
self._img_extension = config.camera.picture_quality
self._light_requests: int = 0
self._light_request_lock: threading.Lock = threading.Lock()
if self._flip_vertically and self._flip_horizontally:
self._flip = -1
elif self._flip_horizontally:
self._flip = 1
elif self._flip_vertically:
self._flip = 0
self._rotate_code: int
if config.camera.rotate == "90_cw":
self._rotate_code = cv2.ROTATE_90_CLOCKWISE
elif config.camera.rotate == "90_ccw":
self._rotate_code = cv2.ROTATE_90_COUNTERCLOCKWISE
elif config.camera.rotate == "180":
self._rotate_code = cv2.ROTATE_180
else:
self._rotate_code = -10
if logging_handler:
logger.addHandler(logging_handler)
if config.bot.debug:
logger.setLevel(logging.DEBUG)
logger.debug(cv2.getBuildInformation())
os.environ["OPENCV_VIDEOIO_DEBUG"] = "1"
# Fixme: deprecated! use T-API https://learnopencv.com/opencv-transparent-api/
if cv2.ocl.haveOpenCL():
logger.debug("OpenCL is available")
cv2.ocl.setUseOpenCL(True)
logger.debug("OpenCL in OpenCV is enabled: %s",
cv2.ocl.useOpenCL())
self._cv2_params: List = config.camera.cv2_params
cv2.setNumThreads(self._threads)
self.cam_cam = cv2.VideoCapture()
self._set_cv2_params()
def get_noise_fft(Y, noise_range=[0.25, 0.5], noise_method='logmexp', max_num_samples_fft=3072,
opencv=True):
"""Estimate the noise level for each pixel by averaging the power spectral density.
Inputs:
-------
Y: np.ndarray
Input movie data with time in the last axis
noise_range: np.ndarray [2 x 1] between 0 and 0.5
Range of frequencies compared to Nyquist rate over which the power spectrum is averaged
default: [0.25,0.5]
noise method: string
method of averaging the noise.
Choices:
'mean': Mean
'median': Median
'logmexp': Exponential of the mean of the logarithm of PSD (default)
Output:
------
sn: np.ndarray
Noise level for each pixel
"""
T = Y.shape[-1]
# Y=np.array(Y,dtype=np.float64)
if T > max_num_samples_fft:
Y = np.concatenate((Y[..., 1:max_num_samples_fft // 3 + 1],
Y[..., np.int(T // 2 - max_num_samples_fft / 3 / 2)
:np.int(T // 2 + max_num_samples_fft / 3 / 2)],
Y[..., -max_num_samples_fft // 3:]), axis=-1)
T = np.shape(Y)[-1]
# we create a map of what is the noise on the FFT space
ff = np.arange(0, 0.5 + 1. / T, 1. / T)
ind1 = ff > noise_range[0]
ind2 = ff <= noise_range[1]
ind = np.logical_and(ind1, ind2)
# we compute the mean of the noise spectral density s
if Y.ndim > 1:
if opencv:
import cv2
try:
cv2.setNumThreads(0)
except:
pass
psdx = []
for y in Y.reshape(-1, T):
dft = cv2.dft(y, flags=cv2.DFT_COMPLEX_OUTPUT).squeeze()[
:len(ind)][ind]
psdx.append(np.sum(1. / T * dft * dft, 1))
psdx = np.reshape(psdx, Y.shape[:-1] + (-1,))
else:
xdft = np.fft.rfft(Y, axis=-1)
xdft = xdft[..., ind[:xdft.shape[-1]]]
psdx = 1. / T * abs(xdft)**2
psdx *= 2
sn = mean_psd(psdx, method=noise_method)
else:
xdft = np.fliplr(np.fft.rfft(Y))
psdx = 1. / T * (xdft**2)
psdx[1:] *= 2
sn = mean_psd(psdx[ind[:psdx.shape[0]]], method=noise_method)
return sn, psdx
import argparse
import logging
import os
import shutil
import time
import timeit
import numpy as np
import cv2 as cv
cv.setNumThreads(0)
import torch
import torch.backends.cudnn as cudnn
import torch.distributed as torch_dist
import torch.nn.functional as F
from torch import nn, optim
from torch.utils import data
from torchvision.utils import save_image
from tqdm import tqdm
from config import get_cfg_defaults
from dataset.VMD import VideoMattingDataset
from models.model import FullModel_VMD
from utils.utils import OPT_DICT, STR_DICT, \
AverageMeter, create_logger, torch_barrier, reduce_tensor
def write_image(outdir, out, step, max_batch=4):
with torch.no_grad():
scaled_imgs, scaled_tris, alphas, comps, gts, fgs, bgs = out
b, s, _, h, w = scaled_imgs.shape
b = max_batch if b > max_batch else b
def transform_img(base_img,
augment_trans=np.array([0,0,0]),
augment_eulers=np.array([0,0,0]),
from_intr=eon_intrinsics,
to_intr=eon_intrinsics,
output_size=None,
pretransform=None,
top_hacks=False,
yuv=False,
alpha=1.0,
beta=0,
blur=0):
import cv2 # pylint: disable=no-name-in-module, import-error
cv2.setNumThreads(1)
if yuv:
base_img = cv2.cvtColor(base_img, cv2.COLOR_YUV2RGB_I420)
size = base_img.shape[:2]
if not output_size:
output_size = size[::-1]
cy = from_intr[1,2]
def get_M(h=1.22):
quadrangle = np.array([[0, cy + 20],
[size[1]-1, cy + 20],
[0, size[0]-1],
[size[1]-1, size[0]-1]], dtype=np.float32)
quadrangle_norm = np.hstack((normalize(quadrangle, intrinsics=from_intr), np.ones((4,1))))
quadrangle_world = np.column_stack((h*quadrangle_norm[:,0]/quadrangle_norm[:,1],
h*np.ones(4),
h/quadrangle_norm[:,1]))
rot = orient.rot_from_euler(augment_eulers)
to_extrinsics = np.hstack((rot.T, -augment_trans[:,None]))
to_KE = to_intr.dot(to_extrinsics)
warped_quadrangle_full = np.einsum('jk,ik->ij', to_KE, np.hstack((quadrangle_world, np.ones((4,1)))))
warped_quadrangle = np.column_stack((warped_quadrangle_full[:,0]/warped_quadrangle_full[:,2],
warped_quadrangle_full[:,1]/warped_quadrangle_full[:,2])).astype(np.float32)
M = cv2.getPerspectiveTransform(quadrangle, warped_quadrangle.astype(np.float32))
return M
M = get_M()
if pretransform is not None:
M = M.dot(pretransform)
augmented_rgb = cv2.warpPerspective(base_img, M, output_size, borderMode=cv2.BORDER_REPLICATE)
if top_hacks:
cyy = int(math.ceil(to_intr[1,2]))
M = get_M(1000)
if pretransform is not None:
M = M.dot(pretransform)
augmented_rgb[:cyy] = cv2.warpPerspective(base_img, M, (output_size[0], cyy), borderMode=cv2.BORDER_REPLICATE)
# brightness and contrast augment
augmented_rgb = np.clip((float(alpha)*augmented_rgb + beta), 0, 255).astype(np.uint8)
# gaussian blur
if blur > 0:
augmented_rgb = cv2.GaussianBlur(augmented_rgb,(blur*2+1,blur*2+1),cv2.BORDER_DEFAULT)
if yuv:
augmented_img = cv2.cvtColor(augmented_rgb, cv2.COLOR_RGB2YUV_I420)
else:
augmented_img = augmented_rgb
return augmented_img
import distutils.util import os import sys import pickle import resource import traceback import logging from collections import defaultdict import pynvml import numpy as np import yaml import torch from torch.autograd import Variable import torch.nn as nn import cv2 cv2.setNumThreads(0) # pytorch issue 1355: possible deadlock in dataloader import _init_paths # pylint: disable=unused-import import nn as mynn import utils.net as net_utils import utils.misc as misc_utils from core.config import cfg, cfg_from_file, cfg_from_list, assert_and_infer_cfg from datasets.roidb import combined_roidb_for_training, combined_roidb_for_training_semseg from modeling.model_builder import Generalized_RCNN from modeling.model_builder_semseg import Generalized_SEMSEG from modeling.model_builder_3DSD import Generalized_3DSD from modeling.model_builder_disp import Generalized_SEGDISP from roi_data.loader import RoiDataLoader, MinibatchSampler, collate_minibatch, collate_minibatch_semseg from utils.detectron_weight_helper import load_detectron_weight from utils.logging import log_stats from utils.timer import Timer
Use "l" to display less rects, 'm' to display more rects, "q" to quit.
'''
import sys
import cv2
if __name__ == '__main__':
# If image path and f/q is not passed as command
# line arguments, quit and display help message
if len(sys.argv) < 3:
print(__doc__)
sys.exit(1)
# speed-up using multithreads
cv2.setUseOptimized(True);
cv2.setNumThreads(4);
# read image
im = cv2.imread(sys.argv[1])
# resize image
newHeight = 800
newWidth = int(im.shape[1]*200/im.shape[0])
im = cv2.resize(im, (newWidth, newHeight))
# create Selective Search Segmentation Object using default parameters
ss = cv2.ximgproc.segmentation.createSelectiveSearchSegmentation()
# set input image on which we will run segmentation
ss.setBaseImage(im)
# Switch to fast but low recall Selective Search method
def __init__(self, config=None, split=None, purpose=None, preload=False):
super(_Coco, self).__init__()
self.split = split
self.purpose = purpose
self.root = config.dataset_root
self.single_mode = hasattr(config,
"single_mode") and config.single_mode
# always used (labels fields used to make relevancy mask for train)
self.gt_k = config.gt_k
self.pre_scale_all = config.pre_scale_all
self.pre_scale_factor = config.pre_scale_factor
self.input_sz = config.input_sz
self.include_rgb = config.include_rgb
self.no_sobel = config.no_sobel
assert ((not hasattr(config, "mask_input")) or (not config.mask_input))
self.mask_input = False
# only used if purpose is train
if purpose == "train":
self.use_random_scale = config.use_random_scale
if self.use_random_scale:
self.scale_max = config.scale_max
self.scale_min = config.scale_min
self.jitter_tf = tvt.ColorJitter(
brightness=config.jitter_brightness,
contrast=config.jitter_contrast,
saturation=config.jitter_saturation,
hue=config.jitter_hue)
self.flip_p = config.flip_p # 0.5
self.use_random_affine = config.use_random_affine
if self.use_random_affine:
self.aff_min_rot = config.aff_min_rot
self.aff_max_rot = config.aff_max_rot
self.aff_min_shear = config.aff_min_shear
self.aff_max_shear = config.aff_max_shear
self.aff_min_scale = config.aff_min_scale
self.aff_max_scale = config.aff_max_scale
assert (not preload)
self.files = []
self.images = []
self.labels = []
"""
if not osp.exists(config.fine_to_coarse_dict):
generate_fine_to_coarse(config.fine_to_coarse_dict)
with open(config.fine_to_coarse_dict, "rb") as dict_f:
d = pickle.load(dict_f)
self._fine_to_coarse_dict = d["fine_index_to_coarse_index"]
"""
cv2.setNumThreads(0)
in Jasper R. R. Uijlings, Koen E. A. van de Sande, Theo Gevers, Arnold W. M. Smeulders:
"Selective Search for Object Recognition"
International Journal of Computer Vision, Volume 104 (2), page 154-171, 2013
Usage:
./selectivesearchsegmentation_demo.py input_image (single|fast|quality)
Use "a" to display less rects, 'd' to display more rects, "q" to quit.
'''
import cv2 as cv
import sys
if __name__ == '__main__':
img = cv.imread(sys.argv[1])
cv.setUseOptimized(True)
cv.setNumThreads(8)
gs = cv.ximgproc.segmentation.createSelectiveSearchSegmentation()
gs.setBaseImage(img)
if (sys.argv[2][0] == 's'):
gs.switchToSingleStrategy()
elif (sys.argv[2][0] == 'f'):
gs.switchToSelectiveSearchFast()
elif (sys.argv[2][0] == 'q'):
gs.switchToSelectiveSearchQuality()
else:
print(__doc__)
sys.exit(1)
def correlate(img,
weights,
output=None,
mode='reflect',
cval=0.0,
origin=0,
backend=None,
delta=0,
threads=None):
"""Multidimensional correlation.
Parameters
---------
see scipy.ndimage.correlate
Additional Parameters
--------------------
backend : {None, 'ndimage', 'opencv'}, optional
If None, defaults to OpenCV for 2D images when possible. If OpenCV is
not available or input.ndim != 2, ndimage is always used.
delta : float, optional
Add this value to the filtered output.
threads : int or None, optional
The number of threads the OpenCV backend will use. If None, the number
of threads is not set internally (the value returned by
cv2.getNumThreads() is used). ``threads=-1`` can be used to specify
that all available threads should be used.
See Also
--------
cv2.filter2D
"""
backend = _get_backend(img.ndim, backend)
if mode == 'wrap' and backend == 'opencv':
warnings.warn("mode='wrap' is unsupported by the underlying OpenCV "
"function... falling back to ndimage")
backend = 'ndimage'
if backend == 'opencv':
if threads is not None:
if threads < 1 and threads != -1:
raise ValueError(
"Invalid number of threads: {}".format(threads))
threads_orig = cv2.getNumThreads()
cv2.setNumThreads(threads)
try:
opencv_mode = _get_opencv_mode(mode, cval)
anchor = _get_opencv_anchor(origin, weights.shape)
if np.isscalar(origin):
origin = (origin, origin)
if origin[0] != origin[1]:
# TODO: fix: does not match ndimage if origin[0] != origin[1]
raise NotImplementedError(
"origin[0] != origin[1] is not supported in opencv mode")
kernel = weights
# TODO: why is this coordinate swap necessary for correlate, but not
# for convolve?
anchor = (anchor[1], anchor[0])
result = cv2.filter2D(img,
dst=output,
ddepth=-1,
kernel=kernel,
anchor=anchor,
delta=delta,
borderType=opencv_mode)
finally:
if threads is not None:
cv2.setNumThreads(threads_orig)
elif backend == 'ndimage':
result = ndi.correlate(img,
weights,
output=output,
mode=mode,
cval=cval,
origin=origin)
if delta != 0:
result += delta
return result
def gaussian_filter(img,
sigma,
order=0,
output=None,
mode='reflect',
cval=0.0,
truncate=4.0,
backend=None,
threads=None):
"""Multidimensional Gaussian filter.
Parameters
---------
see scipy.ndimage.gaussian_filter
Additional Parameters
--------------------
backend : {None, 'ndimage', 'opencv'}, optional
If None, defaults to OpenCV for 2D images when possible. If OpenCV is
not available or input.ndim != 2, ndimage is always used.
threads : int or None, optional
The number of threads the OpenCV backend will use. If None, the number
of threads is not set internally (the value returned by
cv2.getNumThreads() is used). ``threads=-1`` can be used to specify
that all available threads should be used.
Notes
-----
cv2.GaussianBlur implemented for CV_8U, CV_16U, CV_16S, CV_32F, CV_64F and
for any number of channels.
See Also
--------
cv2.GaussianBlur
"""
backend = _get_backend(img.ndim, backend)
if backend == 'opencv':
if mode == 'wrap':
warnings.warn(
"mode == 'wrap' is unsupported by the underlying OpenCV "
"function... falling back to ndimage")
backend = 'ndimage'
if order != 0:
warnings.warn("order != 0 is unsupported by the underlying OpenCV "
"function... falling back to ndimage")
backend = 'ndimage'
if backend == 'opencv':
if threads is not None:
if threads < 1 and threads != -1:
raise ValueError(
"Invalid number of threads: {}".format(threads))
threads_orig = cv2.getNumThreads()
cv2.setNumThreads(threads)
try:
opencv_mode = _get_opencv_mode(mode, cval)
if np.isscalar(sigma):
sigma = (sigma, sigma)
if np.isscalar(truncate):
truncate = (truncate, truncate)
# determine ksize from sigma & truncate
# the equation used is from scipy.ndimage.gaussian_filter1d
wx = (2 * int(truncate[1] * sigma[1] + 0.5) + 1)
wy = (2 * int(truncate[0] * sigma[0] + 0.5) + 1)
result = cv2.GaussianBlur(img,
dst=output,
ksize=(wx, wy),
sigmaX=sigma[1],
sigmaY=sigma[0],
borderType=opencv_mode)
finally:
if threads is not None:
cv2.setNumThreads(threads_orig)
elif backend == 'ndimage':
result = ndi.gaussian_filter(img,
sigma,
order=order,
output=output,
mode=mode,
cval=cval,
truncate=truncate)
return result
def convolve(img,
weights,
output=None,
mode='reflect',
cval=0.0,
origin=0,
backend=None,
delta=0,
threads=None):
"""Multidimensional convolution.
Parameters
---------
see scipy.ndimage.convolve
Additional Parameters
--------------------
backend : {None, 'ndimage', 'opencv'}, optional
If None, defaults to OpenCV for 2D images when possible. If OpenCV is
not available or input.ndim != 2, ndimage is always used.
delta : float, optional
Add this value to the filtered output.
threads : int or None, optional
The number of threads the OpenCV backend will use. If None, the number
of threads is not set internally (the value returned by
cv2.getNumThreads() is used). ``threads=-1`` can be used to specify
that all available threads should be used.
Notes
-----
cv2.filter2D supports
CV_8U input to CV_16S, CV_32F or CV_64F output
CV_16U or CV_16S input to CV_32F or CV_64F output
CV_32F input to CV_32F or CV_64F output
CV_64F input to CV_64F output
User-defined ddepth is not yet suppported in this wrapper, so the
output will have the autoselected output depth given by ``ddepth=-1``.
See Also
--------
cv2.filter2D
"""
backend = _get_backend(img.ndim, backend)
if mode == 'wrap' and backend == 'opencv':
warnings.warn("mode='wrap' is unsupported by the underlying OpenCV "
"function... falling back to ndimage")
backend = 'ndimage'
if backend == 'opencv':
if threads is not None:
if threads < 1 and threads != -1:
raise ValueError(
"Invalid number of threads: {}".format(threads))
threads_orig = cv2.getNumThreads()
cv2.setNumThreads(threads)
try:
opencv_mode = _get_opencv_mode(mode, cval)
anchor = _get_opencv_anchor(origin, weights.shape)
if np.isscalar(origin):
origin = (origin, origin)
if origin[0] != origin[1]:
# TODO: fix: does not match ndimage if origin[0] != origin[1]
raise NotImplementedError(
"origin[0] != origin[1] is not supported in opencv mode")
"""
It is necessary to adjust the kernel and anchor for the fact that
filter2D actually performs correlation, not convolution.
To get a true convolution, we must flip the kernel and adjust the
anchor point as described in the OpenCV documentation of filter2D.
"""
kernel = weights[::-1, ::-1]
anchor = (kernel.shape[1] - anchor[1] - 1,
kernel.shape[0] - anchor[0] - 1)
result = cv2.filter2D(img,
dst=output,
ddepth=-1,
kernel=kernel,
anchor=anchor,
delta=delta,
borderType=opencv_mode)
finally:
if threads is not None:
cv2.setNumThreads(threads_orig)
elif backend == 'ndimage':
result = ndi.convolve(img,
weights,
output=output,
mode=mode,
cval=cval,
origin=origin)
if delta != 0:
result += delta
return result
def median_filter(img,
size=3,
footprint=None,
output=None,
mode='reflect',
cval=0.0,
origin=0,
backend=None,
threads=None):
"""Multi-dimensional median filter.
Parameters
---------
see scipy.ndimage.median_filter
Additional Parameters
--------------------
backend : {None, 'ndimage', 'opencv'}, optional
If None, defaults to OpenCV for 2D images when possible. If OpenCV is
not available or input.ndim != 2, ndimage is always used.
threads : int or None, optional
The number of threads the OpenCV backend will use. If None, the number
of threads is not set internally (the value returned by
cv2.getNumThreads() is used). ``threads=-1`` can be used to specify
that all available threads should be used.
Notes
-----
The OpenCV backend only supports odd-integer ``size`` and does not support
``footprint``. When ``size`` is 3 or 5, filtering for uint8, uint16 and
float32 is available. For other sizes, only uint8 filtering can be
performed.
See Also
--------
cv2.medianBlur (opeates on uint8, uint16 or float32)
"""
backend = _get_backend(img.ndim, backend)
if backend == 'opencv':
dtype_in = img.dtype
if footprint is not None:
if (np.all(footprint == 1)
and (footprint.shape[0] == footprint.shape[1])):
size = footprint.shape[0]
footprint = None
else:
warnings.warn(
"footprint is unsupported by the underlying OpenCV "
"function... falling back to ndimage")
backend = 'ndimage'
if not np.isscalar(size):
if size[0] == size[1]:
size = size[0]
else:
warnings.warn(
"non-square size is unsupported by the underlying "
"OpenCV function... falling back to ndimage")
backend = 'ndimage'
# check for odd kernel size
if size % 2 == 0:
raise ValueError("OpenCV medianBlur requires odd size")
# check for or convert to compatible dtype
if size == 3 or size == 5:
# uint16 and float32 only available for kernel sizes of 3 and 5
if dtype_in in [np.uint8, np.uint16, np.float32]:
dtype = dtype_in
else:
warnings.warn(
"OpenCV median filtering will be performed using float32 "
"dtype")
dtype = np.float32
else:
if dtype_in in [
np.uint8,
]:
dtype = dtype_in
else:
raise ValueError(
("OpenCV median filter with size={} can only be performed "
"for uint8 dtype").format(size))
img = np.asarray(img, dtype=dtype)
opencv_mode = _get_opencv_mode(mode, cval)
if opencv_mode != cv2.BORDER_REFLECT:
warnings.warn(
"only mode == 'reflect' is supported by the underlying "
"OpenCV function... falling back to ndimage")
backend = 'ndimage'
if not np.all(np.asarray(origin) == 0):
warnings.warn("non-zero origin is unsupported by the underlying "
"OpenCV function... falling back to ndimage")
backend = 'ndimage'
if backend == 'opencv':
if threads is not None:
if threads < 1 and threads != -1:
raise ValueError(
"Invalid number of threads: {}".format(threads))
threads_orig = cv2.getNumThreads()
cv2.setNumThreads(threads)
try:
result = cv2.medianBlur(img, ksize=size, dst=output)
finally:
if threads is not None:
cv2.setNumThreads(threads_orig)
elif backend == 'ndimage':
result = ndi.median_filter(img,
size=size,
footprint=footprint,
output=output,
mode=mode,
cval=cval,
origin=origin)
return result
def uniform_filter(img,
size=3,
output=None,
mode='reflect',
cval=0.0,
origin=0,
backend=None,
normalize=True,
threads=None,
squared=False):
"""Multi-dimensional uniform filter.
Parameters
---------
see scipy.ndimage.uniform_filter
Additional Parameters
--------------------
backend : {None, 'ndimage', 'opencv'}, optional
If None, defaults to OpenCV for 2D images when possible. If OpenCV is
not available or input.ndim != 2, ndimage is always used.
normalize : bool, optional
Controls whether or not the uniform filter coefficients are normalized
so that they sum to one.
threads : int or None, optional
The number of threads the OpenCV backend will use. If None, the number
of threads is not set internally (the value returned by
cv2.getNumThreads() is used). ``threads=-1`` can be used to specify
that all available threads should be used.
squared : bool, optional
If True, this returns uniform_filter(img**2, ...).
Notes
-----
cv2.boxFilter when `squared == False`
cv2.sqrBoxFilter when `squared == True`
cv2.blur correspnds to `normalize == True` and `squared == False`
Underlying OpenCV functions are defined for dtypes CV_8U, CV_16U, CV_16S,
CV_32F or CV_64F.
See Also
--------
cv2.boxFilter, cv2.sqrBoxFilter
"""
backend = _get_backend(img.ndim, backend)
if mode == 'wrap' and backend == 'opencv':
warnings.warn("mode='wrap' is unsupported by the underlying OpenCV "
"function... falling back to ndimage")
backend = 'ndimage'
if backend == 'opencv':
if threads is not None:
if threads < 1 and threads != -1:
raise ValueError(
"Invalid number of threads: {}".format(threads))
threads_orig = cv2.getNumThreads()
cv2.setNumThreads(threads)
try:
opencv_mode = _get_opencv_mode(mode, cval)
if np.isscalar(size):
size = (size, size)
else:
if len(size) != 2:
raise ValueError(
"size doesn't match number of image dimensions")
size = (size[1], size[0])
if squared:
func = cv2.sqrBoxFilter
kwargs = dict(_dst=output)
else:
func = cv2.boxFilter
kwargs = dict(dst=output)
result = func(img,
ddepth=-1,
ksize=size,
anchor=_get_opencv_anchor(origin, size),
normalize=normalize,
borderType=opencv_mode,
**kwargs)
finally:
if threads is not None:
cv2.setNumThreads(threads_orig)
elif backend == 'ndimage':
if squared:
img = img * img
result = ndi.uniform_filter(img,
size=size,
output=output,
mode=mode,
cval=cval,
origin=origin)
if not normalize:
# multiply output by the kernel size
if np.isscalar(size):
result *= size**img.ndim
else:
result *= np.prod(size)
return result
def _create_timelapse(
self, printing_filename: str, gcode_name: str,
info_mess: Message) -> Tuple[BytesIO, BytesIO, int, int, str, str]:
if not printing_filename:
raise ValueError("Gcode file name is empty")
while self.light_need_off:
time.sleep(1)
lapse_dir = f"{self._base_dir}/{printing_filename}"
lock_file = Path(f"{lapse_dir}/lapse.lock")
if not lock_file.is_file():
lock_file.touch()
# Todo: check for nonempty photos!
photos = glob.glob(f"{glob.escape(lapse_dir)}/*.{self._img_extension}")
photos.sort(key=os.path.getmtime)
photo_count = len(photos)
if photo_count == 0:
raise ValueError(
f"Empty photos list for {printing_filename} in lapse path {lapse_dir}"
)
info_mess.edit_text(text="Creating thumbnail")
last_photo = photos[-1]
img = cv2.imread(last_photo)
height, width, layers = img.shape
thumb_bio = self._create_thumb(img)
video_filename = Path(printing_filename).name
video_filepath = f"{lapse_dir}/{video_filename}.mp4"
if Path(video_filepath).is_file():
os.remove(video_filepath)
lapse_fps = self._calculate_fps(photo_count)
with self._camera_lock:
cv2.setNumThreads(
self._threads) # TOdo: check self set and remove!
out = cv2.VideoWriter(
video_filepath,
fourcc=cv2.VideoWriter_fourcc(*self._fourcc),
fps=lapse_fps,
frameSize=(width, height),
)
info_mess.edit_text(text="Images recoding")
last_update_time = time.time()
for fnum, filename in enumerate(photos):
if time.time() >= last_update_time + 3:
info_mess.edit_text(
text=f"Images recoded {fnum}/{photo_count}")
last_update_time = time.time()
out.write(cv2.imread(filename))
info_mess.edit_text(
text=
f"Repeating last image for {self._last_frame_duration} seconds"
)
for _ in range(lapse_fps * self._last_frame_duration):
out.write(img)
out.release()
cv2.destroyAllWindows()
del out
del photos, img, layers
# Todo: some error handling?
video_bio = BytesIO()
video_bio.name = f"{video_filename}.mp4"
target_video_file = f"{self._ready_dir}/{printing_filename}.mp4"
with open(video_filepath, "rb") as fh:
video_bio.write(fh.read())
if self._ready_dir and os.path.isdir(self._ready_dir):
info_mess.edit_text(text="Copy lapse to target ditectory")
Path(target_video_file).parent.mkdir(parents=True, exist_ok=True)
with open(target_video_file, "wb") as cpf:
cpf.write(video_bio.getvalue())
video_bio.seek(0)
os.remove(f"{lapse_dir}/lapse.lock")
return video_bio, thumb_bio, width, height, video_filepath, gcode_name
def main():
cv2.setNumThreads(0) # important for pytorch dataloaders
x = UnetV2Trainer() # load model & prepare all
x.train()
def main(train_root, train_csv, val_root, val_csv, test_root, test_csv,
epochs, aug, model_name, batch_size, num_workers, val_samples,
test_samples, early_stopping_patience, limit_data, images_per_epoch,
split_id, _run):
assert(model_name in ('inceptionv4', 'resnet152', 'densenet161',
'senet154', 'pnasnet5large', 'nasnetalarge',
'xception', 'squeezenet', 'resnext', 'dpn',
'inceptionresnetv2', 'mobilenetv2'))
cv2.setNumThreads(0)
AUGMENTED_IMAGES_DIR = os.path.join(fs_observer.dir, 'images')
CHECKPOINTS_DIR = os.path.join(fs_observer.dir, 'checkpoints')
BEST_MODEL_PATH = os.path.join(CHECKPOINTS_DIR, 'model_best.pth')
LAST_MODEL_PATH = os.path.join(CHECKPOINTS_DIR, 'model_last.pth')
RESULTS_CSV_PATH = os.path.join('results', 'results.csv')
EXP_NAME = _run.meta_info['options']['--name']
EXP_ID = _run._id
for directory in (AUGMENTED_IMAGES_DIR, CHECKPOINTS_DIR):
os.makedirs(directory)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if model_name == 'inceptionv4':
model = ptm.inceptionv4(num_classes=1000, pretrained='imagenet')
model.last_linear = nn.Linear(model.last_linear.in_features, 2)
aug['size'] = 299
aug['mean'] = model.mean
aug['std'] = model.std
elif model_name == 'resnet152':
model = models.resnet152(pretrained=True)
model.fc = nn.Linear(model.fc.in_features, 2)
aug['size'] = 224
aug['mean'] = [0.485, 0.456, 0.406]
aug['std'] = [0.229, 0.224, 0.225]
elif model_name == 'densenet161':
model = models.densenet161(pretrained=True)
model.classifier = nn.Linear(model.classifier.in_features, 2)
aug['size'] = 224
aug['mean'] = [0.485, 0.456, 0.406]
aug['std'] = [0.229, 0.224, 0.225]
elif model_name == 'senet154':
model = ptm.senet154(num_classes=1000, pretrained='imagenet')
model.last_linear = nn.Linear(model.last_linear.in_features, 2)
aug['size'] = model.input_size[1]
aug['mean'] = model.mean
aug['std'] = model.std
elif model_name == 'squeezenet':
model = ptm.squeezenet1_1(num_classes=1000, pretrained='imagenet')
model.last_conv = nn.Conv2d(
512, 2, kernel_size=(1, 1), stride=(1, 1))
aug['size'] = model.input_size[1]
aug['mean'] = model.mean
aug['std'] = model.std
elif model_name == 'pnasnet5large':
model = ptm.pnasnet5large(num_classes=1000, pretrained='imagenet')
model.last_linear = nn.Linear(model.last_linear.in_features, 2)
aug['size'] = model.input_size[1]
aug['mean'] = model.mean
aug['std'] = model.std
elif model_name == 'nasnetalarge':
model = ptm.nasnetalarge(num_classes=1000, pretrained='imagenet')
model.last_linear = nn.Linear(model.last_linear.in_features, 2)
aug['size'] = model.input_size[1]
aug['mean'] = model.mean
aug['std'] = model.std
elif model_name == 'xception':
model = ptm.xception(num_classes=1000, pretrained='imagenet')
model.last_linear = nn.Linear(model.last_linear.in_features, 2)
aug['size'] = model.input_size[1]
aug['mean'] = model.mean
aug['std'] = model.std
elif model_name == 'dpn':
model = ptm.dpn131(num_classes=1000, pretrained='imagenet')
model.last_linear = nn.Conv2d(model.last_linear.in_channels, 2,
kernel_size=1, bias=True)
aug['size'] = model.input_size[1]
aug['mean'] = model.mean
aug['std'] = model.std
elif model_name == 'resnext':
model = ptm.resnext101_64x4d(num_classes=1000, pretrained='imagenet')
model.last_linear = nn.Linear(model.last_linear.in_features, 2)
aug['size'] = model.input_size[1]
aug['mean'] = model.mean
aug['std'] = model.std
elif model_name == 'inceptionresnetv2':
model = ptm.inceptionresnetv2(num_classes=1000, pretrained='imagenet')
model.last_linear = nn.Linear(model.last_linear.in_features, 2)
aug['size'] = model.input_size[1]
aug['mean'] = model.mean
aug['std'] = model.std
elif model_name == 'mobilenetv2':
model = MobileNetV2()
model.load_state_dict(torch.load('./auglib/models/mobilenet_v2.pth'))
model.classifier = nn.Sequential(
nn.Dropout(0.2),
nn.Linear(model.last_channel, 2),
)
aug['size'] = 224
aug['mean'] = [0.485, 0.456, 0.406]
aug['std'] = [0.229, 0.224, 0.225]
model.to(device)
augs = Augmentations(**aug)
model.aug_params = aug
datasets = {
'samples': CSVDataset(train_root, train_csv, 'image_id', 'melanoma',
transform=augs.tf_augment, add_extension='.jpg',
limit=(400, 433)),
'train': CSVDataset(train_root, train_csv, 'image_id', 'melanoma',
transform=augs.tf_transform, add_extension='.jpg',
random_subset_size=limit_data),
'val': CSVDatasetWithName(
val_root, val_csv, 'image_id', 'melanoma',
transform=augs.tf_transform, add_extension='.jpg'),
'test': CSVDatasetWithName(
test_root, test_csv, 'image_id', 'melanoma',
transform=augs.tf_transform, add_extension='.jpg'),
'test_no_aug': CSVDatasetWithName(
test_root, test_csv, 'image_id', 'melanoma',
transform=augs.no_augmentation, add_extension='.jpg'),
'test_144': CSVDatasetWithName(
test_root, test_csv, 'image_id', 'melanoma',
transform=augs.inception_crop, add_extension='.jpg'),
}
dataloaders = {
'train': DataLoader(datasets['train'], batch_size=batch_size,
shuffle=True, num_workers=num_workers,
worker_init_fn=set_seeds),
'samples': DataLoader(datasets['samples'], batch_size=batch_size,
shuffle=False, num_workers=num_workers,
worker_init_fn=set_seeds),
}
save_images(datasets['samples'], to=AUGMENTED_IMAGES_DIR, n=32)
sample_batch, _ = next(iter(dataloaders['samples']))
save_image(make_grid(sample_batch, padding=0),
os.path.join(AUGMENTED_IMAGES_DIR, 'grid.jpg'))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=0.001,
momentum=0.9,
weight_decay=0.001)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1,
min_lr=1e-5,
patience=8)
metrics = {
'train': pd.DataFrame(columns=['epoch', 'loss', 'acc', 'auc']),
'val': pd.DataFrame(columns=['epoch', 'loss', 'acc', 'auc'])
}
best_val_auc = 0.0
best_epoch = 0
epochs_without_improvement = 0
if images_per_epoch:
batches_per_epoch = images_per_epoch // batch_size
else:
batches_per_epoch = None
for epoch in range(epochs):
print('train epoch {}/{}'.format(epoch+1, epochs))
epoch_train_result = train_epoch(
device, model, dataloaders, criterion, optimizer,
batches_per_epoch)
metrics['train'] = metrics['train'].append(
{**epoch_train_result, 'epoch': epoch}, ignore_index=True)
print('train', epoch_train_result)
epoch_val_result, _ = test_with_augmentation(
model, datasets['val'], device, num_workers, val_samples)
metrics['val'] = metrics['val'].append(
{**epoch_val_result, 'epoch': epoch}, ignore_index=True)
print('val', epoch_val_result)
print('-' * 40)
scheduler.step(epoch_val_result['loss'])
if epoch_val_result['auc'] > best_val_auc:
best_val_auc = epoch_val_result['auc']
best_val_result = epoch_val_result
best_epoch = epoch
epochs_without_improvement = 0
torch.save(model, BEST_MODEL_PATH)
else:
epochs_without_improvement += 1
if epochs_without_improvement > early_stopping_patience:
last_val_result = epoch_val_result
torch.save(model, LAST_MODEL_PATH)
break
if epoch == (epochs-1):
last_val_result = epoch_val_result
torch.save(model, LAST_MODEL_PATH)
for phase in ['train', 'val']:
metrics[phase].epoch = metrics[phase].epoch.astype(int)
metrics[phase].to_csv(os.path.join(fs_observer.dir, phase + '.csv'),
index=False)
# Run testing
# TODO: reduce code repetition
test_result, preds = test_with_augmentation(
torch.load(BEST_MODEL_PATH), datasets['test'], device,
num_workers, test_samples)
print('[best] test', test_result)
test_noaug_result, preds_noaug = test_with_augmentation(
torch.load(BEST_MODEL_PATH), datasets['test_no_aug'], device,
num_workers, 1)
print('[best] test (no augmentation)', test_noaug_result)
test_result_last, preds_last = test_with_augmentation(
torch.load(LAST_MODEL_PATH), datasets['test'], device,
num_workers, test_samples)
print('[last] test', test_result_last)
test_noaug_result_last, preds_noaug_last = test_with_augmentation(
torch.load(LAST_MODEL_PATH), datasets['test_no_aug'], device,
num_workers, 1)
print('[last] test (no augmentation)', test_noaug_result_last)
# Save predictions
preds.to_csv(os.path.join(fs_observer.dir, 'test-aug-best.csv'),
index=False, columns=['image', 'label', 'score'])
preds_noaug.to_csv(os.path.join(fs_observer.dir, 'test-noaug-best.csv'),
index=False, columns=['image', 'label', 'score'])
preds_last.to_csv(os.path.join(fs_observer.dir, 'test-aug-last.csv'),
index=False, columns=['image', 'label', 'score'])
preds_noaug_last.to_csv(os.path.join(fs_observer.dir, 'test-noaug-last.csv'),
index=False, columns=['image', 'label', 'score'])
# TODO: Avoid repetition.
# use ordereddict, or create a pandas df before saving
with open(RESULTS_CSV_PATH, 'a') as file:
file.write(','.join((
EXP_NAME,
str(EXP_ID),
str(split_id),
str(best_epoch),
str(best_val_result['loss']),
str(best_val_result['acc']),
str(best_val_result['auc']),
str(best_val_result['avp']),
str(best_val_result['sens']),
str(best_val_result['spec']),
str(last_val_result['loss']),
str(last_val_result['acc']),
str(last_val_result['auc']),
str(last_val_result['avp']),
str(last_val_result['sens']),
str(last_val_result['spec']),
str(best_val_auc),
str(test_result['auc']),
str(test_result_last['auc']),
str(test_result['acc']),
str(test_result_last['acc']),
str(test_result['spec']),
str(test_result_last['spec']),
str(test_result['sens']),
str(test_result_last['sens']),
str(test_result['avp']),
str(test_result_last['avp']),
str(test_noaug_result['auc']),
str(test_noaug_result_last['auc']),
str(test_noaug_result['acc']),
str(test_noaug_result_last['acc']),
str(test_noaug_result['spec']),
str(test_noaug_result_last['spec']),
str(test_noaug_result['sens']),
str(test_noaug_result_last['sens']),
str(test_noaug_result['avp']),
str(test_noaug_result_last['avp']),
)) + '\n')
return (test_noaug_result['auc'],
test_result['auc'],
)
def create_yolo_dataset(image_dir,
anno_path,
batch_size,
max_epoch,
device_num,
rank,
config=None,
is_training=True,
shuffle=True):
"""Create dataset for YOLOV3."""
cv2.setNumThreads(0)
if is_training:
filter_crowd = True
remove_empty_anno = True
else:
filter_crowd = False
remove_empty_anno = False
yolo_dataset = COCOYoloDataset(
root=image_dir,
ann_file=anno_path,
filter_crowd_anno=filter_crowd,
remove_images_without_annotations=remove_empty_anno,
is_training=is_training)
distributed_sampler = DistributedSampler(len(yolo_dataset),
device_num,
rank,
shuffle=shuffle)
hwc_to_chw = CV.HWC2CHW()
config.dataset_size = len(yolo_dataset)
num_parallel_workers1 = int(64 / device_num)
num_parallel_workers2 = int(16 / device_num)
if is_training:
multi_scale_trans = MultiScaleTrans(config, device_num)
if device_num != 8:
ds = de.GeneratorDataset(
yolo_dataset,
column_names=["image", "annotation"],
num_parallel_workers=num_parallel_workers1,
sampler=distributed_sampler)
ds = ds.batch(batch_size,
per_batch_map=multi_scale_trans,
input_columns=['image', 'annotation'],
num_parallel_workers=num_parallel_workers2,
drop_remainder=True)
else:
ds = de.GeneratorDataset(yolo_dataset,
column_names=["image", "annotation"],
sampler=distributed_sampler)
ds = ds.batch(batch_size,
per_batch_map=multi_scale_trans,
input_columns=['image', 'annotation'],
num_parallel_workers=8,
drop_remainder=True)
else:
ds = de.GeneratorDataset(yolo_dataset,
column_names=["image", "img_id"],
sampler=distributed_sampler)
compose_map_func = (
lambda image, img_id: reshape_fn(image, img_id, config))
ds = ds.map(operations=compose_map_func,
input_columns=["image", "img_id"],
output_columns=["image", "image_shape", "img_id"],
column_order=["image", "image_shape", "img_id"],
num_parallel_workers=8)
ds = ds.map(operations=hwc_to_chw,
input_columns=["image"],
num_parallel_workers=8)
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.repeat(max_epoch)
return ds, len(yolo_dataset)
def calibrate(input_folder, output_file):
"""
Function to calibrate fish eye camera from a set of images
:param input_folder : folder containing the images to be used for the calibration
:param output_file : path where to write the krabby calibration file
"""
cv2.setNumThreads(4)
CHECKERBOARD = (7, 9)
subpix_criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30,
0.1)
calibration_flags = cv2.fisheye.CALIB_RECOMPUTE_EXTRINSIC + cv2.fisheye.CALIB_CHECK_COND + cv2.fisheye.CALIB_FIX_SKEW
objp = np.zeros((1, CHECKERBOARD[0] * CHECKERBOARD[1], 3), np.float32)
objp[0, :, :2] = np.mgrid[0:CHECKERBOARD[0],
0:CHECKERBOARD[1]].T.reshape(-1, 2)
_img_shape = None
objpoints = [] # 3d point in real world space
imgpoints = [] # 2d points in image plane.
images = glob.glob(f"{input_folder}/*.png")
images.extend(glob.glob(f"{input_folder}/*.jpg"))
for fname in images:
img = cv2.imread(fname)
if _img_shape == None:
_img_shape = img.shape[:2]
else:
assert _img_shape == img.shape[:
2], "All images must share the same size."
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Find the chess board corners
ret, corners = cv2.findChessboardCorners(
gray, CHECKERBOARD, cv2.CALIB_CB_ADAPTIVE_THRESH +
cv2.CALIB_CB_FAST_CHECK + cv2.CALIB_CB_NORMALIZE_IMAGE)
# If found, add object points, image points (after refining them)
if ret == True:
objpoints.append(objp)
cv2.cornerSubPix(gray, corners, (3, 3), (-1, -1), subpix_criteria)
imgpoints.append(corners)
N_OK = len(objpoints)
K = np.zeros((3, 3))
D = np.zeros((4, 1))
rvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
tvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
print("Found " + str(N_OK) + " valid images for calibration")
print("DIM=" + str(_img_shape[::-1]))
rms, _, _, _, _ = \
cv2.fisheye.calibrate(
objpoints,
imgpoints,
gray.shape[::-1],
K,
D,
rvecs,
tvecs,
calibration_flags,
(cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_MAX_ITER, 30, 1e-6)
)
print("K=np.array(" + str(K.tolist()) + ")")
print("D=np.array(" + str(D.tolist()) + ")")
cv_file = cv2.FileStorage(output_file, cv2.FILE_STORAGE_WRITE)
cv_file.write("K", K)
cv_file.write("D", np.transpose(D))
cv_file.write("shape", _img_shape[::-1])
cv_file.release()
print(f"calibration saved in {output_file}")
import tarfile
import numpy as np
import tensorflow as tf
flags = tf.app.flags
flags.DEFINE_string('image_tar_file', '', '')
flags.DEFINE_string('output_path', '', '')
tf.flags.DEFINE_string('parts', '', '')
FLAGS = flags.FLAGS
tf.logging.set_verbosity(tf.logging.DEBUG)
cv2.setUseOptimized(True)
cv2.setNumThreads(8)
ss = cv2.ximgproc.segmentation.createSelectiveSearchSegmentation()
def selective_search(data,
dataset_directory,
label_map_dict,
ignore_difficult_instances=False,
image_subdirectory='JPEGImages'):
"""Convert XML derived dict to tf.Example proto.
Notice that this function normalizes the bounding box coordinates provided
by the raw data.
Args:
data: dict holding PASCAL XML fields for a single image (obtained by
def get_noise_fft(Y,
noise_range=[0.25, 0.5],
noise_method='logmexp',
max_num_samples_fft=3072,
opencv=True):
"""Estimate the noise level for each pixel by averaging the power spectral density.
Args:
Y: np.ndarray
Input movie data with time in the last axis
noise_range: np.ndarray [2 x 1] between 0 and 0.5
Range of frequencies compared to Nyquist rate over which the power spectrum is averaged
default: [0.25,0.5]
noise method: string
method of averaging the noise.
Choices:
'mean': Mean
'median': Median
'logmexp': Exponential of the mean of the logarithm of PSD (default)
Returns:
sn: np.ndarray
Noise level for each pixel
"""
T = Y.shape[-1]
# Y=np.array(Y,dtype=np.float64)
if T > max_num_samples_fft:
Y = np.concatenate(
(Y[..., 1:max_num_samples_fft // 3 + 1],
Y[...,
np.int(T // 2 - max_num_samples_fft / 3 /
2):np.int(T // 2 + max_num_samples_fft / 3 / 2)],
Y[..., -max_num_samples_fft // 3:]),
axis=-1)
T = np.shape(Y)[-1]
# we create a map of what is the noise on the FFT space
ff = np.arange(0, 0.5 + 1. / T, 1. / T)
ind1 = ff > noise_range[0]
ind2 = ff <= noise_range[1]
ind = np.logical_and(ind1, ind2)
# we compute the mean of the noise spectral density s
if Y.ndim > 1:
if opencv:
import cv2
try:
cv2.setNumThreads(0)
except:
pass
psdx_list = []
for y in Y.reshape(-1, T):
dft = cv2.dft(
y, flags=cv2.DFT_COMPLEX_OUTPUT).squeeze()[:len(ind)][ind]
psdx_list.append(np.sum(1. / T * dft * dft, 1))
psdx = np.reshape(psdx_list, Y.shape[:-1] + (-1, ))
else:
xdft = np.fft.rfft(Y, axis=-1)
xdft = xdft[..., ind[:xdft.shape[-1]]]
psdx = 1. / T * abs(xdft)**2
psdx *= 2
sn = mean_psd(psdx, method=noise_method)
else:
xdft = np.fliplr(np.fft.rfft(Y))
psdx = 1. / T * (xdft**2)
psdx[1:] *= 2
sn = mean_psd(psdx[ind[:psdx.shape[0]]], method=noise_method)
return sn, psdx
def main(mode=None, config=None):
torch.cuda.empty_cache()
r"""starts the model
Args:
mode (int): 1: train, 2: test, 3: eval, reads from config file if not specified
4: demo_patch,
"""
if mode == 4:
config = load_config_demo(mode, config=config)
else:
config = load_config(mode)
# init environment
if (config.DEVICE == 1
or config.DEVICE is None) and torch.cuda.is_available():
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(
str(e) for e in config.GPU)
config.DEVICE = torch.device("cuda")
torch.backends.cudnn.benchmark = True # cudnn auto-tuner
else:
config.DEVICE = torch.device("cpu")
# print(torch.cuda.is_available())
print('DEVICE is:', config.DEVICE)
# set cv2 running threads to 1 (prevents deadlocks with pytorch dataloader)
cv2.setNumThreads(0)
# initialize random seed
torch.manual_seed(config.SEED)
torch.cuda.manual_seed_all(config.SEED)
np.random.seed(config.SEED)
random.seed(config.SEED)
# enable the cudnn auto-tuner for hardware.
torch.backends.cudnn.benchmark = True
# build the model and initialize
model = EdgeConnect(config)
model.load()
# model training
if config.MODE == 1:
config.print()
print('\nstart training...\n')
model.train()
# model test
elif config.MODE == 2:
print('\nstart testing...\n')
# import time
# start = time.time()
with torch.no_grad():
model.test()
# print(time.time() - start)
# eval mode
elif config.MODE == 3:
print('\nstart eval...\n')
with torch.no_grad():
model.eval()
elif config.MODE == 4:
if config.DEBUG:
config.print()
print('model prepared.')
return model
import torch
import torchvision
import yaml
from utils.google_utils import gsutil_getsize
from utils.metrics import fitness
from utils.torch_utils import init_torch_seeds
# Settings
torch.set_printoptions(linewidth=320, precision=5, profile='long')
np.set_printoptions(linewidth=320,
formatter={'float_kind': '{:11.5g}'.format
}) # format short g, %precision=5
pd.options.display.max_columns = 10
cv2.setNumThreads(
0
) # prevent OpenCV from multithreading (incompatible with PyTorch DataLoader)
os.environ['NUMEXPR_MAX_THREADS'] = str(min(os.cpu_count(),
8)) # NumExpr max threads
class timeout(contextlib.ContextDecorator):
# Usage: @timeout(seconds) decorator or 'with timeout(seconds):' context manager
def __init__(self,
seconds,
*,
timeout_msg='',
suppress_timeout_errors=True):
self.seconds = int(seconds)
self.timeout_message = timeout_msg
self.suppress = bool(suppress_timeout_errors)
import cv2
time_to_setup = cv2.getTickCount()
# letting the user now there's an inital hang because of setup
print("setting up...")
import os
import sys
import numpy as np
import utils
#potential additional imports later found under "Model Settings" section
# </section>End of Imports
# <section>~~~~~~~~~~~~~~~~~~~~~~~~OpenCV settings~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# optimize when possible
cv2.setUseOptimized(True)
# try using multithreading when possible
cv2.setNumThreads(4)
# </section>End of Disparity Settings
# <section>~~~~~~~~~~~~~~~~~~~~~~~Directory Settings~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
master_path_to_dataset = "../Data/TTBB-durham-02-10-17-sub10" # where is the data
directory_to_cycle_left = "left-images" # where are the left images
directory_to_cycle_right = "right-images" # where are the right images
# pass name of file to start cycling from
skip_forward_file_pattern = sys.argv[2] # pass it via terminal
if skip_forward_file_pattern == "start": # passing in "start"
skip_forward_file_pattern = "" # means from the start
# resolve full directory location of data set for left / right images
full_path_directory_left = os.path.join(master_path_to_dataset,
directory_to_cycle_left)
#\copyright GNU General Public License v2.0
#\date Created on june 2017
#\author: Jremie KALFON
from __future__ import division
from __future__ import print_function
from builtins import str
from builtins import range
import matplotlib
from caiman.utils.utils import download_demo
import cv2
import glob
try:
cv2.setNumThreads(1)
except:
print('Open CV is naturally single threaded')
try:
if __IPYTHON__:
print((1))
# this is used for debugging purposes only. allows to reload classes
# when changed
get_ipython().magic('load_ext autoreload')
get_ipython().magic('autoreload 2')
except NameError:
print('Not IPYTHON')
pass
import caiman as cm
import numpy as np
from __future__ import absolute_import, division, print_function
import os
import cv2
import numpy as np
import torch
from torch.utils.data import DataLoader
from layers import disp_to_depth
from utils import readlines
from options import MonodepthOptions
import datasets
import networks
cv2.setNumThreads(0) # This speeds up evaluation 5x on our unix systems (OpenCV 3.3.1)
splits_dir = os.path.join(os.path.dirname(__file__), "splits")
# Models which were trained with stereo supervision were trained with a nominal
# baseline of 0.1 units. The KITTI rig has a baseline of 54cm. Therefore,
# to convert our stereo predictions to real-world scale we multiply our depths by 5.4.
STEREO_SCALE_FACTOR = 5.4
def compute_errors(gt, pred):
"""Computation of error metrics between predicted and ground truth depths
"""
thresh = np.maximum((gt / pred), (pred / gt))
a1 = (thresh < 1.25 ).mean()
use_qual=config.use_qual)
if isinstance(scheduler, CosineAnnealingLR):
scheduler.step()
else:
scheduler.step(metric)
if metric > max_score:
max_score = metric
model.eval()
torch.save({'st_d': model.state_dict()},
os.path.join(models_path, 'best.pth'))
write2log(os.path.join(experiment_path, log_name), epoch, metric,
optimizer.state_dict()['param_groups'][0]['lr'])
if __name__ == '__main__':
torch.set_num_threads(config.n_work)
cv2.setNumThreads(0)
make_os_settings(config.cuda_num)
make_logdirs_if_needit(config.experiments_root, config.experiment_name)
train_dataset, valid_dataset = get_train_val_datasets(config)
train_loader = DataLoader(train_dataset,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.n_work)
valid_loader = DataLoader(valid_dataset,
batch_size=config.batch_size,
shuffle=False,
num_workers=config.n_work)
train(config.model, train_loader, valid_loader, config)
Use "l" to display less rects, 'm' to display more rects, "q" to quit.
'''
import sys
import cv2
if __name__ == '__main__':
# If image path and f/q is not passed as command
# line arguments, quit and display help message
if len(sys.argv) < 3:
print(__doc__)
sys.exit(1)
# speed-up using multithreads
cv2.setUseOptimized(True);
cv2.setNumThreads(4);
# read image
im = cv2.imread(sys.argv[1])
# resize image
newHeight = 200
newWidth = int(im.shape[1]*200/im.shape[0])
im = cv2.resize(im, (newWidth, newHeight))
# create Selective Search Segmentation Object using default parameters
ss = cv2.ximgproc.segmentation.createSelectiveSearchSegmentation()
# set input image on which we will run segmentation
ss.setBaseImage(im)
# Switch to fast but low recall Selective Search method
in Jasper R. R. Uijlings, Koen E. A. van de Sande, Theo Gevers, Arnold W. M. Smeulders:
"Selective Search for Object Recognition"
International Journal of Computer Vision, Volume 104 (2), page 154-171, 2013
Usage:
./selectivesearchsegmentation_demo.py input_image (single|fast|quality)
Use "a" to display less rects, 'd' to display more rects, "q" to quit.
'''
import cv2
import sys
if __name__ == '__main__':
img = cv2.imread(sys.argv[1])
cv2.setUseOptimized(True)
cv2.setNumThreads(8)
gs = cv2.ximgproc.segmentation.createSelectiveSearchSegmentation()
gs.setBaseImage(img)
if (sys.argv[2][0] == 's'):
gs.switchToSingleStrategy()
elif (sys.argv[2][0] == 'f'):
gs.switchToSelectiveSearchFast()
elif (sys.argv[2][0] == 'q'):
gs.switchToSelectiveSearchQuality()
else:
print(__doc__)
sys.exit(1)
from scipy.optimize import linear_sum_assignment
import json
from skimage.filters import sobel
from skimage.morphology import watershed
from scipy import ndimage as ndi
from skimage.draw import polygon
import matplotlib.pyplot as pl
import matplotlib.patches as mpatches
import zipfile
import tempfile
import shutil
import os
from ..motion_correction import tile_and_correct
try:
cv2.setNumThreads(0)
except:
pass
#%%
def com(A, d1, d2, d3=None):
"""Calculation of the center of mass for spatial components
Inputs:
------
A: np.ndarray
matrix of spatial components (d x K)
d1: int
import argparse import os import sys import pickle import resource import traceback import logging from collections import defaultdict import numpy as np import yaml import torch from torch.autograd import Variable import torch.nn as nn import cv2 cv2.setNumThreads(0) # pytorch issue 1355: possible deadlock in dataloader import _init_paths # pylint: disable=unused-import import nn as mynn import utils.net as net_utils import utils.misc as misc_utils from core.config import cfg, cfg_from_file, cfg_from_list, assert_and_infer_cfg from datasets.roidb import combined_roidb_for_training from roi_data.loader import RoiDataLoader, MinibatchSampler, BatchSampler, collate_minibatch from modeling.model_builder import Generalized_RCNN from utils.detectron_weight_helper import load_detectron_weight from utils.logging import setup_logging from utils.timer import Timer from utils.training_stats import TrainingStats # Set up logging and load config options
def take_video(self) -> Tuple[BytesIO, BytesIO, int, int]:
def process_video_frame(frame_local):
if self._flip_vertically or self._flip_horizontally:
if self._hw_accel:
frame_loc_ = cv2.UMat(frame_local)
frame_loc_ = cv2.flip(frame_loc_, self._flip)
frame_local = cv2.UMat.get(frame_loc_)
del frame_loc_
else:
frame_local = cv2.flip(frame_local, self._flip)
# Todo: check memory leaks
if self._rotate_code > -10:
frame_local = cv2.rotate(frame_local,
rotateCode=self._rotate_code)
return frame_local
def write_video():
cv2.setNumThreads(self._threads)
out = cv2.VideoWriter(
filepath,
fourcc=cv2.VideoWriter_fourcc(*self._fourcc),
fps=fps_cam,
frameSize=(width, height),
)
while video_lock.locked():
try:
frame_local = frame_queue.get(block=False)
except Exception as ex:
logger.warning("Reading video frames queue exception %s",
ex)
frame_local = frame_queue.get()
out.write(process_video_frame(frame_local))
frame_local = None
del frame_local
while not frame_queue.empty():
frame_local = frame_queue.get()
out.write(process_video_frame(frame_local))
frame_local = None
del frame_local
out.release()
video_written_event.set()
with self._camera_lock:
cv2.setNumThreads(
self._threads) # TOdo: check self set and remove!
self.cam_cam.open(self._host)
self._set_cv2_params()
success, frame = self.cam_cam.read()
if not success:
logger.debug("failed to get camera frame for video")
# Todo: get picture from imgs?
frame = process_video_frame(frame)
height, width, channels = frame.shape
thumb_bio = self._create_thumb(frame)
del frame, channels
fps_cam = self.cam_cam.get(
cv2.CAP_PROP_FPS
) if self._stream_fps == 0 else self._stream_fps
filepath = os.path.join("/tmp/", "video.mp4")
frame_queue: Queue = Queue(fps_cam * self._video_buffer_size)
video_lock = threading.Lock()
video_written_event = threading.Event()
video_written_event.clear()
with video_lock:
threading.Thread(target=write_video, args=()).start()
t_end = time.time() + self._video_duration
while success and time.time() <= t_end:
success, frame_loc = self.cam_cam.read()
try:
frame_queue.put(frame_loc, block=False)
except Exception as ex:
logger.warning(
"Writing video frames queue exception %s",
ex.with_traceback)
frame_queue.put(frame_loc)
frame_loc = None
del frame_loc
video_written_event.wait()
self.cam_cam.release()
video_bio = BytesIO()
video_bio.name = "video.mp4"
with open(filepath, "rb") as video_file:
video_bio.write(video_file.read())
os.remove(filepath)
video_bio.seek(0)
return video_bio, thumb_bio, width, height
import time
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np
from packaging import version
try:
from tqdm import tqdm
except ImportError:
print('Warning: tqdm not found. Install it to see the progress bar.')
def tqdm(x): return x
import cv2
cv2.setNumThreads(0) # Prevent opencv from spawning too many threads in the data loaders
import kaolin as kal
from rendering.renderer import Renderer
from rendering.utils import qrot, qmul, circpad, symmetrize_texture, adjust_poles
from rendering.mesh_template import MeshTemplate
from utils.losses import loss_flat
from models.reconstruction import ReconstructionNetwork, DatasetParams
parser = argparse.ArgumentParser()
parser.add_argument('--name', type=str, required=True)
parser.add_argument('--dataset', type=str, required=True, help='(p3d|cub)')
parser.add_argument('--mesh_path', type=str, default='autodetect')
parser.add_argument('--batch_size', type=int, default=50)
parser.add_argument('--image_resolution', type=int, default=256)
def train():
"""
Replicates parameters from https://github.com/kuangliu/pytorch-cifar
The following is a table of kuangliu's reported accuracy and our measured
accuracy for each model.
The first column is kuangliu's reported accuracy, the second column is me
running kuangliu's code, and the final column is using my own training
harness (handles logging and whatnot) called netharn.
model | kuangliu | rerun-kuangliu | netharn |
-------------------------------------------------------
ResNet50 | 93.62% | 95.370% | 95.72% | <- how did that happen?
DenseNet121 | 95.04% | 95.420% | 94.47% |
DPN92 | 95.16% | 95.410% | 94.92% |
"""
import random
import torchvision
from torchvision import transforms
np.random.seed(1031726816 % 4294967295)
torch.manual_seed(137852547 % 4294967295)
random.seed(2497950049 % 4294967295)
# batch_size = int(ub.argval('--batch_size', default=128))
batch_size = int(ub.argval('--batch_size', default=64))
workers = int(ub.argval('--workers', default=2))
model_key = ub.argval('--model', default='densenet121')
xpu = nh.XPU.cast('argv')
lr = 0.1
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
workdir = ub.ensure_app_cache_dir('netharn')
datasets = {
'train':
torchvision.datasets.CIFAR10(root=workdir,
train=True,
download=True,
transform=transform_train),
'test':
torchvision.datasets.CIFAR10(root=workdir,
train=False,
download=True,
transform=transform_test),
}
# For some reason the torchvision objects dont have the label names
CIFAR10_CLASSNAMES = [
'airplane',
'automobile',
'bird',
'cat',
'deer',
'dog',
'frog',
'horse',
'ship',
'truck',
]
datasets['train'].class_names = CIFAR10_CLASSNAMES
datasets['test'].class_names = CIFAR10_CLASSNAMES
n_classes = 10 # hacked in
loaders = {
key: torch.utils.data.DataLoader(dset,
shuffle=key == 'train',
num_workers=workers,
batch_size=batch_size,
pin_memory=True)
for key, dset in datasets.items()
}
if workers > 0:
import cv2
cv2.setNumThreads(0)
initializer_ = (nh.initializers.KaimingNormal, {
'param': 0,
'mode': 'fan_in'
})
# initializer_ = (initializers.LSUV, {})
available_models = {
'densenet121': (nh.models.densenet.DenseNet, {
'nblocks': [6, 12, 24, 16],
'growth_rate': 12,
'reduction': 0.5,
'num_classes': n_classes,
}),
'resnet50': (nh.models.resnet.ResNet, {
'num_blocks': [3, 4, 6, 3],
'num_classes': n_classes,
'block': 'Bottleneck',
}),
'dpn26': (nh.models.dual_path_net.DPN,
dict(
cfg={
'in_planes': (96, 192, 384, 768),
'out_planes': (256, 512, 1024, 2048),
'num_blocks': (2, 2, 2, 2),
'dense_depth': (16, 32, 24, 128),
'num_classes': n_classes,
})),
'dpn92': (nh.models.dual_path_net.DPN,
dict(
cfg={
'in_planes': (96, 192, 384, 768),
'out_planes': (256, 512, 1024, 2048),
'num_blocks': (3, 4, 20, 3),
'dense_depth': (16, 32, 24, 128),
'num_classes': n_classes,
})),
}
model_ = available_models[model_key]
hyper = nh.HyperParams(
datasets=datasets,
nice='cifar10_' + model_key,
loaders=loaders,
workdir=workdir,
xpu=xpu,
model=model_,
optimizer=(torch.optim.SGD, {
'lr': lr,
'weight_decay': 5e-4,
'momentum': 0.9,
'nesterov': True,
}),
scheduler=(nh.schedulers.ListedLR, {
'points': {
0: lr,
150: lr * 0.1,
250: lr * 0.01,
},
'interpolate': False
}),
monitor=(nh.Monitor, {
'minimize': ['loss'],
'patience': 350,
'max_epoch': 350,
}),
initializer=initializer_,
criterion=(torch.nn.CrossEntropyLoss, {}),
# Specify anything else that is special about your hyperparams here
# Especially if you make a custom_batch_runner
# TODO: type of augmentation as a parameter dependency
# augment=str(datasets['train'].augmenter),
# other=ub.dict_union({
# # 'colorspace': datasets['train'].output_colorspace,
# }, datasets['train'].center_inputs.__dict__),
)
harn = CIFAR_FitHarn(hyper=hyper)
harn.initialize()
harn.run()
import numpy as np
import torch
import torchvision
import cv2
from torch.autograd import Variable
import time
import pandas as pd
import hpat
from hpat import prange
hpat.multithread_mode = True
cv2.setNumThreads(0) # we use threading across images
model = torchvision.models.resnet18(True)
@hpat.jit(locals={'fdata:return': 'distributed'})
def read_data():
#fname = "/export/intel/lustre/etotoni/BXP5401-front-camera_2017.dat"
fname = "img2.dat"
blob = np.fromfile(fname, np.uint8)
# reshape to images
n_channels = 3
height = 800
width = 1280
n_images = len(blob) // (n_channels * height * width)
data = blob.reshape(n_images, height, width, n_channels)
# select every 5 image
data = data[::5, :, :, :]
n_images = len(data)
def get_img_prediction_bounding_box(path, model, dim, edge = False, model_type = 'normal'):
'''This function will create a bounding box over what it believes is a weapon given the image path, dimensions, and model used to detect the weapon. Dimensions can be found within the Var.py file. This function is still being used as I need to apply non-max suppresion to create only one bounding box'''
img = get_image_value(path, dim, edge = edge)
if edge == True:
img = img.reshape(1, img.shape[0], img.shape[1], 1)
else:
img = img.reshape(1, img.shape[0], img.shape[1], 3)
pred = model.predict(img)[0]
category_dict = {0: 'No Weapon', 1: 'Handgun', 2: 'Rifle'}
cat_index = np.argmax(pred)
cat = category_dict[cat_index]
print(f'{path}\t\tPrediction: {cat}\t{int(pred.max()*100)}% Confident')
#speed up cv2
cv2.setUseOptimized(True)
cv2.setNumThreads(10)
img = cv2.imread(path)
clone = img.copy()
clone2 = img.copy()
# if cat_index != 0:
ss = cv2.ximgproc.segmentation.createSelectiveSearchSegmentation()
ss.setBaseImage(img)
# ss.switchToSelectiveSearchQuality()
ss.switchToSelectiveSearchFast()
rects = ss.process()
windows = []
locations = []
print(f'Creating Bounding Boxes for {path}')
for x, y, w,h in rects[:1001]:
startx = x
starty = y
endx = x+w
endy = y+h
roi = img[starty:endy, startx:endx]
if edge == True:
roi = get_edged(roi, dim = dim)
roi = cv2.resize(roi, dsize =dim, interpolation = cv2.INTER_CUBIC)
windows.append(roi)
locations.append((startx, starty, endx, endy))
windows = np.array(windows)
if edge == True:
windows = windows.reshape(windows.shape[0], windows.shape[1], windows.shape[2], 1)
else:
windows = windows.reshape(windows.shape[0], windows.shape[1], windows.shape[2], 3)
windows = np.array(windows)
locations = np.array(locations)
# if model_type == 'mobilenet':
# from keras.applications.mobilenet import preprocess_input
# windows = preprocess_input(windows)
predictions = model.predict(windows)
nms = non_max_suppression(locations)
bounding_cnt = 0
for idx in nms:
if np.argmax(predictions[idx]) != cat_index:
continue
# print(np.argmax(predictions[idx]), predictions[idx].max())
startx, starty, endx, endy = locations[idx]
cv2.rectangle(clone, (startx, starty), (endx, endy), (0,0,255), 2)
text = f'{category_dict[np.argmax(predictions[idx])]}: {int(predictions[idx].max()*100)}%'
cv2.putText(clone, text, (startx, starty+15), cv2.FONT_HERSHEY_SIMPLEX, .5, (0,255,0),2)
bounding_cnt += 1
if bounding_cnt == 0:
pred_idx= [idx for idx, i in enumerate(predictions) if np.argmax(i) == cat_index]
cat_locations = np.array([locations[i] for i in pred_idx])
nms = non_max_suppression(cat_locations)
if len(nms)==0:
cat_predictions = predictions[:,cat_index]
pred_max_idx = np.argmax(cat_predictions)
pred_max = cat_predictions[pred_max_idx]
pred_max_window = locations[pred_max_idx]
startx, starty, endx, endy = pred_max_window
cv2.rectangle(clone, (startx, starty), (endx, endy), (0,0,255),2)
text = f'{category_dict[cat_index]}: {int(pred_max*100)}%'
cv2.putText(clone, text, (startx, starty+15), cv2.FONT_HERSHEY_SIMPLEX, .5, (0,255,0),2)
for idx in nms:
startx, starty, endx, endy = cat_locations[idx]
cv2.rectangle(clone, (startx, starty), (endx, endy), (0,0,255), 2)
text = f'{category_dict[np.argmax(predictions[pred_idx[idx]])]}: {int(predictions[pred_idx[idx]].max()*100)}%'
cv2.putText(clone, text, (startx, starty+15), cv2.FONT_HERSHEY_SIMPLEX, .5, (0,255,0),2)
# pick = func.non_max_suppression(locations, probs = None)
# for idx in pick:
# startx, startx, endx, endy = locations[idx]
# cv2.rectangle(clone, (startx, starty), (endx, endy), (0,0,255), 2)
print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
cv2.imshow(f'Test', np.hstack([clone, clone2]))
cv2.waitKey(0)
ss.clear()
return predictions
# def get_lime_predictions(base_folder, model, dim, save_name= None, iter = 3500):
# from lime import lime_image
# '''This function will take a base folder containing images that will be run through the LIME package. It will save the figure if a
# save_name is passed. '''
# lime_images = []
# original_images = []
# for file in os.listdir(base_folder):
# print(f'Creating Lime Image for {file}')
# path = f'{base_folder}/{file}'
# img = get_image_value(path, dim)
# original_images.append(img)
# explainer = lime_image.LimeImageExplainer()
# explanation = explainer.explain_instance(img, model.predict, top_labels = 5, hide_color = 0, num_samples = iter)
# temp, mask = explanation.get_image_and_mask(explanation.top_labels[0], positive_only = False, num_features = 10,
# hide_rest = False)
# lime_img = mark_boundaries(temp/2 + .5, mask)
# lime_images.append(lime_img)
# lime_images = np.hstack(lime_images)
# original_images = np.hstack(original_images)
# joined_images = np.vstack([original_images, lime_images])
# # cv2.imshow('Lime', joined_images)
# # cv2.waitKey(0)
# # cv2.destroyAllWindows()
# plt.figure(figsize = (13,13))
# plt.imshow(joined_images)
# if save_name:
# plt.savefig(f'{save_name}')
# # cv2.imwrite(save_name, joined_images)
# def get_vid_frames(path, model, dim, vid_name, edge = False):
# '''This function will take a path to an mp4 file. After splitting the video into separate frames, it will run each frame through the model that is provided and create bounding boxes around each area the model believes is a weapon. Once the bounding boxes are made, the function will combine each frame back into a video and save it to the path specified above. This function is used for creating bounding boxes within a video'''
# from tqdm import tqdm
# vid = cv2.VideoCapture(path)
# total_frames = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
# pbar = tqdm(total = total_frames, desc = f'Splitting Video Into {total_frames} Frames')
# images = []
# sucess =1
# while True:
# try:
# success, img = vid.read()
# img = cv2.resize(img, dim, interpolation = cv2.INTER_CUBIC)
# images.append(img)
# pbar.update(1)
# except:
# break
# pbar.close()
# images = np.array(images)
# clones = []
# print(f'Creating {vid_name}.mp4')
# for img in tqdm(images, desc = 'Getting Base Prediction and Extracting Sliding Window... Sit Back, This Will Take A While'):
# if edge == True:
# img2 = img.reshape(1, img.shape[0], img.shape[1], 1)
# else:
# img2 = img.reshape(1, img.shape[0], img.shape[1], 3)
# clone = img.copy()
# pred = model.predict(img2)[0]
# category_dict = {0: 'No Weapon', 1: 'Handgun', 2: 'Rifle'}
# cat_index = np.argmax(pred)
# cat = category_dict[cat_index]
# #if the prediction is non_weapon then continue to the next without creating a bounding box
# if cat_index == 0:
# clones.append(clone)
# continue
# if cat in ['Handgun', 'Rifle']:
# cat = 'Weapon'
# ss = cv2.ximgproc.segmentation.createSelectiveSearchSegmentation()
# ss.setBaseImage(img)
# # ss.switchToSelectiveSearchQuality()
# ss.switchToSelectiveSearchFast()
# rects = ss.process()
# windows = []
# locations = []
# for x, y, w,h in rects[:1001]:
# startx = x
# starty = y
# endx = x+w
# endy = y+h
# roi = img[starty:endy, startx:endx]
# if edge == True:
# roi = get_edged(roi, dim = dim)
# roi = cv2.resize(roi, dsize =dim, interpolation = cv2.INTER_CUBIC)
# windows.append(roi)
# locations.append((startx, starty, endx, endy))
# windows = np.array(windows)
# if edge == True:
# windows = windows.reshape(windows.shape[0], windows.shape[1], windows.shape[2], 1)
# else:
# windows = windows.reshape(windows.shape[0], windows.shape[1], windows.shape[2], 3)
# windows = np.array(windows)
# locations = np.array(locations)
# predictions = model.predict(windows)
# nms = non_max_suppression(locations)
# bounding_cnt = 0
# for idx in nms:
# if np.argmax(predictions[idx]) != cat_index:
# continue
# startx, starty, endx, endy = locations[idx]
# cv2.rectangle(clone, (startx, starty), (endx, endy), (0,0,255), 2)
# text = f'{cat}: {int(predictions[idx].max()*100)}%'
# cv2.putText(clone, text, (startx, starty+15), cv2.FONT_HERSHEY_SIMPLEX, .5, (0,255,0),2)
# bounding_cnt += 1
# if bounding_cnt == 0:
# pred_idx= [idx for idx, i in enumerate(predictions) if np.argmax(i) == cat_index]
# cat_locations = np.array([locations[i] for i in pred_idx])
# nms = non_max_suppression(cat_locations)
# if len(nms)==0:
# cat_predictions = predictions[:,cat_index]
# pred_max_idx = np.argmax(cat_predictions)
# pred_max = cat_predictions[pred_max_idx]
# pred_max_window = locations[pred_max_idx]
# startx, starty, endx, endy = pred_max_window
# cv2.rectangle(clone, (startx, starty), (endx, endy), (0,0,255),2)
# text = f'{category_dict[cat_index]}: {int(pred_max*100)}%'
# cv2.putText(clone, text, (startx, starty+15), cv2.FONT_HERSHEY_SIMPLEX, .5, (0,255,0),2)
# for idx in nms:
# startx, starty, endx, endy = cat_locations[idx]
# cv2.rectangle(clone, (startx, starty), (endx, endy), (0,0,255), 2)
# text = f'{cat}: {int(predictions[pred_idx[idx]].max()*100)}%'
# cv2.putText(clone, text, (startx, starty+15), cv2.FONT_HERSHEY_SIMPLEX, .5, (0,255,0),2)
# clones.append(clone)
vid_dim = dim
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter(f'Tests/VideoOutput/{vid_name}.mp4',fourcc, 10, vid_dim) #change the filename to whatever you would like
out_writer = [out.write(i) for i in clones]
out.release()
cv2.destroyAllWindows()
return clones
#\date Created on Mon Nov 21 15:53:15 2016
#\author agiovann
# toclean
from __future__ import division
from __future__ import print_function
from builtins import zip
from builtins import str
from builtins import map
from builtins import range
from past.utils import old_div
import cv2
import glob
try:
cv2.setNumThreads(1)
except:
print('Open CV is naturally single threaded')
try:
if __IPYTHON__:
print(1)
# this is used for debugging purposes only. allows to reload classes
# when changed
get_ipython().magic('load_ext autoreload')
get_ipython().magic('autoreload 2')
except NameError:
print('Not launched under iPython')
import caiman as cm
import numpy as np
approx = cv2.approxPolyDP(cnt, 0.01 * cv2.arcLength(cnt, True), True)
if len(approx) == 5:
cv2.drawContours(findContours, [cnt], 0, 255, -1)
elif len(approx) == 3:
cv2.drawContours(findContours, [cnt], 0, (0, 255, 0), -1)
elif len(approx) == 4:
cv2.drawContours(findContours, [cnt], 0, (0, 0, 255), -1)
else:
cv2.drawContours(findContours, [cnt], 0, (255, 0, 255), -1)
# Print pictures
cv2.imshow('findContours', findContours)
if __name__ == "__main__":
cv2.setNumThreads(4)
cap = cv2.VideoCapture(0)
numberOfFrames = 0
while(1):
if numberOfFrames == 0:
startT1 = time.time()
pointToLongerDistance(cap)
findObject(cap)
numberOfFrames = numberOfFrames + 1
if numberOfFrames > 80:
endT1 = time.time()
