def test_select(self): with deepstar_path(): VideoModel().insert('test1', 'test2') frame_set_model = FrameSetModel() frame_set_model.insert(1) result = frame_set_model.select(1) self.assertEqual(result, (1, 1))
def select_export_dir(self, frame_set_ids, target_dir, opts={}): """ This method exports frame sets to a directory. :param list frame_set_ids: The list of frame set IDs to export. :param str target_dir: The directory to which to export the frame sets. :param dict opts: The dict of options. :raises: CommandLineRouteHandlerError :rtype: None """ frame_set_model = FrameSetModel() for frame_set_id in frame_set_ids: result = frame_set_model.select(frame_set_id) if result is None: raise CommandLineRouteHandlerError( f'Frame set with ID {frame_set_id:08d} not found') frame_model = FrameModel() length = int(os.environ.get('MODEL_LIST_LENGTH', '100')) count = 0 for fid in frame_set_ids: offset = 0 frame_set_path = FrameSetSubDir.path(fid) while True: results = frame_model.list(fid, length=length, offset=offset, rejected=False) if not results: break for frame_id, _, _ in results: file_path = FrameFile().path(frame_set_path, frame_id, 'jpg') if 'format' in opts: filename = opts['format'] % frame_id else: filename = os.path.basename(file_path) target_path = os.path.join(target_dir, filename) shutil.copy(file_path, target_path) count += 1 debug( f'Frame with ID {frame_id:08d} at {file_path} ' f'exported to {target_path}', 4) offset += length debug(f'{count} frames were successfully exported to {target_dir}', 3)
def test_delete_one(self): with deepstar_path(): with mock.patch.dict(os.environ, {'DEBUG_LEVEL': '0'}): route_handler = VideoCommandLineRouteHandler() video_0001 = os.path.dirname(os.path.realpath(__file__)) + '/../../support/video_0001.mp4' # noqa route_handler.insert_file(video_0001) route_handler.select_extract([1]) # files self.assertTrue(os.path.exists(FrameSetSubDir.path(1))) # db frame_set_model = FrameSetModel() self.assertIsNotNone(frame_set_model.select(1)) frame_model = FrameModel() self.assertIsNotNone(frame_model.select(1)) self.assertIsNotNone(frame_model.select(2)) self.assertIsNotNone(frame_model.select(3)) self.assertIsNotNone(frame_model.select(4)) self.assertIsNotNone(frame_model.select(5)) args = ['main.py', 'delete', 'frame_sets', '1'] opts = {} route_handler = FrameSetCommandLineRouteHandler() try: sys.stdout = StringIO() route_handler.handle(args, opts) actual = sys.stdout.getvalue().strip() finally: sys.stdout = sys.__stdout__ # stdout self.assertEqual(actual, 'Frame set 1 was successfully deleted') # db self.assertIsNone(frame_set_model.select(1)) self.assertIsNone(frame_model.select(1)) self.assertIsNone(frame_model.select(2)) self.assertIsNone(frame_model.select(3)) self.assertIsNone(frame_model.select(4)) self.assertIsNone(frame_model.select(5)) # files self.assertFalse(os.path.exists(FrameSetSubDir.path(1)))
def test_insert_images_does_not_insert_non_images(self): with deepstar_path(): tmpdir = os.environ['DEEPSTAR_PATH'] + '/test' os.mkdir(tmpdir) with open(os.path.join(tmpdir, 'test'), 'w') as file_: file_.write('test') with open(os.path.join(tmpdir, 'test.txt'), 'w') as file_: file_.write('test') args = ['main.py', 'insert', 'frame_sets', 'images', tmpdir] opts = {} route_handler = FrameSetCommandLineRouteHandler() try: sys.stdout = StringIO() route_handler.handle(args, opts) actual = sys.stdout.getvalue().strip() finally: sys.stdout = sys.__stdout__ # stdout self.assertEqual(actual, 'frame_set_id=1, fk_videos=None') # db result = FrameSetModel().select(1) self.assertEqual(result, (1, None)) result = FrameModel().list(0)
def test_list(self): with deepstar_path(): VideoModel().insert('test1', 'test2') FrameSetModel().insert(1) frame_model = FrameModel() frame_model.insert(1, 0) frame_model.insert(1, 1) frame_model.insert(1, 0) result = frame_model.list(1) self.assertEqual(len(result), 3) self.assertEqual(result[0], (1, 1, 0)) self.assertEqual(result[1], (2, 1, 1)) self.assertEqual(result[2], (3, 1, 0)) result = frame_model.list(1, length=2) self.assertEqual(len(result), 2) self.assertEqual(result[0], (1, 1, 0)) self.assertEqual(result[1], (2, 1, 1)) result = frame_model.list(1, offset=1) self.assertEqual(len(result), 2) self.assertEqual(result[0], (2, 1, 1)) self.assertEqual(result[1], (3, 1, 0)) result = frame_model.list(1, length=1, offset=1) self.assertEqual(len(result), 1) self.assertEqual(result[0], (2, 1, 1)) result = frame_model.list(1, rejected=False) self.assertEqual(len(result), 2) self.assertEqual(result[0], (1, 1, 0)) self.assertEqual(result[1], (3, 1, 0))
def test_fk_transform_sets_fk_frames_constraint(self): with deepstar_path(): FrameSetModel().insert(None) TransformSetModel().insert('test', 1) with self.assertRaises(sqlite3.IntegrityError): TransformModel().insert(1, 1, '{}', 0)
def test_video_select_extract(self): with deepstar_path(): video_0001 = os.path.dirname(os.path.realpath( __file__)) + '/../../support/video_0001.mp4' # noqa shutil.copyfile(video_0001, VideoFile.path('video_0001.mp4')) VideoModel().insert('test', 'video_0001.mp4') frame_set_id = DefaultVideoSelectExtractPlugin( ).video_select_extract(1) # noqa self.assertEqual(frame_set_id, 1) # db result = FrameSetModel().select(1) self.assertEqual(result, (1, 1)) result = FrameModel().list(1) self.assertEqual(len(result), 5) self.assertEqual(result[0], (1, 1, 0)) self.assertEqual(result[1], (2, 1, 0)) self.assertEqual(result[2], (3, 1, 0)) self.assertEqual(result[3], (4, 1, 0)) self.assertEqual(result[4], (5, 1, 0)) # files p1 = FrameSetSubDir.path(1) # frames self.assertIsInstance(cv2.imread(FrameFile.path(p1, 1, 'jpg')), np.ndarray) # noqa self.assertIsInstance(cv2.imread(FrameFile.path(p1, 2, 'jpg')), np.ndarray) # noqa self.assertIsInstance(cv2.imread(FrameFile.path(p1, 3, 'jpg')), np.ndarray) # noqa self.assertIsInstance(cv2.imread(FrameFile.path(p1, 4, 'jpg')), np.ndarray) # noqa self.assertIsInstance(cv2.imread(FrameFile.path(p1, 5, 'jpg')), np.ndarray) # noqa # thumbnails self.assertEqual( cv2.imread(FrameFile.path(p1, 1, 'jpg', '192x192')).shape[1], 192) # noqa self.assertEqual( cv2.imread(FrameFile.path(p1, 2, 'jpg', '192x192')).shape[1], 192) # noqa self.assertEqual( cv2.imread(FrameFile.path(p1, 3, 'jpg', '192x192')).shape[1], 192) # noqa self.assertEqual( cv2.imread(FrameFile.path(p1, 4, 'jpg', '192x192')).shape[1], 192) # noqa self.assertEqual( cv2.imread(FrameFile.path(p1, 5, 'jpg', '192x192')).shape[1], 192) # noqa
def test_select_clone_one(self): with deepstar_path(): with mock.patch.dict(os.environ, {'DEBUG_LEVEL': '0'}): route_handler = VideoCommandLineRouteHandler() video_0001 = os.path.dirname(os.path.realpath(__file__)) + '/../../support/video_0001.mp4' # noqa route_handler.insert_file(video_0001) route_handler.select_extract([1]) args = ['main.py', 'select', 'frame_sets', '1', 'clone'] opts = {} route_handler = FrameSetCommandLineRouteHandler() try: sys.stdout = StringIO() with mock.patch.dict(os.environ, {'MODEL_LIST_LENGTH': '4'}): route_handler.handle(args, opts) actual = sys.stdout.getvalue().strip() finally: sys.stdout = sys.__stdout__ # stdout self.assertEqual(actual, 'frame_set_id=2, fk_videos=1') # db result = FrameSetModel().select(2) self.assertEqual(result, (2, 1)) result = FrameModel().list(2) self.assertEqual(len(result), 5) self.assertEqual(result[0], (6, 2, 0)) self.assertEqual(result[1], (7, 2, 0)) self.assertEqual(result[2], (8, 2, 0)) self.assertEqual(result[3], (9, 2, 0)) self.assertEqual(result[4], (10, 2, 0)) # files p1 = FrameSetSubDir.path(2) # frames self.assertTrue(os.path.isfile(FrameFile.path(p1, 6, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 7, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 8, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 9, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 10, 'jpg'))) # thumbnails self.assertTrue(os.path.isfile(FrameFile.path(p1, 6, 'jpg', '192x192'))) # noqa self.assertTrue(os.path.isfile(FrameFile.path(p1, 7, 'jpg', '192x192'))) # noqa self.assertTrue(os.path.isfile(FrameFile.path(p1, 8, 'jpg', '192x192'))) # noqa self.assertTrue(os.path.isfile(FrameFile.path(p1, 9, 'jpg', '192x192'))) # noqa self.assertTrue(os.path.isfile(FrameFile.path(p1, 10, 'jpg', '192x192'))) # noqa
def test_select(self): with deepstar_path(): VideoModel().insert('test1', 'test2') FrameSetModel().insert(1) transform_set_model = TransformSetModel() transform_set_model.insert('test', 1) result = transform_set_model.select(1) self.assertEqual(result, (1, 'test', 1, None))
def test_count_rejected_false(self): with deepstar_path(): VideoModel().insert('test1', 'test2') FrameSetModel().insert(1) frame_model = FrameModel() frame_model.insert(1, 0) frame_model.insert(1, 1) frame_model.insert(1, 0) self.assertEqual(frame_model.count(1, rejected=False), 2)
def test_fk_transform_sets_on_delete_cascade(self): with deepstar_path(): FrameSetModel().insert(None) FrameModel().insert(1, 0) transform_set_model = TransformSetModel() transform_set_model.insert('test', 1) transform_model = TransformModel() transform_model.insert(1, 1, '{}', 0) self.assertEqual(transform_model.list(1), [(1, 1, 1, '{}', 0)]) transform_set_model.delete(1) self.assertIsNone(transform_model.list(1))
def test_delete(self): with deepstar_path(): frame_set_model = FrameSetModel() frame_set_model.insert(1) result = frame_set_model.select(1) self.assertEqual(result, (1, 1)) result = frame_set_model.delete(1) self.assertTrue(result) result = frame_set_model.select(1) self.assertIsNone(result)
def select_clone(self, frame_set_ids): """ This method clones frame sets to new frame sets. :param list(int) frame_set_ids: The frame set IDs. :raises: CommandLineRouteHandlerError :rtype: None """ frame_set_model = FrameSetModel() for frame_set_id in frame_set_ids: result = frame_set_model.select(frame_set_id) if result is None: raise CommandLineRouteHandlerError( f'Frame set with ID {frame_set_id:08d} not found') for frame_set_id in frame_set_ids: result = frame_set_model.select(frame_set_id) self.select_merge([frame_set_id], result[1], rejected=True)
def test_select(self): with deepstar_path(): FrameSetModel().insert(None) FrameModel().insert(1, 0) TransformSetModel().insert('test', 1) transform_model = TransformModel() transform_model.insert(1, 1, '{}', 0) result = transform_model.select(1) self.assertEqual(result, (1, 1, 1, '{}', 0))
def test_count(self): with deepstar_path(): VideoModel().insert('test1', 'test2') FrameSetModel().insert(1) frame_model = FrameModel() frame_model.insert(1, 0) frame_model.insert(1, 0) frame_model.insert(1, 0) self.assertEqual(frame_model.count(1), 3)
def test_list(self): with deepstar_path(): video_model = VideoModel() video_model.insert('test1', 'test2') video_model.insert('test3', 'test4') video_model.insert('test5', 'test6') frame_set_model = FrameSetModel() frame_set_model.insert(1) frame_set_model.insert(2) frame_set_model.insert(3) args = ['main.py', 'list', 'frame_sets'] opts = {} route_handler = FrameSetCommandLineRouteHandler() try: sys.stdout = StringIO() route_handler.handle(args, opts) actual = sys.stdout.getvalue().strip() finally: sys.stdout = sys.__stdout__ # stdout expected = textwrap.dedent(''' 3 results id | fk_videos ------------- 1 | 1 2 | 2 3 | 3''').strip() self.assertEqual(actual, expected)
def test_select_extract_one(self): with deepstar_path(): video_0001 = os.path.dirname(os.path.realpath(__file__)) + '/../../support/video_0001.mp4' # noqa shutil.copyfile(video_0001, VideoFile.path('video_0001.mp4')) VideoModel().insert('test', 'video_0001.mp4') args = ['main.py', 'select', 'videos', '1', 'extract'] opts = {} route_handler = VideoCommandLineRouteHandler() try: sys.stdout = StringIO() route_handler.handle(args, opts) actual = sys.stdout.getvalue().strip() finally: sys.stdout = sys.__stdout__ # stdout self.assertEqual(actual, 'frame_set_id=1, video_id=1') # db result = FrameSetModel().select(1) self.assertEqual(result, (1, 1)) result = FrameModel().list(1) self.assertEqual(len(result), 5) self.assertEqual(result[0], (1, 1, 0)) self.assertEqual(result[1], (2, 1, 0)) self.assertEqual(result[2], (3, 1, 0)) self.assertEqual(result[3], (4, 1, 0)) self.assertEqual(result[4], (5, 1, 0)) # files p1 = FrameSetSubDir.path(1) # frames self.assertTrue(os.path.isfile(FrameFile.path(p1, 1, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 2, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 3, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 4, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 5, 'jpg'))) # thumbnails self.assertTrue(os.path.isfile(FrameFile.path(p1, 1, 'jpg', '192x192'))) # noqa self.assertTrue(os.path.isfile(FrameFile.path(p1, 2, 'jpg', '192x192'))) # noqa self.assertTrue(os.path.isfile(FrameFile.path(p1, 3, 'jpg', '192x192'))) # noqa self.assertTrue(os.path.isfile(FrameFile.path(p1, 4, 'jpg', '192x192'))) # noqa self.assertTrue(os.path.isfile(FrameFile.path(p1, 5, 'jpg', '192x192'))) # noqa
def insert_images(self, images_path, opts): """ This method inserts a frame set from a directory of images. :param str images_path: The path to the directory containing the images to insert. :param dict opts: The dict of options. :raises: CommandLineRouteHandlerError :rtype: None """ if not os.path.isdir(images_path): raise CommandLineRouteHandlerError( f'The path at {images_path} is not a directory') frame_set_id = FrameSetModel().insert(None) p1 = FrameSetSubDir.path(frame_set_id) os.makedirs(p1) frame_model = FrameModel() for image_path in glob.glob(os.path.join(images_path, '*')): ext = os.path.splitext(image_path)[1].lower() if ext not in ['.jpg', '.png']: debug( f"Skipped image at {image_path} because it does not " f"have a '.jpg' or '.png' file extension", 4) continue image = cv2.imread(image_path) frame_id = frame_model.insert(frame_set_id, 0) p2 = FrameFile.path(p1, frame_id, 'jpg') cv2.imwrite(p2, image, [cv2.IMWRITE_JPEG_QUALITY, 100]) thumbnail = imutils.resize(image, width=192, height=192) p3 = FrameFile.path(p1, frame_id, 'jpg', '192x192') cv2.imwrite(p3, thumbnail, [cv2.IMWRITE_JPEG_QUALITY, 100]) debug( f'Image at {image_path} inserted with ID {frame_id:08d} at ' f'{p2} and {p3}', 4) debug(f'frame_set_id={frame_set_id}, fk_videos=None', 3)
def list(self, frame_set_id, length=-1, offset=None, rejected=True): """ This method performs a list operation. :param int frame_set_id: The frame set ID. :param int length: The optional length. :param int offset: The optional offset. :param bool rejected: True if should include rejected frames else False if should not. The defalut value is True. :rtype: tuple """ result = FrameSetModel().select(frame_set_id) if result is None: return None params = (frame_set_id, ) query = """ SELECT id, fk_frame_sets, rejected FROM frames WHERE fk_frame_sets = ? """ if rejected is False: query += ' AND rejected = 0' if length is not None: query += ' LIMIT ?' params += (length, ) if offset is not None: query += ' OFFSET ?' params += (offset, ) result = Model.execute(query, params) return result.fetchall()
def mock_transform_set(self): image_0006 = os.path.dirname(os.path.realpath( __file__)) + '/../../support/image_0006.jpg' # noqa FrameSetModel().insert('1') FrameModel().insert(1, 0) p1 = FrameSetSubDir.path(1) os.mkdir(p1) shutil.copy(image_0006, FrameFile.path(p1, 1, 'jpg')) MTCNNFrameSetSelectExtractPlugin().frame_set_select_extract(1, {})
def delete(self, frame_set_ids): """ This method deletes a frame set from the frame set collection. :param list(int) frame_set_ids: The frame set IDs. :raises: CommandLineRouteHandlerError :rtype: None """ frame_set_model = FrameSetModel() for frame_set_id in frame_set_ids: result = frame_set_model.select(frame_set_id) if result is None: raise CommandLineRouteHandlerError( f'Frame set with ID {frame_set_id:08d} not found') for frame_set_id in frame_set_ids: frame_set_model.delete(frame_set_id) shutil.rmtree(FrameSetSubDir.path(frame_set_id)) debug(f'Frame set {frame_set_id} was successfully deleted', 3)
def test_fk_frame_sets_on_delete_cascade(self): with deepstar_path(): frame_set_model = FrameSetModel() frame_set_model.insert(1) frame_model = FrameModel() frame_model.insert(1, 0) self.assertEqual(frame_model.list(1), [(1, 1, 0)]) frame_set_model.delete(1) self.assertIsNone(frame_model.list(1))
def select_extract(self, frame_set_ids, name, opts): """ This method extracts portions of frames to transform sets. :param list(int) frame_set_ids: The frame set IDs. :param str name: The name of the frame set extraction plugin to use. :param dict opts: The dict of options. :raises: CommandLineRouteHandlerError :rtype: None """ frame_set_model = FrameSetModel() for frame_set_id in frame_set_ids: result = frame_set_model.select(frame_set_id) if result is None: raise CommandLineRouteHandlerError( f'Frame set with ID {frame_set_id:08d} not found') plugin = Plugin.get('frame_set_select_extract', name) if plugin is None: raise CommandLineRouteHandlerError( f"'{name}' is not a valid frame set extraction plugin name") transform_set_model = TransformSetModel() for frame_set_id in frame_set_ids: transform_set_id = plugin().frame_set_select_extract( frame_set_id, opts) result = transform_set_model.select(transform_set_id) debug( f'transform_set_id={result[0]}, name={result[1]}, ' f'fk_frame_sets={result[2]}, ' f'fk_prev_transform_sets={result[3]}', 3)
def list(self): """ This method lists frame sets in the frame set collection. :rtype: None """ result = FrameSetModel().list() debug(f'{len(result)} results', 3) debug('id | fk_videos', 3) debug('-------------', 3) for r in result: debug(f'{r[0]} | {r[1]}', 3)
def test_list(self): with deepstar_path(): VideoModel().insert('test1', 'test2') frame_set_model = FrameSetModel() frame_set_model.insert(1) frame_set_model.insert(1) frame_set_model.insert(1) result = frame_set_model.list() self.assertEqual(len(result), 3) self.assertEqual(result[0], (1, 1)) self.assertEqual(result[1], (2, 1)) self.assertEqual(result[2], (3, 1))
def test_list(self): with deepstar_path(): VideoModel().insert('test1', 'test2') FrameSetModel().insert(1) transform_set_model = TransformSetModel() transform_set_model.insert('test', 1) transform_set_model.insert('test', 1) transform_set_model.insert('test', 1) result = transform_set_model.list() self.assertEqual(len(result), 3) self.assertEqual(result[0], (1, 'test', 1, None)) self.assertEqual(result[1], (2, 'test', 1, None)) self.assertEqual(result[2], (3, 'test', 1, None))
def list(self, frame_set_id): """ This method lists frames in the frame collection for a frame set. :param int frame_set_id: The frame set ID. :raises: CommandLineRouteHandlerError :rtype: None """ result = FrameSetModel().select(frame_set_id) if result is None: raise CommandLineRouteHandlerError( f'Frame set with ID {frame_set_id:08d} not found') frame_model = FrameModel() count = frame_model.count(frame_set_id) debug(f'{count} results', 3) debug('id | fk_frame_sets | rejected | (width | height)', 3) debug('------------------------------------------------', 3) if count == 0: return length = int(os.environ.get('MODEL_LIST_LENGTH', '100')) offset = 0 p1 = FrameSetSubDir.path(frame_set_id) while True: frames = frame_model.list(frame_set_id, length=length, offset=offset) if not frames: break for frame in frames: p2 = FrameFile.path(p1, frame[0], 'jpg') height, width, _ = cv2.imread(p2).shape debug( f'{frame[0]} | {frame[1]} | {frame[2]} | ({width} | ' f'{height})', 3) offset += length
def test_insert_images(self): with deepstar_path(): image_0001 = os.path.dirname(os.path.realpath(__file__)) + '/../../support/image_0001.jpg' # noqa image_0007 = os.path.dirname(os.path.realpath(__file__)) + '/../../support/image_0007.png' # noqa tmpdir = os.environ['DEEPSTAR_PATH'] + '/test' os.mkdir(tmpdir) shutil.copy(image_0001, os.path.join(tmpdir, 'image_0001.jpg')) shutil.copy(image_0001, os.path.join(tmpdir, 'image_0002.jpg')) shutil.copy(image_0007, os.path.join(tmpdir, 'image_0003.png')) shutil.copy(image_0007, os.path.join(tmpdir, 'image_0004.png')) args = ['main.py', 'insert', 'frame_sets', 'images', tmpdir] opts = {} route_handler = FrameSetCommandLineRouteHandler() try: sys.stdout = StringIO() route_handler.handle(args, opts) actual = sys.stdout.getvalue().strip() finally: sys.stdout = sys.__stdout__ # stdout self.assertEqual(actual, 'frame_set_id=1, fk_videos=None') # db result = FrameSetModel().select(1) self.assertEqual(result, (1, None)) result = FrameModel().list(1) self.assertEqual(len(result), 4) self.assertEqual(result[0], (1, 1, 0)) self.assertEqual(result[1], (2, 1, 0)) self.assertEqual(result[2], (3, 1, 0)) self.assertEqual(result[3], (4, 1, 0)) # files p1 = FrameSetSubDir.path(1) # frames self.assertTrue(os.path.isfile(FrameFile.path(p1, 1, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 2, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 3, 'jpg'))) self.assertTrue(os.path.isfile(FrameFile.path(p1, 4, 'jpg')))
def test_count(self): with deepstar_path(): FrameSetModel().insert(None) frame_model = FrameModel() frame_model.insert(1, 0) frame_model.insert(1, 0) frame_model.insert(1, 0) TransformSetModel().insert('test', 1) transform_model = TransformModel() transform_model.insert(1, 1, '{}', 0) transform_model.insert(1, 2, '{}', 0) transform_model.insert(1, 3, '{}', 0) self.assertEqual(frame_model.count(1), 3)
def test_count_rejected_false(self): with deepstar_path(): FrameSetModel().insert(None) frame_model = FrameModel() frame_model.insert(1, 0) frame_model.insert(1, 0) frame_model.insert(1, 0) TransformSetModel().insert('test', 1) transform_model = TransformModel() transform_model.insert(1, 1, '{}', 0) transform_model.insert(1, 2, '{}', 1) transform_model.insert(1, 3, '{}', 0) self.assertEqual(transform_model.count(1, rejected=False), 2)