def test_interface_create_dir_fail(self): dummy_class = get_descriptor_modules()['Dummy'] dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir) dirpath = osp.join(osp.dirname(__file__), 'work/existing') tools.assert_true(osp.isfile(dirpath)) dummy_inst.create_dir(dirpath)
def test_interface_get_prefix_abs(self): dummy_class = get_descriptor_modules()['Dummy'] dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir) # Abs path pfx, key = dummy_inst.get_video_prefix('/home/user/data/HVC904576.flv') tools.assert_equal(pfx, '576') tools.assert_equal(key, '904576')
def test_interface_get_prefix_short(self): dummy_class = get_descriptor_modules()['Dummy'] dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir) # not 6-digit key pfx, key = dummy_inst.get_video_prefix('HVC5761.video') tools.assert_equal(pfx, '761') tools.assert_equal(key, '5761')
def test_interface_get_prefix_rel(self): dummy_class = get_descriptor_modules()['Dummy'] dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir) # Relative path pfx, key = dummy_inst.get_video_prefix('HVC764768.mp4') tools.assert_equal(pfx, '768') tools.assert_equal(key, '764768')
def test_plugin_importer(self): """ testing plugin gather method, which tests that all plugins are structured correctly that retrieval of a known dummy class works. """ descr_modules = get_descriptor_modules() tools.assert_in('Dummy', descr_modules.keys()) tools.assert_true( issubclass(descr_modules['Dummy'], VCDWorkerInterface))
def test_plugin_importer(self): """ testing plugin gather method, which tests that all plugins are structured correctly that retrieval of a known dummy class works. """ descr_modules = get_descriptor_modules() tools.assert_in('Dummy', descr_modules.keys()) tools.assert_true(issubclass(descr_modules['Dummy'], VCDWorkerInterface))
def test_interface_create_dir(self): dummy_class = get_descriptor_modules()['Dummy'] dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir) dirpath = osp.join(osp.dirname(__file__), 'work/example_dir') tools.assert_false(osp.isdir(dirpath)) p = dummy_inst.create_dir(dirpath) tools.assert_true(osp.isabs(p)) tools.assert_true(osp.isdir(dirpath)) tools.assert_true(osp.isdir(p)) os.rmdir(dirpath) # clean-up
def test_frame_predictor(self): dummy_class = get_descriptor_modules()['Dummy'] dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir) frames = dummy_inst._get_frames_for_interval(10, 1, 0, 1.0) tools.assert_equal(frames, range(10)) frames = dummy_inst._get_frames_for_interval(10, 1, 0, 1.0, 3) tools.assert_equal(frames, range(3)) frames = dummy_inst._get_frames_for_interval(600, 20, 1.5, 1.0, 10.0) tools.assert_equal(frames, [30, 50, 70, 90, 110, 130, 150, 170, 190])
def test_dummy(self): """ Test creating dummy instance and methods, including interface methods. Default configuration parameters should be functional. """ dummy_class = get_descriptor_modules()['Dummy'] dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir) elem = dummy_inst.process_video('/foo/bar/HVC123456.mp4') self.log.info("VCDStoreElement type: %s", VCDStoreElement) tools.assert_is_instance(elem, VCDStoreElement) tools.assert_equal(elem.descriptor_id, "Dummy") tools.assert_equal(elem.video_id, 123456) #noinspection PyTypeChecker # reason -> because this is how numpy works tools.assert_true(all(elem.feat_vec == numpy.array((1, 2, 3, 4, 5))))
def process_video(a): """ a is the string ObjectId of the video to be processed """ # Setup the task from SMQTK_Backend.VCDWorkers import descriptor_modules metastr = {'state': 1, 'label': "Inspection done", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr # Prepare to fetch the file from database (local pymongo database) conn = pymongo.Connection() datadb = conn["files"] gf = gridfs.GridFS(datadb, "video") fileobj = gf.get(bson.ObjectId(a)) # Create the directory structure for the file filename = osp.join(app.config["WORK_DIR"], a, "HVC123456.mp4") directory = os.path.dirname(filename) if not os.path.exists(directory): os.makedirs(directory) os.makedirs(directory + "/work") os.makedirs(directory + "/images") # Download the file here for fout = open(filename, "wb") fout.write(fileobj.read()) metastr = {'state': 2, 'label': "File Saved", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr # Setup color descriptors mods = descriptor_modules.get_descriptor_modules() cd = mods["colordescriptor"] cdconf = cd.generate_config() cdobj = cd(cdconf, osp.join(app.config["WORK_DIR"], a, "work"), osp.join(app.config["WORK_DIR"], a, "images")) # Generate frames cdobj.generate_frames(filename) metastr = {'state': 3, 'label': "Frame extraction complete", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr # Create sprite create_sprite(osp.join(app.config["WORK_DIR"], a, "images/456/123456")) metastr = {'state': 4, 'label': "Sprite creation complete", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr # Aggregate features cdobj.descriptor_generation(filename) metastr = {'state': 5, 'label': "Features generated", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr results = cdobj.process_video(filename) # print results for aresult in results: save_feature_vector(a, aresult) # Save results metastr = {'state': 6, 'label': "Features aggregated", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print "Done ", a return a
def generate_config(config=None): """ Generate a sample configuration object. :param config: An optional configuration object to add to. :type config: None or SafeConfigCommentParser :return: Generated configuration object :rtype: SafeConfigCommentParser """ log = logging.getLogger() if config is None: log.debug('generating new config object') config = SafeConfigCommentParser() log.info("Adding vcd_scheduler section...") sect = MPIS_CONFIG_SECT config.add_section(sect) config.set( sect, 'image_directory', '# REQUIRED', "The root directory where imagery will be extracted and stored. " "This is a common structure across any descriptors that want to " "extract imagery.") config.set(sect, 'run_work_directory', '# REQUIRED', "The base working directory for this run.") config.set( sect, 'video_work_list', '# REQUIRED', "A test time with a new-line separated list of video files to " "process.") config.set(sect, 'log_dir', "%(run_work_directory)s/logs", "Directory where log files from the run are to be stored.") config.set( sect, 'target_vcd_store', 'SQLiteVCDStore.db', "The sqlite3 database file results will be accumulated into. If " "this is given as a relative path it will be considered " "relative to the provided work directory.") config.set( sect, 'vcd_store_overwrite', 'false', "When the server node attempts to store VCDStoreElement " "objects, overwrite existing entries if found. Else, if this is " "false, favor the existing entries and roll back the insertion " "attempt.") # TODO: [generate_config] (see below) # This tells the vcd_scheduler which descriptor client module to load # This should also tell us what VCDWorker implementation to use. This # requires that we shift over to using the VCDWorker implementation # structure instead of the very similar function thing we currently have # going on... # Should be straight forward to convert current stuff to VCDWorker impl. # Need to modify VCDWorker impls with lessons learned here. # avail_modes = get_descriptor_client_methods().keys() descr_modules = get_descriptor_modules() config.set( sect, 'descriptor_mode', '# REQUIRED', "The descriptor mode to run in. This defined which descriptor " "will process the videos listed in the given work list file.\n" "\n" "The available modes are as follows:\n" " - '%s'" % "'\n - '".join(descr_modules.keys())) # Append descriptor module configs to this one for name, m in descr_modules.items(): log.info("Adding '%s' module section...", name) m.generate_config(config) return config
def client_side(config, dry_run=False): """ Client side implementation """ global file_count log_print("Client starting") # Get pertinent configuration values work_dir = config.get(MPIS_CONFIG_SECT, 'run_work_directory') image_dir = config.get(MPIS_CONFIG_SECT, 'image_directory') descr_mode = config.get(MPIS_CONFIG_SECT, 'descriptor_mode') # get configured module descr_module = get_descriptor_modules()[descr_mode] descriptor = descr_module(config, work_dir, image_dir) log_print("module name: %s" % descr_module.__name__) while True: log_print("sending request for work") comm.send('feed-me', dest=0) reply = comm.recv(source=0) task_orders = reply.strip().split(' ') # end of input if task_orders[0] == 'die-sucka': log_print("Received terminate command") break # video file to process elif task_orders[0] == 'proc-file': video_file = task_orders[1] if not dry_run: log_print("generating store elements via descriptor") vcd_store_elems = descriptor.process_video(video_file) # if any elements were returned at all... if vcd_store_elems: assert isinstance(vcd_store_elems, _abcoll.Iterable), \ "Module process_file function did not return an " \ "iterable!" # Store elements to file in a sub-location within the work # directory, sending the paths to those files back to the # server node. dump_file = get_vcd_element_file(descr_mode, video_file, work_dir) log_print("dumping serialized store elements -> %s" % dump_file) with open(dump_file, 'w') as f: cPickle.dump(tuple(vcd_store_elems), f, cPickle.HIGHEST_PROTOCOL) # Send back to the server node the serialized files for # storage log_print("Sending serialized file path to server") comm.send('complete %s' % dump_file, dest=0) else: log_print("Dry run on video: %s" % video_file) file_count += 1 # add other commands here log_print("**** Client side exiting") print_client_stats()
def client_side(config, dry_run=False): """ Client side implementation """ global file_count log_print("Client starting") # Get pertinent configuration values work_dir = config.get(MPIS_CONFIG_SECT, "run_work_directory") image_dir = config.get(MPIS_CONFIG_SECT, "image_directory") descr_mode = config.get(MPIS_CONFIG_SECT, "descriptor_mode") # get configured module descr_module = get_descriptor_modules()[descr_mode] descriptor = descr_module(config, work_dir, image_dir) log_print("module name: %s" % descr_module.__name__) while True: log_print("sending request for work") comm.send("feed-me", dest=0) reply = comm.recv(source=0) task_orders = reply.strip().split(" ") # end of input if task_orders[0] == "die-sucka": log_print("Received terminate command") break # video file to process elif task_orders[0] == "proc-file": video_file = task_orders[1] if not dry_run: log_print("generating store elements via descriptor") vcd_store_elems = descriptor.process_video(video_file) # if any elements were returned at all... if vcd_store_elems: assert isinstance(vcd_store_elems, _abcoll.Iterable), ( "Module process_file function did not return an " "iterable!" ) # Store elements to file in a sub-location within the work # directory, sending the paths to those files back to the # server node. dump_file = get_vcd_element_file(descr_mode, video_file, work_dir) log_print("dumping serialized store elements -> %s" % dump_file) with open(dump_file, "w") as f: cPickle.dump(tuple(vcd_store_elems), f, cPickle.HIGHEST_PROTOCOL) # Send back to the server node the serialized files for # storage log_print("Sending serialized file path to server") comm.send("complete %s" % dump_file, dest=0) else: log_print("Dry run on video: %s" % video_file) file_count += 1 # add other commands here log_print("**** Client side exiting") print_client_stats()
def process_video(a): """ a is the string ObjectId of the video to be processed """ # Setup the task from SMQTK_Backend.VCDWorkers import descriptor_modules metastr = {'state': 1, 'label': "Inspection done", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr # Prepare to fetch the file from database (local pymongo database) conn = pymongo.Connection() datadb = conn["files"] gf = gridfs.GridFS(datadb, "video") fileobj = gf.get(bson.ObjectId(a)) # Create the directory structure for the file filename = osp.join(app.config["WORK_DIR"], a, "HVC123456.mp4") directory = os.path.dirname(filename) if not os.path.exists(directory): os.makedirs(directory) os.makedirs(directory + "/work") os.makedirs(directory + "/images") # Download the file here for fout = open(filename, "wb") fout.write(fileobj.read()) metastr = {'state': 2, 'label': "File Saved", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr # Setup color descriptors mods = descriptor_modules.get_descriptor_modules() cd = mods["colordescriptor"] cdconf = cd.generate_config() cdobj = cd(cdconf, osp.join(app.config["WORK_DIR"], a, "work"), osp.join(app.config["WORK_DIR"], a, "images") ) # Generate frames cdobj.generate_frames(filename) metastr = {'state': 3, 'label': "Frame extraction complete", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr # Create sprite create_sprite(osp.join(app.config["WORK_DIR"], a, "images/456/123456")) metastr = {'state': 4, 'label': "Sprite creation complete", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr # Aggregate features cdobj.descriptor_generation(filename) metastr = {'state': 5, 'label': "Features generated", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print metastr results = cdobj.process_video(filename) # print results for aresult in results: save_feature_vector(a, aresult) # Save results metastr = {'state': 6, 'label': "Features aggregated", 'id': a} celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr)) print "Done ", a return a
def generate_config(config=None): """ Generate a sample configuration object. :param config: An optional configuration object to add to. :type config: None or SafeConfigCommentParser :return: Generated configuration object :rtype: SafeConfigCommentParser """ log = logging.getLogger() if config is None: log.debug("generating new config object") config = SafeConfigCommentParser() log.info("Adding vcd_scheduler section...") sect = MPIS_CONFIG_SECT config.add_section(sect) config.set( sect, "image_directory", "# REQUIRED", "The root directory where imagery will be extracted and stored. " "This is a common structure across any descriptors that want to " "extract imagery.", ) config.set(sect, "run_work_directory", "# REQUIRED", "The base working directory for this run.") config.set( sect, "video_work_list", "# REQUIRED", "A test time with a new-line separated list of video files to " "process.", ) config.set( sect, "log_dir", "%(run_work_directory)s/logs", "Directory where log files from the run are to be stored." ) config.set( sect, "target_vcd_store", "SQLiteVCDStore.db", "The sqlite3 database file results will be accumulated into. If " "this is given as a relative path it will be considered " "relative to the provided work directory.", ) config.set( sect, "vcd_store_overwrite", "false", "When the server node attempts to store VCDStoreElement " "objects, overwrite existing entries if found. Else, if this is " "false, favor the existing entries and roll back the insertion " "attempt.", ) # TODO: [generate_config] (see below) # This tells the vcd_scheduler which descriptor client module to load # This should also tell us what VCDWorker implementation to use. This # requires that we shift over to using the VCDWorker implementation # structure instead of the very similar function thing we currently have # going on... # Should be straight forward to convert current stuff to VCDWorker impl. # Need to modify VCDWorker impls with lessons learned here. # avail_modes = get_descriptor_client_methods().keys() descr_modules = get_descriptor_modules() config.set( sect, "descriptor_mode", "# REQUIRED", "The descriptor mode to run in. This defined which descriptor " "will process the videos listed in the given work list file.\n" "\n" "The available modes are as follows:\n" " - '%s'" % "'\n - '".join(descr_modules.keys()), ) # Append descriptor module configs to this one for name, m in descr_modules.items(): log.info("Adding '%s' module section...", name) m.generate_config(config) return config
def test_interface_get_prefix_fail(self): dummy_class = get_descriptor_modules()['Dummy'] dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir) # Expects an HVC formatted file name dummy_inst.get_video_prefix("foobat123456.mp4")
def test_interface_tempify_filename(self): dummy_class = get_descriptor_modules()['Dummy'] dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir) t = dummy_inst.tempify_filename('foo.txt') tools.assert_equal(t, 'foo.txt.TEMP')