def test_interface_create_dir_fail(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
dirpath = osp.join(osp.dirname(__file__), 'work/existing')
tools.assert_true(osp.isfile(dirpath))
dummy_inst.create_dir(dirpath)
def test_interface_create_dir_fail(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
dirpath = osp.join(osp.dirname(__file__), 'work/existing')
tools.assert_true(osp.isfile(dirpath))
dummy_inst.create_dir(dirpath)
def test_interface_get_prefix_abs(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
# Abs path
pfx, key = dummy_inst.get_video_prefix('/home/user/data/HVC904576.flv')
tools.assert_equal(pfx, '576')
tools.assert_equal(key, '904576')
def test_interface_get_prefix_short(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
# not 6-digit key
pfx, key = dummy_inst.get_video_prefix('HVC5761.video')
tools.assert_equal(pfx, '761')
tools.assert_equal(key, '5761')
def test_interface_get_prefix_abs(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
# Abs path
pfx, key = dummy_inst.get_video_prefix('/home/user/data/HVC904576.flv')
tools.assert_equal(pfx, '576')
tools.assert_equal(key, '904576')
def test_interface_get_prefix_short(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
# not 6-digit key
pfx, key = dummy_inst.get_video_prefix('HVC5761.video')
tools.assert_equal(pfx, '761')
tools.assert_equal(key, '5761')
def test_interface_get_prefix_rel(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
# Relative path
pfx, key = dummy_inst.get_video_prefix('HVC764768.mp4')
tools.assert_equal(pfx, '768')
tools.assert_equal(key, '764768')
def test_interface_get_prefix_rel(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
# Relative path
pfx, key = dummy_inst.get_video_prefix('HVC764768.mp4')
tools.assert_equal(pfx, '768')
tools.assert_equal(key, '764768')
def test_plugin_importer(self):
"""
testing plugin gather method, which tests that all plugins are
structured correctly that retrieval of a known dummy class works.
"""
descr_modules = get_descriptor_modules()
tools.assert_in('Dummy', descr_modules.keys())
tools.assert_true(
issubclass(descr_modules['Dummy'], VCDWorkerInterface))
def test_plugin_importer(self):
"""
testing plugin gather method, which tests that all plugins are
structured correctly that retrieval of a known dummy class works.
"""
descr_modules = get_descriptor_modules()
tools.assert_in('Dummy', descr_modules.keys())
tools.assert_true(issubclass(descr_modules['Dummy'],
VCDWorkerInterface))
def test_interface_create_dir(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
dirpath = osp.join(osp.dirname(__file__), 'work/example_dir')
tools.assert_false(osp.isdir(dirpath))
p = dummy_inst.create_dir(dirpath)
tools.assert_true(osp.isabs(p))
tools.assert_true(osp.isdir(dirpath))
tools.assert_true(osp.isdir(p))
os.rmdir(dirpath) # clean-up
def test_interface_create_dir(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
dirpath = osp.join(osp.dirname(__file__), 'work/example_dir')
tools.assert_false(osp.isdir(dirpath))
p = dummy_inst.create_dir(dirpath)
tools.assert_true(osp.isabs(p))
tools.assert_true(osp.isdir(dirpath))
tools.assert_true(osp.isdir(p))
os.rmdir(dirpath) # clean-up
def test_frame_predictor(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
frames = dummy_inst._get_frames_for_interval(10, 1, 0, 1.0)
tools.assert_equal(frames, range(10))
frames = dummy_inst._get_frames_for_interval(10, 1, 0, 1.0, 3)
tools.assert_equal(frames, range(3))
frames = dummy_inst._get_frames_for_interval(600, 20, 1.5, 1.0, 10.0)
tools.assert_equal(frames, [30, 50, 70, 90, 110, 130, 150, 170, 190])
def test_frame_predictor(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
frames = dummy_inst._get_frames_for_interval(10, 1, 0, 1.0)
tools.assert_equal(frames, range(10))
frames = dummy_inst._get_frames_for_interval(10, 1, 0, 1.0, 3)
tools.assert_equal(frames, range(3))
frames = dummy_inst._get_frames_for_interval(600, 20, 1.5, 1.0, 10.0)
tools.assert_equal(frames, [30, 50, 70, 90, 110, 130, 150, 170, 190])
def test_dummy(self):
"""
Test creating dummy instance and methods, including interface methods.
Default configuration parameters should be functional.
"""
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
elem = dummy_inst.process_video('/foo/bar/HVC123456.mp4')
self.log.info("VCDStoreElement type: %s", VCDStoreElement)
tools.assert_is_instance(elem, VCDStoreElement)
tools.assert_equal(elem.descriptor_id, "Dummy")
tools.assert_equal(elem.video_id, 123456)
#noinspection PyTypeChecker
# reason -> because this is how numpy works
tools.assert_true(all(elem.feat_vec == numpy.array((1, 2, 3, 4, 5))))
def test_dummy(self):
"""
Test creating dummy instance and methods, including interface methods.
Default configuration parameters should be functional.
"""
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
elem = dummy_inst.process_video('/foo/bar/HVC123456.mp4')
self.log.info("VCDStoreElement type: %s", VCDStoreElement)
tools.assert_is_instance(elem, VCDStoreElement)
tools.assert_equal(elem.descriptor_id, "Dummy")
tools.assert_equal(elem.video_id, 123456)
#noinspection PyTypeChecker
# reason -> because this is how numpy works
tools.assert_true(all(elem.feat_vec == numpy.array((1, 2, 3, 4, 5))))
def process_video(a):
"""
a is the string ObjectId of the video to be processed
"""
# Setup the task
from SMQTK_Backend.VCDWorkers import descriptor_modules
metastr = {'state': 1, 'label': "Inspection done", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS',
meta=json.dumps(metastr))
print metastr
# Prepare to fetch the file from database (local pymongo database)
conn = pymongo.Connection()
datadb = conn["files"]
gf = gridfs.GridFS(datadb, "video")
fileobj = gf.get(bson.ObjectId(a))
# Create the directory structure for the file
filename = osp.join(app.config["WORK_DIR"], a, "HVC123456.mp4")
directory = os.path.dirname(filename)
if not os.path.exists(directory):
os.makedirs(directory)
os.makedirs(directory + "/work")
os.makedirs(directory + "/images")
# Download the file here for
fout = open(filename, "wb")
fout.write(fileobj.read())
metastr = {'state': 2, 'label': "File Saved", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS',
meta=json.dumps(metastr))
print metastr
# Setup color descriptors
mods = descriptor_modules.get_descriptor_modules()
cd = mods["colordescriptor"]
cdconf = cd.generate_config()
cdobj = cd(cdconf, osp.join(app.config["WORK_DIR"], a, "work"),
osp.join(app.config["WORK_DIR"], a, "images"))
# Generate frames
cdobj.generate_frames(filename)
metastr = {'state': 3, 'label': "Frame extraction complete", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS',
meta=json.dumps(metastr))
print metastr
# Create sprite
create_sprite(osp.join(app.config["WORK_DIR"], a, "images/456/123456"))
metastr = {'state': 4, 'label': "Sprite creation complete", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS',
meta=json.dumps(metastr))
print metastr
# Aggregate features
cdobj.descriptor_generation(filename)
metastr = {'state': 5, 'label': "Features generated", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS',
meta=json.dumps(metastr))
print metastr
results = cdobj.process_video(filename)
# print results
for aresult in results:
save_feature_vector(a, aresult)
# Save results
metastr = {'state': 6, 'label': "Features aggregated", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS',
meta=json.dumps(metastr))
print "Done ", a
return a
def generate_config(config=None):
"""
Generate a sample configuration object.
:param config: An optional configuration object to add to.
:type config: None or SafeConfigCommentParser
:return: Generated configuration object
:rtype: SafeConfigCommentParser
"""
log = logging.getLogger()
if config is None:
log.debug('generating new config object')
config = SafeConfigCommentParser()
log.info("Adding vcd_scheduler section...")
sect = MPIS_CONFIG_SECT
config.add_section(sect)
config.set(
sect, 'image_directory', '# REQUIRED',
"The root directory where imagery will be extracted and stored. "
"This is a common structure across any descriptors that want to "
"extract imagery.")
config.set(sect, 'run_work_directory', '# REQUIRED',
"The base working directory for this run.")
config.set(
sect, 'video_work_list', '# REQUIRED',
"A test time with a new-line separated list of video files to "
"process.")
config.set(sect, 'log_dir', "%(run_work_directory)s/logs",
"Directory where log files from the run are to be stored.")
config.set(
sect, 'target_vcd_store', 'SQLiteVCDStore.db',
"The sqlite3 database file results will be accumulated into. If "
"this is given as a relative path it will be considered "
"relative to the provided work directory.")
config.set(
sect, 'vcd_store_overwrite', 'false',
"When the server node attempts to store VCDStoreElement "
"objects, overwrite existing entries if found. Else, if this is "
"false, favor the existing entries and roll back the insertion "
"attempt.")
# TODO: [generate_config] (see below)
# This tells the vcd_scheduler which descriptor client module to load
# This should also tell us what VCDWorker implementation to use. This
# requires that we shift over to using the VCDWorker implementation
# structure instead of the very similar function thing we currently have
# going on...
# Should be straight forward to convert current stuff to VCDWorker impl.
# Need to modify VCDWorker impls with lessons learned here.
# avail_modes = get_descriptor_client_methods().keys()
descr_modules = get_descriptor_modules()
config.set(
sect, 'descriptor_mode', '# REQUIRED',
"The descriptor mode to run in. This defined which descriptor "
"will process the videos listed in the given work list file.\n"
"\n"
"The available modes are as follows:\n"
" - '%s'" % "'\n - '".join(descr_modules.keys()))
# Append descriptor module configs to this one
for name, m in descr_modules.items():
log.info("Adding '%s' module section...", name)
m.generate_config(config)
return config
def client_side(config, dry_run=False):
"""
Client side implementation
"""
global file_count
log_print("Client starting")
# Get pertinent configuration values
work_dir = config.get(MPIS_CONFIG_SECT, 'run_work_directory')
image_dir = config.get(MPIS_CONFIG_SECT, 'image_directory')
descr_mode = config.get(MPIS_CONFIG_SECT, 'descriptor_mode')
# get configured module
descr_module = get_descriptor_modules()[descr_mode]
descriptor = descr_module(config, work_dir, image_dir)
log_print("module name: %s" % descr_module.__name__)
while True:
log_print("sending request for work")
comm.send('feed-me', dest=0)
reply = comm.recv(source=0)
task_orders = reply.strip().split(' ')
# end of input
if task_orders[0] == 'die-sucka':
log_print("Received terminate command")
break
# video file to process
elif task_orders[0] == 'proc-file':
video_file = task_orders[1]
if not dry_run:
log_print("generating store elements via descriptor")
vcd_store_elems = descriptor.process_video(video_file)
# if any elements were returned at all...
if vcd_store_elems:
assert isinstance(vcd_store_elems, _abcoll.Iterable), \
"Module process_file function did not return an " \
"iterable!"
# Store elements to file in a sub-location within the work
# directory, sending the paths to those files back to the
# server node.
dump_file = get_vcd_element_file(descr_mode, video_file,
work_dir)
log_print("dumping serialized store elements -> %s" %
dump_file)
with open(dump_file, 'w') as f:
cPickle.dump(tuple(vcd_store_elems), f,
cPickle.HIGHEST_PROTOCOL)
# Send back to the server node the serialized files for
# storage
log_print("Sending serialized file path to server")
comm.send('complete %s' % dump_file, dest=0)
else:
log_print("Dry run on video: %s" % video_file)
file_count += 1
# add other commands here
log_print("**** Client side exiting")
print_client_stats()
def client_side(config, dry_run=False):
"""
Client side implementation
"""
global file_count
log_print("Client starting")
# Get pertinent configuration values
work_dir = config.get(MPIS_CONFIG_SECT, "run_work_directory")
image_dir = config.get(MPIS_CONFIG_SECT, "image_directory")
descr_mode = config.get(MPIS_CONFIG_SECT, "descriptor_mode")
# get configured module
descr_module = get_descriptor_modules()[descr_mode]
descriptor = descr_module(config, work_dir, image_dir)
log_print("module name: %s" % descr_module.__name__)
while True:
log_print("sending request for work")
comm.send("feed-me", dest=0)
reply = comm.recv(source=0)
task_orders = reply.strip().split(" ")
# end of input
if task_orders[0] == "die-sucka":
log_print("Received terminate command")
break
# video file to process
elif task_orders[0] == "proc-file":
video_file = task_orders[1]
if not dry_run:
log_print("generating store elements via descriptor")
vcd_store_elems = descriptor.process_video(video_file)
# if any elements were returned at all...
if vcd_store_elems:
assert isinstance(vcd_store_elems, _abcoll.Iterable), (
"Module process_file function did not return an " "iterable!"
)
# Store elements to file in a sub-location within the work
# directory, sending the paths to those files back to the
# server node.
dump_file = get_vcd_element_file(descr_mode, video_file, work_dir)
log_print("dumping serialized store elements -> %s" % dump_file)
with open(dump_file, "w") as f:
cPickle.dump(tuple(vcd_store_elems), f, cPickle.HIGHEST_PROTOCOL)
# Send back to the server node the serialized files for
# storage
log_print("Sending serialized file path to server")
comm.send("complete %s" % dump_file, dest=0)
else:
log_print("Dry run on video: %s" % video_file)
file_count += 1
# add other commands here
log_print("**** Client side exiting")
print_client_stats()
def process_video(a):
"""
a is the string ObjectId of the video to be processed
"""
# Setup the task
from SMQTK_Backend.VCDWorkers import descriptor_modules
metastr = {'state': 1, 'label': "Inspection done", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr))
print metastr
# Prepare to fetch the file from database (local pymongo database)
conn = pymongo.Connection()
datadb = conn["files"]
gf = gridfs.GridFS(datadb, "video")
fileobj = gf.get(bson.ObjectId(a))
# Create the directory structure for the file
filename = osp.join(app.config["WORK_DIR"], a, "HVC123456.mp4")
directory = os.path.dirname(filename)
if not os.path.exists(directory):
os.makedirs(directory)
os.makedirs(directory + "/work")
os.makedirs(directory + "/images")
# Download the file here for
fout = open(filename, "wb")
fout.write(fileobj.read())
metastr = {'state': 2, 'label': "File Saved", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr))
print metastr
# Setup color descriptors
mods = descriptor_modules.get_descriptor_modules()
cd = mods["colordescriptor"]
cdconf = cd.generate_config()
cdobj = cd(cdconf, osp.join(app.config["WORK_DIR"], a, "work"),
osp.join(app.config["WORK_DIR"], a, "images")
)
# Generate frames
cdobj.generate_frames(filename)
metastr = {'state': 3, 'label': "Frame extraction complete", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr))
print metastr
# Create sprite
create_sprite(osp.join(app.config["WORK_DIR"], a, "images/456/123456"))
metastr = {'state': 4, 'label': "Sprite creation complete", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr))
print metastr
# Aggregate features
cdobj.descriptor_generation(filename)
metastr = {'state': 5, 'label': "Features generated", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr))
print metastr
results = cdobj.process_video(filename)
# print results
for aresult in results:
save_feature_vector(a, aresult)
# Save results
metastr = {'state': 6, 'label': "Features aggregated", 'id': a}
celeryapp.current_task.update_state(state='PROGRESS', meta=json.dumps(metastr))
print "Done ", a
return a
def generate_config(config=None):
"""
Generate a sample configuration object.
:param config: An optional configuration object to add to.
:type config: None or SafeConfigCommentParser
:return: Generated configuration object
:rtype: SafeConfigCommentParser
"""
log = logging.getLogger()
if config is None:
log.debug("generating new config object")
config = SafeConfigCommentParser()
log.info("Adding vcd_scheduler section...")
sect = MPIS_CONFIG_SECT
config.add_section(sect)
config.set(
sect,
"image_directory",
"# REQUIRED",
"The root directory where imagery will be extracted and stored. "
"This is a common structure across any descriptors that want to "
"extract imagery.",
)
config.set(sect, "run_work_directory", "# REQUIRED", "The base working directory for this run.")
config.set(
sect,
"video_work_list",
"# REQUIRED",
"A test time with a new-line separated list of video files to " "process.",
)
config.set(
sect, "log_dir", "%(run_work_directory)s/logs", "Directory where log files from the run are to be stored."
)
config.set(
sect,
"target_vcd_store",
"SQLiteVCDStore.db",
"The sqlite3 database file results will be accumulated into. If "
"this is given as a relative path it will be considered "
"relative to the provided work directory.",
)
config.set(
sect,
"vcd_store_overwrite",
"false",
"When the server node attempts to store VCDStoreElement "
"objects, overwrite existing entries if found. Else, if this is "
"false, favor the existing entries and roll back the insertion "
"attempt.",
)
# TODO: [generate_config] (see below)
# This tells the vcd_scheduler which descriptor client module to load
# This should also tell us what VCDWorker implementation to use. This
# requires that we shift over to using the VCDWorker implementation
# structure instead of the very similar function thing we currently have
# going on...
# Should be straight forward to convert current stuff to VCDWorker impl.
# Need to modify VCDWorker impls with lessons learned here.
# avail_modes = get_descriptor_client_methods().keys()
descr_modules = get_descriptor_modules()
config.set(
sect,
"descriptor_mode",
"# REQUIRED",
"The descriptor mode to run in. This defined which descriptor "
"will process the videos listed in the given work list file.\n"
"\n"
"The available modes are as follows:\n"
" - '%s'" % "'\n - '".join(descr_modules.keys()),
)
# Append descriptor module configs to this one
for name, m in descr_modules.items():
log.info("Adding '%s' module section...", name)
m.generate_config(config)
return config
def test_interface_get_prefix_fail(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
# Expects an HVC formatted file name
dummy_inst.get_video_prefix("foobat123456.mp4")
def test_interface_tempify_filename(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
t = dummy_inst.tempify_filename('foo.txt')
tools.assert_equal(t, 'foo.txt.TEMP')
def test_interface_tempify_filename(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
t = dummy_inst.tempify_filename('foo.txt')
tools.assert_equal(t, 'foo.txt.TEMP')
def test_interface_get_prefix_fail(self):
dummy_class = get_descriptor_modules()['Dummy']
dummy_inst = dummy_class(dummy_class.generate_config(), self.work_dir)
# Expects an HVC formatted file name
dummy_inst.get_video_prefix("foobat123456.mp4")
