def main():
parser = optparse.OptionParser()
parser.add_option('-c',
'--config',
type=str,
help='Path to the configuration file.')
parser.add_option('-v',
'--verbose',
action='store_true',
default=False,
help='Add debugging log messages.')
opts, args = parser.parse_args()
config_file = opts.config
assert config_file is not None, \
"Not given a configuration file for the server!"
assert osp.exists(config_file), \
"Given config file path does not exist."
assert not osp.isdir(config_file), \
"Given config file is a directory!"
config = SafeConfigCommentParser()
parsed = config.read(config_file)
assert parsed, "Configuration file not parsed!"
section = 'server'
assert config.has_section(section), \
"No server section found!"
assert config.has_option(section, 'port'), \
"No port option in config!"
assert config.has_option(section, 'authkey'), \
"No authkey option in config!"
port = config.getint(section, 'port')
authkey = config.get(section, 'authkey')
# Setup logging
log_format_str = '%(levelname)7s - %(asctime)s - %(name)s.%(funcName)s - ' \
'%(message)s'
if opts.verbose:
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.getLogger().setLevel(log_level)
logging.basicConfig(format=log_format_str)
mgr = FeatureManager(('', port), authkey)
mgr.get_server().serve_forever()
def main():
parser = optparse.OptionParser()
parser.add_option('-c', '--config', type=str,
help='Path to the configuration file.')
parser.add_option('-v', '--verbose', action='store_true', default=False,
help='Add debugging log messages.')
opts, args = parser.parse_args()
config_file = opts.config
assert config_file is not None, \
"Not given a configuration file for the server!"
assert osp.exists(config_file), \
"Given config file path does not exist."
assert not osp.isdir(config_file), \
"Given config file is a directory!"
config = SafeConfigCommentParser()
parsed = config.read(config_file)
assert parsed, "Configuration file not parsed!"
section = 'server'
assert config.has_section(section), \
"No server section found!"
assert config.has_option(section, 'port'), \
"No port option in config!"
assert config.has_option(section, 'authkey'), \
"No authkey option in config!"
port = config.getint(section, 'port')
authkey = config.get(section, 'authkey')
# Setup logging
log_format_str = '%(levelname)7s - %(asctime)s - %(name)s.%(funcName)s - ' \
'%(message)s'
if opts.verbose:
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.getLogger().setLevel(log_level)
logging.basicConfig(format=log_format_str)
mgr = FeatureManager(('', port), authkey)
mgr.get_server().serve_forever()
def generate_config(config=None):
"""
Generate a sample configuration object.
:param config: An optional configuration object to add to.
:type config: None or SafeConfigCommentParser
:return: Generated configuration object
:rtype: SafeConfigCommentParser
"""
log = logging.getLogger()
if config is None:
log.debug('generating new config object')
config = SafeConfigCommentParser()
log.info("Adding vcd_scheduler section...")
sect = MPIS_CONFIG_SECT
config.add_section(sect)
config.set(
sect, 'image_directory', '# REQUIRED',
"The root directory where imagery will be extracted and stored. "
"This is a common structure across any descriptors that want to "
"extract imagery.")
config.set(sect, 'run_work_directory', '# REQUIRED',
"The base working directory for this run.")
config.set(
sect, 'video_work_list', '# REQUIRED',
"A test time with a new-line separated list of video files to "
"process.")
config.set(sect, 'log_dir', "%(run_work_directory)s/logs",
"Directory where log files from the run are to be stored.")
config.set(
sect, 'target_vcd_store', 'SQLiteVCDStore.db',
"The sqlite3 database file results will be accumulated into. If "
"this is given as a relative path it will be considered "
"relative to the provided work directory.")
config.set(
sect, 'vcd_store_overwrite', 'false',
"When the server node attempts to store VCDStoreElement "
"objects, overwrite existing entries if found. Else, if this is "
"false, favor the existing entries and roll back the insertion "
"attempt.")
# TODO: [generate_config] (see below)
# This tells the vcd_scheduler which descriptor client module to load
# This should also tell us what VCDWorker implementation to use. This
# requires that we shift over to using the VCDWorker implementation
# structure instead of the very similar function thing we currently have
# going on...
# Should be straight forward to convert current stuff to VCDWorker impl.
# Need to modify VCDWorker impls with lessons learned here.
# avail_modes = get_descriptor_client_methods().keys()
descr_modules = get_descriptor_modules()
config.set(
sect, 'descriptor_mode', '# REQUIRED',
"The descriptor mode to run in. This defined which descriptor "
"will process the videos listed in the given work list file.\n"
"\n"
"The available modes are as follows:\n"
" - '%s'" % "'\n - '".join(descr_modules.keys()))
# Append descriptor module configs to this one
for name, m in descr_modules.items():
log.info("Adding '%s' module section...", name)
m.generate_config(config)
return config
# ================================================================
# Main entry
# - Start the program
#
# TODO: Functionalize this and move this implementation into an importable
# location, creating a wrapper script for bin.
# - Will probably want to just use this as the VCD run system within the
# SmqtkController/VCDSController instead of making two discrete run
# subsystems.
if __name__ == '__main__':
# process options
opts, _ = parse_options()
# initially load config files if any given (potentially for output config)
config = SafeConfigCommentParser()
config.read(opts.config_files)
###
# Config file output if requested
#
if opts.output_config:
logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.WARNING - (10 * opts.verbosity))
log.info("Outputting configuration file@ %s", opts.output_config)
if osp.exists(opts.output_config) and not opts.overwrite:
log.error("Target already exists. Not overwriting.")
exit(2)
def generate_config(config=None):
"""
Generate a sample configuration object.
:param config: An optional configuration object to add to.
:type config: None or SafeConfigCommentParser
:return: Generated configuration object
:rtype: SafeConfigCommentParser
"""
log = logging.getLogger()
if config is None:
log.debug("generating new config object")
config = SafeConfigCommentParser()
log.info("Adding vcd_scheduler section...")
sect = MPIS_CONFIG_SECT
config.add_section(sect)
config.set(
sect,
"image_directory",
"# REQUIRED",
"The root directory where imagery will be extracted and stored. "
"This is a common structure across any descriptors that want to "
"extract imagery.",
)
config.set(sect, "run_work_directory", "# REQUIRED", "The base working directory for this run.")
config.set(
sect,
"video_work_list",
"# REQUIRED",
"A test time with a new-line separated list of video files to " "process.",
)
config.set(
sect, "log_dir", "%(run_work_directory)s/logs", "Directory where log files from the run are to be stored."
)
config.set(
sect,
"target_vcd_store",
"SQLiteVCDStore.db",
"The sqlite3 database file results will be accumulated into. If "
"this is given as a relative path it will be considered "
"relative to the provided work directory.",
)
config.set(
sect,
"vcd_store_overwrite",
"false",
"When the server node attempts to store VCDStoreElement "
"objects, overwrite existing entries if found. Else, if this is "
"false, favor the existing entries and roll back the insertion "
"attempt.",
)
# TODO: [generate_config] (see below)
# This tells the vcd_scheduler which descriptor client module to load
# This should also tell us what VCDWorker implementation to use. This
# requires that we shift over to using the VCDWorker implementation
# structure instead of the very similar function thing we currently have
# going on...
# Should be straight forward to convert current stuff to VCDWorker impl.
# Need to modify VCDWorker impls with lessons learned here.
# avail_modes = get_descriptor_client_methods().keys()
descr_modules = get_descriptor_modules()
config.set(
sect,
"descriptor_mode",
"# REQUIRED",
"The descriptor mode to run in. This defined which descriptor "
"will process the videos listed in the given work list file.\n"
"\n"
"The available modes are as follows:\n"
" - '%s'" % "'\n - '".join(descr_modules.keys()),
)
# Append descriptor module configs to this one
for name, m in descr_modules.items():
log.info("Adding '%s' module section...", name)
m.generate_config(config)
return config
# ================================================================
# Main entry
# - Start the program
#
# TODO: Functionalize this and move this implementation into an importable
# location, creating a wrapper script for bin.
# - Will probably want to just use this as the VCD run system within the
# SmqtkController/VCDSController instead of making two discrete run
# subsystems.
if __name__ == "__main__":
# process options
opts, _ = parse_options()
# initially load config files if any given (potentially for output config)
config = SafeConfigCommentParser()
config.read(opts.config_files)
###
# Config file output if requested
#
if opts.output_config:
logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.WARNING - (10 * opts.verbosity))
log.info("Outputting configuration file@ %s", opts.output_config)
if osp.exists(opts.output_config) and not opts.overwrite:
log.error("Target already exists. Not overwriting.")
exit(2)
def test_controller_interaction(self):
print 'poke'
# Dummy classes representing function of intended components
class Controller (Process):
""" Reads input from queue, exiting when receives None
"""
def __init__(self, data_queue):
super(Controller, self).__init__(name="controller")
self._queue = data_queue
def run(self):
running = True
while running:
print "[Controller] waiting for data..."
elem = self._queue.get()
print "[Controller] received:", type(elem), elem
if elem is None:
running = False
class DummyAsw (Process):
""" sends dk to queue on separate process
"""
def __init__(self, dk, data_queue):
super(DummyAsw, self).__init__()
self._dk = dk
self._queue = data_queue
def run(self):
self._queue.put(self._dk)
def generate_dks(mgr):
""" generated some distance kernels to test with given a connected
FeatureManager
"""
c_fmm = mgr.get_common_fmm()
if 'test' in c_fmm.get_feature_types():
c_fmm.remove('test')
c_fmm.initialize_from_files('test', self.sym_cids_file,
self.sym_bg_flags,
self.sym_feat_mat_f,
self.sym_dk_mat_f)
dk1 = c_fmm.get_distance_kernel('test')
dk2 = mgr.symmetric_dk_from_file(self.sym_cids_file,
self.sym_dk_mat_f,
self.sym_bg_flags)
return dk1, dk2
# Set-up and start the extra-process feature manager server
host = 'localhost'
port = 54321
addr = (host, port)
auth_key = "foobar"
config_file = osp.join(self.work_dir,
'controller_interaction_server.ini')
config = SafeConfigCommentParser()
config.add_section('server')
config.set('server', 'port', str(port))
config.set('server', 'authkey', auth_key)
with open(config_file, 'w') as ofile:
# noinspection PyTypeChecker
config.write(ofile)
p = subprocess.Popen(['FeatureManagerServer', '-c', config_file])
# wait a sec for the server to start/fail
time.sleep(1) # should be sufficient...
self.assertIsNone(p.poll(), "Server process shutdown prematurely. "
"Check reported error on console.")
initFeatureManagerConnection(addr, auth_key)
mgr = getFeatureManager(addr)
# Initialize and start dummy controller process
q = Queue()
c = Controller(q)
c.start()
# generated DistanceKernels and dummy ASW processes
dk1, dk2 = generate_dks(mgr)
asw1 = DummyAsw(dk1, q)
asw2 = DummyAsw(dk2, q)
# Running the two dummy asw processes should cause no errors in either
# dummy ASW or the dummy Controller
asw1.start()
asw1.join()
asw2.start()
asw2.join()
# shutdown controller
q.put(None)
c.join()
print "C exitcode:", c.exitcode
# Clean-up
# Need to call del on some things else we would get hung up in decref
# call to remove server at function/process exit.
del dk1, dk2, asw1, asw2
os.remove(config_file)
removeFeatureManagerConnection(addr)
p.terminate()
p.poll()
self.assertEqual(c.exitcode, 0, "Controller dummy did not exit cleanly")
del c
def test_fm_external_server(self):
print "==================================================="
print "Testing use of external server with update function"
print "==================================================="
host = 'localhost'
port = 54321
addr = (host, port)
auth_key = "foobar"
# Trying to connect to these right away should result in a timeout
# in the underlying proxy.
self.assertRaises(
Exception,
initFeatureManagerConnection,
addr, auth_key
)
# Start the server at the specified addr
config_file = osp.join(self.work_dir, 'test_server_config.ini')
config = SafeConfigCommentParser()
config.add_section('server')
config.set('server', 'port', str(port))
config.set('server', 'authkey', auth_key)
with open(config_file, 'w') as ofile:
# noinspection PyTypeChecker
config.write(ofile)
p = subprocess.Popen(['FeatureManagerServer', '-c', config_file])
# wait a sec for the server to start/fail
time.sleep(1) # should be sufficient...
self.assertIsNone(p.poll(), "Server process shutdown prematurely. "
"Check reported error on console.")
# Now we should be able to connect, create a FeatureMemory[Map] and
# successfully go through the update process.
from multiprocessing.connection import AuthenticationError
self.assertRaises(
AuthenticationError,
initFeatureManagerConnection,
addr, "not the right key"
)
# Shouldn't be in there yet as its not initialized
self.assertRaises(
KeyError,
getFeatureManager,
addr
)
initFeatureManagerConnection(addr, auth_key)
# should get a ValueError when trying to init the same address more than
# once.
self.assertRaises(
ValueError,
initFeatureManagerConnection,
addr, auth_key
)
mgr = getFeatureManager(addr)
# Create a feature map->featureMemory and pass through update process
f_type = 'test'
cid_file = osp.join(self.data_dir, 'symmetric_clipids.txt')
bg_flags_file = osp.join(self.data_dir, 'symmetric_bgflags.txt')
feature_file = osp.join(self.data_dir, 'symmetric_feature.npy')
kernel_file = osp.join(self.data_dir, 'symmetric_distance_kernel.npy')
#: :type: FeatureMemoryMap
fmm = mgr.get_common_fmm()
fmm.initialize_from_files(f_type, cid_file, bg_flags_file, feature_file,
kernel_file)
fm = fmm.get_feature_memory(f_type)
self._update_test_helper(fm)
# Clean up
# Need to call del else we would get hung up in decref call to remove
# server at function/process exit.
del fm, fmm
p.terminate()
os.remove(config_file)
removeFeatureManagerConnection(addr)
def generate_config(cls, config=None):
"""
Generate, define and return a configuration object for this descriptor
worker.
:param config: An optionally existing configuration object to update.
:type config: None or SafeConfigCommentParser
:return: Updated configuration object with this descriptor's specific
configuration parameters.
:rtype: SafeConfigCommentParser
"""
if config is None:
config = SafeConfigCommentParser()
# If the config sections for the utility methods are not present, add
# them and their options.
if not config.has_section(cls.EXTRACTOR_CONFIG_SECT):
sect = cls.EXTRACTOR_CONFIG_SECT
config.add_section(sect,
"Options controlling how frames are extracted "
"from videos when requested.\n"
"\n"
"The following are standard frame extraction "
"configurations for configurable descriptor "
"modules (in offset,interval,max_duration "
"value order):\n"
" - colordescriptor -> (0, 2, 10)\n"
" - subattributes -> (3, 4, 0)\n")
config.set(sect, 'frame_extractor_exe', 'frame_extractor',
"The frame_extractor executable to use")
config.set(sect, 'start_offset_seconds', '0.0',
"The number of seconds after the start of the video to "
"start extracting frames. This may be 0 or not provided "
"to indicate no offset.")
config.set(sect, 'second_interval', '2.0',
"Extract frames, after the provided offset, every N "
"seconds, where N is the value provided here. This may "
"be floating point. This may be 0 or not provided to "
"extract all frames after the given offset.")
config.set(sect, 'max_duration_minutes', '10.0',
"Maximum time which we are not to extract frames past. "
"This should be provided in decimal minutes. This may "
"be not provided (no value given), or set to 0, to "
"indicate no maximum duration limit. ")
if not config.has_section(cls.PROBE_CONFIG_SECT):
sect = cls.PROBE_CONFIG_SECT
config.add_section(sect)
config.set(sect, 'ffprobe_exe', 'ffprobe',
"The ffmpeg ffprobe executable to use when probing for "
"video metadata.")
config.set(sect, 'use_TBR_fallback', 'false',
"Use or don't use the TBR value when the FPS value is "
"not present when probing video metadata.")
return config
def generate_config(cls, config=None):
"""
Generate, define and return a configuration object for this descriptor
worker.
:param config: An optionally existing configuration object to update.
:type config: None or SafeConfigCommentParser
:return: Updated configuration object with this descriptor's specific
configuration parameters.
:rtype: SafeConfigCommentParser
"""
if config is None:
config = SafeConfigCommentParser()
# If the config sections for the utility methods are not present, add
# them and their options.
if not config.has_section(cls.EXTRACTOR_CONFIG_SECT):
sect = cls.EXTRACTOR_CONFIG_SECT
config.add_section(
sect, "Options controlling how frames are extracted "
"from videos when requested.\n"
"\n"
"The following are standard frame extraction "
"configurations for configurable descriptor "
"modules (in offset,interval,max_duration "
"value order):\n"
" - colordescriptor -> (0, 2, 10)\n"
" - subattributes -> (3, 4, 0)\n")
config.set(sect, 'frame_extractor_exe', 'frame_extractor',
"The frame_extractor executable to use")
config.set(
sect, 'start_offset_seconds', '0.0',
"The number of seconds after the start of the video to "
"start extracting frames. This may be 0 or not provided "
"to indicate no offset.")
config.set(
sect, 'second_interval', '2.0',
"Extract frames, after the provided offset, every N "
"seconds, where N is the value provided here. This may "
"be floating point. This may be 0 or not provided to "
"extract all frames after the given offset.")
config.set(
sect, 'max_duration_minutes', '10.0',
"Maximum time which we are not to extract frames past. "
"This should be provided in decimal minutes. This may "
"be not provided (no value given), or set to 0, to "
"indicate no maximum duration limit. ")
if not config.has_section(cls.PROBE_CONFIG_SECT):
sect = cls.PROBE_CONFIG_SECT
config.add_section(sect)
config.set(
sect, 'ffprobe_exe', 'ffprobe',
"The ffmpeg ffprobe executable to use when probing for "
"video metadata.")
config.set(
sect, 'use_TBR_fallback', 'false',
"Use or don't use the TBR value when the FPS value is "
"not present when probing video metadata.")
return config
