def setUpClass(cls):
# local FeatureManager server may have already been initialized by
# another unittest
try:
initFeatureManagerConnection()
except ValueError:
pass
data_dir = osp.abspath(osp.join(osp.dirname(__file__), "data"))
classifier_config_file = osp.join(data_dir, "classifier_config.json")
cls.classifier_config = jsmin(open(classifier_config_file).read())
symmetric_clipids = np.loadtxt(osp.join(data_dir, "symmetric_clipids.txt"))
symmetric_bgflags = np.loadtxt(osp.join(data_dir, "symmetric_bgflags.txt"))
# noinspection PyCallingNonCallable
symmetric_dk_mat = np.matrix(np.load(osp.join(data_dir, "symmetric_distance_kernel.npy")))
symmetric_bgclipids = np.array([symmetric_clipids[i] for i, e in enumerate(symmetric_bgflags) if e])
# noinspection PyUnresolvedReferences
#: :type: DistanceKernel
cls.dk_iqr = getFeatureManager().DistanceKernel(
symmetric_clipids, symmetric_clipids, symmetric_dk_mat, symmetric_bgclipids
)
asymmetric_row_ids = np.loadtxt(osp.join(data_dir, "asymmetric_clipids_rows.txt"))
asymmetric_col_ids = np.loadtxt(osp.join(data_dir, "asymmetric_clipids_cols.txt"))
# noinspection PyCallingNonCallable
asymmetric_dk_mat = np.matrix(np.load(osp.join(data_dir, "asymmetric_distance_kernel.npy")))
# noinspection PyUnresolvedReferences
#: :type: DistanceKernel
cls.dk_arc = getFeatureManager().DistanceKernel(asymmetric_row_ids, asymmetric_col_ids, asymmetric_dk_mat)
cls.mdb_info = DatabaseInfo("localhost", 27017, "RSS_Test")
register_mdb_loc(cls.mdb_info.host, cls.mdb_info.port)
def setUp(self):
# uses default database location (localhost, 27017)
gc_config = SmqtkController.generate_config()
# Modifying VCDController database location so we can safely remove it
# when we're done
self._vcd_store_loc = osp.join(osp.dirname(osp.abspath(__file__)),
'work/vcd_store')
gc_config.set(VCDController.CONFIG_SECT, 'store_name',
self._vcd_store_loc)
# TODO: change the smqtk_controller:mongo_database
self.gc = SmqtkController(gc_config)
data_dir = osp.abspath(osp.join(osp.dirname(__file__), 'data'))
print "Data dir:", data_dir
symmetric_clipids = np.loadtxt(osp.join(data_dir,
'symmetric_clipids.txt'))
symmetric_bgflags = np.loadtxt(osp.join(data_dir,
'symmetric_bgflags.txt'))
# noinspection PyCallingNonCallable
symmetric_dk_mat = \
np.matrix(np.load(osp.join(data_dir,
'symmetric_distance_kernel.npy')))
symmetric_bgclipids = np.array([symmetric_clipids[i]
for i, e in enumerate(symmetric_bgflags)
if e])
# noinspection PyUnresolvedReferences
#: :type: DistanceKernel
self.dk_iqr = getFeatureManager().DistanceKernel(symmetric_clipids,
symmetric_clipids,
symmetric_dk_mat,
symmetric_bgclipids)
asymmetric_row_ids = np.loadtxt(osp.join(data_dir,
'asymmetric_clipids_rows.txt'))
asymmetric_col_ids = np.loadtxt(osp.join(data_dir,
'asymmetric_clipids_cols.txt'))
# noinspection PyCallingNonCallable
asymmetric_dk_mat = \
np.matrix(np.load(osp.join(data_dir,
'asymmetric_distance_kernel.npy')))
# noinspection PyUnresolvedReferences
#: :type: DistanceKernel
self.dk_arc = getFeatureManager().DistanceKernel(asymmetric_row_ids,
asymmetric_col_ids,
asymmetric_dk_mat)
def setUp(self):
# uses default database location (localhost, 27017)
gc_config = SmqtkController.generate_config()
# Modifying VCDController database location so we can safely remove it
# when we're done
self._vcd_store_loc = osp.join(osp.dirname(osp.abspath(__file__)),
'work/vcd_store')
gc_config.set(VCDController.CONFIG_SECT, 'store_name',
self._vcd_store_loc)
# TODO: change the smqtk_controller:mongo_database
self.gc = SmqtkController(gc_config)
data_dir = osp.abspath(osp.join(osp.dirname(__file__), 'data'))
print "Data dir:", data_dir
symmetric_clipids = np.loadtxt(
osp.join(data_dir, 'symmetric_clipids.txt'))
symmetric_bgflags = np.loadtxt(
osp.join(data_dir, 'symmetric_bgflags.txt'))
# noinspection PyCallingNonCallable
symmetric_dk_mat = \
np.matrix(np.load(osp.join(data_dir,
'symmetric_distance_kernel.npy')))
symmetric_bgclipids = np.array([
symmetric_clipids[i] for i, e in enumerate(symmetric_bgflags) if e
])
# noinspection PyUnresolvedReferences
#: :type: DistanceKernel
self.dk_iqr = getFeatureManager().DistanceKernel(
symmetric_clipids, symmetric_clipids, symmetric_dk_mat,
symmetric_bgclipids)
asymmetric_row_ids = np.loadtxt(
osp.join(data_dir, 'asymmetric_clipids_rows.txt'))
asymmetric_col_ids = np.loadtxt(
osp.join(data_dir, 'asymmetric_clipids_cols.txt'))
# noinspection PyCallingNonCallable
asymmetric_dk_mat = \
np.matrix(np.load(osp.join(data_dir,
'asymmetric_distance_kernel.npy')))
# noinspection PyUnresolvedReferences
#: :type: DistanceKernel
self.dk_arc = getFeatureManager().DistanceKernel(
asymmetric_row_ids, asymmetric_col_ids, asymmetric_dk_mat)
def setUpClass(cls):
# local FeatureManager server may have already been initialized by
# another unittest
try:
initFeatureManagerConnection()
except ValueError:
pass
data_dir = osp.abspath(osp.join(osp.dirname(__file__), 'data'))
classifier_config_file = osp.join(data_dir, 'classifier_config.json')
cls.classifier_config = jsmin(open(classifier_config_file).read())
symmetric_clipids = np.loadtxt(
osp.join(data_dir, 'symmetric_clipids.txt'))
symmetric_bgflags = np.loadtxt(
osp.join(data_dir, 'symmetric_bgflags.txt'))
# noinspection PyCallingNonCallable
symmetric_dk_mat = \
np.matrix(np.load(osp.join(data_dir,
'symmetric_distance_kernel.npy')))
symmetric_bgclipids = np.array([
symmetric_clipids[i] for i, e in enumerate(symmetric_bgflags) if e
])
# noinspection PyUnresolvedReferences
#: :type: DistanceKernel
cls.dk_iqr = getFeatureManager().DistanceKernel(
symmetric_clipids, symmetric_clipids, symmetric_dk_mat,
symmetric_bgclipids)
asymmetric_row_ids = np.loadtxt(
osp.join(data_dir, 'asymmetric_clipids_rows.txt'))
asymmetric_col_ids = np.loadtxt(
osp.join(data_dir, 'asymmetric_clipids_cols.txt'))
# noinspection PyCallingNonCallable
asymmetric_dk_mat = \
np.matrix(np.load(osp.join(data_dir,
'asymmetric_distance_kernel.npy')))
# noinspection PyUnresolvedReferences
#: :type: DistanceKernel
cls.dk_arc = getFeatureManager().DistanceKernel(
asymmetric_row_ids, asymmetric_col_ids, asymmetric_dk_mat)
cls.mdb_info = DatabaseInfo('localhost', 27017, 'RSS_Test')
register_mdb_loc(cls.mdb_info.host, cls.mdb_info.port)
def setUp(self):
self.mgr = getFeatureManager()
self.N = 1024 * 4 # 4096
self.M = 1024 * 6 # 6144
# self.N = 1024 * 8
# self.M = 1024 * 16
self.id_vec = np.array(range(self.N))
self.bg_ids = np.array(range(self.N//10))
self.feature_mat = np.matrix(np.random.random((self.N, self.M)))
self.dk_mat = np.matrix(np.random.random((self.N, self.N)))
def test_common_fm(lock):
lock.acquire()
lock.release()
lcl_common_fmm = get_common_fmm()
assert len(lcl_common_fmm.get_feature_types()) == 0, \
"local fmm had something in it"
#: :type: FeatureMemoryMap
gbl_common_fmm = getFeatureManager().get_common_fmm()
assert len(gbl_common_fmm.get_feature_types()) >= 1, \
"Unexpected number of features in global fmm: %s" \
% gbl_common_fmm.get_feature_types()
assert expected_type in gbl_common_fmm.get_feature_types(), \
"Expected feature not present in global fmm. Current " \
"features: %s" % gbl_common_fmm.get_feature_types()
gbl_common_fmm.initialize_from_files(
str(current_process().pid),
cid_file, bg_flags_file, feature_file, kernel_file
)
from WebUI import app
from SMQTK_Backend.FeatureMemory import \
initFeatureManagerConnection, \
getFeatureManager, \
ReadWriteLock, \
DummyRWLock
import logging
logger = logging.getLogger("WebUI.Cache")
logger.setLevel(logging.INFO)
import os
# Connect with the feature memory server
mgr_srvr_addr = ('127.0.0.1', 30000)
initFeatureManagerConnection(mgr_srvr_addr, 'test')
mgr = getFeatureManager(mgr_srvr_addr)
# Get timed cache
# noinspection PyUnresolvedReferences
#: :type: SMQTK_Backend.FeatureMemory.TimedCache
tc = mgr.get_common_tc()
data_path = app.config['DATA_DIR']
# Load the distance kernel matrices
known_features = ["csift", "hog_2x2", "isa", "mfcc_4096",
"OB_max_avg_positive"]
def load_feature_dk(f_name):
try:
def test_controller_interaction(self):
print 'poke'
# Dummy classes representing function of intended components
class Controller (Process):
""" Reads input from queue, exiting when receives None
"""
def __init__(self, data_queue):
super(Controller, self).__init__(name="controller")
self._queue = data_queue
def run(self):
running = True
while running:
print "[Controller] waiting for data..."
elem = self._queue.get()
print "[Controller] received:", type(elem), elem
if elem is None:
running = False
class DummyAsw (Process):
""" sends dk to queue on separate process
"""
def __init__(self, dk, data_queue):
super(DummyAsw, self).__init__()
self._dk = dk
self._queue = data_queue
def run(self):
self._queue.put(self._dk)
def generate_dks(mgr):
""" generated some distance kernels to test with given a connected
FeatureManager
"""
c_fmm = mgr.get_common_fmm()
if 'test' in c_fmm.get_feature_types():
c_fmm.remove('test')
c_fmm.initialize_from_files('test', self.sym_cids_file,
self.sym_bg_flags,
self.sym_feat_mat_f,
self.sym_dk_mat_f)
dk1 = c_fmm.get_distance_kernel('test')
dk2 = mgr.symmetric_dk_from_file(self.sym_cids_file,
self.sym_dk_mat_f,
self.sym_bg_flags)
return dk1, dk2
# Set-up and start the extra-process feature manager server
host = 'localhost'
port = 54321
addr = (host, port)
auth_key = "foobar"
config_file = osp.join(self.work_dir,
'controller_interaction_server.ini')
config = SafeConfigCommentParser()
config.add_section('server')
config.set('server', 'port', str(port))
config.set('server', 'authkey', auth_key)
with open(config_file, 'w') as ofile:
# noinspection PyTypeChecker
config.write(ofile)
p = subprocess.Popen(['FeatureManagerServer', '-c', config_file])
# wait a sec for the server to start/fail
time.sleep(1) # should be sufficient...
self.assertIsNone(p.poll(), "Server process shutdown prematurely. "
"Check reported error on console.")
initFeatureManagerConnection(addr, auth_key)
mgr = getFeatureManager(addr)
# Initialize and start dummy controller process
q = Queue()
c = Controller(q)
c.start()
# generated DistanceKernels and dummy ASW processes
dk1, dk2 = generate_dks(mgr)
asw1 = DummyAsw(dk1, q)
asw2 = DummyAsw(dk2, q)
# Running the two dummy asw processes should cause no errors in either
# dummy ASW or the dummy Controller
asw1.start()
asw1.join()
asw2.start()
asw2.join()
# shutdown controller
q.put(None)
c.join()
print "C exitcode:", c.exitcode
# Clean-up
# Need to call del on some things else we would get hung up in decref
# call to remove server at function/process exit.
del dk1, dk2, asw1, asw2
os.remove(config_file)
removeFeatureManagerConnection(addr)
p.terminate()
p.poll()
self.assertEqual(c.exitcode, 0, "Controller dummy did not exit cleanly")
del c
def test_fm_external_server(self):
print "==================================================="
print "Testing use of external server with update function"
print "==================================================="
host = 'localhost'
port = 54321
addr = (host, port)
auth_key = "foobar"
# Trying to connect to these right away should result in a timeout
# in the underlying proxy.
self.assertRaises(
Exception,
initFeatureManagerConnection,
addr, auth_key
)
# Start the server at the specified addr
config_file = osp.join(self.work_dir, 'test_server_config.ini')
config = SafeConfigCommentParser()
config.add_section('server')
config.set('server', 'port', str(port))
config.set('server', 'authkey', auth_key)
with open(config_file, 'w') as ofile:
# noinspection PyTypeChecker
config.write(ofile)
p = subprocess.Popen(['FeatureManagerServer', '-c', config_file])
# wait a sec for the server to start/fail
time.sleep(1) # should be sufficient...
self.assertIsNone(p.poll(), "Server process shutdown prematurely. "
"Check reported error on console.")
# Now we should be able to connect, create a FeatureMemory[Map] and
# successfully go through the update process.
from multiprocessing.connection import AuthenticationError
self.assertRaises(
AuthenticationError,
initFeatureManagerConnection,
addr, "not the right key"
)
# Shouldn't be in there yet as its not initialized
self.assertRaises(
KeyError,
getFeatureManager,
addr
)
initFeatureManagerConnection(addr, auth_key)
# should get a ValueError when trying to init the same address more than
# once.
self.assertRaises(
ValueError,
initFeatureManagerConnection,
addr, auth_key
)
mgr = getFeatureManager(addr)
# Create a feature map->featureMemory and pass through update process
f_type = 'test'
cid_file = osp.join(self.data_dir, 'symmetric_clipids.txt')
bg_flags_file = osp.join(self.data_dir, 'symmetric_bgflags.txt')
feature_file = osp.join(self.data_dir, 'symmetric_feature.npy')
kernel_file = osp.join(self.data_dir, 'symmetric_distance_kernel.npy')
#: :type: FeatureMemoryMap
fmm = mgr.get_common_fmm()
fmm.initialize_from_files(f_type, cid_file, bg_flags_file, feature_file,
kernel_file)
fm = fmm.get_feature_memory(f_type)
self._update_test_helper(fm)
# Clean up
# Need to call del else we would get hung up in decref call to remove
# server at function/process exit.
del fm, fmm
p.terminate()
os.remove(config_file)
removeFeatureManagerConnection(addr)
def test_fmm_singleton(self):
# test that the correct singleton is being accessed across disjoint
# processes
gbl_lock = Lock()
expected_type = 'expected_type'
cid_file = osp.join(self.data_dir, 'symmetric_clipids.txt')
bg_flags_file = osp.join(self.data_dir, 'symmetric_bgflags.txt')
feature_file = osp.join(self.data_dir, 'symmetric_feature.npy')
kernel_file = osp.join(self.data_dir, 'symmetric_distance_kernel.npy')
def test_common_fm(lock):
lock.acquire()
lock.release()
lcl_common_fmm = get_common_fmm()
assert len(lcl_common_fmm.get_feature_types()) == 0, \
"local fmm had something in it"
#: :type: FeatureMemoryMap
gbl_common_fmm = getFeatureManager().get_common_fmm()
assert len(gbl_common_fmm.get_feature_types()) >= 1, \
"Unexpected number of features in global fmm: %s" \
% gbl_common_fmm.get_feature_types()
assert expected_type in gbl_common_fmm.get_feature_types(), \
"Expected feature not present in global fmm. Current " \
"features: %s" % gbl_common_fmm.get_feature_types()
gbl_common_fmm.initialize_from_files(
str(current_process().pid),
cid_file, bg_flags_file, feature_file, kernel_file
)
gbl_lock.acquire()
print '[main] acquire'
p1 = Process(name='p1', target=test_common_fm, args=(gbl_lock,))
p2 = Process(name='p2', target=test_common_fm, args=(gbl_lock,))
p3 = Process(name='p3', target=test_common_fm, args=(gbl_lock,))
print '[main] starting processes'
p1.start()
p2.start()
p3.start()
print '[main] adding expected feature to global FMM'
#: :type: FeatureMemoryMap
gbl_fmm = getFeatureManager().get_common_fmm()
gbl_fmm.initialize_from_files(expected_type, cid_file, bg_flags_file,
feature_file, kernel_file)
print '[main] release'
gbl_lock.release()
p1.join(timeout=1)
print '[main] checking p1'
self.assertIsNotNone(p1.exitcode, "p1 timed out")
self.assertEquals(p1.exitcode, 0, "p1 returned non-0 exitcode")
p2.join(timeout=1)
print '[main] checking p2'
self.assertIsNotNone(p2.exitcode, "p2 timed out")
self.assertEquals(p2.exitcode, 0, "p2 returned non-0 exitcode")
p3.join(timeout=1)
print '[main] checking p3'
self.assertIsNotNone(p3.exitcode, "p3 timed out")
self.assertEquals(p3.exitcode, 0, "p3 returned non-0 exitcode")
print "[main] checking post-processing expected keys"
self.assertEqual(set(gbl_fmm.get_feature_types()),
set((expected_type, str(p1.pid),
str(p2.pid), str(p3.pid))))