def setUpClass(cls): # local FeatureManager server may have already been initialized by # another unittest try: initFeatureManagerConnection() except ValueError: pass data_dir = osp.abspath(osp.join(osp.dirname(__file__), "data")) classifier_config_file = osp.join(data_dir, "classifier_config.json") cls.classifier_config = jsmin(open(classifier_config_file).read()) symmetric_clipids = np.loadtxt(osp.join(data_dir, "symmetric_clipids.txt")) symmetric_bgflags = np.loadtxt(osp.join(data_dir, "symmetric_bgflags.txt")) # noinspection PyCallingNonCallable symmetric_dk_mat = np.matrix(np.load(osp.join(data_dir, "symmetric_distance_kernel.npy"))) symmetric_bgclipids = np.array([symmetric_clipids[i] for i, e in enumerate(symmetric_bgflags) if e]) # noinspection PyUnresolvedReferences #: :type: DistanceKernel cls.dk_iqr = getFeatureManager().DistanceKernel( symmetric_clipids, symmetric_clipids, symmetric_dk_mat, symmetric_bgclipids ) asymmetric_row_ids = np.loadtxt(osp.join(data_dir, "asymmetric_clipids_rows.txt")) asymmetric_col_ids = np.loadtxt(osp.join(data_dir, "asymmetric_clipids_cols.txt")) # noinspection PyCallingNonCallable asymmetric_dk_mat = np.matrix(np.load(osp.join(data_dir, "asymmetric_distance_kernel.npy"))) # noinspection PyUnresolvedReferences #: :type: DistanceKernel cls.dk_arc = getFeatureManager().DistanceKernel(asymmetric_row_ids, asymmetric_col_ids, asymmetric_dk_mat) cls.mdb_info = DatabaseInfo("localhost", 27017, "RSS_Test") register_mdb_loc(cls.mdb_info.host, cls.mdb_info.port)
def setUp(self): # uses default database location (localhost, 27017) gc_config = SmqtkController.generate_config() # Modifying VCDController database location so we can safely remove it # when we're done self._vcd_store_loc = osp.join(osp.dirname(osp.abspath(__file__)), 'work/vcd_store') gc_config.set(VCDController.CONFIG_SECT, 'store_name', self._vcd_store_loc) # TODO: change the smqtk_controller:mongo_database self.gc = SmqtkController(gc_config) data_dir = osp.abspath(osp.join(osp.dirname(__file__), 'data')) print "Data dir:", data_dir symmetric_clipids = np.loadtxt(osp.join(data_dir, 'symmetric_clipids.txt')) symmetric_bgflags = np.loadtxt(osp.join(data_dir, 'symmetric_bgflags.txt')) # noinspection PyCallingNonCallable symmetric_dk_mat = \ np.matrix(np.load(osp.join(data_dir, 'symmetric_distance_kernel.npy'))) symmetric_bgclipids = np.array([symmetric_clipids[i] for i, e in enumerate(symmetric_bgflags) if e]) # noinspection PyUnresolvedReferences #: :type: DistanceKernel self.dk_iqr = getFeatureManager().DistanceKernel(symmetric_clipids, symmetric_clipids, symmetric_dk_mat, symmetric_bgclipids) asymmetric_row_ids = np.loadtxt(osp.join(data_dir, 'asymmetric_clipids_rows.txt')) asymmetric_col_ids = np.loadtxt(osp.join(data_dir, 'asymmetric_clipids_cols.txt')) # noinspection PyCallingNonCallable asymmetric_dk_mat = \ np.matrix(np.load(osp.join(data_dir, 'asymmetric_distance_kernel.npy'))) # noinspection PyUnresolvedReferences #: :type: DistanceKernel self.dk_arc = getFeatureManager().DistanceKernel(asymmetric_row_ids, asymmetric_col_ids, asymmetric_dk_mat)
def setUp(self): # uses default database location (localhost, 27017) gc_config = SmqtkController.generate_config() # Modifying VCDController database location so we can safely remove it # when we're done self._vcd_store_loc = osp.join(osp.dirname(osp.abspath(__file__)), 'work/vcd_store') gc_config.set(VCDController.CONFIG_SECT, 'store_name', self._vcd_store_loc) # TODO: change the smqtk_controller:mongo_database self.gc = SmqtkController(gc_config) data_dir = osp.abspath(osp.join(osp.dirname(__file__), 'data')) print "Data dir:", data_dir symmetric_clipids = np.loadtxt( osp.join(data_dir, 'symmetric_clipids.txt')) symmetric_bgflags = np.loadtxt( osp.join(data_dir, 'symmetric_bgflags.txt')) # noinspection PyCallingNonCallable symmetric_dk_mat = \ np.matrix(np.load(osp.join(data_dir, 'symmetric_distance_kernel.npy'))) symmetric_bgclipids = np.array([ symmetric_clipids[i] for i, e in enumerate(symmetric_bgflags) if e ]) # noinspection PyUnresolvedReferences #: :type: DistanceKernel self.dk_iqr = getFeatureManager().DistanceKernel( symmetric_clipids, symmetric_clipids, symmetric_dk_mat, symmetric_bgclipids) asymmetric_row_ids = np.loadtxt( osp.join(data_dir, 'asymmetric_clipids_rows.txt')) asymmetric_col_ids = np.loadtxt( osp.join(data_dir, 'asymmetric_clipids_cols.txt')) # noinspection PyCallingNonCallable asymmetric_dk_mat = \ np.matrix(np.load(osp.join(data_dir, 'asymmetric_distance_kernel.npy'))) # noinspection PyUnresolvedReferences #: :type: DistanceKernel self.dk_arc = getFeatureManager().DistanceKernel( asymmetric_row_ids, asymmetric_col_ids, asymmetric_dk_mat)
def setUpClass(cls): # local FeatureManager server may have already been initialized by # another unittest try: initFeatureManagerConnection() except ValueError: pass data_dir = osp.abspath(osp.join(osp.dirname(__file__), 'data')) classifier_config_file = osp.join(data_dir, 'classifier_config.json') cls.classifier_config = jsmin(open(classifier_config_file).read()) symmetric_clipids = np.loadtxt( osp.join(data_dir, 'symmetric_clipids.txt')) symmetric_bgflags = np.loadtxt( osp.join(data_dir, 'symmetric_bgflags.txt')) # noinspection PyCallingNonCallable symmetric_dk_mat = \ np.matrix(np.load(osp.join(data_dir, 'symmetric_distance_kernel.npy'))) symmetric_bgclipids = np.array([ symmetric_clipids[i] for i, e in enumerate(symmetric_bgflags) if e ]) # noinspection PyUnresolvedReferences #: :type: DistanceKernel cls.dk_iqr = getFeatureManager().DistanceKernel( symmetric_clipids, symmetric_clipids, symmetric_dk_mat, symmetric_bgclipids) asymmetric_row_ids = np.loadtxt( osp.join(data_dir, 'asymmetric_clipids_rows.txt')) asymmetric_col_ids = np.loadtxt( osp.join(data_dir, 'asymmetric_clipids_cols.txt')) # noinspection PyCallingNonCallable asymmetric_dk_mat = \ np.matrix(np.load(osp.join(data_dir, 'asymmetric_distance_kernel.npy'))) # noinspection PyUnresolvedReferences #: :type: DistanceKernel cls.dk_arc = getFeatureManager().DistanceKernel( asymmetric_row_ids, asymmetric_col_ids, asymmetric_dk_mat) cls.mdb_info = DatabaseInfo('localhost', 27017, 'RSS_Test') register_mdb_loc(cls.mdb_info.host, cls.mdb_info.port)
def setUp(self): self.mgr = getFeatureManager() self.N = 1024 * 4 # 4096 self.M = 1024 * 6 # 6144 # self.N = 1024 * 8 # self.M = 1024 * 16 self.id_vec = np.array(range(self.N)) self.bg_ids = np.array(range(self.N//10)) self.feature_mat = np.matrix(np.random.random((self.N, self.M))) self.dk_mat = np.matrix(np.random.random((self.N, self.N)))
def test_common_fm(lock): lock.acquire() lock.release() lcl_common_fmm = get_common_fmm() assert len(lcl_common_fmm.get_feature_types()) == 0, \ "local fmm had something in it" #: :type: FeatureMemoryMap gbl_common_fmm = getFeatureManager().get_common_fmm() assert len(gbl_common_fmm.get_feature_types()) >= 1, \ "Unexpected number of features in global fmm: %s" \ % gbl_common_fmm.get_feature_types() assert expected_type in gbl_common_fmm.get_feature_types(), \ "Expected feature not present in global fmm. Current " \ "features: %s" % gbl_common_fmm.get_feature_types() gbl_common_fmm.initialize_from_files( str(current_process().pid), cid_file, bg_flags_file, feature_file, kernel_file )
from WebUI import app from SMQTK_Backend.FeatureMemory import \ initFeatureManagerConnection, \ getFeatureManager, \ ReadWriteLock, \ DummyRWLock import logging logger = logging.getLogger("WebUI.Cache") logger.setLevel(logging.INFO) import os # Connect with the feature memory server mgr_srvr_addr = ('127.0.0.1', 30000) initFeatureManagerConnection(mgr_srvr_addr, 'test') mgr = getFeatureManager(mgr_srvr_addr) # Get timed cache # noinspection PyUnresolvedReferences #: :type: SMQTK_Backend.FeatureMemory.TimedCache tc = mgr.get_common_tc() data_path = app.config['DATA_DIR'] # Load the distance kernel matrices known_features = ["csift", "hog_2x2", "isa", "mfcc_4096", "OB_max_avg_positive"] def load_feature_dk(f_name): try:
def test_controller_interaction(self): print 'poke' # Dummy classes representing function of intended components class Controller (Process): """ Reads input from queue, exiting when receives None """ def __init__(self, data_queue): super(Controller, self).__init__(name="controller") self._queue = data_queue def run(self): running = True while running: print "[Controller] waiting for data..." elem = self._queue.get() print "[Controller] received:", type(elem), elem if elem is None: running = False class DummyAsw (Process): """ sends dk to queue on separate process """ def __init__(self, dk, data_queue): super(DummyAsw, self).__init__() self._dk = dk self._queue = data_queue def run(self): self._queue.put(self._dk) def generate_dks(mgr): """ generated some distance kernels to test with given a connected FeatureManager """ c_fmm = mgr.get_common_fmm() if 'test' in c_fmm.get_feature_types(): c_fmm.remove('test') c_fmm.initialize_from_files('test', self.sym_cids_file, self.sym_bg_flags, self.sym_feat_mat_f, self.sym_dk_mat_f) dk1 = c_fmm.get_distance_kernel('test') dk2 = mgr.symmetric_dk_from_file(self.sym_cids_file, self.sym_dk_mat_f, self.sym_bg_flags) return dk1, dk2 # Set-up and start the extra-process feature manager server host = 'localhost' port = 54321 addr = (host, port) auth_key = "foobar" config_file = osp.join(self.work_dir, 'controller_interaction_server.ini') config = SafeConfigCommentParser() config.add_section('server') config.set('server', 'port', str(port)) config.set('server', 'authkey', auth_key) with open(config_file, 'w') as ofile: # noinspection PyTypeChecker config.write(ofile) p = subprocess.Popen(['FeatureManagerServer', '-c', config_file]) # wait a sec for the server to start/fail time.sleep(1) # should be sufficient... self.assertIsNone(p.poll(), "Server process shutdown prematurely. " "Check reported error on console.") initFeatureManagerConnection(addr, auth_key) mgr = getFeatureManager(addr) # Initialize and start dummy controller process q = Queue() c = Controller(q) c.start() # generated DistanceKernels and dummy ASW processes dk1, dk2 = generate_dks(mgr) asw1 = DummyAsw(dk1, q) asw2 = DummyAsw(dk2, q) # Running the two dummy asw processes should cause no errors in either # dummy ASW or the dummy Controller asw1.start() asw1.join() asw2.start() asw2.join() # shutdown controller q.put(None) c.join() print "C exitcode:", c.exitcode # Clean-up # Need to call del on some things else we would get hung up in decref # call to remove server at function/process exit. del dk1, dk2, asw1, asw2 os.remove(config_file) removeFeatureManagerConnection(addr) p.terminate() p.poll() self.assertEqual(c.exitcode, 0, "Controller dummy did not exit cleanly") del c
def test_fm_external_server(self): print "===================================================" print "Testing use of external server with update function" print "===================================================" host = 'localhost' port = 54321 addr = (host, port) auth_key = "foobar" # Trying to connect to these right away should result in a timeout # in the underlying proxy. self.assertRaises( Exception, initFeatureManagerConnection, addr, auth_key ) # Start the server at the specified addr config_file = osp.join(self.work_dir, 'test_server_config.ini') config = SafeConfigCommentParser() config.add_section('server') config.set('server', 'port', str(port)) config.set('server', 'authkey', auth_key) with open(config_file, 'w') as ofile: # noinspection PyTypeChecker config.write(ofile) p = subprocess.Popen(['FeatureManagerServer', '-c', config_file]) # wait a sec for the server to start/fail time.sleep(1) # should be sufficient... self.assertIsNone(p.poll(), "Server process shutdown prematurely. " "Check reported error on console.") # Now we should be able to connect, create a FeatureMemory[Map] and # successfully go through the update process. from multiprocessing.connection import AuthenticationError self.assertRaises( AuthenticationError, initFeatureManagerConnection, addr, "not the right key" ) # Shouldn't be in there yet as its not initialized self.assertRaises( KeyError, getFeatureManager, addr ) initFeatureManagerConnection(addr, auth_key) # should get a ValueError when trying to init the same address more than # once. self.assertRaises( ValueError, initFeatureManagerConnection, addr, auth_key ) mgr = getFeatureManager(addr) # Create a feature map->featureMemory and pass through update process f_type = 'test' cid_file = osp.join(self.data_dir, 'symmetric_clipids.txt') bg_flags_file = osp.join(self.data_dir, 'symmetric_bgflags.txt') feature_file = osp.join(self.data_dir, 'symmetric_feature.npy') kernel_file = osp.join(self.data_dir, 'symmetric_distance_kernel.npy') #: :type: FeatureMemoryMap fmm = mgr.get_common_fmm() fmm.initialize_from_files(f_type, cid_file, bg_flags_file, feature_file, kernel_file) fm = fmm.get_feature_memory(f_type) self._update_test_helper(fm) # Clean up # Need to call del else we would get hung up in decref call to remove # server at function/process exit. del fm, fmm p.terminate() os.remove(config_file) removeFeatureManagerConnection(addr)
def test_fmm_singleton(self): # test that the correct singleton is being accessed across disjoint # processes gbl_lock = Lock() expected_type = 'expected_type' cid_file = osp.join(self.data_dir, 'symmetric_clipids.txt') bg_flags_file = osp.join(self.data_dir, 'symmetric_bgflags.txt') feature_file = osp.join(self.data_dir, 'symmetric_feature.npy') kernel_file = osp.join(self.data_dir, 'symmetric_distance_kernel.npy') def test_common_fm(lock): lock.acquire() lock.release() lcl_common_fmm = get_common_fmm() assert len(lcl_common_fmm.get_feature_types()) == 0, \ "local fmm had something in it" #: :type: FeatureMemoryMap gbl_common_fmm = getFeatureManager().get_common_fmm() assert len(gbl_common_fmm.get_feature_types()) >= 1, \ "Unexpected number of features in global fmm: %s" \ % gbl_common_fmm.get_feature_types() assert expected_type in gbl_common_fmm.get_feature_types(), \ "Expected feature not present in global fmm. Current " \ "features: %s" % gbl_common_fmm.get_feature_types() gbl_common_fmm.initialize_from_files( str(current_process().pid), cid_file, bg_flags_file, feature_file, kernel_file ) gbl_lock.acquire() print '[main] acquire' p1 = Process(name='p1', target=test_common_fm, args=(gbl_lock,)) p2 = Process(name='p2', target=test_common_fm, args=(gbl_lock,)) p3 = Process(name='p3', target=test_common_fm, args=(gbl_lock,)) print '[main] starting processes' p1.start() p2.start() p3.start() print '[main] adding expected feature to global FMM' #: :type: FeatureMemoryMap gbl_fmm = getFeatureManager().get_common_fmm() gbl_fmm.initialize_from_files(expected_type, cid_file, bg_flags_file, feature_file, kernel_file) print '[main] release' gbl_lock.release() p1.join(timeout=1) print '[main] checking p1' self.assertIsNotNone(p1.exitcode, "p1 timed out") self.assertEquals(p1.exitcode, 0, "p1 returned non-0 exitcode") p2.join(timeout=1) print '[main] checking p2' self.assertIsNotNone(p2.exitcode, "p2 timed out") self.assertEquals(p2.exitcode, 0, "p2 returned non-0 exitcode") p3.join(timeout=1) print '[main] checking p3' self.assertIsNotNone(p3.exitcode, "p3 timed out") self.assertEquals(p3.exitcode, 0, "p3 returned non-0 exitcode") print "[main] checking post-processing expected keys" self.assertEqual(set(gbl_fmm.get_feature_types()), set((expected_type, str(p1.pid), str(p2.pid), str(p3.pid))))