def setUpClass(cls): # local FeatureManager server may have already been initialized by # another unittest try: initFeatureManagerConnection() except ValueError: pass
def setUpClass(cls): # local FeatureManager server may have already been initialized by # another unittest try: initFeatureManagerConnection() except ValueError: pass data_dir = osp.abspath(osp.join(osp.dirname(__file__), "data")) classifier_config_file = osp.join(data_dir, "classifier_config.json") cls.classifier_config = jsmin(open(classifier_config_file).read()) symmetric_clipids = np.loadtxt(osp.join(data_dir, "symmetric_clipids.txt")) symmetric_bgflags = np.loadtxt(osp.join(data_dir, "symmetric_bgflags.txt")) # noinspection PyCallingNonCallable symmetric_dk_mat = np.matrix(np.load(osp.join(data_dir, "symmetric_distance_kernel.npy"))) symmetric_bgclipids = np.array([symmetric_clipids[i] for i, e in enumerate(symmetric_bgflags) if e]) # noinspection PyUnresolvedReferences #: :type: DistanceKernel cls.dk_iqr = getFeatureManager().DistanceKernel( symmetric_clipids, symmetric_clipids, symmetric_dk_mat, symmetric_bgclipids ) asymmetric_row_ids = np.loadtxt(osp.join(data_dir, "asymmetric_clipids_rows.txt")) asymmetric_col_ids = np.loadtxt(osp.join(data_dir, "asymmetric_clipids_cols.txt")) # noinspection PyCallingNonCallable asymmetric_dk_mat = np.matrix(np.load(osp.join(data_dir, "asymmetric_distance_kernel.npy"))) # noinspection PyUnresolvedReferences #: :type: DistanceKernel cls.dk_arc = getFeatureManager().DistanceKernel(asymmetric_row_ids, asymmetric_col_ids, asymmetric_dk_mat) cls.mdb_info = DatabaseInfo("localhost", 27017, "RSS_Test") register_mdb_loc(cls.mdb_info.host, cls.mdb_info.port)
def setUpClass(cls): # local FeatureManager server may have already been initialized by # another unittest try: initFeatureManagerConnection() except ValueError: pass cls.data_dir = osp.abspath(osp.join(osp.dirname(__file__), 'data')) cls.work_dir = osp.abspath(osp.join(osp.dirname(__file__), 'work')) cls.sym_cids_file = osp.join(cls.data_dir, "symmetric_clipids.txt") cls.sym_bg_flags = osp.join(cls.data_dir, "symmetric_bgflags.txt") cls.sym_dk_mat_f = osp.join(cls.data_dir, "symmetric_distance_kernel.npy") cls.sym_feat_mat_f = osp.join(cls.data_dir, "symmetric_feature.npy")
def setUpClass(cls): # local FeatureManager server may have already been initialized by # another unittest try: initFeatureManagerConnection() except ValueError: pass data_dir = osp.abspath(osp.join(osp.dirname(__file__), 'data')) classifier_config_file = osp.join(data_dir, 'classifier_config.json') cls.classifier_config = jsmin(open(classifier_config_file).read()) symmetric_clipids = np.loadtxt( osp.join(data_dir, 'symmetric_clipids.txt')) symmetric_bgflags = np.loadtxt( osp.join(data_dir, 'symmetric_bgflags.txt')) # noinspection PyCallingNonCallable symmetric_dk_mat = \ np.matrix(np.load(osp.join(data_dir, 'symmetric_distance_kernel.npy'))) symmetric_bgclipids = np.array([ symmetric_clipids[i] for i, e in enumerate(symmetric_bgflags) if e ]) # noinspection PyUnresolvedReferences #: :type: DistanceKernel cls.dk_iqr = getFeatureManager().DistanceKernel( symmetric_clipids, symmetric_clipids, symmetric_dk_mat, symmetric_bgclipids) asymmetric_row_ids = np.loadtxt( osp.join(data_dir, 'asymmetric_clipids_rows.txt')) asymmetric_col_ids = np.loadtxt( osp.join(data_dir, 'asymmetric_clipids_cols.txt')) # noinspection PyCallingNonCallable asymmetric_dk_mat = \ np.matrix(np.load(osp.join(data_dir, 'asymmetric_distance_kernel.npy'))) # noinspection PyUnresolvedReferences #: :type: DistanceKernel cls.dk_arc = getFeatureManager().DistanceKernel( asymmetric_row_ids, asymmetric_col_ids, asymmetric_dk_mat) cls.mdb_info = DatabaseInfo('localhost', 27017, 'RSS_Test') register_mdb_loc(cls.mdb_info.host, cls.mdb_info.port)
from WebUI import app from SMQTK_Backend.FeatureMemory import \ initFeatureManagerConnection, \ getFeatureManager, \ ReadWriteLock, \ DummyRWLock import logging logger = logging.getLogger("WebUI.Cache") logger.setLevel(logging.INFO) import os # Connect with the feature memory server mgr_srvr_addr = ('127.0.0.1', 30000) initFeatureManagerConnection(mgr_srvr_addr, 'test') mgr = getFeatureManager(mgr_srvr_addr) # Get timed cache # noinspection PyUnresolvedReferences #: :type: SMQTK_Backend.FeatureMemory.TimedCache tc = mgr.get_common_tc() data_path = app.config['DATA_DIR'] # Load the distance kernel matrices known_features = ["csift", "hog_2x2", "isa", "mfcc_4096", "OB_max_avg_positive"] def load_feature_dk(f_name): try:
def test_controller_interaction(self): print 'poke' # Dummy classes representing function of intended components class Controller (Process): """ Reads input from queue, exiting when receives None """ def __init__(self, data_queue): super(Controller, self).__init__(name="controller") self._queue = data_queue def run(self): running = True while running: print "[Controller] waiting for data..." elem = self._queue.get() print "[Controller] received:", type(elem), elem if elem is None: running = False class DummyAsw (Process): """ sends dk to queue on separate process """ def __init__(self, dk, data_queue): super(DummyAsw, self).__init__() self._dk = dk self._queue = data_queue def run(self): self._queue.put(self._dk) def generate_dks(mgr): """ generated some distance kernels to test with given a connected FeatureManager """ c_fmm = mgr.get_common_fmm() if 'test' in c_fmm.get_feature_types(): c_fmm.remove('test') c_fmm.initialize_from_files('test', self.sym_cids_file, self.sym_bg_flags, self.sym_feat_mat_f, self.sym_dk_mat_f) dk1 = c_fmm.get_distance_kernel('test') dk2 = mgr.symmetric_dk_from_file(self.sym_cids_file, self.sym_dk_mat_f, self.sym_bg_flags) return dk1, dk2 # Set-up and start the extra-process feature manager server host = 'localhost' port = 54321 addr = (host, port) auth_key = "foobar" config_file = osp.join(self.work_dir, 'controller_interaction_server.ini') config = SafeConfigCommentParser() config.add_section('server') config.set('server', 'port', str(port)) config.set('server', 'authkey', auth_key) with open(config_file, 'w') as ofile: # noinspection PyTypeChecker config.write(ofile) p = subprocess.Popen(['FeatureManagerServer', '-c', config_file]) # wait a sec for the server to start/fail time.sleep(1) # should be sufficient... self.assertIsNone(p.poll(), "Server process shutdown prematurely. " "Check reported error on console.") initFeatureManagerConnection(addr, auth_key) mgr = getFeatureManager(addr) # Initialize and start dummy controller process q = Queue() c = Controller(q) c.start() # generated DistanceKernels and dummy ASW processes dk1, dk2 = generate_dks(mgr) asw1 = DummyAsw(dk1, q) asw2 = DummyAsw(dk2, q) # Running the two dummy asw processes should cause no errors in either # dummy ASW or the dummy Controller asw1.start() asw1.join() asw2.start() asw2.join() # shutdown controller q.put(None) c.join() print "C exitcode:", c.exitcode # Clean-up # Need to call del on some things else we would get hung up in decref # call to remove server at function/process exit. del dk1, dk2, asw1, asw2 os.remove(config_file) removeFeatureManagerConnection(addr) p.terminate() p.poll() self.assertEqual(c.exitcode, 0, "Controller dummy did not exit cleanly") del c
def test_fm_external_server(self): print "===================================================" print "Testing use of external server with update function" print "===================================================" host = 'localhost' port = 54321 addr = (host, port) auth_key = "foobar" # Trying to connect to these right away should result in a timeout # in the underlying proxy. self.assertRaises( Exception, initFeatureManagerConnection, addr, auth_key ) # Start the server at the specified addr config_file = osp.join(self.work_dir, 'test_server_config.ini') config = SafeConfigCommentParser() config.add_section('server') config.set('server', 'port', str(port)) config.set('server', 'authkey', auth_key) with open(config_file, 'w') as ofile: # noinspection PyTypeChecker config.write(ofile) p = subprocess.Popen(['FeatureManagerServer', '-c', config_file]) # wait a sec for the server to start/fail time.sleep(1) # should be sufficient... self.assertIsNone(p.poll(), "Server process shutdown prematurely. " "Check reported error on console.") # Now we should be able to connect, create a FeatureMemory[Map] and # successfully go through the update process. from multiprocessing.connection import AuthenticationError self.assertRaises( AuthenticationError, initFeatureManagerConnection, addr, "not the right key" ) # Shouldn't be in there yet as its not initialized self.assertRaises( KeyError, getFeatureManager, addr ) initFeatureManagerConnection(addr, auth_key) # should get a ValueError when trying to init the same address more than # once. self.assertRaises( ValueError, initFeatureManagerConnection, addr, auth_key ) mgr = getFeatureManager(addr) # Create a feature map->featureMemory and pass through update process f_type = 'test' cid_file = osp.join(self.data_dir, 'symmetric_clipids.txt') bg_flags_file = osp.join(self.data_dir, 'symmetric_bgflags.txt') feature_file = osp.join(self.data_dir, 'symmetric_feature.npy') kernel_file = osp.join(self.data_dir, 'symmetric_distance_kernel.npy') #: :type: FeatureMemoryMap fmm = mgr.get_common_fmm() fmm.initialize_from_files(f_type, cid_file, bg_flags_file, feature_file, kernel_file) fm = fmm.get_feature_memory(f_type) self._update_test_helper(fm) # Clean up # Need to call del else we would get hung up in decref call to remove # server at function/process exit. del fm, fmm p.terminate() os.remove(config_file) removeFeatureManagerConnection(addr)