def test_resume_session(get_interface_params): """ Test resume session. Args: get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params local_interface = LocalInterface(config_name, config_directory) session_id = local_interface.session_request( ["OND.54011215.0000.1236"], "OND", "image_classification", "0.1.1", [], 0.5 ) local_interface.complete_test(session_id, "OND.54011215.0000.1236") finished_test = local_interface.resume_session(session_id) assert finished_test == ["OND.54011215.0000.1236"] # Testing with hints session_id = local_interface.session_request( ["OND.54011215.0000.1236"], "OND", "image_classification", "0.1.1", ["red_light"], 0.4, ) local_interface.complete_test(session_id, "OND.54011215.0000.1236") finished_test = local_interface.resume_session(session_id) assert finished_test == ["OND.54011215.0000.1236"]
def test_create_feedback_instance(domain, test_id, get_interface_params, expected): """ Test for creating metric instance. Args: protocol: Name of the protocol domain: Name of the domain gt_dict: Parameters for the class created by metric expected: Expected Output Class Returns: None """ config_directory, config_name = get_interface_params local_interface = LocalInterface(config_name, config_directory) protocol_name = "OND" session_id = local_interface.session_request([test_id], protocol_name, domain, "0.1.1", (), 0.5) feedback_dict = { "first_budget": 10, "income_per_batch": 10, "maximum_budget": 10, "interface": local_interface, "session_id": session_id, "test_id": test_id, "feedback_type": "classification", } feedback_obj = create_feedback_instance(domain, feedback_dict) assert isinstance(feedback_obj, expected)
def test_transcripts_evaluate(get_transcripts_interface_params): """ Test evaluate for transcripts. Args: get_transcripts_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_transcripts_interface_params local_interface = LocalInterface(config_name, config_directory) session_id = _initialize_session(local_interface, "OND", "transcripts") result_folder = os.path.join( os.path.dirname(__file__), "mock_results", "transcripts" ) detection_file_id = os.path.join( result_folder, "OND.0.90001.8714062_PreComputedDetector_detection.csv" ) classification_file_id = os.path.join( result_folder, "OND.0.90001.8714062_PreComputedDetector_classification.csv" ) characterization_file_id = os.path.join( result_folder, "OND.0.90001.8714062_PreComputedDetector_characterization.csv" ) results = { "detection": detection_file_id, "classification": classification_file_id, "characterization": characterization_file_id, } baseline_session_id = _initialize_session(local_interface, "OND", "transcripts") local_interface.post_results(results, "OND.0.90001.8714062", 0, session_id) local_interface.evaluate("OND.0.90001.8714062", 0, session_id) baseline_classification_file_id = os.path.join( result_folder, "OND.0.90001.8714062_BaselinePreComputedDetector_classification.csv", ) baseline_result = { "classification": baseline_classification_file_id, } local_interface.post_results( baseline_result, "OND.0.90001.8714062", 0, baseline_session_id ) local_interface.evaluate("OND.0.90001.8714062", 0, session_id, baseline_session_id)
def test_terminate_session(get_interface_params): """ Test terminate session request. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params local_interface = LocalInterface(config_name, config_directory) session_id = _initialize_session(local_interface, "OND") local_interface.terminate_session(session_id)
def test_feature_extraction( server_setup, get_interface_params, discoverable_plugins, ond_config_with_feature_extraction, ): """ Test feature extraction. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface discoverable_plugins (dict): Dictionary with the plugins ond_config_with_feature_extraction (str): Path to json file Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) ond = SailOn(discoverable_plugins, "", par_interface, ond_config_with_feature_extraction) ond.run_protocol() local_interface = LocalInterface(config_name, config_directory) SailOn(discoverable_plugins, "", local_interface, ond_config_with_feature_extraction) ond.run_protocol()
def test_complete_test(get_interface_params): """ Test complete test request. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params local_interface = LocalInterface(config_name, config_directory) session_id = _initialize_session(local_interface, "OND") local_interface.complete_test(session_id, "OND.10.90001.2100554")
def test_image_classification_evaluate(get_ic_interface_params): """ Test evaluate with rounds. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_ic_interface_params local_interface = LocalInterface(config_name, config_directory) session_id = _initialize_session(local_interface, "OND", "image_classification") baseline_session_id = _initialize_session( local_interface, "OND", "image_classification" ) result_folder = os.path.join( os.path.dirname(__file__), "mock_results", "image_classification" ) detection_file_id = os.path.join( result_folder, "OND.54011215.0000.1236_PreComputedDetector_detection.csv" ) classification_file_id = os.path.join( result_folder, "OND.54011215.0000.1236_PreComputedDetector_classification.csv" ) baseline_classification_file_id = os.path.join( result_folder, "OND.54011215.0000.1236_BaselinePreComputedDetector_classification.csv", ) results = { "detection": detection_file_id, "classification": classification_file_id, } baseline_result = { "classification": baseline_classification_file_id, } local_interface.post_results(results, "OND.54011215.0000.1236", 0, session_id) local_interface.post_results( baseline_result, "OND.54011215.0000.1236", 0, baseline_session_id ) local_interface.evaluate("OND.54011215.0000.1236", 0, session_id) local_interface.evaluate( "OND.54011215.0000.1236", 0, session_id, baseline_session_id )
def test_evaluate(get_interface_params): """ Test evaluate with rounds. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params data_dir = f"{os.path.dirname(__file__)}/data" local_interface = LocalInterface(config_name, config_directory) session_id = _initialize_session(local_interface, "OND") response = local_interface.evaluate("OND.1.1.1234", 0, session_id) expected = os.path.join(data_dir, "evaluation.csv") assert expected == response
def test_feedback_request(get_interface_params, feedback_mapping, protocol_name): """ Tests for feedback request. Args: get_interface_params (tuple): Tuple to configure local interface feedback_mapping (dict): Dict with mapping for feedback protocol_name (str): Name of the protocol (options: OND and CONDDA) Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params local_interface = LocalInterface(config_name, config_directory) session_id = _initialize_session(local_interface, protocol_name) # Post results before posting result_files = {} protocol_constant = feedback_mapping[0] required_files = feedback_mapping[1] for required_file in required_files: result_files[required_file] = os.path.join( os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv" ) local_interface.post_results( result_files, f"{protocol_name}.1.1.1234", 0, session_id ) # Get feedback for detection response = local_interface.get_feedback_request( ["n01484850_18013.JPEG", "n01484850_24624.JPEG"], protocol_constant, f"{protocol_name}.1.1.1234", 0, session_id, ) expected = os.path.join( local_interface.result_directory, "feedback", f"{session_id}.{protocol_name}.1.1.1234.0_{protocol_constant}.csv", ) assert expected == response
def test_session_request(get_interface_params): """ Test session request. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params data_dir = f"{os.path.dirname(__file__)}/data" local_interface = LocalInterface(config_name, config_directory) test_id_path = os.path.join(data_dir, "OND", "image_classification", TEST_ID_NAME) test_ids = list(map(str.strip, open(test_id_path, "r").readlines())) # Testing if session was sucessfully initalized local_interface.session_request( test_ids, "OND", "image_classification", "0.1.1", [], 0.5 ) # Testing with hints local_interface.session_request( test_ids, "OND", "image_classification", "0.1.1", ["red_light"], 0.5 )
def test_get_metadata(get_interface_params): """ Test get metadata. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params local_interface = LocalInterface(config_name, config_directory) session_id = _initialize_session(local_interface, "OND") metadata = local_interface.get_test_metadata(session_id, "OND.1.1.1234") assert "OND" == metadata["protocol"] assert 3 == metadata["known_classes"] session_id = _initialize_session(local_interface, "OND", ["red_light"]) metadata = local_interface.get_test_metadata(session_id, "OND.1.1.1234") assert "n01484850_4515.JPEG" == metadata["red_light"]
def test_post_results(get_interface_params, protocol_constant, protocol_name): """ Tests for post results. Args: get_interface_params (tuple): Tuple to configure local interface protocol_constant (str): Constants used by the server to identifying results protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params local_interface = LocalInterface(config_name, config_directory) session_id = _initialize_session(local_interface, protocol_name) result_files = { protocol_constant: os.path.join(os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv") } local_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0, session_id)
def test_dataset_request(get_interface_params): """ Tests for dataset request. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params local_interface = LocalInterface(config_name, config_directory) session_id = _initialize_session(local_interface, "OND") # Test correct dataset request filename = local_interface.dataset_request("OND.1.1.1234", 0, session_id) expected = os.path.join( local_interface.result_directory, f"{session_id}.OND.1.1.1234.0.csv" ) assert expected == filename expected_image_ids = _read_image_ids(expected) assert expected_image_ids == ["n01484850_18013.JPEG", "n01484850_24624.JPEG"]
def test_test_ids_request(get_interface_params): """ Test request for test ids. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params data_dir = f"{os.path.dirname(__file__)}/data" local_interface = LocalInterface(config_name, config_directory) assumptions_path = os.path.join(os.path.dirname(__file__), "assumptions.json") filename = local_interface.test_ids_request( "OND", "image_classification", "5678", assumptions_path ) expected = os.path.join(data_dir, "OND", "image_classification", TEST_ID_NAME) assert os.stat(expected).st_size > 5 assert expected == filename
def test_initialize(get_interface_params): """ Test local interface initialization. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ from sail_on_client.protocol.localinterface import LocalInterface config_directory, config_name = get_interface_params LocalInterface(config_name, config_directory)
def test_update_harness_paramerters(get_interface_params): """ Test to update harness parameters. Args: get_interface_params (tuple): Tuple to configure local interface Return: None """ with TemporaryDirectory() as updated_data_dir: config_directory, config_name = get_interface_params local_interface = LocalInterface(config_name, config_directory) assert local_interface.data_dir != updated_data_dir local_interface = update_harness_parameters( local_interface, {"data_dir": updated_data_dir} ) assert local_interface.data_dir == updated_data_dir
def test_initialize(server_setup, get_interface_params, discoverable_plugins, ond_config): """ Test ond protocol initialization. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface discoverable_plugins (dict): Dictionary with the plugins ond_config (str): Path to json file Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) SailOn(discoverable_plugins, "", par_interface, ond_config) local_interface = LocalInterface(config_name, config_directory) SailOn(discoverable_plugins, "", local_interface, ond_config)
def test_run_protocol(server_setup, get_interface_params, discoverable_plugins, condda_config): """ Test running protocol. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface discoverable_plugins (dict): Dictionary with the plugins condda_config (str): Path to json file Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) condda = Condda(discoverable_plugins, "", par_interface, condda_config) condda.run_protocol() local_interface = LocalInterface(config_name, config_directory) condda = Condda(discoverable_plugins, "", local_interface, condda_config) condda.run_protocol()
def test_reaction_baseline( server_setup, get_ar_interface_params, discoverable_plugins, ond_config_with_reaction_baseline, ): """ Test reaction baseline with a detector. Args: server_setup (tuple): Tuple containing url and result directory get_ar_interface_params (tuple): Tuple to configure par interface discoverable_plugins (dict): Dictionary with the plugins ond_config_with_reaction_baseline(str): Path to json file Return: None """ config_directory, config_name = get_ar_interface_params local_interface = LocalInterface(config_name, config_directory) ond = SailOn(discoverable_plugins, "", local_interface, ond_config_with_reaction_baseline) ond.run_protocol()
def run_protocol( self, config: Dict[str, Any], extra_plugins: Dict[str, Any] = dict()) -> None: """Run the protocol by printout out the config. Args: Config passed in uses 3 parameters to control the launching of the protocols - protocol: either 'ond' or 'condda' to define which protocol to run - harness: either 'local' or 'par' to define which harness to use - workdir: a directory to save all the information from the run including - Config - Output of algorithm Example: >>> from sailon_tinker_launcher.main import * >>> dpath = ub.ensure_app_cache_dir('tinker/tests') >>> config = get_debug_config() >>> self = LaunchSailonProtocol() >>> self.run_protocol(config) >>> assert(self.working_folder.exists()) >>> ub.delete(str(self.working_folder), verbose=False) """ # Setup working folder and create new config for this run self.working_folder, working_config_fp, privileged_config, config = self.setup_experiment( config) # Now experiment setup, start a new logger for this fh = logging.FileHandler( self.working_folder / f'{datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p")}.log') fh.setLevel(logging.DEBUG) formatter = logging.Formatter( '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s') fh.setFormatter(formatter) logging.getLogger().addHandler(fh) log.info(f'Config Filepath: {working_config_fp}') log.info(f'Config: \n{json.dumps(config, indent=4)}') # Load the harness # This config is not used but will throw error if not pointed at harnness_config_path = Path(protocol_folder.__file__).parent if privileged_config['harness'] == 'local': log.info('Loading Local Harness') harness = LocalInterface('configuration.json', str(harnness_config_path)) harness.result_directory = config['detectors']['csv_folder'] harness.file_provider.results_folder = config['detectors'][ 'csv_folder'] elif privileged_config['harness'] == 'par': log.info('Loading Par Harness') harness = ParInterface('configuration.json', str(harnness_config_path)) harness.folder = config['detectors']['csv_folder'] else: raise AttributeError( f'Valid harnesses "local" or "par". ' f'Given harness "{privileged_config["harness"]}" ') # Get the plugins plugins = ub.dict_union(discoverable_plugins('tinker'), extra_plugins) log.debug('Plugins found:') log.debug(plugins) # Load the protocol if privileged_config['protocol'] == 'ond': log.info('Running OND Protocol') run_protocol = OND(discovered_plugins=plugins, algorithmsdirectory='', harness=harness, config_file=str(working_config_fp)) elif privileged_config['protocol'] == 'condda': log.info('Running Condda Protocol') run_protocol = Condda(discovered_plugins=plugins, algorithmsdirectory='', harness=harness, config_file=str(working_config_fp)) else: raise AttributeError( f'Please set protocol to either "ond" or "condda". ' f'"{privileged_config["protocol"]}" in the config files') # Run the protocol run_protocol.run_protocol() log.info('Protocol Finished') logging.getLogger().removeHandler(fh)