def test_resume_session(server_setup, get_interface_params): """ Test resume session. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id = par_interface.session_request(["OND.54011215.0000.1236"], "OND", "image_classification", "0.1.1", [], 0.5) par_interface.complete_test(session_id, "OND.54011215.0000.1236") finished_test = par_interface.resume_session(session_id) assert finished_test == ["OND.54011215.0000.1236"] # Testing with hints session_id = par_interface.session_request( ["OND.54011215.0000.1236"], "OND", "image_classification", "0.1.1", ["red_light"], 0.4, ) par_interface.complete_test(session_id, "OND.54011215.0000.1236") finished_test = par_interface.resume_session(session_id) assert finished_test == ["OND.54011215.0000.1236"]
def test_dataset_request(server_setup, get_interface_params): """ Tests for dataset request. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id = _initialize_session(par_interface, "OND") # Test correct dataset request filename = par_interface.dataset_request("OND.1.1.1234", 0, session_id) expected = os.path.join(config_directory, f"{session_id}.OND.1.1.1234.0.csv") assert expected == filename expected_image_ids = _read_image_ids(expected) assert expected_image_ids == [ "n01484850_18013.JPEG", "n01484850_24624.JPEG" ] # Test incorrect dataset request with pytest.raises(ProtocolError): par_interface.dataset_request(f"{session_id}", "OND.1.1.1234", 3)
def test_initialize( server_setup, get_interface_params, feedback_mapping, protocol_name ): """ Test feedback initialization. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface feedback_mapping (dict): Dict with mapping for feedback protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id, test_id = _initialize_session(par_interface, protocol_name) result_files = {} result_folder = os.path.join( os.path.dirname(__file__), "mock_results", "transcripts" ) protocol_constant = feedback_mapping[0] required_files = feedback_mapping[1] for required_file in required_files: result_files[required_file] = os.path.join( result_folder, f"{test_id}_PreComputedDetector_{required_file}.csv" ) par_interface.post_results(result_files, f"{test_id}", 0, session_id) DocumentTranscriptionFeedback( 10, 10, 10, par_interface, session_id, test_id, protocol_constant )
def test_get_feedback( server_setup, get_interface_params, feedback_mapping, protocol_name ): """ Test get feedback. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface feedback_mapping (dict): Dict with mapping for feedback protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id, test_ids = _initialize_session(par_interface, protocol_name) result_files = {} protocol_constant = feedback_mapping[0] required_files = feedback_mapping[1] for required_file in required_files: result_files[required_file] = os.path.join( os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv" ) par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0, session_id) ic_feedback = ImageClassificationFeedback( 2, 2, 2, par_interface, session_id, test_ids[0], protocol_constant ) df_feedback = ic_feedback.get_feedback( 0, [0, 1], ["n01484850_18013.JPEG", "n01484850_24624.JPEG"] ) expected_list = [["n01484850_18013.JPEG", 1], ["n01484850_24624.JPEG", 2]] assert df_feedback.values.tolist() == expected_list
def test_get_metadata(server_setup, get_interface_params): """ Test get metadata. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id = _initialize_session(par_interface, "OND") metadata = par_interface.get_test_metadata(session_id, "OND.1.1.1234") assert "OND" == metadata["protocol"] assert 3 == metadata["known_classes"] session_id = _initialize_session(par_interface, "OND", ["red_light"]) metadata = par_interface.get_test_metadata(session_id, "OND.1.1.1234") assert "n01484850_4515.JPEG" == metadata["red_light"]
def test_initialize( server_setup, get_interface_params, feedback_mapping, protocol_name ): """ Test feedback initialization. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface feedback_mapping (dict): Dict with mapping for feedback protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id, test_ids = _initialize_session(par_interface, protocol_name) result_files = {} protocol_constant = feedback_mapping[0] required_files = feedback_mapping[1] for required_file in required_files: result_files[required_file] = os.path.join( os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv" ) par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0, session_id) ImageClassificationFeedback( 2, 2, 2, par_interface, session_id, test_ids[0], protocol_constant )
def test_test_ids_request(server_setup, get_interface_params): """ Test request for test ids. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) assumptions_path = os.path.join(os.path.dirname(__file__), "assumptions.json") filename = par_interface.test_ids_request("OND", "image_classification", "5678", assumptions_path) expected = os.path.join(config_directory, "OND.image_classification.5678.csv") assert os.stat(expected).st_size > 5 assert expected == filename
def test_get_feedback(server_setup, get_interface_params, feedback_mapping, protocol_name): """ Test get feedback. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface feedback_mapping (dict): Dict with mapping for feedback protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id, test_id = _initialize_session(par_interface, protocol_name) result_files = {} result_folder = os.path.join(os.path.dirname(__file__), "mock_results", "image_classification") protocol_constant = feedback_mapping[0] required_files = feedback_mapping[1] for required_file in required_files: result_files[required_file] = os.path.join( result_folder, f"{test_id}_{required_file}.csv") par_interface.post_results(result_files, f"{test_id}", 0, session_id) feedback = ImageClassificationFeedback(10, 10, 10, par_interface, session_id, test_id, protocol_constant) feedback.get_feedback(0, list(range(10)), feedback_image_ids)
def test_post_results(server_setup, get_interface_params, protocol_constant, protocol_name): """ Tests for post results. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface protocol_constant (str): Constants used by the server to identifying results protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id = _initialize_session(par_interface, protocol_name) result_files = { protocol_constant: os.path.join(os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv") } par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0, session_id)
def test_feature_extraction( server_setup, get_interface_params, discoverable_plugins, ond_config_with_feature_extraction, ): """ Test feature extraction. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface discoverable_plugins (dict): Dictionary with the plugins ond_config_with_feature_extraction (str): Path to json file Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) ond = SailOn(discoverable_plugins, "", par_interface, ond_config_with_feature_extraction) ond.run_protocol() local_interface = LocalInterface(config_name, config_directory) SailOn(discoverable_plugins, "", local_interface, ond_config_with_feature_extraction) ond.run_protocol()
def test_terminate_session(server_setup, get_interface_params): """ Test terminate session request. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id = _initialize_session(par_interface, "OND") par_interface.terminate_session(session_id)
def test_feedback_request(server_setup, get_interface_params, feedback_mapping, protocol_name): """ Tests for feedback request. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface feedback_mapping (dict): Dict with mapping for feedback protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id = _initialize_session(par_interface, protocol_name) # Post results before posting result_files = {} protocol_constant = feedback_mapping[0] required_files = feedback_mapping[1] for required_file in required_files: result_files[required_file] = os.path.join( os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv") par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0, session_id) # Get feedback for detection response = par_interface.get_feedback_request( ["n01484850_18013.JPEG", "n01484850_24624.JPEG"], protocol_constant, f"{protocol_name}.1.1.1234", 0, session_id, ) expected = os.path.join( config_directory, f"{session_id}.{protocol_name}.1.1.1234.0_{protocol_constant}.csv", ) assert expected == response
def test_evaluate(server_setup, get_interface_params): """ Test evaluate with rounds. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id = _initialize_session(par_interface, "OND") response = par_interface.evaluate("OND.1.1.1234", 0, session_id) expected = os.path.join(config_directory, f"{session_id}.OND.1.1.1234.0_evaluation.csv") assert expected == response
def test_evaluate_roundwise(server_setup, get_interface_params): """ Test evaluate with rounds. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id = _initialize_session(par_interface, "OND") session_id = _initialize_session(par_interface, "OND") result_folder = os.path.join(os.path.dirname(__file__), "mock_results", "image_classification") detection_file_id = os.path.join(result_folder, "OND.54011215.0000.1236_detection.csv") classification_file_id = os.path.join( result_folder, "OND.54011215.0000.1236_classification.csv") results = { "detection": detection_file_id, "classification": classification_file_id, } par_interface.post_results(results, "OND.54011215.0000.1236", 0, session_id) with pytest.raises(NotImplementedError): par_interface.evaluate_round_wise("OND.54011215.0000.1236", 0, session_id)
def test_session_request(server_setup, get_interface_params): """ Test session request. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.parinterface import ParInterface url, result_dir = server_setup config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) test_id_path = os.path.join(os.path.dirname(__file__), "data", "OND", "image_classification", "test_ids.csv") test_ids = list(map(str.strip, open(test_id_path, "r").readlines())) # Testing if session was sucessfully initalized par_interface.session_request(test_ids, "OND", "image_classification", "0.1.1", []) # Testing with hints par_interface.session_request(test_ids, "OND", "image_classification", "0.1.1", ["red_light"])
def test_initialize(server_setup, get_interface_params): """ Test par interface initialization. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface Return: None """ from sail_on_client.protocol.parinterface import ParInterface config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) assert par_interface.api_url == server_setup[0]
def test_initialize(server_setup, get_interface_params, discoverable_plugins, ond_config): """ Test ond protocol initialization. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface discoverable_plugins (dict): Dictionary with the plugins ond_config (str): Path to json file Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) SailOn(discoverable_plugins, "", par_interface, ond_config) local_interface = LocalInterface(config_name, config_directory) SailOn(discoverable_plugins, "", local_interface, ond_config)
def test_initialize(server_setup, get_interface_params, feedback_mapping, protocol_name): """ Test feedback initialization. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface feedback_mapping (dict): Dict with mapping for feedback protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id, test_id = _initialize_session(par_interface, protocol_name) protocol_constant = feedback_mapping[0] ImageClassificationFeedback(10, 10, 10, par_interface, session_id, test_id, protocol_constant)
def test_run_protocol(server_setup, get_interface_params, discoverable_plugins, condda_config): """ Test running protocol. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface discoverable_plugins (dict): Dictionary with the plugins condda_config (str): Path to json file Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) condda = Condda(discoverable_plugins, "", par_interface, condda_config) condda.run_protocol() local_interface = LocalInterface(config_name, config_directory) condda = Condda(discoverable_plugins, "", local_interface, condda_config) condda.run_protocol()
def test_get_budget( server_setup, get_interface_params, feedback_mapping, protocol_name ): """ Test get budget. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface feedback_mapping (dict): Dict with mapping for feedback protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id, test_id = _initialize_session(par_interface, protocol_name) protocol_constant = feedback_mapping[0] dt_feedback = DocumentTranscriptionFeedback( 10, 10, 10, par_interface, session_id, test_id, protocol_constant ) assert dt_feedback.get_budget() == 10
def test_deposit_income(server_setup, get_interface_params, feedback_mapping, protocol_name): """ Test deposit income. Args: server_setup (tuple): Tuple containing url and result directory get_interface_params (tuple): Tuple to configure par interface feedback_mapping (dict): Dict with mapping for feedback protocol_name (str): Name of the protocol ( options: OND and CONDDA) Return: None """ config_directory, config_name = get_interface_params par_interface = ParInterface(config_name, config_directory) session_id, test_id = _initialize_session(par_interface, protocol_name) protocol_constant = feedback_mapping[0] ar_feedback = ActivityRecognitionFeedback(10, 10, 10, par_interface, session_id, test_id, protocol_constant) ar_feedback.deposit_income() assert ar_feedback.budget == 10
def run_protocol( self, config: Dict[str, Any], extra_plugins: Dict[str, Any] = dict()) -> None: """Run the protocol by printout out the config. Args: Config passed in uses 3 parameters to control the launching of the protocols - protocol: either 'ond' or 'condda' to define which protocol to run - harness: either 'local' or 'par' to define which harness to use - workdir: a directory to save all the information from the run including - Config - Output of algorithm Example: >>> from sailon_tinker_launcher.main import * >>> dpath = ub.ensure_app_cache_dir('tinker/tests') >>> config = get_debug_config() >>> self = LaunchSailonProtocol() >>> self.run_protocol(config) >>> assert(self.working_folder.exists()) >>> ub.delete(str(self.working_folder), verbose=False) """ # Setup working folder and create new config for this run self.working_folder, working_config_fp, privileged_config, config = self.setup_experiment( config) # Now experiment setup, start a new logger for this fh = logging.FileHandler( self.working_folder / f'{datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p")}.log') fh.setLevel(logging.DEBUG) formatter = logging.Formatter( '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s') fh.setFormatter(formatter) logging.getLogger().addHandler(fh) log.info(f'Config Filepath: {working_config_fp}') log.info(f'Config: \n{json.dumps(config, indent=4)}') # Load the harness # This config is not used but will throw error if not pointed at harnness_config_path = Path(protocol_folder.__file__).parent if privileged_config['harness'] == 'local': log.info('Loading Local Harness') harness = LocalInterface('configuration.json', str(harnness_config_path)) harness.result_directory = config['detectors']['csv_folder'] harness.file_provider.results_folder = config['detectors'][ 'csv_folder'] elif privileged_config['harness'] == 'par': log.info('Loading Par Harness') harness = ParInterface('configuration.json', str(harnness_config_path)) harness.folder = config['detectors']['csv_folder'] else: raise AttributeError( f'Valid harnesses "local" or "par". ' f'Given harness "{privileged_config["harness"]}" ') # Get the plugins plugins = ub.dict_union(discoverable_plugins('tinker'), extra_plugins) log.debug('Plugins found:') log.debug(plugins) # Load the protocol if privileged_config['protocol'] == 'ond': log.info('Running OND Protocol') run_protocol = OND(discovered_plugins=plugins, algorithmsdirectory='', harness=harness, config_file=str(working_config_fp)) elif privileged_config['protocol'] == 'condda': log.info('Running Condda Protocol') run_protocol = Condda(discovered_plugins=plugins, algorithmsdirectory='', harness=harness, config_file=str(working_config_fp)) else: raise AttributeError( f'Please set protocol to either "ond" or "condda". ' f'"{privileged_config["protocol"]}" in the config files') # Run the protocol run_protocol.run_protocol() log.info('Protocol Finished') logging.getLogger().removeHandler(fh)