def test_resume_session(server_setup, get_interface_params):
"""
Test resume session.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = par_interface.session_request(["OND.54011215.0000.1236"],
"OND", "image_classification",
"0.1.1", [], 0.5)
par_interface.complete_test(session_id, "OND.54011215.0000.1236")
finished_test = par_interface.resume_session(session_id)
assert finished_test == ["OND.54011215.0000.1236"]
# Testing with hints
session_id = par_interface.session_request(
["OND.54011215.0000.1236"],
"OND",
"image_classification",
"0.1.1",
["red_light"],
0.4,
)
par_interface.complete_test(session_id, "OND.54011215.0000.1236")
finished_test = par_interface.resume_session(session_id)
assert finished_test == ["OND.54011215.0000.1236"]
def test_dataset_request(server_setup, get_interface_params):
"""
Tests for dataset request.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, "OND")
# Test correct dataset request
filename = par_interface.dataset_request("OND.1.1.1234", 0, session_id)
expected = os.path.join(config_directory,
f"{session_id}.OND.1.1.1234.0.csv")
assert expected == filename
expected_image_ids = _read_image_ids(expected)
assert expected_image_ids == [
"n01484850_18013.JPEG", "n01484850_24624.JPEG"
]
# Test incorrect dataset request
with pytest.raises(ProtocolError):
par_interface.dataset_request(f"{session_id}", "OND.1.1.1234", 3)
def test_initialize(
server_setup, get_interface_params, feedback_mapping, protocol_name
):
"""
Test feedback initialization.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_id = _initialize_session(par_interface, protocol_name)
result_files = {}
result_folder = os.path.join(
os.path.dirname(__file__), "mock_results", "transcripts"
)
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
result_folder, f"{test_id}_PreComputedDetector_{required_file}.csv"
)
par_interface.post_results(result_files, f"{test_id}", 0, session_id)
DocumentTranscriptionFeedback(
10, 10, 10, par_interface, session_id, test_id, protocol_constant
)
def test_get_feedback(
server_setup, get_interface_params, feedback_mapping, protocol_name
):
"""
Test get feedback.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_ids = _initialize_session(par_interface, protocol_name)
result_files = {}
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv"
)
par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0, session_id)
ic_feedback = ImageClassificationFeedback(
2, 2, 2, par_interface, session_id, test_ids[0], protocol_constant
)
df_feedback = ic_feedback.get_feedback(
0, [0, 1], ["n01484850_18013.JPEG", "n01484850_24624.JPEG"]
)
expected_list = [["n01484850_18013.JPEG", 1], ["n01484850_24624.JPEG", 2]]
assert df_feedback.values.tolist() == expected_list
def test_get_metadata(server_setup, get_interface_params):
"""
Test get metadata.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, "OND")
metadata = par_interface.get_test_metadata(session_id, "OND.1.1.1234")
assert "OND" == metadata["protocol"]
assert 3 == metadata["known_classes"]
session_id = _initialize_session(par_interface, "OND", ["red_light"])
metadata = par_interface.get_test_metadata(session_id, "OND.1.1.1234")
assert "n01484850_4515.JPEG" == metadata["red_light"]
def test_initialize(
server_setup, get_interface_params, feedback_mapping, protocol_name
):
"""
Test feedback initialization.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_ids = _initialize_session(par_interface, protocol_name)
result_files = {}
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv"
)
par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0, session_id)
ImageClassificationFeedback(
2, 2, 2, par_interface, session_id, test_ids[0], protocol_constant
)
def test_test_ids_request(server_setup, get_interface_params):
"""
Test request for test ids.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
assumptions_path = os.path.join(os.path.dirname(__file__),
"assumptions.json")
filename = par_interface.test_ids_request("OND", "image_classification",
"5678", assumptions_path)
expected = os.path.join(config_directory,
"OND.image_classification.5678.csv")
assert os.stat(expected).st_size > 5
assert expected == filename
def test_get_feedback(server_setup, get_interface_params, feedback_mapping,
protocol_name):
"""
Test get feedback.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_id = _initialize_session(par_interface, protocol_name)
result_files = {}
result_folder = os.path.join(os.path.dirname(__file__), "mock_results",
"image_classification")
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
result_folder, f"{test_id}_{required_file}.csv")
par_interface.post_results(result_files, f"{test_id}", 0, session_id)
feedback = ImageClassificationFeedback(10, 10, 10, par_interface,
session_id, test_id,
protocol_constant)
feedback.get_feedback(0, list(range(10)), feedback_image_ids)
def test_post_results(server_setup, get_interface_params, protocol_constant,
protocol_name):
"""
Tests for post results.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
protocol_constant (str): Constants used by the server to identifying results
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, protocol_name)
result_files = {
protocol_constant:
os.path.join(os.path.dirname(__file__),
f"test_results_{protocol_name}.1.1.1234.csv")
}
par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0,
session_id)
def test_feature_extraction(
server_setup,
get_interface_params,
discoverable_plugins,
ond_config_with_feature_extraction,
):
"""
Test feature extraction.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
discoverable_plugins (dict): Dictionary with the plugins
ond_config_with_feature_extraction (str): Path to json file
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
ond = SailOn(discoverable_plugins, "", par_interface,
ond_config_with_feature_extraction)
ond.run_protocol()
local_interface = LocalInterface(config_name, config_directory)
SailOn(discoverable_plugins, "", local_interface,
ond_config_with_feature_extraction)
ond.run_protocol()
def test_terminate_session(server_setup, get_interface_params):
"""
Test terminate session request.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, "OND")
par_interface.terminate_session(session_id)
def test_feedback_request(server_setup, get_interface_params, feedback_mapping,
protocol_name):
"""
Tests for feedback request.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, protocol_name)
# Post results before posting
result_files = {}
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
os.path.dirname(__file__),
f"test_results_{protocol_name}.1.1.1234.csv")
par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0,
session_id)
# Get feedback for detection
response = par_interface.get_feedback_request(
["n01484850_18013.JPEG", "n01484850_24624.JPEG"],
protocol_constant,
f"{protocol_name}.1.1.1234",
0,
session_id,
)
expected = os.path.join(
config_directory,
f"{session_id}.{protocol_name}.1.1.1234.0_{protocol_constant}.csv",
)
assert expected == response
def test_evaluate(server_setup, get_interface_params):
"""
Test evaluate with rounds.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, "OND")
response = par_interface.evaluate("OND.1.1.1234", 0, session_id)
expected = os.path.join(config_directory,
f"{session_id}.OND.1.1.1234.0_evaluation.csv")
assert expected == response
def test_evaluate_roundwise(server_setup, get_interface_params):
"""
Test evaluate with rounds.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, "OND")
session_id = _initialize_session(par_interface, "OND")
result_folder = os.path.join(os.path.dirname(__file__), "mock_results",
"image_classification")
detection_file_id = os.path.join(result_folder,
"OND.54011215.0000.1236_detection.csv")
classification_file_id = os.path.join(
result_folder, "OND.54011215.0000.1236_classification.csv")
results = {
"detection": detection_file_id,
"classification": classification_file_id,
}
par_interface.post_results(results, "OND.54011215.0000.1236", 0,
session_id)
with pytest.raises(NotImplementedError):
par_interface.evaluate_round_wise("OND.54011215.0000.1236", 0,
session_id)
def test_session_request(server_setup, get_interface_params):
"""
Test session request.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
test_id_path = os.path.join(os.path.dirname(__file__), "data", "OND",
"image_classification", "test_ids.csv")
test_ids = list(map(str.strip, open(test_id_path, "r").readlines()))
# Testing if session was sucessfully initalized
par_interface.session_request(test_ids, "OND", "image_classification",
"0.1.1", [])
# Testing with hints
par_interface.session_request(test_ids, "OND", "image_classification",
"0.1.1", ["red_light"])
def test_initialize(server_setup, get_interface_params):
"""
Test par interface initialization.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
assert par_interface.api_url == server_setup[0]
def test_initialize(server_setup, get_interface_params, discoverable_plugins,
ond_config):
"""
Test ond protocol initialization.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
discoverable_plugins (dict): Dictionary with the plugins
ond_config (str): Path to json file
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
SailOn(discoverable_plugins, "", par_interface, ond_config)
local_interface = LocalInterface(config_name, config_directory)
SailOn(discoverable_plugins, "", local_interface, ond_config)
def test_initialize(server_setup, get_interface_params, feedback_mapping,
protocol_name):
"""
Test feedback initialization.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_id = _initialize_session(par_interface, protocol_name)
protocol_constant = feedback_mapping[0]
ImageClassificationFeedback(10, 10, 10, par_interface, session_id, test_id,
protocol_constant)
def test_run_protocol(server_setup, get_interface_params, discoverable_plugins,
condda_config):
"""
Test running protocol.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
discoverable_plugins (dict): Dictionary with the plugins
condda_config (str): Path to json file
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
condda = Condda(discoverable_plugins, "", par_interface, condda_config)
condda.run_protocol()
local_interface = LocalInterface(config_name, config_directory)
condda = Condda(discoverable_plugins, "", local_interface, condda_config)
condda.run_protocol()
def test_get_budget(
server_setup, get_interface_params, feedback_mapping, protocol_name
):
"""
Test get budget.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_id = _initialize_session(par_interface, protocol_name)
protocol_constant = feedback_mapping[0]
dt_feedback = DocumentTranscriptionFeedback(
10, 10, 10, par_interface, session_id, test_id, protocol_constant
)
assert dt_feedback.get_budget() == 10
def test_deposit_income(server_setup, get_interface_params, feedback_mapping,
protocol_name):
"""
Test deposit income.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_id = _initialize_session(par_interface, protocol_name)
protocol_constant = feedback_mapping[0]
ar_feedback = ActivityRecognitionFeedback(10, 10, 10, par_interface,
session_id, test_id,
protocol_constant)
ar_feedback.deposit_income()
assert ar_feedback.budget == 10
def run_protocol(
self, config: Dict[str, Any],
extra_plugins: Dict[str, Any] = dict()) -> None:
"""Run the protocol by printout out the config.
Args:
Config passed in uses 3 parameters to control the launching of the protocols
- protocol: either 'ond' or 'condda' to define which protocol to run
- harness: either 'local' or 'par' to define which harness to use
- workdir: a directory to save all the information from the run including
- Config
- Output of algorithm
Example:
>>> from sailon_tinker_launcher.main import *
>>> dpath = ub.ensure_app_cache_dir('tinker/tests')
>>> config = get_debug_config()
>>> self = LaunchSailonProtocol()
>>> self.run_protocol(config)
>>> assert(self.working_folder.exists())
>>> ub.delete(str(self.working_folder), verbose=False)
"""
# Setup working folder and create new config for this run
self.working_folder, working_config_fp, privileged_config, config = self.setup_experiment(
config)
# Now experiment setup, start a new logger for this
fh = logging.FileHandler(
self.working_folder /
f'{datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p")}.log')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s][%(name)s][%(levelname)s] - %(message)s')
fh.setFormatter(formatter)
logging.getLogger().addHandler(fh)
log.info(f'Config Filepath: {working_config_fp}')
log.info(f'Config: \n{json.dumps(config, indent=4)}')
# Load the harness
# This config is not used but will throw error if not pointed at
harnness_config_path = Path(protocol_folder.__file__).parent
if privileged_config['harness'] == 'local':
log.info('Loading Local Harness')
harness = LocalInterface('configuration.json',
str(harnness_config_path))
harness.result_directory = config['detectors']['csv_folder']
harness.file_provider.results_folder = config['detectors'][
'csv_folder']
elif privileged_config['harness'] == 'par':
log.info('Loading Par Harness')
harness = ParInterface('configuration.json',
str(harnness_config_path))
harness.folder = config['detectors']['csv_folder']
else:
raise AttributeError(
f'Valid harnesses "local" or "par". '
f'Given harness "{privileged_config["harness"]}" ')
# Get the plugins
plugins = ub.dict_union(discoverable_plugins('tinker'), extra_plugins)
log.debug('Plugins found:')
log.debug(plugins)
# Load the protocol
if privileged_config['protocol'] == 'ond':
log.info('Running OND Protocol')
run_protocol = OND(discovered_plugins=plugins,
algorithmsdirectory='',
harness=harness,
config_file=str(working_config_fp))
elif privileged_config['protocol'] == 'condda':
log.info('Running Condda Protocol')
run_protocol = Condda(discovered_plugins=plugins,
algorithmsdirectory='',
harness=harness,
config_file=str(working_config_fp))
else:
raise AttributeError(
f'Please set protocol to either "ond" or "condda". '
f'"{privileged_config["protocol"]}" in the config files')
# Run the protocol
run_protocol.run_protocol()
log.info('Protocol Finished')
logging.getLogger().removeHandler(fh)
