def test_image_classification_evaluate_roundwise(get_ic_interface_params):
"""
Test evaluate with rounds.
Args:
get_interface_params (tuple): Tuple to configure local interface
Return:
None
"""
from sail_on_client.protocol.localinterface import LocalInterface
config_directory, config_name = get_ic_interface_params
local_interface = LocalInterface(config_name, config_directory)
session_id = _initialize_session(local_interface, "OND", "image_classification")
result_folder = os.path.join(
os.path.dirname(__file__), "mock_results", "image_classification"
)
detection_file_id = os.path.join(
result_folder, "OND.54011215.0000.1236_PreComputedDetector_detection.csv"
)
classification_file_id = os.path.join(
result_folder, "OND.54011215.0000.1236_PreComputedDetector_classification.csv"
)
results = {
"detection": detection_file_id,
"classification": classification_file_id,
}
local_interface.post_results(results, "OND.54011215.0000.1236", 0, session_id)
local_interface.evaluate_round_wise("OND.54011215.0000.1236", 0, session_id)
def test_transcripts_evaluate(get_transcripts_interface_params):
"""
Test evaluate for transcripts.
Args:
get_transcripts_interface_params (tuple): Tuple to configure local interface
Return:
None
"""
from sail_on_client.protocol.localinterface import LocalInterface
config_directory, config_name = get_transcripts_interface_params
local_interface = LocalInterface(config_name, config_directory)
session_id = _initialize_session(local_interface, "OND", "transcripts")
result_folder = os.path.join(
os.path.dirname(__file__), "mock_results", "transcripts"
)
detection_file_id = os.path.join(
result_folder, "OND.0.90001.8714062_PreComputedDetector_detection.csv"
)
classification_file_id = os.path.join(
result_folder, "OND.0.90001.8714062_PreComputedDetector_classification.csv"
)
characterization_file_id = os.path.join(
result_folder, "OND.0.90001.8714062_PreComputedDetector_characterization.csv"
)
results = {
"detection": detection_file_id,
"classification": classification_file_id,
"characterization": characterization_file_id,
}
baseline_session_id = _initialize_session(local_interface, "OND", "transcripts")
local_interface.post_results(results, "OND.0.90001.8714062", 0, session_id)
local_interface.evaluate("OND.0.90001.8714062", 0, session_id)
baseline_classification_file_id = os.path.join(
result_folder,
"OND.0.90001.8714062_BaselinePreComputedDetector_classification.csv",
)
baseline_result = {
"classification": baseline_classification_file_id,
}
local_interface.post_results(
baseline_result, "OND.0.90001.8714062", 0, baseline_session_id
)
local_interface.evaluate("OND.0.90001.8714062", 0, session_id, baseline_session_id)
def test_feedback_request(get_interface_params, feedback_mapping, protocol_name):
"""
Tests for feedback request.
Args:
get_interface_params (tuple): Tuple to configure local interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol (options: OND and CONDDA)
Return:
None
"""
from sail_on_client.protocol.localinterface import LocalInterface
config_directory, config_name = get_interface_params
local_interface = LocalInterface(config_name, config_directory)
session_id = _initialize_session(local_interface, protocol_name)
# Post results before posting
result_files = {}
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv"
)
local_interface.post_results(
result_files, f"{protocol_name}.1.1.1234", 0, session_id
)
# Get feedback for detection
response = local_interface.get_feedback_request(
["n01484850_18013.JPEG", "n01484850_24624.JPEG"],
protocol_constant,
f"{protocol_name}.1.1.1234",
0,
session_id,
)
expected = os.path.join(
local_interface.result_directory,
"feedback",
f"{session_id}.{protocol_name}.1.1.1234.0_{protocol_constant}.csv",
)
assert expected == response
def test_post_results(get_interface_params, protocol_constant, protocol_name):
"""
Tests for post results.
Args:
get_interface_params (tuple): Tuple to configure local interface
protocol_constant (str): Constants used by the server to identifying results
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
from sail_on_client.protocol.localinterface import LocalInterface
config_directory, config_name = get_interface_params
local_interface = LocalInterface(config_name, config_directory)
session_id = _initialize_session(local_interface, protocol_name)
result_files = {
protocol_constant:
os.path.join(os.path.dirname(__file__),
f"test_results_{protocol_name}.1.1.1234.csv")
}
local_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0,
session_id)
