def test_post_results(server_setup, get_interface_params, protocol_constant,
protocol_name):
"""
Tests for post results.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
protocol_constant (str): Constants used by the server to identifying results
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, protocol_name)
result_files = {
protocol_constant:
os.path.join(os.path.dirname(__file__),
f"test_results_{protocol_name}.1.1.1234.csv")
}
par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0,
session_id)
def test_initialize(
server_setup, get_interface_params, feedback_mapping, protocol_name
):
"""
Test feedback initialization.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_id = _initialize_session(par_interface, protocol_name)
result_files = {}
result_folder = os.path.join(
os.path.dirname(__file__), "mock_results", "transcripts"
)
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
result_folder, f"{test_id}_PreComputedDetector_{required_file}.csv"
)
par_interface.post_results(result_files, f"{test_id}", 0, session_id)
DocumentTranscriptionFeedback(
10, 10, 10, par_interface, session_id, test_id, protocol_constant
)
def test_get_feedback(
server_setup, get_interface_params, feedback_mapping, protocol_name
):
"""
Test get feedback.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_ids = _initialize_session(par_interface, protocol_name)
result_files = {}
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv"
)
par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0, session_id)
ic_feedback = ImageClassificationFeedback(
2, 2, 2, par_interface, session_id, test_ids[0], protocol_constant
)
df_feedback = ic_feedback.get_feedback(
0, [0, 1], ["n01484850_18013.JPEG", "n01484850_24624.JPEG"]
)
expected_list = [["n01484850_18013.JPEG", 1], ["n01484850_24624.JPEG", 2]]
assert df_feedback.values.tolist() == expected_list
def test_initialize(
server_setup, get_interface_params, feedback_mapping, protocol_name
):
"""
Test feedback initialization.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_ids = _initialize_session(par_interface, protocol_name)
result_files = {}
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
os.path.dirname(__file__), f"test_results_{protocol_name}.1.1.1234.csv"
)
par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0, session_id)
ImageClassificationFeedback(
2, 2, 2, par_interface, session_id, test_ids[0], protocol_constant
)
def test_get_feedback(server_setup, get_interface_params, feedback_mapping,
protocol_name):
"""
Test get feedback.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id, test_id = _initialize_session(par_interface, protocol_name)
result_files = {}
result_folder = os.path.join(os.path.dirname(__file__), "mock_results",
"image_classification")
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
result_folder, f"{test_id}_{required_file}.csv")
par_interface.post_results(result_files, f"{test_id}", 0, session_id)
feedback = ImageClassificationFeedback(10, 10, 10, par_interface,
session_id, test_id,
protocol_constant)
feedback.get_feedback(0, list(range(10)), feedback_image_ids)
def test_evaluate_roundwise(server_setup, get_interface_params):
"""
Test evaluate with rounds.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, "OND")
session_id = _initialize_session(par_interface, "OND")
result_folder = os.path.join(os.path.dirname(__file__), "mock_results",
"image_classification")
detection_file_id = os.path.join(result_folder,
"OND.54011215.0000.1236_detection.csv")
classification_file_id = os.path.join(
result_folder, "OND.54011215.0000.1236_classification.csv")
results = {
"detection": detection_file_id,
"classification": classification_file_id,
}
par_interface.post_results(results, "OND.54011215.0000.1236", 0,
session_id)
with pytest.raises(NotImplementedError):
par_interface.evaluate_round_wise("OND.54011215.0000.1236", 0,
session_id)
def test_feedback_request(server_setup, get_interface_params, feedback_mapping,
protocol_name):
"""
Tests for feedback request.
Args:
server_setup (tuple): Tuple containing url and result directory
get_interface_params (tuple): Tuple to configure par interface
feedback_mapping (dict): Dict with mapping for feedback
protocol_name (str): Name of the protocol ( options: OND and CONDDA)
Return:
None
"""
from sail_on_client.protocol.parinterface import ParInterface
url, result_dir = server_setup
config_directory, config_name = get_interface_params
par_interface = ParInterface(config_name, config_directory)
session_id = _initialize_session(par_interface, protocol_name)
# Post results before posting
result_files = {}
protocol_constant = feedback_mapping[0]
required_files = feedback_mapping[1]
for required_file in required_files:
result_files[required_file] = os.path.join(
os.path.dirname(__file__),
f"test_results_{protocol_name}.1.1.1234.csv")
par_interface.post_results(result_files, f"{protocol_name}.1.1.1234", 0,
session_id)
# Get feedback for detection
response = par_interface.get_feedback_request(
["n01484850_18013.JPEG", "n01484850_24624.JPEG"],
protocol_constant,
f"{protocol_name}.1.1.1234",
0,
session_id,
)
expected = os.path.join(
config_directory,
f"{session_id}.{protocol_name}.1.1.1234.0_{protocol_constant}.csv",
)
assert expected == response