def test_save_load_update_model_manager():
# Local imports are recommended when using moto
from emmaa.model_tests import ModelManager, save_model_manager_to_s3, \
load_model_manager_from_s3, update_model_manager_on_s3
from emmaa.util import find_number_of_files_on_s3
client = setup_bucket(add_model=True)
# Should be None if no model manager
assert find_number_of_files_on_s3(TEST_BUCKET_NAME,
'results/test/model_manager_',
'.pkl') == 0
loaded_mm = load_model_manager_from_s3(model_name='test',
bucket=TEST_BUCKET_NAME)
assert loaded_mm is None
# Save a model manager and load it back
model = create_model()
mm = ModelManager(model)
save_model_manager_to_s3('test', mm, bucket=TEST_BUCKET_NAME)
loaded_mm = load_model_manager_from_s3(model_name='test',
bucket=TEST_BUCKET_NAME)
assert loaded_mm
assert find_number_of_files_on_s3(TEST_BUCKET_NAME,
'results/test/model_manager_',
'.pkl') == 1
# Update should create a new file if there's at least one second difference
time.sleep(1)
update_model_manager_on_s3('test', TEST_BUCKET_NAME)
assert find_number_of_files_on_s3(TEST_BUCKET_NAME,
'results/test/model_manager_',
'.pkl') == 2
def test_generate_stats_on_s3():
# Local imports are recommended when using moto
from emmaa.analyze_tests_results import generate_stats_on_s3
from emmaa.util import find_number_of_files_on_s3, make_date_str
from emmaa.model_tests import update_model_manager_on_s3
# Try with only one set of results first (as for new model/test)
client = setup_bucket(add_results=True, add_mm=True, add_model=True)
msg = generate_stats_on_s3('test',
'model',
upload_stats=True,
bucket=TEST_BUCKET_NAME)
assert msg.latest_round
assert not msg.previous_round
assert not msg.previous_json_stats
assert find_number_of_files_on_s3(TEST_BUCKET_NAME,
'model_stats/test/model_stats_') == 1
tsg = generate_stats_on_s3('test',
'tests',
'simple_tests',
upload_stats=True,
bucket=TEST_BUCKET_NAME)
assert tsg.latest_round
assert not tsg.previous_round
assert not tsg.previous_json_stats
assert find_number_of_files_on_s3(TEST_BUCKET_NAME,
'stats/test/test_stats_') == 1
# Now add new results and new mm
time.sleep(1)
update_model_manager_on_s3('test', TEST_BUCKET_NAME)
client.put_object(
Body=json.dumps(previous_results, indent=1),
Bucket=TEST_BUCKET_NAME,
Key=f'results/test/results_simple_tests_{make_date_str()}.json')
msg = generate_stats_on_s3('test',
'model',
upload_stats=True,
bucket=TEST_BUCKET_NAME)
assert msg.latest_round
assert msg.previous_round
assert msg.previous_json_stats
assert find_number_of_files_on_s3(TEST_BUCKET_NAME,
'model_stats/test/model_stats_') == 2
tsg = generate_stats_on_s3('test',
'tests',
'simple_tests',
upload_stats=True,
bucket=TEST_BUCKET_NAME)
assert tsg.latest_round
assert tsg.previous_round
assert tsg.previous_json_stats
assert find_number_of_files_on_s3(TEST_BUCKET_NAME,
'stats/test/test_stats_') == 2