def test_unpacker_read_manifest_metadata_v2(config, mocker, path_map_mock):
"""Test that _read_manifest_metadata_v2 will try to read metadata."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
with patch("builtins.open", mock_open(read_data='{"some": "object"}')) as metadata_mock:
assert p._read_manifest_metadata_v2("0869ea50-e437-443f-8cdb-31a350f88e57") == {"some": "object"}
metadata_mock.assert_called_with(mocker.ANY)
async def test_unpacker_run(config, mocker, path_map_mock):
"""Test the Unpacker does the work the unpacker should do."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
p._do_work = AsyncMock()
await p.run()
p._do_work.assert_called()
def test_unpacker_clean_outbox_directory_bail_early(config, mocker, path_map_mock):
"""Test that _clean_outbox_directory will bail when configured not to clean."""
logger_mock = mocker.MagicMock()
config["CLEAN_OUTBOX"] = "FALSE"
mock_os_scandir = mocker.patch("os.scandir")
p = Unpacker(config, logger_mock)
p._clean_outbox_directory()
mock_os_scandir.assert_not_called()
def test_unpacker_read_manifest_metadata_v3_no_throw(file_open_mock, config,
mocker, path_map_mock):
"""Test that _read_manifest_metadata_v3 will not throw when unable to read metadata."""
logger_mock = mocker.MagicMock()
file_open_mock.side_effect = NameError
p = Unpacker(config, logger_mock)
assert not p._read_manifest_metadata_v3(
"0869ea50-e437-443f-8cdb-31a350f88e57")
def test_unpacker_delete_manifest_metadata_v3(config, mocker, path_map_mock):
"""Test that _delete_manifest_metadata will delete metadata of either version."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
mock_os_remove = mocker.patch("os.remove")
mock_os_remove.side_effect = [NameError, None]
p._delete_manifest_metadata("0869ea50-e437-443f-8cdb-31a350f88e57")
mock_os_remove.assert_called_with("/tmp/lta/testing/unpacker/outbox/0869ea50-e437-443f-8cdb-31a350f88e57.metadata.ndjson")
async def test_unpacker_run_exception(config, mocker, path_map_mock):
"""Test an error doesn't kill the Unpacker."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
p.last_work_end_timestamp = None
p._do_work = AsyncMock()
p._do_work.side_effect = [Exception("bad thing happen!")]
await p.run()
p._do_work.assert_called()
assert p.last_work_end_timestamp
def test_unpacker_read_manifest_metadata_for_v3(config, mocker, path_map_mock):
"""Test that _read_manifest_metadata will read v3 metadata."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
mock_v2 = mocker.patch("lta.unpacker.Unpacker._read_manifest_metadata_v2")
mock_v2.side_effect = [None]
mock_v3 = mocker.patch("lta.unpacker.Unpacker._read_manifest_metadata_v3")
mock_v3.side_effect = [{"some": "object"}]
p._read_manifest_metadata("0869ea50-e437-443f-8cdb-31a350f88e57")
mock_v2.assert_called_with("0869ea50-e437-443f-8cdb-31a350f88e57")
mock_v3.assert_called_with("0869ea50-e437-443f-8cdb-31a350f88e57")
def test_unpacker_clean_outbox_directory_empty(config, mocker, path_map_mock):
"""Test that _clean_outbox_directory will bail when configured not to clean."""
logger_mock = mocker.MagicMock()
mock_os_scandir = mocker.patch("os.scandir")
mock_os_scandir.return_value.__enter__.return_value = []
mock_os_remove = mocker.patch("os.remove")
mock_shutil_rmtree = mocker.patch("shutil.rmtree")
p = Unpacker(config, logger_mock)
p._clean_outbox_directory()
mock_os_remove.assert_not_called()
mock_shutil_rmtree.assert_not_called()
def test_unpacker_read_manifest_metadata_unknown(config, mocker, path_map_mock):
"""Test that _read_manifest_metadata will throw on an unknown version."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
mock_v2 = mocker.patch("lta.unpacker.Unpacker._read_manifest_metadata_v2")
mock_v2.side_effect = [None]
mock_v3 = mocker.patch("lta.unpacker.Unpacker._read_manifest_metadata_v3")
mock_v3.side_effect = [None]
with pytest.raises(Exception):
p._read_manifest_metadata("0869ea50-e437-443f-8cdb-31a350f88e57")
mock_v2.assert_called_with("0869ea50-e437-443f-8cdb-31a350f88e57")
mock_v3.assert_called_with("0869ea50-e437-443f-8cdb-31a350f88e57")
def test_unpacker_read_manifest_metadata_v3(config, mocker, path_map_mock):
"""Test that _read_manifest_metadata_v3 will try to read metadata."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
read_data = """{}
{"some": "object"}"""
result = {"files": [{"some": "object"}]}
with patch("builtins.open",
mock_open(read_data=read_data)) as metadata_mock:
assert p._read_manifest_metadata_v3(
"0869ea50-e437-443f-8cdb-31a350f88e57") == result
metadata_mock.assert_called_with(mocker.ANY)
def test_constructor_missing_logging():
"""Fail with a TypeError if a logging object isn't provided."""
with pytest.raises(TypeError):
config = {
"PAN_GALACTIC_GARGLE_BLASTER": "Yummy"
}
Unpacker(config)
def test_constructor_config_poison_values(config, mocker):
"""Fail with a ValueError if the configuration object is missing required configuration variables."""
unpacker_config = config.copy()
unpacker_config["LTA_REST_URL"] = None
logger_mock = mocker.MagicMock()
with pytest.raises(ValueError):
Unpacker(unpacker_config, logger_mock)
async def test_unpacker_do_work_no_results(config, mocker, path_map_mock):
"""Test that _do_work goes on vacation when the LTA DB has no work."""
logger_mock = mocker.MagicMock()
claim_mock = mocker.patch("lta.unpacker.Unpacker._do_work_claim", new_callable=AsyncMock)
claim_mock.return_value = False
p = Unpacker(config, logger_mock)
await p._do_work()
async def test_unpacker_quarantine_bundle_with_reason(config, mocker, path_map_mock):
"""Test that _do_work_claim attempts to quarantine a Bundle that fails to get unpacked."""
logger_mock = mocker.MagicMock()
lta_rc_mock = mocker.patch("rest_tools.client.RestClient", new_callable=AsyncMock)
p = Unpacker(config, logger_mock)
await p._quarantine_bundle(lta_rc_mock, {"uuid": "c4b345e4-2395-4f9e-b0eb-9cc1c9cdf003"}, "Rucio caught fire, then we roasted marshmellows.")
lta_rc_mock.request.assert_called_with("PATCH", "/Bundles/c4b345e4-2395-4f9e-b0eb-9cc1c9cdf003", mocker.ANY)
async def test_unpacker_add_location_to_file_catalog(config, mocker,
path_map_mock):
"""Test that _add_location_to_file_catalog adds a location in the File Catalog."""
logger_mock = mocker.MagicMock()
fc_rc_mock = mocker.patch("rest_tools.client.RestClient.request",
new_callable=AsyncMock)
p = Unpacker(config, logger_mock)
bundle_file = {
"checksum": {
"sha512":
"09de7c539b724dee9543669309f978b172f6c7449d0269fecbb57d0c9cf7db51713fed3a94573c669fe0aa08fa122b41f84a0ea107c62f514b1525efbd08846b",
},
"file_size": 105311728,
"logical_name":
"/data/exp/IceCube/2013/filtered/PFFilt/1109/PFFilt_PhysicsFiltering_Run00123231_Subrun00000000_00000066.tar.bz2",
"meta_modify_date": "2020-02-20 22:47:25.180303",
"uuid": "2f0cb3c8-6cba-49b1-8eeb-13e13fed41dd",
}
dest_path = bundle_file["logical_name"]
assert await p._add_location_to_file_catalog(bundle_file, dest_path)
fc_rc_mock.assert_called_with(
"POST", "/api/files/2f0cb3c8-6cba-49b1-8eeb-13e13fed41dd/locations", {
"locations": [{
"site":
"WIPAC",
"path":
"/data/exp/IceCube/2013/filtered/PFFilt/1109/PFFilt_PhysicsFiltering_Run00123231_Subrun00000000_00000066.tar.bz2",
}]
})
async def test_unpacker_update_bundle_in_lta_db(config, mocker, path_map_mock):
"""Test that _update_bundle_in_lta_db updates the status of the bundle in the LTA DB."""
logger_mock = mocker.MagicMock()
lta_rc_mock = mocker.patch("rest_tools.client.RestClient", new_callable=AsyncMock)
p = Unpacker(config, logger_mock)
assert await p._update_bundle_in_lta_db(lta_rc_mock, {"uuid": "c4b345e4-2395-4f9e-b0eb-9cc1c9cdf003"})
lta_rc_mock.request.assert_called_with("PATCH", "/Bundles/c4b345e4-2395-4f9e-b0eb-9cc1c9cdf003", mocker.ANY)
def test_constructor_config_missing_values(mocker):
"""Fail with a ValueError if the configuration object is missing required configuration variables."""
config = {
"PAN_GALACTIC_GARGLE_BLASTER": "Yummy"
}
logger_mock = mocker.MagicMock()
with pytest.raises(ValueError):
Unpacker(config, logger_mock)
async def test_unpacker_do_work_pop_exception(config, mocker, path_map_mock):
"""Test that _do_work raises when the RestClient can't pop."""
logger_mock = mocker.MagicMock()
lta_rc_mock = mocker.patch("rest_tools.client.RestClient.request", new_callable=AsyncMock)
lta_rc_mock.side_effect = HTTPError(500, "LTA DB on fire. Again.")
p = Unpacker(config, logger_mock)
with pytest.raises(HTTPError):
await p._do_work()
lta_rc_mock.assert_called_with("POST", '/Bundles/actions/pop?source=NERSC&dest=WIPAC&status=unpacking', mocker.ANY)
def test_constructor_config_sleep_type_int(config, mocker, path_map_mock):
"""Ensure that sleep seconds can also be provided as an integer."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
assert p.heartbeat_sleep_duration_seconds == 60
assert p.lta_rest_url == "http://RmMNHdPhHpH2ZxfaFAC9d2jiIbf5pZiHDqy43rFLQiM.com/"
assert p.name == "testing-unpacker"
assert p.work_sleep_duration_seconds == 60
assert p.logger == logger_mock
def test_constructor_config(config, mocker, path_map_mock):
"""Test that a Unpacker can be constructed with a configuration object and a logging object."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
assert p.heartbeat_sleep_duration_seconds == 60
assert p.lta_rest_url == "http://RmMNHdPhHpH2ZxfaFAC9d2jiIbf5pZiHDqy43rFLQiM.com/"
assert p.name == "testing-unpacker"
assert p.work_sleep_duration_seconds == 60
assert p.logger == logger_mock
async def test_unpacker_logs_configuration(mocker, path_map_mock):
"""Test to make sure the Unpacker logs its configuration."""
logger_mock = mocker.MagicMock()
unpacker_config = {
"CLEAN_OUTBOX": "true",
"COMPONENT_NAME": "logme-testing-unpacker",
"DEST_SITE": "WIPAC",
"FILE_CATALOG_REST_TOKEN": "fake-file-catalog-token",
"FILE_CATALOG_REST_URL":
"http://kVj74wBA1AMTDV8zccn67pGuWJqHZzD7iJQHrUJKA.com/",
"HEARTBEAT_PATCH_RETRIES": "1",
"HEARTBEAT_PATCH_TIMEOUT_SECONDS": "20",
"HEARTBEAT_SLEEP_DURATION_SECONDS": "30",
"INPUT_STATUS": "unpacking",
"LTA_REST_TOKEN": "logme-fake-lta-rest-token",
"LTA_REST_URL":
"logme-http://RmMNHdPhHpH2ZxfaFAC9d2jiIbf5pZiHDqy43rFLQiM.com/",
"OUTPUT_STATUS": "completed",
"PATH_MAP_JSON": "logme/tmp/lta/testing/path_map.json",
"RUN_ONCE_AND_DIE": "False",
"SOURCE_SITE": "NERSC",
"UNPACKER_OUTBOX_PATH": "logme/tmp/lta/testing/unpacker/outbox",
"UNPACKER_WORKBOX_PATH": "logme/tmp/lta/testing/unpacker/workbox",
"WORK_RETRIES": "5",
"WORK_SLEEP_DURATION_SECONDS": "70",
"WORK_TIMEOUT_SECONDS": "90",
}
Unpacker(unpacker_config, logger_mock)
EXPECTED_LOGGER_CALLS = [
call("unpacker 'logme-testing-unpacker' is configured:"),
call('CLEAN_OUTBOX = true'),
call('COMPONENT_NAME = logme-testing-unpacker'),
call('DEST_SITE = WIPAC'),
call('FILE_CATALOG_REST_TOKEN = fake-file-catalog-token'),
call(
'FILE_CATALOG_REST_URL = http://kVj74wBA1AMTDV8zccn67pGuWJqHZzD7iJQHrUJKA.com/'
),
call('HEARTBEAT_PATCH_RETRIES = 1'),
call('HEARTBEAT_PATCH_TIMEOUT_SECONDS = 20'),
call('HEARTBEAT_SLEEP_DURATION_SECONDS = 30'),
call('INPUT_STATUS = unpacking'),
call('LTA_REST_TOKEN = logme-fake-lta-rest-token'),
call(
'LTA_REST_URL = logme-http://RmMNHdPhHpH2ZxfaFAC9d2jiIbf5pZiHDqy43rFLQiM.com/'
),
call('OUTPUT_STATUS = completed'),
call('PATH_MAP_JSON = logme/tmp/lta/testing/path_map.json'),
call('RUN_ONCE_AND_DIE = False'),
call('SOURCE_SITE = NERSC'),
call('UNPACKER_OUTBOX_PATH = logme/tmp/lta/testing/unpacker/outbox'),
call('UNPACKER_WORKBOX_PATH = logme/tmp/lta/testing/unpacker/workbox'),
call('WORK_RETRIES = 5'),
call('WORK_SLEEP_DURATION_SECONDS = 70'),
call('WORK_TIMEOUT_SECONDS = 90'),
]
logger_mock.info.assert_has_calls(EXPECTED_LOGGER_CALLS)
async def test_unpacker_do_work_claim_no_results(config, mocker, path_map_mock):
"""Test that _do_work_claim returns False when the LTA DB has no work."""
logger_mock = mocker.MagicMock()
lta_rc_mock = mocker.patch("rest_tools.client.RestClient.request", new_callable=AsyncMock)
lta_rc_mock.return_value = {
"bundle": None
}
p = Unpacker(config, logger_mock)
assert not await p._do_work_claim()
lta_rc_mock.assert_called_with("POST", '/Bundles/actions/pop?source=NERSC&dest=WIPAC&status=unpacking', mocker.ANY)
async def test_unpacker_do_work_bundle_once_and_die(config, mocker, path_map_mock):
"""Test that _do_work goes on vacation when the LTA DB has no work."""
once = config.copy()
once["RUN_ONCE_AND_DIE"] = "True"
logger_mock = mocker.MagicMock()
claim_mock = mocker.patch("lta.unpacker.Unpacker._do_work_claim", new_callable=AsyncMock)
claim_mock.return_value = False
sys_exit_mock = mocker.patch("sys.exit")
p = Unpacker(once, logger_mock)
assert not await p._do_work()
sys_exit_mock.assert_not_called()
async def test_unpacker_do_work_bundle_path_remapping(config, mocker, path_map_mock):
"""Test that _do_work_bundle does the work of preparing an archive."""
logger_mock = mocker.MagicMock()
lta_rc_mock = mocker.patch("rest_tools.client.RestClient.request", new_callable=AsyncMock)
mock_zipfile_init = mocker.patch("zipfile.ZipFile.__init__")
mock_zipfile_init.return_value = None
mock_zipfile_write = mocker.patch("zipfile.ZipFile.extractall")
mock_zipfile_write.return_value = None
mock_json_load = mocker.patch("json.load")
mock_json_load.return_value = {
"files": [
{
"logical_name": "/mnt/lfs7/IceCube/2013/filtered/PFFilt/1109/PFFilt_PhysicsFiltering_Run00123231_Subrun00000000_00000002.tar.bz2",
"file_size": 1234567890,
"checksum": {
"adler32": "89d5efeb",
"sha512": "c919210281b72327c179e26be799b06cdaf48bf6efce56fb9d53f758c1b997099831ad05453fdb1ba65be7b35d0b4c5cebfc439efbdf83317ba0e38bf6f42570",
},
}
]
}
mock_shutil_move = mocker.patch("shutil.move")
mock_shutil_move.return_value = None
mock_lta_checksums = mocker.patch("lta.unpacker.lta_checksums")
mock_lta_checksums.return_value = {
"adler32": "89d5efeb",
"sha512": "c919210281b72327c179e26be799b06cdaf48bf6efce56fb9d53f758c1b997099831ad05453fdb1ba65be7b35d0b4c5cebfc439efbdf83317ba0e38bf6f42570",
}
mock_os_path_getsize = mocker.patch("os.path.getsize")
mock_os_path_getsize.return_value = 1234567890
mock_os_remove = mocker.patch("os.remove")
mock_os_remove.return_value = None
mock_os_scandir = mocker.patch("os.scandir")
mock_os_scandir.return_value.__enter__.return_value = []
altfc_mock = mocker.patch("lta.unpacker.Unpacker._add_location_to_file_catalog", new_callable=AsyncMock)
altfc_mock.return_value = False
p = Unpacker(config, logger_mock)
BUNDLE_OBJ = {
"bundle_path": "/mnt/lfss/jade-lta/bundler_out/9a1cab0a395211eab1cbce3a3da73f88.zip",
"uuid": "f74db80e-9661-40cc-9f01-8d087af23f56",
"source": "NERSC",
"dest": "WIPAC",
"path": "/mnt/lfs7/IceCube/2013/filtered/PFFilt/1109",
"files": [{"logical_name": "/mnt/lfs7/IceCube/2013/filtered/PFFilt/1109/PFFilt_PhysicsFiltering_Run00123231_Subrun00000000_00000002.tar.bz2", }],
}
with patch("builtins.open", mock_open(read_data="data")) as metadata_mock:
await p._do_work_bundle(lta_rc_mock, BUNDLE_OBJ)
metadata_mock.assert_called_with(mocker.ANY)
mock_lta_checksums.assert_called_with("/data/exp/IceCube/2013/filtered/PFFilt/1109/PFFilt_PhysicsFiltering_Run00123231_Subrun00000000_00000002.tar.bz2")
async def test_unpacker_do_work_yes_results(config, mocker, path_map_mock):
"""Test that _do_work_claim processes each Bundle that it gets from the LTA DB."""
BUNDLE_OBJ = {
"uuid": "f74db80e-9661-40cc-9f01-8d087af23f56"
}
logger_mock = mocker.MagicMock()
request_mock = mocker.patch("rest_tools.client.RestClient.request", new_callable=AsyncMock)
request_mock.return_value = {
"bundle": BUNDLE_OBJ,
}
dwb_mock = mocker.patch("lta.unpacker.Unpacker._do_work_bundle", new_callable=AsyncMock)
p = Unpacker(config, logger_mock)
assert await p._do_work_claim()
request_mock.assert_called_with("POST", '/Bundles/actions/pop?source=NERSC&dest=WIPAC&status=unpacking', mocker.ANY)
dwb_mock.assert_called_with(mocker.ANY, BUNDLE_OBJ)
async def test_unpacker_do_work_raise_exception(config, mocker, path_map_mock):
"""Test that _do_work_claim processes each Bundle that it gets from the LTA DB."""
BUNDLE_OBJ = {
"uuid": "f74db80e-9661-40cc-9f01-8d087af23f56"
}
logger_mock = mocker.MagicMock()
request_mock = mocker.patch("rest_tools.client.RestClient.request", new_callable=AsyncMock)
request_mock.return_value = {
"bundle": BUNDLE_OBJ,
}
dwb_mock = mocker.patch("lta.unpacker.Unpacker._do_work_bundle", new_callable=AsyncMock)
dwb_mock.side_effect = Exception("LTA DB started on fire again")
qb_mock = mocker.patch("lta.unpacker.Unpacker._quarantine_bundle", new_callable=AsyncMock)
p = Unpacker(config, logger_mock)
with pytest.raises(Exception):
await p._do_work_claim()
request_mock.assert_called_with("POST", '/Bundles/actions/pop?source=NERSC&dest=WIPAC&status=unpacking', mocker.ANY)
dwb_mock.assert_called_with(mocker.ANY, BUNDLE_OBJ)
qb_mock.assert_called_with(mocker.ANY, BUNDLE_OBJ, "LTA DB started on fire again")
def test_constructor_state(config, mocker, path_map_mock):
"""Verify that the Unpacker has a reasonable state when it is first constructed."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
assert p.last_work_begin_timestamp is p.last_work_end_timestamp
def test_do_status(config, mocker, path_map_mock):
"""Verify that the Unpacker has no additional state to offer."""
logger_mock = mocker.MagicMock()
p = Unpacker(config, logger_mock)
assert p._do_status() == {}
def test_constructor_missing_config():
"""Fail with a TypeError if a configuration object isn't provided."""
with pytest.raises(TypeError):
Unpacker()