async def test_desy_verifier_run(config, mocker): """Test the DesyVerifier does the work the desy_verifier should do.""" logger_mock = mocker.MagicMock() p = DesyVerifier(config, logger_mock) p._do_work = AsyncMock() await p.run() p._do_work.assert_called()
async def test_desy_verifier_run_exception(config, mocker): """Test an error doesn't kill the DesyVerifier.""" logger_mock = mocker.MagicMock() p = DesyVerifier(config, logger_mock) p.last_work_end_timestamp = None p._do_work = AsyncMock() p._do_work.side_effect = [Exception("bad thing happen!")] await p.run() p._do_work.assert_called() assert p.last_work_end_timestamp
async def test_desy_verifier_logs_configuration(mocker): """Test to make sure the DesyVerifier logs its configuration.""" logger_mock = mocker.MagicMock() desy_verifier_config = { "COMPONENT_NAME": "logme-testing-desy_verifier", "DEST_SITE": "DESY", "DESY_CRED_PATH": "/path/to/my/gridftp/cert", "DESY_GSIFTP": "gsiftp://kVj74wBA1AMTDV8zccn67pGuWJqHZzD7iJQHrUJKA.com:2811/path/to/files/at/desy", "FILE_CATALOG_REST_TOKEN": "logme-fake-file-catalog-token", "FILE_CATALOG_REST_URL": "logme-http://kVj74wBA1AMTDV8zccn67pGuWJqHZzD7iJQHrUJKA.com/", "HEARTBEAT_PATCH_RETRIES": "1", "HEARTBEAT_PATCH_TIMEOUT_SECONDS": "20", "HEARTBEAT_SLEEP_DURATION_SECONDS": "30", "INPUT_STATUS": "verifying", "LTA_REST_TOKEN": "logme-fake-lta-rest-token", "LTA_REST_URL": "logme-http://RmMNHdPhHpH2ZxfaFAC9d2jiIbf5pZiHDqy43rFLQiM.com/", "OUTPUT_STATUS": "completed", "RUN_ONCE_AND_DIE": "False", "SOURCE_SITE": "WIPAC", "TAPE_BASE_PATH": "/logme/path/to/hpss", "WORK_RETRIES": "5", "WORK_SLEEP_DURATION_SECONDS": "70", "WORK_TIMEOUT_SECONDS": "90", "WORKBOX_PATH": "/path/to/wipac/workbox/directory", } DesyVerifier(desy_verifier_config, logger_mock) EXPECTED_LOGGER_CALLS = [ call("desy_verifier 'logme-testing-desy_verifier' is configured:"), call('COMPONENT_NAME = logme-testing-desy_verifier'), call('DEST_SITE = DESY'), call('DESY_CRED_PATH = /path/to/my/gridftp/cert'), call( 'DESY_GSIFTP = gsiftp://kVj74wBA1AMTDV8zccn67pGuWJqHZzD7iJQHrUJKA.com:2811/path/to/files/at/desy' ), call('FILE_CATALOG_REST_TOKEN = logme-fake-file-catalog-token'), call( 'FILE_CATALOG_REST_URL = logme-http://kVj74wBA1AMTDV8zccn67pGuWJqHZzD7iJQHrUJKA.com/' ), call('HEARTBEAT_PATCH_RETRIES = 1'), call('HEARTBEAT_PATCH_TIMEOUT_SECONDS = 20'), call('HEARTBEAT_SLEEP_DURATION_SECONDS = 30'), call('INPUT_STATUS = verifying'), call('LTA_REST_TOKEN = logme-fake-lta-rest-token'), call( 'LTA_REST_URL = logme-http://RmMNHdPhHpH2ZxfaFAC9d2jiIbf5pZiHDqy43rFLQiM.com/' ), call('OUTPUT_STATUS = completed'), call('RUN_ONCE_AND_DIE = False'), call('SOURCE_SITE = WIPAC'), call('TAPE_BASE_PATH = /logme/path/to/hpss'), call('WORK_RETRIES = 5'), call('WORK_SLEEP_DURATION_SECONDS = 70'), call('WORK_TIMEOUT_SECONDS = 90'), call('WORKBOX_PATH = /path/to/wipac/workbox/directory'), ] logger_mock.info.assert_has_calls(EXPECTED_LOGGER_CALLS)
async def test_desy_verifier_do_work_yes_results(config, mocker): """Test that _do_work keeps working until the LTA DB has no work.""" logger_mock = mocker.MagicMock() dwc_mock = mocker.patch("lta.desy_verifier.DesyVerifier._do_work_claim", new_callable=AsyncMock) dwc_mock.side_effect = [True, True, False] p = DesyVerifier(config, logger_mock) await p._do_work() assert dwc_mock.call_count == 3
async def test_desy_verifier_do_work_no_results(config, mocker): """Test that _do_work goes on vacation when the LTA DB has no work.""" logger_mock = mocker.MagicMock() dwc_mock = mocker.patch("lta.desy_verifier.DesyVerifier._do_work_claim", new_callable=AsyncMock) dwc_mock.return_value = False p = DesyVerifier(config, logger_mock) await p._do_work() dwc_mock.assert_called()
async def test_desy_verifier_do_work_pop_exception(config, mocker): """Test that _do_work raises when the RestClient can't pop.""" logger_mock = mocker.MagicMock() lta_rc_mock = mocker.patch("rest_tools.client.RestClient.request", new_callable=AsyncMock) lta_rc_mock.side_effect = HTTPError(500, "LTA DB on fire. Again.") p = DesyVerifier(config, logger_mock) with pytest.raises(HTTPError): await p._do_work() lta_rc_mock.assert_called_with( "POST", '/Bundles/actions/pop?source=WIPAC&dest=DESY&status=verifying', {'claimant': f'{p.name}-{p.instance_uuid}'})
async def test_desy_verifier_update_bundle_in_lta_db(config, mocker): """Test that _update_bundle_in_lta_db updates the bundle as verified in the LTA DB.""" logger_mock = mocker.MagicMock() bundle = {"uuid": "7ec8a8f9-fae3-4f25-ae54-c1f66014f5ef"} lta_mock = mocker.MagicMock() lta_rc_mock = mocker.patch("rest_tools.client.RestClient.request", new_callable=AsyncMock) lta_rc_mock.return_value = True lta_mock.request = lta_rc_mock p = DesyVerifier(config, logger_mock) assert await p._update_bundle_in_lta_db(lta_mock, bundle) lta_rc_mock.assert_called_with( "PATCH", '/Bundles/7ec8a8f9-fae3-4f25-ae54-c1f66014f5ef', mocker.ANY)
def test_constructor_config(config, mocker): """Test that a DesyVerifier can be constructed with a configuration object and a logging object.""" logger_mock = mocker.MagicMock() p = DesyVerifier(config, logger_mock) assert p.name == "testing-desy_verifier" assert p.file_catalog_rest_token == "fake-file-catalog-token" assert p.file_catalog_rest_url == "http://kVj74wBA1AMTDV8zccn67pGuWJqHZzD7iJQHrUJKA.com/" assert p.heartbeat_patch_retries == 3 assert p.heartbeat_patch_timeout_seconds == 30 assert p.heartbeat_sleep_duration_seconds == 60 assert p.lta_rest_token == "fake-lta-rest-token" assert p.lta_rest_url == "http://RmMNHdPhHpH2ZxfaFAC9d2jiIbf5pZiHDqy43rFLQiM.com/" assert p.source_site == "WIPAC" assert p.tape_base_path == "/path/to/hpss" assert p.work_retries == 3 assert p.work_sleep_duration_seconds == 60 assert p.work_timeout_seconds == 30 assert p.logger == logger_mock
async def test_desy_verifier_do_work_claim_yes_result_update_fc_and_lta( config, mocker): """Test that _do_work_claim processes the Bundle that it gets from the LTA DB.""" logger_mock = mocker.MagicMock() lta_rc_mock = mocker.patch("rest_tools.client.RestClient.request", new_callable=AsyncMock) lta_rc_mock.return_value = { "bundle": { "one": 1, }, } abtfc_mock = mocker.patch( "lta.desy_verifier.DesyVerifier._add_bundle_to_file_catalog", new_callable=AsyncMock) ubild_mock = mocker.patch( "lta.desy_verifier.DesyVerifier._update_bundle_in_lta_db", new_callable=AsyncMock) p = DesyVerifier(config, logger_mock) assert await p._do_work_claim() lta_rc_mock.assert_called_with( "POST", '/Bundles/actions/pop?source=WIPAC&dest=DESY&status=verifying', {'claimant': f'{p.name}-{p.instance_uuid}'}) abtfc_mock.assert_called_with(mocker.ANY, {"one": 1}) ubild_mock.assert_called_with(mocker.ANY, {"one": 1})
def test_do_status(config, mocker): """Verify that the DesyVerifier has no additional state to offer.""" logger_mock = mocker.MagicMock() p = DesyVerifier(config, logger_mock) assert p._do_status() == {}
async def test_desy_verifier_add_bundle_to_file_catalog_patch_after_post_error( config, mocker): """Test that _add_bundle_to_file_catalog patches the record for the bundle already in the file catalog.""" logger_mock = mocker.MagicMock() bundle = { "uuid": "7ec8a8f9-fae3-4f25-ae54-c1f66014f5ef", "path": "/data/exp/IceCube/2019/filtered/PFFilt/1109", "bundle_path": "/path/to/source/rse/7ec8a8f9-fae3-4f25-ae54-c1f66014f5ef.zip", "checksum": { "sha512": "97de2a6ad728f50a381eb1be6ecf015019887fac27e8bf608334fb72caf8d3f654fdcce68c33b0f0f27de499b84e67b8357cd81ef7bba3cdaa9e23a648f43ad2", }, "size": 12345, } fc_rc_mock = mocker.patch("rest_tools.client.RestClient.request", new_callable=AsyncMock) fc_rc_mock.side_effect = [ Exception("409 conflict"), # POST /api/files - bundle record already exists!! True, # PATCH /api/files/UUID - bundle record gets updated { # GET /api/files/UUID - get the file record "uuid": "e0d15152-fd73-4e98-9aea-a9e5fdd8618e", "logical_name": "/data/exp/IceCube/2019/filtered/PFFilt/1109/file1.tar.gz", }, True, # POST /api/files/UUID/locations - add the location { # GET /api/files/UUID - get the file record "uuid": "e107a8e8-8a86-41d6-9d4d-b6c8bc3797c4", "logical_name": "/data/exp/IceCube/2019/filtered/PFFilt/1109/file2.tar.gz", }, True, # POST /api/files/UUID/locations - add the location { # GET /api/files/UUID - get the file record "uuid": "93bcd96e-0110-4064-9a79-b5bdfa3effb4", "logical_name": "/data/exp/IceCube/2019/filtered/PFFilt/1109/file3.tar.gz", }, True, # POST /api/files/UUID/locations - add the location ] metadata_uuid0 = uuid1().hex metadata_uuid1 = uuid1().hex metadata_uuid2 = uuid1().hex lta_rc_mock = mocker.patch("rest_tools.client.RestClient", new_callable=AsyncMock) lta_rc_mock.request.side_effect = [ { # GET /Metadata?bundle_uuid={bundle_uuid}&limit={limit} "results": [ {"uuid": metadata_uuid0, "file_catalog_uuid": "e0d15152-fd73-4e98-9aea-a9e5fdd8618e"}, {"uuid": metadata_uuid1, "file_catalog_uuid": "e107a8e8-8a86-41d6-9d4d-b6c8bc3797c4"}, {"uuid": metadata_uuid2, "file_catalog_uuid": "93bcd96e-0110-4064-9a79-b5bdfa3effb4"}, ] }, { # POST /Metadata/actions/bulk_delete "metadata": [metadata_uuid0, metadata_uuid1, metadata_uuid2], "count": 3, }, { "results": [] }, ] p = DesyVerifier(config, logger_mock) assert await p._add_bundle_to_file_catalog(lta_rc_mock, bundle) assert lta_rc_mock.request.call_count == 3 lta_rc_mock.request.assert_called_with( "GET", '/Metadata?bundle_uuid=7ec8a8f9-fae3-4f25-ae54-c1f66014f5ef&limit=1000' ) assert fc_rc_mock.call_count == 8 fc_rc_mock.assert_called_with( "POST", '/api/files/93bcd96e-0110-4064-9a79-b5bdfa3effb4/locations', mocker.ANY)