def test_fetcher_s3_to_s3(mock_http_to_s3, mock_s3_to_s3):
cfg = FetcherJobConfig(S3_SRC, DST)
retrying_fetch(cfg)
mock_http_to_s3.assert_not_called()
mock_s3_to_s3.assert_called_once()
def test_fetcher_http_runtime_error(mock_http_to_s3):
cfg = FetcherJobConfig(HTTP_SRC, DST)
mock_http_to_s3.side_effect = ValueError("Something bad")
with pytest.raises(ValueError):
retrying_fetch(cfg)
def test_fetcher_updates_zk_fail(mock_http_to_s3_client_error, mock_update_zk_node):
cfg = FetcherJobConfig(HTTP_SRC, DST, zk_node_path=ZK_NODE_PATH, zookeeper_ensemble_hosts=ZK_ENSEMBLE)
retrying_fetch(cfg)
mock_update_zk_node.assert_called_with(
ZK_NODE_PATH, ZK_ENSEMBLE, FetcherResult(status=FetcherStatus.FAILED, message=FILE_NOT_FOUND)
)
def test_fetcher_updates_zk(mock_http_to_s3, mock_update_zk_node):
cfg = FetcherJobConfig(HTTP_SRC, DST, zk_node_path=ZK_NODE_PATH, zookeeper_ensemble_hosts=ZK_ENSEMBLE)
retrying_fetch(cfg)
mock_update_zk_node.assert_called_with(
ZK_NODE_PATH, ZK_ENSEMBLE, FetcherResult(FetcherStatus.DONE, FetchedType.FILE, SUCCESS_MESSAGE)
)
def test_fetcher_updates_zk_with_directory(mock_s3_to_s3, mock_update_zk_node):
# It's a directory
mock_s3_to_s3.return_value = FetchedType.DIRECTORY
cfg = FetcherJobConfig(S3_SRC, DST, zk_node_path=ZK_NODE_PATH, zookeeper_ensemble_hosts=ZK_ENSEMBLE)
retrying_fetch(cfg)
mock_update_zk_node.assert_called_with(
ZK_NODE_PATH, ZK_ENSEMBLE, FetcherResult(FetcherStatus.DONE, FetchedType.DIRECTORY, SUCCESS_MESSAGE)
)
def test_fetcher_updates_zk_once(mock_http_to_s3_server_error, mock_update_zk_node):
cfg = FetcherJobConfig(
HTTP_SRC,
DST,
zk_node_path=ZK_NODE_PATH,
zookeeper_ensemble_hosts=ZK_ENSEMBLE,
retry=RetryConfig(max_attempts=1),
)
retrying_fetch(cfg)
mock_update_zk_node.assert_called_with(
ZK_NODE_PATH, ZK_ENSEMBLE, FetcherResult(status=FetcherStatus.FAILED, message=SERVER_ERROR)
)
def test_get_fetcher_job_args():
cfg: FetcherJobConfig = get_fetcher_job_args(
"--src=SRC --dst=DST --md5=42 --zk-node-path=/zk/path",
{
"LOGGING_LEVEL": "WARN",
"ZOOKEEPER_ENSEMBLE_HOSTS": "Z1",
"RETRY_MAX_ATTEMPTS": "11",
"RETRY_EXP_MAX": "12000",
"RETRY_EXP_MULTIPLIER": "200",
"TMP_DIR": "/tmp/something",
},
)
assert cfg == FetcherJobConfig(
src="SRC",
dst="DST",
md5="42",
zk_node_path="/zk/path",
logging_level="WARN",
zookeeper_ensemble_hosts="Z1",
retry=RetryConfig(max_attempts=11, exp_max=12000, exp_multiplier=200),
tmp_dir="/tmp/something",
)