def test_results_dir_fix_perms(self, m_fix_perms, m_run_in_subproc,
run_type, host_name, config):
parsed_args = SimpleNamespace(
host_name=host_name,
run_type=run_type,
dest_host="localhost",
run_date=arrow.get("2018-05-22"),
)
download_results.download_results(parsed_args, config)
assert m_fix_perms.call_args_list[0][0] == (Path(
"SalishSea", run_type, "22may18"), )
assert m_fix_perms.call_args_list[0][1] == {
"mode": int(lib.FilePerms(user="******", group="rwx", other="rx")),
"grp_name": "allen",
}
def test_checklist_agrif(self, m_fix_perms, m_run_in_subproc, config):
parsed_args = SimpleNamespace(
host_name="orcinus-nowcast-agrif",
run_type="nowcast-agrif",
dest_host="localhost",
run_date=arrow.get("2018-12-07"),
)
p_glob = patch(
"nowcast.workers.download_results.Path.glob",
side_effect=[
[],
[],
[Path("1_Salishsea_1h_20180522_20180522_grid_T.nc")],
[
Path("1_Salishsea_1d_20180522_20180522_grid_T.nc"),
Path("Salishsea_1d_20180522_20180522_grid_T.nc"),
],
],
)
with p_glob:
checklist = download_results.download_results(parsed_args, config)
assert checklist == {
"nowcast-agrif": {
"run date":
"2018-12-07",
"1h": ["1_Salishsea_1h_20180522_20180522_grid_T.nc"],
"1d": [
"1_Salishsea_1d_20180522_20180522_grid_T.nc",
"Salishsea_1d_20180522_20180522_grid_T.nc",
],
}
}
def test_checklist(self, m_fix_perms, m_run_in_subproc, run_type,
host_name, config):
parsed_args = SimpleNamespace(
host_name=host_name,
run_type=run_type,
dest_host="localhost",
run_date=arrow.get("2018-05-22"),
)
p_glob = patch(
"nowcast.workers.download_results.Path.glob",
side_effect=[
[],
[Path("Salishsea_1h_20180522_20180522_grid_T.nc")],
[Path("Salishsea_1d_20180522_20180522_grid_T.nc")],
] if run_type == "hindcast" else [
[],
[],
[Path("Salishsea_1h_20180522_20180522_grid_T.nc")],
[Path("Salishsea_1d_20180522_20180522_grid_T.nc")],
],
)
with p_glob:
checklist = download_results.download_results(parsed_args, config)
assert checklist == {
run_type: {
"run date": "2018-05-22",
"1h": ["Salishsea_1h_20180522_20180522_grid_T.nc"],
"1d": ["Salishsea_1d_20180522_20180522_grid_T.nc"],
}
}
def test_scp_to_localhost_subprocess(self, m_fix_perms, m_run_in_subproc,
run_type, host_name, config):
parsed_args = SimpleNamespace(
host_name=host_name,
run_type=run_type,
dest_host="localhost",
run_date=arrow.get("2018-05-22"),
)
download_results.download_results(parsed_args, config)
m_run_in_subproc.assert_called_once_with(
shlex.split(
f"scp -pr {host_name}:SalishSea/{run_type}/22may18 SalishSea/{run_type}"
),
download_results.logger.debug,
download_results.logger.error,
)
def test_results_files_fix_perms(self, m_fix_perms, m_run_in_subproc,
run_type, host_name, config):
parsed_args = SimpleNamespace(
host_name=host_name,
run_type=run_type,
dest_host="localhost",
run_date=arrow.get("2018-05-22"),
)
p_glob = patch(
"nowcast.workers.download_results.Path.glob",
side_effect=[[Path("namelist_cfg")], [], []] if run_type
== "hindcast" else [[], [Path("namelist_cfg")], [], []],
)
with p_glob:
download_results.download_results(parsed_args, config)
assert m_fix_perms.call_args_list[1][0] == (Path("namelist_cfg"), )
assert m_fix_perms.call_args_list[1][1] == {"grp_name": "allen"}
def test_hindcast_not_unlink_fvcom_boundary_files(
self,
m_fix_perms,
m_run_in_subproc,
host_name,
dest_host,
config,
tmp_path,
monkeypatch,
):
def mock_glob(*args):
return []
monkeypatch.setattr(download_results.Path, "glob", mock_glob)
def mock_tidy_dest_host(*args):
pass
monkeypatch.setattr(download_results, "_tidy_dest_host",
mock_tidy_dest_host)
fvcom_t = tmp_path / "FVCOM_T.nc"
fvcom_t.write_bytes(b"")
fvcom_u = tmp_path / "FVCOM_U.nc"
fvcom_u.write_bytes(b"")
fvcom_v = tmp_path / "FVCOM_V.nc"
fvcom_v.write_bytes(b"")
fvcom_w = tmp_path / "FVCOM_W.nc"
fvcom_w.write_bytes(b"")
parsed_args = SimpleNamespace(
host_name=host_name,
run_type="hindcast",
dest_host=dest_host,
run_date=arrow.get("2019-09-03"),
)
download_results.download_results(parsed_args, config)
assert fvcom_t.exists()
assert fvcom_u.exists()
assert fvcom_v.exists()
assert fvcom_w.exists()
def test_unrecognized_host(self, m_fix_perms, m_run_in_subproc, run_type,
config, caplog, monkeypatch):
parsed_args = SimpleNamespace(
host_name="foo",
run_type=run_type,
dest_host="localhost",
run_date=arrow.get("2018-05-22"),
)
monkeypatch.setitem(config, "run",
{"hindcast hosts": {
"optimum-hindcast": {}
}})
monkeypatch.setitem(config, "run",
{"enabled hosts": {
"arbutus.cloud-nowcast": {}
}})
caplog.set_level(logging.CRITICAL)
with pytest.raises(nemo_nowcast.WorkerError):
download_results.download_results(parsed_args, config)
assert caplog.records[0].levelname == "CRITICAL"
assert caplog.messages[0] == "unrecognized host: foo"
def test_scp_to_dest_host_subprocess(self, m_fix_perms, m_run_in_subproc,
config, monkeypatch):
def mock_tidy_dest_host(*args):
pass
monkeypatch.setattr(download_results, "_tidy_dest_host",
mock_tidy_dest_host)
parsed_args = SimpleNamespace(
host_name="sockeye-hindcast",
run_type="hindcast",
dest_host="graham-dtn",
run_date=arrow.get("2019-09-03"),
)
download_results.download_results(parsed_args, config)
m_run_in_subproc.assert_called_once_with(
shlex.split(
"scp -pr sockeye-hindcast:SalishSea/hindcast/03sep19 graham-dtn:nearline/SalishSea/hindcast"
),
download_results.logger.debug,
download_results.logger.error,
)
def test_unlink_fvcom_boundary_files(
self,
m_fix_perms,
m_run_in_subproc,
run_type,
host_name,
config,
tmp_path,
monkeypatch,
):
fvcom_t = tmp_path / "FVCOM_T.nc"
fvcom_t.write_bytes(b"")
fvcom_u = tmp_path / "FVCOM_U.nc"
fvcom_u.write_bytes(b"")
fvcom_v = tmp_path / "FVCOM_V.nc"
fvcom_v.write_bytes(b"")
fvcom_w = tmp_path / "FVCOM_W.nc"
fvcom_w.write_bytes(b"")
def mock_glob(path, pattern):
return ([fvcom_t, fvcom_u, fvcom_v, fvcom_w]
if pattern.startswith("FVCOM") else [])
monkeypatch.setattr(download_results.Path, "glob", mock_glob)
parsed_args = SimpleNamespace(
host_name=host_name,
run_type=run_type,
dest_host="localhost",
run_date=arrow.get("2018-05-22"),
)
download_results.download_results(parsed_args, config)
assert not fvcom_t.exists()
assert not fvcom_u.exists()
assert not fvcom_v.exists()
assert not fvcom_w.exists()