def upload_file(sftp_client, host_name, localpath, remotepath):
sftp_client.put(localpath, remotepath)
sftp_client.chmod(remotepath,
mode=int(lib.FilePerms(user="******", group="rw",
other="r")))
logger.debug("{local} uploaded to {host} at {remote}".format(
local=localpath, host=host_name, remote=remotepath))
def upload_file(sftp_client, host, localpath, remotepath, logger):
"""Upload the file at localpath to remotepath on host_name via SFTP.
:param sftp_client: SFTP client instance to use for upload.
:type sftp_client: :py:class:`paramiko.sftp_client.SFTPClient`
:param str host: Name of the host to upload the file to.
:param localpath: Local path and file name of file to upload.
:type localpath: :py:class:`pathlib.Path`
:param remotepath: Path and file name to upload file to on remote host.
:type localpath: :py:class:`pathlib.Path`
:param logger: Logger object to send debug message to.
:type logger: :py:class:`logging.Logger`
"""
sftp_client.put(os.fspath(localpath), os.fspath(remotepath))
try:
sftp_client.chmod(os.fspath(remotepath),
int(lib.FilePerms(user="******", group="rw", other="r")))
except PermissionError:
# We're probably trying to change permissions on a file owned by
# another user. We can live with not being able to do that.
pass
logger.debug(f"{localpath} uploaded to {host} at {remotepath}")
def _create_run_script(
run_date, model_config, run_type, tmp_run_dir, run_desc_file_path, config
):
"""
:param :py:class:`arrow.Arrow` run_date:
:param str model_config:
:param str run_type:
:param :py:class:`pathlib.Path` tmp_run_dir:
:param :py:class:`pathlib.Path` run_desc_file_path:
:param :py:class:`nemo_nowcast.Config` config:
:return: Run script file path
:rtype: :py:class:`pathlib.Path`
"""
results_dir = Path(
config["vhfr fvcom runs"]["run types"][f"{run_type} {model_config}"]["results"]
)
ddmmmyy = run_date.format("DDMMMYY").lower()
script = _build_script(
tmp_run_dir, run_desc_file_path, results_dir / ddmmmyy, model_config, config
)
run_script_path = tmp_run_dir / "VHFR_FVCOM.sh"
with run_script_path.open("wt") as f:
f.write(script)
lib.fix_perms(
run_script_path, mode=int(lib.FilePerms(user="******", group="rwx", other="r"))
)
logger.debug(f"{run_type}: run script: {run_script_path}")
return run_script_path
def _create_run_script(run_date, run_type, run_dir, run_desc_filepath,
host_name, config):
host_config = config["run"]["enabled hosts"][host_name]
dmy = run_date.format("DDMMMYY").lower()
results_dir = Path(host_config["run types"][run_type]["results"])
script = _build_script(run_dir, run_type, run_desc_filepath,
results_dir / dmy, host_name, config)
run_script_filepath = run_dir / "SalishSeaNEMO.sh"
with run_script_filepath.open("wt") as f:
f.write(script)
lib.fix_perms(run_script_filepath,
mode=int(lib.FilePerms(user="******", group="rwx", other="r")))
logger.debug(f"{run_type}: run script: {run_script_filepath}")
return run_script_filepath
def _tidy_localhost(run_type, dest, results_dir, config):
results_archive_dir = dest / results_dir
if not run_type == "hindcast":
# Keep FVCOM boundary slab files from hindcast runs so that we can do FVCOM hindcast runs
for filepath in results_archive_dir.glob("FVCOM_[TUVW].nc"):
filepath.unlink()
lib.fix_perms(
results_archive_dir,
mode=int(lib.FilePerms(user="******", group="rwx", other="rx")),
grp_name=config["file group"],
)
for filepath in results_archive_dir.glob("*"):
lib.fix_perms(filepath, grp_name=config["file group"])
return results_archive_dir
def test_results_dir_fix_perms(self, m_fix_perms, m_run_in_subproc,
run_type, host_name, config):
parsed_args = SimpleNamespace(
host_name=host_name,
run_type=run_type,
dest_host="localhost",
run_date=arrow.get("2018-05-22"),
)
download_results.download_results(parsed_args, config)
assert m_fix_perms.call_args_list[0][0] == (Path(
"SalishSea", run_type, "22may18"), )
assert m_fix_perms.call_args_list[0][1] == {
"mode": int(lib.FilePerms(user="******", group="rwx", other="rx")),
"grp_name": "allen",
}
def _write_run_script(run_type, script, run_dir_path):
"""
:param str run_type:
:param str script:
:param :py:class:`pathlib.Path` run_dir_path:
:return: wwatch3 run set-up and execution script path
:rtype: :py:class:`pathlib.Path`
"""
run_script_path = run_dir_path / "SoGWW3.sh"
with run_script_path.open("wt") as f:
f.write(script)
lib.fix_perms(run_script_path,
int(lib.FilePerms(user="******", group="rwx", other="r")))
logger.debug(f"wwatch3-{run_type}: run script: {run_script_path}")
return run_script_path
def download_wwatch3_results(parsed_args, config, *args):
host_name = parsed_args.host_name
run_type = parsed_args.run_type
run_date = parsed_args.run_date
results_dir = run_date.format("DDMMMYY").lower()
run_type_results = Path(config["wave forecasts"]["results"][run_type])
src = f"{host_name}:{run_type_results / results_dir}"
dest = Path(config["wave forecasts"]["results archive"][run_type])
cmd = shlex.split(f"scp -Cpr {src} {dest}")
lib.run_in_subprocess(cmd, logger.debug, logger.error)
results_archive_dir = dest / results_dir
lib.fix_perms(
dest / results_dir,
mode=int(lib.FilePerms(user="******", group="rwx", other="rx")),
grp_name=config["file group"],
)
for filepath in results_archive_dir.glob("*"):
lib.fix_perms(filepath, grp_name=config["file group"])
checklist = {
run_type:
list(map(os.fspath, results_archive_dir.glob(f"SoG_ww3_*.nc")))
}
return checklist
def download_fvcom_results(parsed_args, config, *args):
"""
:param :py:class:`argparse.Namespace` parsed_args:
:param :py:class:`nemo_nowcast.Config` config:
:return: Nowcast system checklist items
:rtype: dict
"""
host_name = parsed_args.host_name
model_config = parsed_args.model_config
run_type = parsed_args.run_type
run_date = parsed_args.run_date
results_dir = run_date.format("DDMMMYY").lower()
run_type_results = Path(config["vhfr fvcom runs"]["run types"]
[f"{run_type} {model_config}"]["results"])
src = f"{host_name}:{run_type_results / results_dir}"
dest = Path(config["vhfr fvcom runs"]["results archive"]
[f"{run_type} {model_config}"])
cmd = shlex.split(f"scp -Cpr {src} {dest}")
lib.run_in_subprocess(cmd, logger.debug, logger.error)
results_archive_dir = dest / results_dir
lib.fix_perms(
dest / results_dir,
mode=int(lib.FilePerms(user="******", group="rwx", other="rx")),
grp_name=config["file group"],
)
for filepath in results_archive_dir.glob("*"):
lib.fix_perms(filepath, grp_name=config["file group"])
checklist = {
run_type: {
"host": host_name,
"model config": model_config,
"run date": run_date.format("YYYY-MM-DD"),
"files": list(map(os.fspath, results_archive_dir.glob("vh*.nc"))),
}
}
return checklist
def make_remote_directory(sftp_client, host_name, remote_dir):
sftp_client.mkdir(remote_dir,
mode=int(
lib.FilePerms(user="******", group="rwx", other="rx")))
logger.debug("{remote} directory made on {host}".format(remote=remote_dir,
host=host_name))