def _write_mag_file( # pylint:disable=too-many-arguments
openscmdf, metadata, header, outfile_dir, symlink_dir, fnames, force):
out_file = os.path.join(outfile_dir, fnames[0])
out_file = "{}.MAG".format(os.path.splitext(out_file)[0])
if _skip_file(out_file, force, symlink_dir):
return
writer = MAGICCData(openscmdf)
writer["todo"] = "SET"
time_steps = writer.timeseries().columns[1:] - writer.timeseries(
).columns[:-1]
step_upper = np.timedelta64(32, "D") # pylint:disable=too-many-function-args
step_lower = np.timedelta64(28, "D") # pylint:disable=too-many-function-args
if any((time_steps > step_upper) | (time_steps < step_lower)):
raise ValueError("Please raise an issue at "
"github.com/znicholls/netcdf-scm/issues "
"to discuss how to handle non-monthly data wrangling")
writer.metadata = metadata
writer.metadata["timeseriestype"] = "MONTHLY"
writer.metadata["header"] = header
logger.info("Writing file to %s", out_file)
writer.write(out_file, magicc_version=7)
symlink_file = os.path.join(symlink_dir, os.path.basename(out_file))
logger.info("Making symlink to %s", symlink_file)
os.symlink(out_file, symlink_file)
def test_wrangling_in_file_operations(tmpdir, test_cmip6_crunch_output, caplog,
out_format_in_file):
runner = CliRunner()
input_dir = join(test_cmip6_crunch_output,
"CMIP/IPSL/IPSL-CM6A-LR/piControl")
output_dir = str(tmpdir)
caplog.clear()
with caplog.at_level("INFO"):
result_raw = runner.invoke(
wrangle_netcdf_scm_ncs,
[
input_dir,
output_dir,
"test",
"--drs",
"CMIP6Output",
"--number-workers",
1,
"--out-format",
"magicc-input-files",
"--regexp",
".*tas.*",
],
)
assert result_raw.exit_code == 0, result_raw.stdout
# also a global file but don't worry about that
expected_file_raw = join(
output_dir,
"CMIP6/CMIP/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Amon/tas/gr/v20181123/TAS_PICONTROL_IPSL-CM6A-LR_R1I1P1F1_284001-285912_FOURBOX_SURFACE_TEMP.IN",
)
res_raw = MAGICCData(expected_file_raw)
res_raw_resampled = _get_expected_wrangled_ts(res_raw, out_format_in_file)
caplog.clear()
with caplog.at_level("INFO"):
result = runner.invoke(
wrangle_netcdf_scm_ncs,
[
input_dir,
output_dir,
"test",
"--drs",
"CMIP6Output",
"--out-format",
out_format_in_file,
"--regexp",
".*tas.*",
"--number-workers",
1,
],
)
assert result.exit_code == 0, result.stdout
os.listdir(
join(
output_dir,
"CMIP6/CMIP/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Amon/tas/gr/v20181123/",
))
expected_file = join(
output_dir,
"CMIP6/CMIP/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Amon/tas/gr/v20181123/TAS_PICONTROL_IPSL-CM6A-LR_R1I1P1F1_2840-2859_FOURBOX_SURFACE_TEMP.IN",
)
res = MAGICCData(expected_file)
np.testing.assert_allclose(res_raw_resampled,
res.timeseries(),
rtol=2 * 1e-3)
with open(expected_file) as f:
content = f.read()
assert ("timeseriestype: {}".format(
out_format_in_file.replace("magicc-input-files-",
"").replace("-", "_").upper()) in content)
def test_wrangling_mag_file_operations(tmpdir, test_cmip6_crunch_output,
caplog, out_format_mag):
runner = CliRunner()
input_dir = join(test_cmip6_crunch_output, "ScenarioMIP/IPSL/IPSL-CM6A-LR")
output_dir = str(tmpdir)
caplog.clear()
with caplog.at_level("INFO"):
result_raw = runner.invoke(
wrangle_netcdf_scm_ncs,
[
input_dir,
output_dir,
"test",
"--drs",
"CMIP6Output",
"--number-workers",
1,
],
)
assert result_raw.exit_code == 0
expected_file_raw = join(
output_dir,
"CMIP6/ScenarioMIP/IPSL/IPSL-CM6A-LR/ssp126/r1i1p1f1/Lmon/cSoilFast/gr/v20190121/netcdf-scm_cSoilFast_Lmon_IPSL-CM6A-LR_ssp126_r1i1p1f1_gr_202501-204012.MAG",
)
res_raw = MAGICCData(expected_file_raw)
res_raw_resampled = _get_expected_wrangled_ts(res_raw, out_format_mag)
caplog.clear()
with caplog.at_level("INFO"):
result = runner.invoke(
wrangle_netcdf_scm_ncs,
[
input_dir,
output_dir,
"test",
"--drs",
"CMIP6Output",
"--out-format",
out_format_mag,
"--number-workers",
1,
],
)
assert result.exit_code == 0
expected_file = join(
output_dir,
"CMIP6/ScenarioMIP/IPSL/IPSL-CM6A-LR/ssp126/r1i1p1f1/Lmon/cSoilFast/gr/v20190121/netcdf-scm_cSoilFast_Lmon_IPSL-CM6A-LR_ssp126_r1i1p1f1_gr_2025-2040.MAG",
)
res = MAGICCData(expected_file)
np.testing.assert_allclose(res_raw_resampled,
res.timeseries(),
rtol=2 * 1e-3)
with open(expected_file) as f:
content = f.read()
assert ("THISFILE_TIMESERIESTYPE = '{}'".format(
out_format_mag.replace("mag-files-", "").replace("-", "_").upper())
in content)
def test_wrangling_units_specs_area_sum(tmpdir, test_cmip6_crunch_output,
caplog):
target_unit = "Gt / yr"
target_units = pd.DataFrame([["fgco2", target_unit], ["tos", "K"]],
columns=["variable", "unit"])
target_units_csv = join(tmpdir, "target_units.csv")
target_units.to_csv(target_units_csv, index=False)
runner = CliRunner()
input_dir = join(test_cmip6_crunch_output, "CMIP/CCCma")
output_dir = str(tmpdir)
result_raw = runner.invoke(
wrangle_netcdf_scm_ncs,
[
input_dir, output_dir, "test", "--drs", "CMIP6Output",
"--number-workers", 1
],
)
expected_file = join(
output_dir,
"CMIP6/CMIP/CCCma/CanESM5/piControl/r1i1p1f1/Omon/fgco2/gn/v20190429/netcdf-scm_fgco2_Omon_CanESM5_piControl_r1i1p1f1_gn_600101-600103.MAG",
)
assert result_raw.exit_code == 0
res_raw = MAGICCData(expected_file)
caplog.clear()
with caplog.at_level("INFO"):
result = runner.invoke(
wrangle_netcdf_scm_ncs,
[
input_dir,
output_dir,
"test",
"--drs",
"CMIP6Output",
"--number-workers",
1,
"--target-units-specs",
target_units_csv,
"--force",
],
)
assert result.exit_code == 0
assert (
"Converting units of fgco2 from kg m^-2 s^-1 to {}".format(target_unit)
in caplog.messages)
res = MAGICCData(expected_file)
assert sorted(res["region"].tolist()) == sorted(res_raw["region"].tolist())
for region, df in res.timeseries().groupby("region"):
for k, v in res.metadata.items():
if "{} (".format(SCMCube._convert_region_to_area_key(region)) in k:
unit = k.split("(")[-1].split(")")[0]
assert unit == "m**2", "assumed unit for test has changed..."
conv_factor = (float(v) * 10**-12 * 3.155695e07
) # area x mass conv x time conv
break
np.testing.assert_allclose(df.values,
res_raw.filter(region=region).values *
conv_factor,
rtol=1e-5)
def test_wrangling_units_specs(tmpdir, test_cmip6_crunch_output, target_unit,
conv_factor, caplog):
target_units = pd.DataFrame([["fgco2", target_unit], ["tos", "K"]],
columns=["variable", "unit"])
target_units_csv = join(tmpdir, "target_units.csv")
target_units.to_csv(target_units_csv, index=False)
runner = CliRunner()
input_dir = join(test_cmip6_crunch_output, "CMIP/CCCma")
output_dir = str(tmpdir)
caplog.clear()
with caplog.at_level("INFO"):
result_raw = runner.invoke(
wrangle_netcdf_scm_ncs,
[
input_dir,
output_dir,
"test",
"--drs",
"CMIP6Output",
"--number-workers",
1,
],
)
assert result_raw.exit_code == 0
assert "Converting units" not in caplog.messages
expected_file = join(
output_dir,
"CMIP6/CMIP/CCCma/CanESM5/piControl/r1i1p1f1/Omon/fgco2/gn/v20190429/netcdf-scm_fgco2_Omon_CanESM5_piControl_r1i1p1f1_gn_600101-600103.MAG",
)
res_raw = MAGICCData(expected_file)
caplog.clear()
with caplog.at_level("INFO"):
result = runner.invoke(
wrangle_netcdf_scm_ncs,
[
input_dir,
output_dir,
"test",
"--drs",
"CMIP6Output",
"--number-workers",
1,
"--target-units-specs",
target_units_csv,
"--force",
],
)
assert result.exit_code == 0
assert (
"Converting units of fgco2 from kg m^-2 s^-1 to {}".format(target_unit)
in caplog.messages)
res = MAGICCData(expected_file)
np.testing.assert_allclose(res_raw.timeseries() * conv_factor,
res.timeseries(),
rtol=1e-5)