def export_era5POS():
# if the working directory is alread ml_drought don't need ../data
if Path('.').absolute().as_posix().split('/')[-1] == 'ml_drought':
data_path = Path('data')
else:
data_path = Path('../data')
exporter = ERA5ExporterPOS(data_path)
exporter.export(variable='precipitation_amount_1hour_Accumulation')
def export_era5POS():
# if the working directory is alread ml_drought don't need ../data
if Path(".").absolute().as_posix().split("/")[-1] == "ml_drought":
data_path = Path("data")
else:
data_path = Path("~/github/ml_drought")
exporter = ERA5ExporterPOS(data_path)
exporter.export(variable="precipitation_amount_1hour_Accumulation", years=[2018])
def export_era5POS():
exporter = ERA5ExporterPOS(get_data_path())
variables = [
"air_temperature_at_2_metres",
"precipitation_amount_1hour_Accumulation",
]
for variable in variables:
exporter.export(variable=variable)
def test_available_years(self, tmp_path):
# setup our fake bucket
era5_bucket = 'era5-pds'
conn = boto3.client('s3')
conn.create_bucket(Bucket=era5_bucket)
files = [f'{year}/main.nc' for year in range(2008, 2019)]
# This will check we don't erroneously add non-year folders
files.append('QA/main.nc')
for file in files:
conn.put_object(Bucket=era5_bucket, Key=file, Body='')
exporter = ERA5ExporterPOS(tmp_path)
years = exporter._get_available_years()
assert years == list(range(2008, 2019)), \
f'Expected exporter to retrieve {list(range(2008, 2019))} years, got {years} instead'
def test_available_variables(self, tmp_path):
# setup our fake bucket
era5_bucket = 'era5-pds'
conn = boto3.client('s3')
conn.create_bucket(Bucket=era5_bucket, ACL='public-read')
expected_variables = {'a', 'b', 'c', 'd', 'e'}
for variable in expected_variables:
key = f'2008/01/data/{variable}.nc'
conn.put_object(Bucket=era5_bucket, Key=key, Body='')
exporter = ERA5ExporterPOS(tmp_path)
returned_variables = exporter.get_variables(2008, 1)
assert len(returned_variables) == len(expected_variables), \
f'Expected {len(expected_variables)} to be returned, got {len(returned_variables)}'
for variable in expected_variables:
assert variable in returned_variables, \
f'Expected to get variable {variable} but did not'
def test_export(self, tmp_path):
# setup our fake bucket
era5_bucket = 'era5-pds'
conn = boto3.client('s3')
conn.create_bucket(Bucket=era5_bucket, ACL='public-read')
variable = 'precipitation'
years = range(2008, 2019)
months = range(1, 12 + 1)
keys = []
expected_files = []
for year, month in product(years, months):
key = f'{year}/{month:02d}/data/{variable}.nc'
keys.append(key)
filename = tmp_path / f'raw/era5POS/{year}/{month:02d}/{variable}.nc'
expected_files.append(filename)
# This will check we don't erroneously add non-year folders
keys.append('QA/main.nc')
for key in keys:
conn.put_object(Bucket=era5_bucket, Key=key, Body='')
exporter = ERA5ExporterPOS(tmp_path)
downloaded_files = exporter.export(variable)
for file in expected_files:
assert file.exists(), f'Expected {file} to be downloaded'
assert len(expected_files) == len(downloaded_files), \
f'Expected {len(expected_files)} files to be downloaded, ' \
f'got {len(downloaded_files)} instead'
for file in expected_files:
assert file in downloaded_files, f'{file} not returned by the export function'
nohup python -c "from src.exporters import ERA5ExporterPOS; e = ERA5ExporterPOS(); e.export('air_pressure_at_mean_sea_level')"> planetOS_airP.out &
nohup python -c "from src.exporters import ERA5ExporterPOS; e = ERA5ExporterPOS(); e.export('sea_surface_temperature')"> planetOS_sst.out &
nohup python -c "from src.exporters import ERA5ExporterPOS; e = ERA5ExporterPOS(); e.export('surface_air_pressure')"> planetOS_slp.out &
air_temperature_at_2_metres
air_temperature_at_2_metres_1hour_Maximum
air_pressure_at_mean_sea_level
sea_surface_temperature
surface_air_pressure
"""
from src.exporters import ERA5ExporterPOS
e = ERA5ExporterPOS()
# download precip for 2010 - 2018
e.export("precipitation_amount_1hour_Accumulation",
years=[y for y in range(2010, 2019)])
variables = [
"air_pressure_at_mean_sea_level",
"air_temperature_at_2_metres",
"air_temperature_at_2_metres_1hour_Maximum",
"air_temperature_at_2_metres_1hour_Minimum",
"dew_point_temperature_at_2_metres",
"eastward_wind_at_100_metres",
"eastward_wind_at_10_metres",
"integral_wrt_time_of_surface_direct_downwelling_shortwave_flux_in_air_1hour_Accumulation",
"lwe_thickness_of_surface_snow_amount",
def export_era5POS():
exporter = ERA5ExporterPOS(get_data_path())
exporter.export(variable="precipitation_amount_1hour_Accumulation")