def export_era5POS(): # if the working directory is alread ml_drought don't need ../data if Path('.').absolute().as_posix().split('/')[-1] == 'ml_drought': data_path = Path('data') else: data_path = Path('../data') exporter = ERA5ExporterPOS(data_path) exporter.export(variable='precipitation_amount_1hour_Accumulation')
def export_era5POS(): # if the working directory is alread ml_drought don't need ../data if Path(".").absolute().as_posix().split("/")[-1] == "ml_drought": data_path = Path("data") else: data_path = Path("~/github/ml_drought") exporter = ERA5ExporterPOS(data_path) exporter.export(variable="precipitation_amount_1hour_Accumulation", years=[2018])
def export_era5POS(): exporter = ERA5ExporterPOS(get_data_path()) variables = [ "air_temperature_at_2_metres", "precipitation_amount_1hour_Accumulation", ] for variable in variables: exporter.export(variable=variable)
def test_available_years(self, tmp_path): # setup our fake bucket era5_bucket = 'era5-pds' conn = boto3.client('s3') conn.create_bucket(Bucket=era5_bucket) files = [f'{year}/main.nc' for year in range(2008, 2019)] # This will check we don't erroneously add non-year folders files.append('QA/main.nc') for file in files: conn.put_object(Bucket=era5_bucket, Key=file, Body='') exporter = ERA5ExporterPOS(tmp_path) years = exporter._get_available_years() assert years == list(range(2008, 2019)), \ f'Expected exporter to retrieve {list(range(2008, 2019))} years, got {years} instead'
def test_available_variables(self, tmp_path): # setup our fake bucket era5_bucket = 'era5-pds' conn = boto3.client('s3') conn.create_bucket(Bucket=era5_bucket, ACL='public-read') expected_variables = {'a', 'b', 'c', 'd', 'e'} for variable in expected_variables: key = f'2008/01/data/{variable}.nc' conn.put_object(Bucket=era5_bucket, Key=key, Body='') exporter = ERA5ExporterPOS(tmp_path) returned_variables = exporter.get_variables(2008, 1) assert len(returned_variables) == len(expected_variables), \ f'Expected {len(expected_variables)} to be returned, got {len(returned_variables)}' for variable in expected_variables: assert variable in returned_variables, \ f'Expected to get variable {variable} but did not'
def test_export(self, tmp_path): # setup our fake bucket era5_bucket = 'era5-pds' conn = boto3.client('s3') conn.create_bucket(Bucket=era5_bucket, ACL='public-read') variable = 'precipitation' years = range(2008, 2019) months = range(1, 12 + 1) keys = [] expected_files = [] for year, month in product(years, months): key = f'{year}/{month:02d}/data/{variable}.nc' keys.append(key) filename = tmp_path / f'raw/era5POS/{year}/{month:02d}/{variable}.nc' expected_files.append(filename) # This will check we don't erroneously add non-year folders keys.append('QA/main.nc') for key in keys: conn.put_object(Bucket=era5_bucket, Key=key, Body='') exporter = ERA5ExporterPOS(tmp_path) downloaded_files = exporter.export(variable) for file in expected_files: assert file.exists(), f'Expected {file} to be downloaded' assert len(expected_files) == len(downloaded_files), \ f'Expected {len(expected_files)} files to be downloaded, ' \ f'got {len(downloaded_files)} instead' for file in expected_files: assert file in downloaded_files, f'{file} not returned by the export function'
nohup python -c "from src.exporters import ERA5ExporterPOS; e = ERA5ExporterPOS(); e.export('air_pressure_at_mean_sea_level')"> planetOS_airP.out & nohup python -c "from src.exporters import ERA5ExporterPOS; e = ERA5ExporterPOS(); e.export('sea_surface_temperature')"> planetOS_sst.out & nohup python -c "from src.exporters import ERA5ExporterPOS; e = ERA5ExporterPOS(); e.export('surface_air_pressure')"> planetOS_slp.out & air_temperature_at_2_metres air_temperature_at_2_metres_1hour_Maximum air_pressure_at_mean_sea_level sea_surface_temperature surface_air_pressure """ from src.exporters import ERA5ExporterPOS e = ERA5ExporterPOS() # download precip for 2010 - 2018 e.export("precipitation_amount_1hour_Accumulation", years=[y for y in range(2010, 2019)]) variables = [ "air_pressure_at_mean_sea_level", "air_temperature_at_2_metres", "air_temperature_at_2_metres_1hour_Maximum", "air_temperature_at_2_metres_1hour_Minimum", "dew_point_temperature_at_2_metres", "eastward_wind_at_100_metres", "eastward_wind_at_10_metres", "integral_wrt_time_of_surface_direct_downwelling_shortwave_flux_in_air_1hour_Accumulation", "lwe_thickness_of_surface_snow_amount",
def export_era5POS(): exporter = ERA5ExporterPOS(get_data_path()) exporter.export(variable="precipitation_amount_1hour_Accumulation")