#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 21 10:29:55 2019
@author: julia_wagemann
"""
import os
from era5_in_gee_functions import createFileList
fileList = createFileList('/Volumes/FREECOM HDD/era5_tp/manifest/2000/',
'./manifest_2000*.json')
for i in fileList:
print(i)
cmd = 'earthengine --use_cloud_api upload image --manifest ' + i
os.system(cmd)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 21 16:42:56 2019
@author: julia_wagemann
"""
import xarray as xr
from era5_in_gee_functions import createFileList
import time
execTime = time.time()
directory = '/Volumes/FREECOM HDD/era5_tp/nc/'
fileList = createFileList(directory, './1981/era5_tp_1981*')
print(fileList)
for file in fileList:
array = xr.open_dataset(file, mask_and_scale=True, decode_times=True)
outFileName = directory + '/daily/1981/' + file[7:-3] + '_daily.nc'
print(outFileName)
array.resample(time='1D').sum().to_netcdf(outFileName,
mode='w',
compute=True)
print("The script took {0} second !".format(time.time() - execTime))
# day=int(tmp1[4]),
# hour=int(hour))
# manifestToJSON(manifest,'./manifest/'+tmp1[2]+'/','manifest_'+assetName)
##Upload to GCP
#print("2nd step - Upload daily files to GCP")
#fileList = createFileList(directory, './tiff/'+year+'/*.tif')
#fileList.sort()
#
#for i in fileList:
# print(i)
# upload_blob(bucket_name, i, i[13:])
#Ingest to EE
print("3rd step - Ingest to EE")
fileList = createFileList(directory, './manifest/' + year + '/*.json')
print(len(fileList))
for i in fileList[3346:]:
print(i)
cmd = 'earthengine --use_cloud_api upload image --manifest ' + i
os.system(cmd)
##Delete files from GCP
#print("6th step - Delete from GCP")
#storage_client = storage.Client()
#bucket = storage_client.get_bucket(bucket_name)
#blob_list = list(bucket.list_blobs(prefix="era5_tp_"+year))
#bucket.delete_blobs(blob_list)
#print('Files from GCP deleted')
# print("2nd step - Convert daily files to tiffs")
# convertFilesToTiff(directory1, 'daily', 't2m', year, 4326)
#Upload to GCP
# print("3rd step - Upload daily files to GCP")
# uploadToGCP(directory1,year,'daily','t2m',bucket)
#Create manifest
print("4th step - Create manifest of daily files")
# Directory where tiff files are stored
directory = directory1 + '/era5_t2m/tiff/daily/' + year + '/'
#Create list of all tiff files that shall be ingested
fileList = createFileList(directory, '*.tif')
print(fileList)
for i in fileList:
# Split name of file and find all numbers
tmp = re.findall('\d+', i)
print(tmp)
# Create name of asset based on year month day info
assetName = tmp[2] + tmp[3] + tmp[4]
print(assetName)
# Get start and end times of the asset in epoch times
ls_epochtimes = getEpochTimes_daily(int(tmp[2]), int(tmp[3]),
int(tmp[4]))
print(ls_epochtimes)
# Define start and end time
directory2 = '/Volumes/FREECOM HDD/era5_tp/'
yearList=['1996','1997','1998','1999','2000','2001','2002','2003','2004','2005','2006','2007','2008','2009','2010','2011','2012','2013','2014','2015','2016','2017','2018']
bucket_t2m = 'era5_t2m_monthly'
bucket_tp = 'era5_tp_monthly'
param1 = 't2m'
param2 = 'tp'
for year in yearList:
print(year)
##############
print('1st step - Create monthly files')
#
month_list = ['01','02','03','04','05','06','07','08','09','10','11','12']
for i in month_list:
fileList_t2m = createFileList(directory1,'./nc/'+year+'/era5_t2m_'+year+'_'+i+'*')
fileList_tp = createFileList(directory2,'./nc/'+year+'/era5_tp_'+year+'_'+i+'*')
fileList_t2m.sort()
fileList_tp.sort()
print(fileList_t2m)
print(fileList_tp)
os.chdir(directory1)
array_t2m = xr.open_mfdataset(fileList_t2m)
outFileName_t2m = directory1+'nc/monthly/'+year+'/era5_'+param1+'_'+year+'_'+i+'.nc'
array_t2m.resample(time='1M').mean().to_netcdf(outFileName_t2m, mode='w', compute=True)
os.chdir(directory2)
array_tp = xr.open_mfdataset(fileList_tp)
outFileName_tp = directory2+'nc/monthly/'+year+'/era5_'+param2+'_'+year+'_'+i+'.nc'
array_tp.resample(time='1M').sum().to_netcdf(outFileName_tp, mode='w', compute=True)
'maximum_2m_temperature_since_previous_post_processing',
bucket_mx2t)
uploadToGCP(directory3, year, time_step, 'tp', bucket_tp)
uploadToGCP(directory2, year, time_step, '2m_dewpoint_temperature',
bucket_2d)
uploadToGCP(directory4, year, time_step, 'surface_pressure', bucket_sp)
uploadToGCP(directory2, year, time_step, 'mean_sea_level_pressure',
bucket_mslp)
uploadToGCP(directory4, year, time_step, '10m_u_component_of_wind',
bucket_u10)
uploadToGCP(directory4, year, time_step, '10m_v_component_of_wind',
bucket_v10)
print("3rd step - Create manifests")
fileList = createListOfLists(directory_list, 'hourly', year)
print(len(fileList))
ncFileList = createFileList(directory1,
'./era5_t2m/nc/hourly/' + year + '/*')
print(len(ncFileList))
createManifestCombined_hourly(fileList, ncFileList, year, bucket_list,
directory_manifest, directory_outfile)
print('4th step - Ingest to EE')
manifest_list = createFileList(directory_outfile + year + '/', '*.json')
print(manifest_list)
ee_ingest(manifest_list)
print("The script took {0} second !".format(time.time() - execTime))
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 21 16:55:51 2019
@author: julia_wagemann
"""
from era5_in_gee_functions import createFileList, ncToTiff
import re
directory = '/Volumes/FREECOM HDD/era5_tp/'
pattern = './nc/daily/1981/*.nc'
fileList = createFileList(directory, pattern)
fileList.sort()
print(fileList)
for file in fileList:
print(file)
tmp1 = re.findall("\d+", file)
year = tmp1[2]
outfile = directory + 'tiff/daily/' + year + '/' + file[16:-3] + '.tif'
ncToTiff(file, 1, year, 4326, outfile)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Feb 25 11:07:50 2019
@author: julia_wagemann
"""
from era5_in_gee_functions import createFileList
import os
directory = '/Volumes/FREECOM HDD/era5_tp/nc/2017/'
fileList = createFileList(directory, './era5_tp_2017_11*')
print(fileList)
for i in fileList:
ls = i[:-3].split('_')
ls[3] = ls[3].zfill(2)
ls[4] = ls[4].zfill(2)
renamed = '_'.join(ls) + '.nc'
os.rename(i, renamed)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 21 10:19:35 2019
@author: julia_wagemann
"""
from era5_in_gee_functions import upload_blob, createFileList
import time
execTime = time.time()
bucket_name = 'era5_tp'
fileList = createFileList('/Volumes/FREECOM HDD/era5_tp/tiff/2000','era5_tp_2000*')
for i in fileList[327:]:
print(i)
upload_blob(bucket_name, i, i)
print("The script took {0} second !".format(time.time() - execTime))
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Feb 22 11:00:05 2019
@author: julia_wagemann
"""
import xarray as xr
from era5_in_gee_functions import createFileList
import time
execTime = time.time()
directory = '/Volumes/FREECOM HDD/era5_tp/nc/1979/'
month_list = [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'
]
for i in month_list:
fileList = createFileList(directory, './era5_tp_1979_' + i + '*')
fileList.sort()
print(fileList)
array = xr.open_mfdataset(fileList)
outFileName = '/Volumes/FREECOM HDD/era5_tp/nc/monthly/1979/era5_tp_1979_' + i + '.nc'
array.resample(time='1M').sum().to_netcdf(outFileName,
mode='w',
compute=True)
print("The script took {0} second !".format(time.time() - execTime))
from era5_in_gee_functions import createFileList, ncToTiff, createManifest_daily, getEpochTimes_daily, manifestToJSON, upload_blob
import time
import re
import os
from google.cloud import storage
execTime = time.time()
year = '1983'
bucket_name = 'era5_tp_daily'
directory = '/Volumes/FREECOM HDD/era5_tp/'
# Create daily files
print("1st step - Create daily files")
fileList = createFileList(directory, './nc/' + year + '/era5_tp_' + year + '*')
fileList.sort()
print(fileList)
for file in fileList:
array = xr.open_dataset(file, mask_and_scale=True, decode_times=True)
outFileName = directory + 'nc/daily/' + year + '/' + file[
10:-3] + '_daily.nc'
print(outFileName)
array.resample(time='1D').sum().to_netcdf(outFileName,
mode='w',
compute=True)
#Convert daily files to tiffs
print("2nd step - Convert daily files to tiffs")
fileList = createFileList(directory,
