#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Thu Feb 21 10:29:55 2019 @author: julia_wagemann """ import os from era5_in_gee_functions import createFileList fileList = createFileList('/Volumes/FREECOM HDD/era5_tp/manifest/2000/', './manifest_2000*.json') for i in fileList: print(i) cmd = 'earthengine --use_cloud_api upload image --manifest ' + i os.system(cmd)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Thu Feb 21 16:42:56 2019 @author: julia_wagemann """ import xarray as xr from era5_in_gee_functions import createFileList import time execTime = time.time() directory = '/Volumes/FREECOM HDD/era5_tp/nc/' fileList = createFileList(directory, './1981/era5_tp_1981*') print(fileList) for file in fileList: array = xr.open_dataset(file, mask_and_scale=True, decode_times=True) outFileName = directory + '/daily/1981/' + file[7:-3] + '_daily.nc' print(outFileName) array.resample(time='1D').sum().to_netcdf(outFileName, mode='w', compute=True) print("The script took {0} second !".format(time.time() - execTime))
# day=int(tmp1[4]), # hour=int(hour)) # manifestToJSON(manifest,'./manifest/'+tmp1[2]+'/','manifest_'+assetName) ##Upload to GCP #print("2nd step - Upload daily files to GCP") #fileList = createFileList(directory, './tiff/'+year+'/*.tif') #fileList.sort() # #for i in fileList: # print(i) # upload_blob(bucket_name, i, i[13:]) #Ingest to EE print("3rd step - Ingest to EE") fileList = createFileList(directory, './manifest/' + year + '/*.json') print(len(fileList)) for i in fileList[3346:]: print(i) cmd = 'earthengine --use_cloud_api upload image --manifest ' + i os.system(cmd) ##Delete files from GCP #print("6th step - Delete from GCP") #storage_client = storage.Client() #bucket = storage_client.get_bucket(bucket_name) #blob_list = list(bucket.list_blobs(prefix="era5_tp_"+year)) #bucket.delete_blobs(blob_list) #print('Files from GCP deleted')
# print("2nd step - Convert daily files to tiffs") # convertFilesToTiff(directory1, 'daily', 't2m', year, 4326) #Upload to GCP # print("3rd step - Upload daily files to GCP") # uploadToGCP(directory1,year,'daily','t2m',bucket) #Create manifest print("4th step - Create manifest of daily files") # Directory where tiff files are stored directory = directory1 + '/era5_t2m/tiff/daily/' + year + '/' #Create list of all tiff files that shall be ingested fileList = createFileList(directory, '*.tif') print(fileList) for i in fileList: # Split name of file and find all numbers tmp = re.findall('\d+', i) print(tmp) # Create name of asset based on year month day info assetName = tmp[2] + tmp[3] + tmp[4] print(assetName) # Get start and end times of the asset in epoch times ls_epochtimes = getEpochTimes_daily(int(tmp[2]), int(tmp[3]), int(tmp[4])) print(ls_epochtimes) # Define start and end time
directory2 = '/Volumes/FREECOM HDD/era5_tp/' yearList=['1996','1997','1998','1999','2000','2001','2002','2003','2004','2005','2006','2007','2008','2009','2010','2011','2012','2013','2014','2015','2016','2017','2018'] bucket_t2m = 'era5_t2m_monthly' bucket_tp = 'era5_tp_monthly' param1 = 't2m' param2 = 'tp' for year in yearList: print(year) ############## print('1st step - Create monthly files') # month_list = ['01','02','03','04','05','06','07','08','09','10','11','12'] for i in month_list: fileList_t2m = createFileList(directory1,'./nc/'+year+'/era5_t2m_'+year+'_'+i+'*') fileList_tp = createFileList(directory2,'./nc/'+year+'/era5_tp_'+year+'_'+i+'*') fileList_t2m.sort() fileList_tp.sort() print(fileList_t2m) print(fileList_tp) os.chdir(directory1) array_t2m = xr.open_mfdataset(fileList_t2m) outFileName_t2m = directory1+'nc/monthly/'+year+'/era5_'+param1+'_'+year+'_'+i+'.nc' array_t2m.resample(time='1M').mean().to_netcdf(outFileName_t2m, mode='w', compute=True) os.chdir(directory2) array_tp = xr.open_mfdataset(fileList_tp) outFileName_tp = directory2+'nc/monthly/'+year+'/era5_'+param2+'_'+year+'_'+i+'.nc' array_tp.resample(time='1M').sum().to_netcdf(outFileName_tp, mode='w', compute=True)
'maximum_2m_temperature_since_previous_post_processing', bucket_mx2t) uploadToGCP(directory3, year, time_step, 'tp', bucket_tp) uploadToGCP(directory2, year, time_step, '2m_dewpoint_temperature', bucket_2d) uploadToGCP(directory4, year, time_step, 'surface_pressure', bucket_sp) uploadToGCP(directory2, year, time_step, 'mean_sea_level_pressure', bucket_mslp) uploadToGCP(directory4, year, time_step, '10m_u_component_of_wind', bucket_u10) uploadToGCP(directory4, year, time_step, '10m_v_component_of_wind', bucket_v10) print("3rd step - Create manifests") fileList = createListOfLists(directory_list, 'hourly', year) print(len(fileList)) ncFileList = createFileList(directory1, './era5_t2m/nc/hourly/' + year + '/*') print(len(ncFileList)) createManifestCombined_hourly(fileList, ncFileList, year, bucket_list, directory_manifest, directory_outfile) print('4th step - Ingest to EE') manifest_list = createFileList(directory_outfile + year + '/', '*.json') print(manifest_list) ee_ingest(manifest_list) print("The script took {0} second !".format(time.time() - execTime))
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Thu Feb 21 16:55:51 2019 @author: julia_wagemann """ from era5_in_gee_functions import createFileList, ncToTiff import re directory = '/Volumes/FREECOM HDD/era5_tp/' pattern = './nc/daily/1981/*.nc' fileList = createFileList(directory, pattern) fileList.sort() print(fileList) for file in fileList: print(file) tmp1 = re.findall("\d+", file) year = tmp1[2] outfile = directory + 'tiff/daily/' + year + '/' + file[16:-3] + '.tif' ncToTiff(file, 1, year, 4326, outfile)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Mon Feb 25 11:07:50 2019 @author: julia_wagemann """ from era5_in_gee_functions import createFileList import os directory = '/Volumes/FREECOM HDD/era5_tp/nc/2017/' fileList = createFileList(directory, './era5_tp_2017_11*') print(fileList) for i in fileList: ls = i[:-3].split('_') ls[3] = ls[3].zfill(2) ls[4] = ls[4].zfill(2) renamed = '_'.join(ls) + '.nc' os.rename(i, renamed)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Thu Feb 21 10:19:35 2019 @author: julia_wagemann """ from era5_in_gee_functions import upload_blob, createFileList import time execTime = time.time() bucket_name = 'era5_tp' fileList = createFileList('/Volumes/FREECOM HDD/era5_tp/tiff/2000','era5_tp_2000*') for i in fileList[327:]: print(i) upload_blob(bucket_name, i, i) print("The script took {0} second !".format(time.time() - execTime))
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Fri Feb 22 11:00:05 2019 @author: julia_wagemann """ import xarray as xr from era5_in_gee_functions import createFileList import time execTime = time.time() directory = '/Volumes/FREECOM HDD/era5_tp/nc/1979/' month_list = [ '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12' ] for i in month_list: fileList = createFileList(directory, './era5_tp_1979_' + i + '*') fileList.sort() print(fileList) array = xr.open_mfdataset(fileList) outFileName = '/Volumes/FREECOM HDD/era5_tp/nc/monthly/1979/era5_tp_1979_' + i + '.nc' array.resample(time='1M').sum().to_netcdf(outFileName, mode='w', compute=True) print("The script took {0} second !".format(time.time() - execTime))
from era5_in_gee_functions import createFileList, ncToTiff, createManifest_daily, getEpochTimes_daily, manifestToJSON, upload_blob import time import re import os from google.cloud import storage execTime = time.time() year = '1983' bucket_name = 'era5_tp_daily' directory = '/Volumes/FREECOM HDD/era5_tp/' # Create daily files print("1st step - Create daily files") fileList = createFileList(directory, './nc/' + year + '/era5_tp_' + year + '*') fileList.sort() print(fileList) for file in fileList: array = xr.open_dataset(file, mask_and_scale=True, decode_times=True) outFileName = directory + 'nc/daily/' + year + '/' + file[ 10:-3] + '_daily.nc' print(outFileName) array.resample(time='1D').sum().to_netcdf(outFileName, mode='w', compute=True) #Convert daily files to tiffs print("2nd step - Convert daily files to tiffs") fileList = createFileList(directory,