print(ls_epochtimes)
# Define start and end time
startTime = int(ls_epochtimes[0])
endTime = int(ls_epochtimes[1])
# Define name of asset folder in Earth Engine
eeCollectionName = 'projects/earthengine-legacy/assets/projects/ecmwf/era5_daily_test/'
uris1 = i
manifest = updateManifest_daily_single_variable(
directory_manifest, eeCollectionName, assetName, startTime,
endTime, bucket_t2m, uris1, int(tmp[2]), int(tmp[3]), int(tmp[4]))
# Create name of individual manifests
outfile = 'manifest_' + assetName + '_daily_single_test'
# Store manifest info as JSON
manifestToJSON(manifest, directory_outfile + year + '/', outfile)
#Ingest to EE
print("5th step - Ingest to EE")
# Create list of manifest files that shall be ingested
manifest_list = createFileList(directory_outfile + year + '/',
'*test.json')
# Ingest assets based on manifest
ee_ingest(manifest_list)
print("The script took {0} second !".format(time.time() - execTime))
import re
directory = "/Volumes/FREECOM HDD/era5_tp/"
pattern = "./tiff/monthly/2000/*.tif"
fileList = createFileList(directory, pattern)
print(fileList)
for file in fileList:
tmp = re.findall("\d+", file)
assetName = tmp[2] + tmp[3] + '_tp'
print(assetName)
ls_epochtimes = getEpochTimes_monthly(int(tmp[2]), int(tmp[3]))
manifest = createManifest_monthly(
eeCollectionName=
"projects/earthengine-legacy/assets/projects/ecmwf/era5_monthly/",
assetName=assetName,
parameter='tp',
bandIndex=0,
startTime=int(ls_epochtimes[0]),
endTime=int(ls_epochtimes[1]),
gs_bucket='gs://era5_tp_monthly/',
uris=file[20:],
year=int(tmp[2]),
month=int(tmp[3]))
outfile = 'manifest_' + assetName + '_monthly'
manifestToJSON(manifest,
'/Volumes/FREECOM HDD/era5_tp/manifest/monthly/2000/',
outfile)
manifest = updateManifest(directory1, eeCollectionName="projects/earthengine-legacy/assets/projects/ecmwf/era5_monthly/",
assetName=assetName,
startTime=int(ls_epochtimes[0]),
endTime=int(ls_epochtimes[1]),
gs_bucket1='gs://'+bucket_t2m+'/',
gs_bucket2='gs://'+bucket_tp+'/',
uris1=file1[20:],
uris2=file2[20:],
year=int(year),
month=int(month))
print(manifest)
outfile='manifest_'+assetName+'_monthly'
print(outfile)
manifestToJSON(manifest,directory1+'manifest/monthly/'+year+'/',outfile)
#####################
#Upload to GCP
print("4th step - Upload monthly files to GCP")
fileList1 = createFileList(directory1, './tiff/monthly/'+year+'/*.tif')
fileList2 = createFileList(directory2, './tiff/monthly/'+year+'/*.tif')
fileList1.sort()
fileList2.sort()
for i,j in zip(fileList1,fileList2):
print(i,j)
os.chdir(directory1)
upload_blob(bucket_t2m, i, i[20:])
os.chdir(directory2)
manifest = createManifest_daily(
eeCollectionName=
"projects/earthengine-legacy/assets/projects/ecmwf/era5_daily/",
assetName=assetName,
parameter='tp',
bandIndex=0,
startTime=int(ls_epochtimes[0]),
endTime=int(ls_epochtimes[1]),
gs_bucket='gs://era5_tp_daily/',
uris=file[18:],
year=int(tmp[2]),
month=int(tmp[3]),
day=int(tmp[4]))
outfile = 'manifest_' + assetName + '_daily'
directory_tiff = '/Volumes/FREECOM HDD/era5_tp/manifest/daily/' + year + '/'
manifestToJSON(manifest, directory_tiff, outfile)
#Upload to GCP
print("4th step - Upload daily files to GCP")
fileList = createFileList(directory, './tiff/daily/' + year + '/*.tif')
fileList.sort()
for i in fileList:
print(i)
upload_blob(bucket_name, i, i[18:])
#Ingest to EE
print("5th step - Ingest to EE")
fileList = createFileList(directory, './manifest/daily/' + year + '/*.json')
for i in fileList: