def checkCreateCollection(VARS):
existing_dates = []
existing_dates_by_var = []
for VAR in VARS:
# For one of the variables, get the date of the most recent data set
# All variables come from the same file
# If we have one for a particular data, we should have them all
collection = EE_COLLECTION_GEN.format(var=VAR)
if not eeUtil.exists(PARENT_FOLDER):
logging.info('{} does not exist, creating'.format(PARENT_FOLDER))
eeUtil.createFolder(PARENT_FOLDER)
if eeUtil.exists(collection):
existing_assets = eeUtil.ls(collection)
dates = [getDate_GEE(a) for a in existing_assets]
existing_dates_by_var.append(dates)
for date in dates:
if date not in existing_dates:
existing_dates.append(date)
else:
existing_dates_by_var.append([])
logging.info('{} does not exist, creating'.format(collection))
eeUtil.createFolder(collection, True)
existing_dates_all_vars = copy.copy(existing_dates)
for date in existing_dates:
count = sum(x.count(date)
for x in existing_dates_by_var) / len(TIME_HOURS)
if count < len(VARS):
existing_dates_all_vars.remove(date)
return existing_dates_all_vars, existing_dates_by_var
def checkCreateCollection(collection):
'''List assests in collection else create new collection'''
if eeUtil.exists(collection):
return eeUtil.ls(collection)
else:
logging.info('{} does not exist, creating'.format(collection))
eeUtil.createFolder(collection, True, public=True)
return []
def checkCreateCollection(collection):
'''
List assests in collection if it exists, else create new collection
INPUT collection: GEE collection to check or create (string)
RETURN list of assets in collection (list of strings)
'''
# if collection exists, return list of assets in collection
if eeUtil.exists(collection):
return eeUtil.ls(collection)
# if collection does not exist, create it and return an empty list (because no assets are in the collection)
else:
logging.info('{} does not exist, creating'.format(collection))
eeUtil.createFolder(collection, True, public=True)
return []
def checkCreateCollection(VARS):
'''
List assets in collection if it exists, else create new collection
INPUT VARS: list variables (as named in netcdf) that we want to check collections for (list of strings)
RETURN existing_dates_all_vars: list of dates, in the format of the DATE_FORMAT variable, that exist for all variable collections in GEE (list of strings)
existing_dates_by_var: list of dates, in the format of the DATE_FORMAT variable, that exist for each individual variable collection in GEE (list containing list of strings for each variable)
'''
# create a master list (not variable-specific) to store the dates for which all variables already have data for
existing_dates = []
# create an empty list to store the dates that we currently have for each AQ variable
# will be used in case the previous script run crashed before completing the data upload for every variable.
existing_dates_by_var = []
# loop through each variables that we want to pull
for var in VARS:
# For one of the variables, get the date of the most recent dataset
# All variables come from the same file
# If we have one for a particular data, we should have them all
collection = getCollectionName(var)
# Check if folder to store GEE collections exists. If not, create it.
# we will make one collection per variable, all stored in the parent folder for the dataset
if not eeUtil.exists(PARENT_FOLDER):
logging.info('{} does not exist, creating'.format(PARENT_FOLDER))
eeUtil.createFolder(PARENT_FOLDER)
# If the GEE collection for a particular variable exists, get a list of existing assets
if eeUtil.exists(collection):
existing_assets = eeUtil.ls(collection)
# get a list of the dates from these existing assets
dates = [getDate_GEE(a) for a in existing_assets]
# append this list of dates to our list of dates by variable
existing_dates_by_var.append(dates)
# for each of the dates that we have for this variable, append the date to the master
# list of which dates we already have data for (if it isn't already in the list)
for date in dates:
if date not in existing_dates:
existing_dates.append(date)
# If the GEE collection does not exist, append an empty list to our list of dates by variable
else:
existing_dates_by_var.append([])
# create a collection for this variable
logging.info('{} does not exist, creating'.format(collection))
eeUtil.createFolder(collection, True)
'''
We want make sure all variables correctly uploaded the data on the last run. To do this, we will
check that we have the correct number of appearances of the data in our GEE collection. If we do
not, we will want to re-upload this date's data.
'''
# Create a copy of the master list of dates that will store the dates that were properly uploaded for all variables.
existing_dates_all_vars = copy.copy(existing_dates)
for date in existing_dates:
# check how many times each date appears in our list of dates by variable
date_count = sum(x.count(date) for x in existing_dates_by_var)
# divide this count by the number of time intervals we have (because the date will be
# repeated for each time)
count = date_count / len(TIME_HOURS)
# If this count is less than the number of variables we have, one of the variables did not finish
# uploading for this date, and we need to re-upload this file.
if count < len(VARS):
# remove this from the list of existing dates for all variables
existing_dates_all_vars.remove(date)
return existing_dates_all_vars, existing_dates_by_var
import eeUtil
eeUtil.initJson()
collection = 'cli_012_co2_concentrations'
print(eeUtil.exists(f'test_{collection}'))
eeUtil.createFolder(f'test_{collection}', True, public=True)
print('hola holita!')
print(eeUtil.exists(f'test_{collection}'))
eeUtil.removeAsset(f'test_{collection}')
print(eeUtil.exists(f'test_{collection}'))
def test_createCollection():
eeUtil.createFolder(f'test_{COLLECTION}', True, public=True)
assert len(
eeUtil.ls(f'test_{COLLECTION}')) == 0, "Response metadata incorrect"
return
