def ee_init(self):
"""Loads the EE credentials and initializes the EE client."""
if self.service_account_file:
credentials = ee.ServiceAccountCredentials(
self.client_email, self.service_account_file)
elif self.account and self.private_key:
credentials = ee.ServiceAccountCredentials(self.account,
self.private_key)
elif self.refresh_token:
credentials = Credentials(None,
refresh_token=self.refresh_token,
token_uri=ee.oauth.TOKEN_URI,
client_id=ee.oauth.CLIENT_ID,
client_secret=ee.oauth.CLIENT_SECRET,
scopes=ee.oauth.SCOPES)
else:
credentials = 'persistent'
# If a --project flag is passed into a command, it supercedes the one set
# by calling the set_project command.
project = self.project
if self.project_override is not None:
project = self.project_override
ee.Initialize(credentials=credentials,
opt_url=self.url,
use_cloud_api=self.use_cloud_api,
cloud_api_key=self.cloud_api_key,
project=project)
def _get_credentials(self):
"""Acquires credentials."""
if self.service_account_file:
return ee.ServiceAccountCredentials(self.client_email,
self.service_account_file)
elif self.account and self.private_key:
return ee.ServiceAccountCredentials(self.account, self.private_key)
elif self.refresh_token:
return Credentials(None,
refresh_token=self.refresh_token,
token_uri=ee.oauth.TOKEN_URI,
client_id=ee.oauth.CLIENT_ID,
client_secret=ee.oauth.CLIENT_SECRET,
scopes=ee.oauth.SCOPES)
else:
return 'persistent'
def main(key=None, state='READY'):
"""Cancel Earth Engine tasks
Parameters
----------
key : str, optional
File path to an Earth Engine json key file.
state : str, {'ALL', 'READY', 'RUNNING'}
Task state (the default is to only cancel 'READY' tasks).
"""
logging.info('\nCancelling {} tasks'.format(state.lower()))
if state == 'ALL':
states = ['READY', 'RUNNING']
else:
states = [state]
logging.info('\nInitializing Earth Engine')
if key:
logging.info(' Using service account key file: {}'.format(key))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('deadbeef', key_file=key),
use_cloud_api=False)
else:
ee.Initialize(use_cloud_api=False)
# Get current task list
tasks = utils.get_ee_tasks(states=states)
logging.info('\nCancelling tasks:')
for k, v in tasks.items():
logging.info(k)
ee.data.cancelTask(v)
def reallyinitEarthEngineService():
util.positional_parameters_enforcement = util.POSITIONAL_IGNORE # avoid the WARNING [util.py:129] new_request() takes at most 1 positional argument (4 given)
OAuthInfo.SCOPE += ' https://www.googleapis.com/auth/fusiontables.readonly'
print OAuthInfo.SCOPE
try:
if os.environ['SERVER_SOFTWARE'].startswith('Development'):
logging.info("Initialising Earth Engine authenticated connection from devserver")
try:
acct = os.environ['MY_SERVICE_ACCOUNT']
key = os.environ['MY_PRIVATE_KEY_FILE']
except KeyError:
logging.warning("Environment variable MY_SERVICE_ACCOUNT and MY_PRIVATE_KEY_FILE not set. Using settings.py")
logging.debug (os.environ)
acct = settings.MY_LOCAL_SERVICE_ACCOUNT
key = settings.MY_LOCAL_PRIVATE_KEY_FILE
EE_CREDENTIALS = ee.ServiceAccountCredentials(acct, key)
else:
logging.info("Initialising Earth Engine authenticated connection from App Engine")
EE_CREDENTIALS = AppAssertionCredentials(OAuthInfo.SCOPE)
ee.Initialize(EE_CREDENTIALS)
#print EE_CREDENTIALS
# permit long wait for response from earth engine
urlfetch.set_default_fetch_deadline(60)
return EE_CREDENTIALS
except Exception, e:
logging.error("Failed to connect to Earth Engine Google API. Exception: %s", e)
return False
def setup_ee():
"""Sets up Earth Engine authentication."""
with open("privatekey.json") as keyfile:
extracted_key_data = keyfile.read()
credentials = ee.ServiceAccountCredentials(config.EE_SERVICE_ACCOUNT,
key_data=extracted_key_data)
ee.Initialize(credentials)
def setUpCredentials(self):
"""Sets up the credentials"""
credentials = ee.ServiceAccountCredentials(os.getenv('GEE_SACCOUNT'),
'geePrivatekey.json')
ee.Initialize(credentials)
storage_client = storage.Client.from_service_account_json(
'gcsPrivatekey.json')
return storage_client.get_bucket(self.meta['gcsBucket'])
def init(service_account=GEE_SERVICE_ACCOUNT,
credential_path=GOOGLE_APPLICATION_CREDENTIALS,
project=GEE_PROJECT,
bucket=GEE_STAGING_BUCKET,
bucket_prefix=GEE_STAGING_BUCKET_PREFIX,
credential_json=GEE_JSON):
'''
Initialize Earth Engine and Google Storage bucket connection.
Defaults to read from environment.
If no service_account is provided, will attempt to use credentials saved by
`earthengine authenticate`, and `gcloud auth application-default login`
utilities.
`service_account` Service account name. Will need access to both GEE and
Storage
`credential_path` Path to json file containing private key
`project` GCP project for earthengine and storage bucket
`bucket` Storage bucket for staging assets for ingestion
`bucket_prefix` Default bucket folder for staging operations
`credential_json` Json-string to use instead of `credential_path`
https://developers.google.com/earth-engine/service_account
'''
global _gs_bucket_prefix
init_opts = {}
if credential_json:
init_opts['credentials'] = ee.ServiceAccountCredentials(
service_account, key_data=credential_json)
elif credential_path:
init_opts['credentials'] = ee.ServiceAccountCredentials(
service_account, key_file=credential_path)
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credential_path
if project:
init_opts['project'] = project
ee.Initialize(**init_opts)
try:
gsbucket.init(bucket, **init_opts)
except Exception as e:
logger.warning(
"Could not authenticate Google Cloud Storage Bucket. Upload and download functions will not work."
)
logger.error(e)
if bucket_prefix:
_gs_bucket_prefix = bucket_prefix
def initialize_ee():
GEE_JSON = os.environ.get("GEE_JSON")
_CREDENTIAL_FILE = 'credentials.json'
GEE_SERVICE_ACCOUNT = os.environ.get("GEE_SERVICE_ACCOUNT")
with open(_CREDENTIAL_FILE, 'w') as f:
f.write(GEE_JSON)
auth = ee.ServiceAccountCredentials(GEE_SERVICE_ACCOUNT, _CREDENTIAL_FILE)
ee.Initialize(auth)
def ee_init(self):
"""Loads the EE credentils and initializes the EE client."""
if self.service_account_file:
credentials = ee.ServiceAccountCredentials(
self.client_email, self.service_account_file)
elif self.account and self.private_key:
credentials = ee.ServiceAccountCredentials(self.account,
self.private_key)
elif self.refresh_token:
credentials = oauth2client.client.OAuth2Credentials(
None, ee.oauth.CLIENT_ID, ee.oauth.CLIENT_SECRET,
self.refresh_token, None,
'https://accounts.google.com/o/oauth2/token', None)
else:
credentials = 'persistent'
ee.Initialize(credentials=credentials, opt_url=self.url)
def _initialize_ee_client(self):
if self.service_account_key is None:
ee.Initialize("persistent")
else:
service_account_name = json.loads(
self.service_account_key)["client_email"]
credentials = ee.ServiceAccountCredentials(
service_account_name, key_data=self.service_account_key)
ee.Initialize(credentials)
def geeAuth(user_path=os.path.expanduser('~')):
# Authenticate Earth Engine user. Ensure that the *.p12 key file
# is in your ~/.ssh/ directory.
key_file = '%s/.ssh/ee-privatekey.p12' % user_path
if os.path.exists(key_file):
acct = '*****@*****.**'
ee.Initialize(ee.ServiceAccountCredentials(acct, key_file))
else:
raise Exception('Ensure GEE key file is in ~/.ssh directory')
def initialize(ee_account='', ee_key_path=''):
try:
if ee_account and ee_key_path and os.path.exists(ee_key_path):
credentials = ee.ServiceAccountCredentials(ee_account, ee_key_path)
ee.Initialize(credentials)
else:
ee.Initialize()
except Exception as e:
print(e)
def initialize_google_ee():
"""Initialize the EE library."""
if GOOGLE_SERVICE_ACCOUNT:
credentials = ee.ServiceAccountCredentials(
None, key_data=GOOGLE_SERVICE_ACCOUNT
)
ee.Initialize(credentials)
else:
ee.Initialize()
def init(user=None):
sa = os.environ.get('SERVICE_ACCOUNT')
if sa:
_out('init', 'SERVICE_ACCOUNT ({})'.format(sa))
ee.Initialize(ee.ServiceAccountCredentials(sa, PEM_PATH))
else:
if user: user_str = '({})'.format(user)
else: user_str = ''
_out('init', 'USER_ACCOUNT {}'.format(user_str))
ee.Initialize()
def get_footprint(nelat, nelng, swlat, swlng):
# initialise EE
credentials = ee.ServiceAccountCredentials(gee_account.service_account,
gee_account.private_key)
ee.Initialize(credentials)
# define footprint
footprint = ee.Geometry.Polygon(
[[[swlng, swlat], [nelng, swlat], [nelng, nelat], [swlng, nelat]]],
None, False)
return footprint
def startup():
service_account = '*****@*****.**'
credentials = ee.ServiceAccountCredentials(service_account, '../private_key.json')
ee.Initialize(credentials)
# eehelpers.init_projections()
distDir = []
for filename in os.listdir('_dist'):
if filename.endswith(".js"):
distDir.append(filename)
else:
continue
return render_template("application.html", distDir=distDir)
def setup_ee():
"""Sets up Earth Engine authentication."""
try:
private_key = os.getenv('EE_PRIVATE_KEY')
service_email = os.getenv('GEO_PREDICTOR_SERVICE_EMAIL')
credentials = ee.ServiceAccountCredentials(email=service_email,
key_data=private_key)
ee.Initialize(credentials=credentials, use_cloud_api=False)
ee.data.setDeadline(60000)
app.logger.info("EE Initialized")
except Exception as err:
return error(status=502, detail=f'{err}')
def initialize_ee():
'''
Initialize eeUtil and ee modules
'''
# get GEE credentials from env file
GEE_JSON = os.environ.get("GEE_JSON")
_CREDENTIAL_FILE = 'credentials.json'
GEE_SERVICE_ACCOUNT = os.environ.get("GEE_SERVICE_ACCOUNT")
with open(_CREDENTIAL_FILE, 'w') as f:
f.write(GEE_JSON)
auth = ee.ServiceAccountCredentials(GEE_SERVICE_ACCOUNT, _CREDENTIAL_FILE)
ee.Initialize(auth)
def testInitEarthEngineService():
try:
logging.info("Initialising Earth Engine authenticated connection")
EE_CREDENTIALS = ee.ServiceAccountCredentials(
settings.MY_LOCAL_SERVICE_ACCOUNT,
settings.MY_LOCAL_PRIVATE_KEY_FILE)
ee.Initialize(EE_CREDENTIALS)
return True
except Exception, e:
logging.error("Failed to connect to Earth Engine. Exception: %s", e)
return False
def get_CO(coordenates):
service_account = 'some_stuff@some_stuff.iam.gserviceaccount.com'
credentials = ee.ServiceAccountCredentials(service_account,
'privatekey.json')
ee.Initialize(credentials)
roi = ee.Geometry.Polygon(coordenates)
#Choosing the satelite and filtrate with GEE functions
collection = ee.ImageCollection('COPERNICUS/S5P/NRTI/L3_CO')
collection = collection.select('CO_column_number_density').filterDate(
'2020-01-01', '2020-01-31').filterBounds(roi)
collection_mean = collection.mean()
mean_cliped = collection_mean.clip(roi)
#Obtaining min, mean, and max values as dictionaries.
minMaxImage = mean_cliped.select("CO_column_number_density").reduceRegion(
reducer=ee.Reducer.minMax(), geometry=roi, scale=10, maxPixels=1e10)
medImage = mean_cliped.select("CO_column_number_density").reduceRegion(
reducer=ee.Reducer.mean(), geometry=roi, scale=10, maxPixels=1e10)
minMaxDic = ee.Dictionary(minMaxImage)
#Retreaving one value out of the dictionary
med = ee.Number(medImage.get("CO_column_number_density"))
min = ee.Number(minMaxDic.get('CO_column_number_density_min'))
max = ee.Number(minMaxDic.get('CO_column_number_density_max'))
#This is the pallet we will be using
band_viz = {
'min': 0,
'max': 0.05,
'palette':
['black', 'blue', 'purple', 'cyan', 'green', 'yellow', 'red']
}
#Data map by url.
ee_image_object = mean_cliped.updateMask(mean_cliped.gt(0))
map_id_dict = ee.Image(ee_image_object).getMapId(band_viz)
tilesURL = map_id_dict['tile_fetcher'].url_format
#All the information obtained in a dictionary
CO_dictionary = {
'id': "CO",
'units': "mol/m^2",
'tileURL': tilesURL,
'min': min.getInfo(),
'med': med.getInfo(),
'max': max.getInfo()
}
return CO_dictionary
def initialize(account=None, key_file=None):
'''Initialize the Earth Engine object, using your authentication credentials.'''
try:
ee.Initialize()
except:
# in the past, EE keys had to be installed manually. We keep this old method for
# backwards compatibility
if account == None:
f = open(__MY_ACCOUNT_FILE, 'r')
account = f.readline().strip()
if key_file == None:
key_file = __MY_PRIVATE_KEY_FILE
ee.Initialize(ee.ServiceAccountCredentials(account, key_file))
def ee_init(self):
"""Loads the EE credentials and initializes the EE client."""
if self.service_account_file:
credentials = ee.ServiceAccountCredentials(
self.client_email, self.service_account_file)
elif self.account and self.private_key:
credentials = ee.ServiceAccountCredentials(self.account,
self.private_key)
elif self.refresh_token:
credentials = Credentials(None,
refresh_token=self.refresh_token,
token_uri=ee.oauth.TOKEN_URI,
client_id=ee.oauth.CLIENT_ID,
client_secret=ee.oauth.CLIENT_SECRET,
scopes=ee.oauth.SCOPES)
else:
credentials = 'persistent'
ee.Initialize(credentials=credentials,
opt_url=self.url,
use_cloud_api=self.use_cloud_api,
cloud_api_key=self.cloud_api_key)
def main(key=None, state='READY', regex=None):
"""Cancel Earth Engine tasks
Parameters
----------
key : str, optional
File path to an Earth Engine json key file.
state : str, {'ALL', 'READY', 'RUNNING'}
Task state (the default is to only cancel 'READY' tasks).
regex : str, optional
"""
logging.info('\nCancelling {} tasks'.format(state.lower()))
if state == 'ALL':
states = ['READY', 'RUNNING']
else:
states = [state]
logging.info('\nInitializing Earth Engine')
if key:
logging.info(' Using service account key file: {}'.format(key))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('deadbeef', key_file=key),
use_cloud_api=True)
else:
ee.Initialize(use_cloud_api=True)
# Get current task list
tasks = utils.get_ee_tasks(states=states)
if regex:
logging.info('\nFiltering tasks:')
logging.info('{}'.format(regex))
tasks = {
task_desc: task_info
for task_desc, task_info in tasks.items()
if re.match(regex, task_desc)
}
logging.info('\nCancelling tasks:')
for task_desc, task_info in tasks.items():
logging.info(task_desc)
logging.debug(task_info)
try:
ee.data.cancelTask(task_info['id'])
# ee.data.cancelOperation(tasks[export_id]['id'])
except Exception as e:
logging.info(' Exception: {}\n Skipping'.format(e))
def ee_init():
"""log in to earthengine using environment variables or local if available"""
if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS'):
credential_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
with open(credential_file) as f:
credential_info = json.load(f)
service_account = credential_info['client_email']
logger.info(
'logging in with service account: {}'.format(service_account))
credentials = ee.ServiceAccountCredentials(service_account,
credential_file)
ee.Initialize(credentials)
else:
# authenticate with user account
logger.info('logging into earthengine with local user')
ee.Initialize()
def main(args=None):
setup_logging()
parser = argparse.ArgumentParser(description='Google Earth Engine Batch Asset Manager')
subparsers = parser.add_subparsers()
parser_delete = subparsers.add_parser('delete', help='Deletes collection and all items inside. Supports Unix-like wildcards.')
parser_delete.add_argument('id', help='Full path to asset for deletion. Recursively removes all folders, collections and images.')
parser_delete.set_defaults(func=delete_collection_from_parser)
parser_delete.add_argument('-s', '--service-account', help='Google Earth Engine service account.')
parser_delete.add_argument('-k', '--private-key', help='Google Earth Engine private key file.')
parser_upload = subparsers.add_parser('upload', help='Batch Asset Uploader.')
required_named = parser_upload.add_argument_group('Required named arguments.')
required_named.add_argument('--source', help='Path to the directory with images for upload.', required=True)
required_named.add_argument('--dest', help='Destination. Full path for upload to Google Earth Engine, e.g. users/pinkiepie/myponycollection', required=True)
optional_named = parser_upload.add_argument_group('Optional named arguments')
optional_named.add_argument('-m', '--metadata', help='Path to CSV with metadata.')
optional_named.add_argument('--large', action='store_true', help='(Advanced) Use multipart upload. Might help if upload of large '
'files is failing on some systems. Might cause other issues.')
optional_named.add_argument('--nodata', type=int, help='The value to burn into the raster as NoData (missing data)')
optional_named.add_argument('--bands', type=_comma_separated_strings, help='Comma-separated list of names to use for the image bands. Spaces'
'or other special characters are not allowed.')
required_named.add_argument('-u', '--user', help='Google account name (gmail address).')
optional_named.add_argument('-s', '--service-account', help='Google Earth Engine service account.')
optional_named.add_argument('-k', '--private-key', help='Google Earth Engine private key file.')
optional_named.add_argument('-b', '--bucket', help='Google Cloud Storage bucket name.')
parser_upload.set_defaults(func=upload_from_parser)
parser_cancel = subparsers.add_parser('cancel', help='Cancel all running tasks')
parser_cancel.set_defaults(func=cancel_all_running_tasks_from_parser)
parser_cancel.add_argument('-s', '--service-account', help='Google Earth Engine service account.')
parser_cancel.add_argument('-k', '--private-key', help='Google Earth Engine private key file.')
args = parser.parse_args()
if args.service_account:
credentials = ee.ServiceAccountCredentials(args.service_account, args.private_key)
ee.Initialize(credentials)
else:
ee.Initialize()
if args.private_key is not None:
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.private_key
args.func(args)
def pytest_configure():
# Called before tests are collected
# https://docs.pytest.org/en/latest/reference.html#_pytest.hookspec.pytest_sessionstart
logging.basicConfig(level=logging.INFO, format='%(message)s')
logging.getLogger('googleapiclient').setLevel(logging.ERROR)
logging.debug('Test Setup')
# On Travis-CI authenticate using private key environment variable
if 'EE_PRIVATE_KEY_B64' in os.environ:
print('Writing privatekey.json from environmental variable ...')
content = base64.b64decode(os.environ['EE_PRIVATE_KEY_B64']).decode('ascii')
GEE_KEY_FILE = 'privatekey.json'
with open(GEE_KEY_FILE, 'w') as f:
f.write(content)
ee.Initialize(ee.ServiceAccountCredentials('', key_file=GEE_KEY_FILE))
else:
ee.Initialize()
def test_init():
logging.basicConfig(level=logging.DEBUG, format='%(message)s')
logging.getLogger('googleapiclient').setLevel(logging.ERROR)
logging.debug('Test Setup')
# On Travis-CI authenticate using private key environment variable
if 'EE_PRIVATE_KEY_B64' in os.environ:
print('Writing privatekey.json from environmental variable ...')
content = base64.b64decode(
os.environ['EE_PRIVATE_KEY_B64']).decode('ascii')
EE_KEY_FILE = 'privatekey.json'
with open(EE_KEY_FILE, 'w') as f:
f.write(content)
ee.Initialize(ee.ServiceAccountCredentials('test',
key_file=EE_KEY_FILE),
use_cloud_api=True)
else:
ee.Initialize(use_cloud_api=True)
def __init__(self):
# print("Please log in to your google earth engine account.")
# ee.Authenticate()
# ee.Initialize()
os.environ[
"GOOGLE_APPLICATION_CREDENTIALS"] = util.path_key + "woven-grail-248317-4b9659d1c442.json"
service_account = '*****@*****.**'
credentials = ee.ServiceAccountCredentials(
service_account, '../key/woven-grail-248317-4b9659d1c442.json')
ee.Initialize(credentials)
self.mapa = ee.Image(
"UMD/hansen/global_forest_change_2019_v1_7").select(
['treecover2000'])
self.loss = ee.Image(
"UMD/hansen/global_forest_change_2019_v1_7").select(['loss'])
self.storage_client = storage.Client()
self.bucket = self.storage_client.get_bucket('earth_engine_forest')
def __init__( self, handler ):
self.url = handler.get_config( 'earth-engine', 'api' )
account = handler.get_config( 'earth-engine', 'ee_account' )
key = handler.get_config( 'earth-engine', 'ee_private_key_file' )
DEBUG_MODE = ('SERVER_SOFTWARE' in os.environ and
os.environ['SERVER_SOFTWARE'].startswith('Dev'))
if DEBUG_MODE:
EE_API_URL = 'https://earthengine.sandbox.google.com'
EE_CREDENTIALS = ee.ServiceAccountCredentials(account, key)
else:
EE_API_URL = 'https://earthengine.googleapis.com'
EE_CREDENTIALS = appengine.AppAssertionCredentials(ee.OAUTH2_SCOPE)
# Initialize the EE API
EE_TILE_SERVER = EE_API_URL + '/map/'
ee.data.DEFAULT_DEADLINE = 60 * 20
logging.info('Initializing with ' + EE_API_URL)
ee.Initialize(EE_CREDENTIALS, EE_API_URL)
def test_init():
logging.basicConfig(level=logging.DEBUG, format='%(message)s')
logging.debug('Test Setup')
# On Travis-CI authenticate using private key environment variable
if 'EE_PRIVATE_KEY_B64' in os.environ:
print('Writing privatekey.json from environmental variable ...')
content = base64.b64decode(
os.environ['EE_PRIVATE_KEY_B64']).decode('ascii')
EE_PRIVATE_KEY_FILE = 'privatekey.json'
with open(EE_PRIVATE_KEY_FILE, 'w') as f:
f.write(content)
EE_CREDENTIALS = ee.ServiceAccountCredentials(
'', key_file=EE_PRIVATE_KEY_FILE)
ee.Initialize(EE_CREDENTIALS)
else:
ee.Initialize()
# Make a simple EE request
logging.debug(ee.Number(1).getInfo())
