def get(cls, username, password):
if username == 'espa_admin':
cp = ConfigurationProvider()
if pbkdf2_sha256.verify(password, cp.espa256):
return username, cp.get('apiemailreceive'), 'espa', 'admin', ''
else:
msg = "ERR validating espa_admin, invalid password "
logger.critical(msg)
raise UserException(msg)
else:
eu = ers.get_user_info(username, password)
return eu['username'], eu['email'], eu['firstName'], eu[
'lastName'], eu['contact_id']
class DefaultProviders(object):
ordering = OrderingProvider()
validation = ValidationProvider()
metrics = MetricsProvider()
inventory = InventoryProvider()
production = ProductionProvider()
configuration = ConfigurationProvider()
reporting = ReportingProvider()
administration = AdministrationProvider()
def get(cls, username, password):
if username == 'espa_admin':
try:
cp = ConfigurationProvider()
if pbkdf2_sha256.verify(password, cp.espa256):
return username, cp.get(
'apiemailreceive'), 'espa', 'admin', ''
else:
raise UserException(
"ERR validating espa_admin, invalid password ")
except:
# try/except added due to trouble sorting out issue when
raise UserException(
"ERR validating espa_admin, traceback: {0}".format(
traceback.format_exc()))
else:
try:
eu = ers.get_user_info(username, password)
return eu['username'], eu['email'], eu['firstName'], eu[
'lastName'], eu['contact_id']
except ERSApiException, e:
raise UserException(
"Error authenticating user in get() with ERS. "
"message:{}".format(e.message))
'''
Purpose: lpdaac services client module
Author: David V. Hill
'''
import requests
import os
from api.domain import sensor
from api import util as utils
from api.providers.configuration.configuration_provider import ConfigurationProvider
from api.system.logger import ilogger as logger
config = ConfigurationProvider()
class LPDAACService(object):
def __init__(self):
self.datapool = config.url_for('modis.datapool')
def verify_products(self, products):
response = {}
if isinstance(products, str):
products = [products]
for product in products:
if isinstance(product, basestring):
product = sensor.instance(product)
def get_system_config():
return ConfigurationProvider()._retrieve_config()
class AdministrationProvider(AdminProviderInterfaceV0):
config = ConfigurationProvider()
db = db_instance()
def orders(self, query=None, cancel=False):
pass
def system(self, key=None, disposition='on'):
pass
def products(self, query=None, resubmit=None):
pass
def access_configuration(self, key=None, value=None, delete=False):
if not key:
return self.config.configuration_keys
elif not delete and not value:
return self.config.get(key)
elif value and not delete:
return self.config.put(key, value)
elif delete and key:
return self.config.delete(key)
# def restore_configuration(self, filepath, clear=False):
def restore_configuration(self, filepath):
# self.config.load(filepath, clear=clear)
self.config.load(filepath)
def backup_configuration(self, path=None):
return self.config.dump(path)
def onlinecache(self,
list_orders=False,
orderid=None,
filename=None,
delete=False):
if delete and orderid and filename:
return OnlineCache().delete(orderid, filename)
elif delete and orderid:
return OnlineCache().delete(orderid)
elif list_orders:
return OnlineCache().list()
elif orderid:
return OnlineCache().list(orderid)
else:
return OnlineCache().capacity()
def jobs(self, jobid=None, stop=False):
if not jobid:
resp = HadoopHandler().list_jobs()
else:
resp = HadoopHandler().kill_job(jobid)
return resp
def error_to(self, orderid, state):
order = Order.find(orderid)
err_scenes = order.scenes({'status': 'error'})
try:
for scene in err_scenes:
scene.update('status', state)
if state == 'submitted':
order.status = 'ordered'
order.completion_email_sent = None
order.save()
return True
except SceneException as e:
logger.debug('ERR admin provider error_to\ntrace: {}'.format(
e.message))
raise AdministrationProviderException('ERR updating with error_to')
@staticmethod
def get_system_status():
sql = "select key, value from ordering_configuration where " \
"key in ('msg.system_message_body', 'msg.system_message_title', 'system.display_system_message');"
with db_instance() as db:
db.select(sql)
if db:
resp_dict = dict(db.fetcharr)
return {
'system_message_body':
resp_dict['msg.system_message_body'],
'system_message_title':
resp_dict['msg.system_message_title'],
'display_system_message':
resp_dict['system.display_system_message']
}
else:
return {'system_message_body': None, 'system_message_title': None}
@staticmethod
def update_system_status(params):
if params.keys().sort() is not [
'system_message_title', 'system_message_body',
'display_system_message'
].sort():
return {
'msg':
'Only 3 params are valid, and they must be present:'
'system_message_title, system_message_body,'
'display_system_message'
}
sql_dict = {
'msg.system_message_title': params['system_message_title'],
'msg.system_message_body': params['system_message_body'],
'system.display_system_message': params['display_system_message']
}
sql = ""
for k, v in sql_dict.iteritems():
sql += "update ordering_configuration set value = '{0}' where key = '{1}';".format(
v, k)
try:
with db_instance() as db:
db.execute(sql)
db.commit()
except DBConnectException as e:
logger.debug("error updating system status: {}".format(e))
return {'msg': "error updating database: {}".format(e.message)}
return True
@staticmethod
def get_system_config():
return ConfigurationProvider()._retrieve_config()
@staticmethod
def admin_whitelist():
return ['127.0.0.1']
class OnlineCache(object):
""" Client code to interact with the LSRD online cache """
config = ConfigurationProvider()
__order_path_key = 'online_cache_orders_dir'
__host_key = 'landsatds.host'
__user_key = 'landsatds.username'
__pw_key = 'landsatds.password'
def __init__(self):
self.orderpath = self.config.get(self.__order_path_key)
if not self.orderpath:
msg = '{} not defined in configurations'.format(self.__order_path_key)
logger.debug(msg)
raise OnlineCacheException(msg)
host, user, pw = self.config.get([self.__host_key,
self.__user_key,
self.__pw_key])
self.client = sshcmd.RemoteHost(host, user, pw, timeout=5)
try:
self.client.execute('ls')
except Exception as e:
logger.debug('No connection to OnlineCache host: {}'.format(e))
raise OnlineCacheException(e)
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
try:
self.execute_command('sudo chattr -fR -i {0};rm -rf {0}'.format(path))
except OnlineCacheException:
# in the event /lustre is mounted to an NFS system
logger.info("onlinecache delete, chattr error, attempting chmod instead...")
self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))
return True
def list(self, orderid=None):
"""
List the orders currently stored on cache, or files listed
insed of a specific order
:param orderid: order name to look inside of
:return: list of folders/files
"""
if orderid:
path = os.path.join(self.orderpath, orderid)
else:
path = self.orderpath
cmd = 'ls {}'.format(path)
result = self.execute_command(cmd)
ret = tuple(x.rstrip() for x in result['stdout'])
return ret
def capacity(self):
"""
Returns the capacity of the online cache
:return: dict
"""
cmd = 'df -mhP {}'.format(self.orderpath)
result = self.execute_command(cmd)
line = result['stdout'][1].split(' ')
clean = [l for l in line if len(l) > 0]
results = {'capacity': clean[1],
'used': clean[2],
'available': clean[3],
'percent_free': clean[4]}
return results
def execute_command(self, cmd):
"""
Execute the given command on the cache
:param cmd: cmd string to execute
:return: results of the command
"""
try:
result = self.client.execute(cmd)
except Exception, exception:
logger.debug('Error executing command: {} '
'Raised exception: {}'.format(cmd, exception))
raise OnlineCacheException(exception)
if 'stderr' in result and result['stderr']:
logger.debug('Error executing command: {} '
'stderror returned: {}'.format(cmd, result['stderr']))
raise OnlineCacheException(result['stderr'])
logger.info('call to {} returned {}'.format(cmd, result))
return result
class OnlineCache(object):
""" Client code to interact with the LSRD online cache """
config = ConfigurationProvider()
__order_path_key = 'online_cache_orders_dir'
__host_key = 'landsatds.host'
__user_key = 'landsatds.username'
__pw_key = 'landsatds.password'
def __init__(self):
self.orderpath = self.config.get(self.__order_path_key)
if not self.orderpath:
msg = '{} not defined in configurations'.format(
self.__order_path_key)
logger.critical(msg)
raise OnlineCacheException(msg)
host, user, pw = self.config.get(
[self.__host_key, self.__user_key, self.__pw_key])
self.client = sshcmd.RemoteHost(host, user, pw, timeout=5)
try:
self.client.execute('ls')
except Exception as e:
logger.critical('No connection to OnlineCache host: {}'.format(e))
raise OnlineCacheException(e)
def exists(self, orderid, filename=None):
""" Check if an order [optional filename] exists on the onlinecache
:param orderid: associated order to check
:param filename: file to check inside of an order
:return: bool
"""
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
try:
result = self.execute_command('ls -d {0}'.format(path),
silent=True)
ret = tuple(x.rstrip() for x in result['stdout'])
return ret[-1] == path
except OnlineCacheException as e:
return False
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if not self.exists(orderid, filename):
msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
logger.critical(msg)
return False
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
# TODO: if storage system supports immutable options
# >>> sudo chattr -fR -i {0};rm -rf {0}
# However, nfs does not support this extended attributes
try:
cmd = 'chmod -R 744 {0};rm -rf {0}'.format(path)
self.execute_command(cmd)
except OnlineCacheException as exc:
logger.critical('Failed to remove files from output cache. '
'Command: {} Error: {}'.format(cmd, exc))
return False
return True
def list(self, orderid=None):
"""
List the orders currently stored on cache, or files listed
insed of a specific order
:param orderid: order name to look inside of
:return: list of folders/files
"""
if orderid:
path = os.path.join(self.orderpath, orderid)
else:
path = self.orderpath
cmd = 'ls {}'.format(path)
result = self.execute_command(cmd)
ret = tuple(x.rstrip() for x in result['stdout'])
return ret
def capacity(self):
"""
Returns the capacity of the online cache
:return: dict
"""
cmd = 'df -mhP {}'.format(self.orderpath)
result = self.execute_command(cmd)
line = result['stdout'][1].split(' ')
clean = [l for l in line if len(l) > 0]
results = {
'capacity': clean[1],
'used': clean[2],
'available': clean[3],
'percent_used': clean[4]
}
return results
def execute_command(self, cmd, silent=False):
"""
Execute the given command on the cache
:param cmd: cmd string to execute
:return: results of the command
"""
try:
result = self.client.execute(cmd)
except Exception, exception:
if not silent:
logger.critical('Error executing command: {} '
'Raised exception: {}'.format(cmd, exception))
raise OnlineCacheException(exception)
if 'stderr' in result and result['stderr']:
if not silent:
logger.critical('Error executing command: {} '
'stderror returned: {}'.format(
cmd, result['stderr']))
raise OnlineCacheException(result['stderr'])
logger.info('call to {} returned {}'.format(cmd, result))
return result