'''
import requests
import collections
import xml.etree.ElementTree as xml
from cStringIO import StringIO
from suds.client import Client as SoapClient
from suds.cache import ObjectCache
from api.domain import sensor
from api.providers.configuration.configuration_provider import ConfigurationProvider
from api.system.logger import ilogger as logger
from api import util as utils
config = ConfigurationProvider()
def check_lta_available():
"""
Simple wrapper to check if lta is up
:return: bool
"""
url = config.url_for('earthexplorer')
return utils.connections.is_reachable(url, timeout=1)
class LTAService(object):
''' Abstract service client for all of LTA services '''
service_name = None
class OnlineCache(object):
""" Client code to interact with the LSRD online cache """
config = ConfigurationProvider()
__order_path_key = 'online_cache_orders_dir'
__host_key = 'landsatds.host'
__user_key = 'landsatds.username'
__pw_key = 'landsatds.password'
def __init__(self):
self.orderpath = self.config.get(self.__order_path_key)
if not self.orderpath:
msg = '{} not defined in configurations'.format(
self.__order_path_key)
logger.critical(msg)
raise OnlineCacheException(msg)
host, user, pw = self.config.get(
[self.__host_key, self.__user_key, self.__pw_key])
self.client = sshcmd.RemoteHost(host, user, pw, timeout=5)
try:
self.client.execute('ls')
except Exception as e:
logger.critical('No connection to OnlineCache host: {}'.format(e))
raise OnlineCacheException(e)
def exists(self, orderid, filename=None):
""" Check if an order [optional filename] exists on the onlinecache
:param orderid: associated order to check
:param filename: file to check inside of an order
:return: bool
"""
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
try:
result = self.execute_command('ls -d {0}'.format(path),
silent=True)
ret = tuple(x.rstrip() for x in result['stdout'])
return ret[-1] == path
except OnlineCacheException as e:
return False
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if not self.exists(orderid, filename):
msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
logger.critical(msg)
return False
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
try:
self.execute_command(
'sudo chattr -fR -i {0};rm -rf {0}'.format(path))
except OnlineCacheException:
# in the event /lustre is mounted to an NFS system
logger.info(
"onlinecache delete, chattr error, attempting chmod instead..."
)
self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))
return True
def list(self, orderid=None):
"""
List the orders currently stored on cache, or files listed
insed of a specific order
:param orderid: order name to look inside of
:return: list of folders/files
"""
if orderid:
path = os.path.join(self.orderpath, orderid)
else:
path = self.orderpath
cmd = 'ls {}'.format(path)
result = self.execute_command(cmd)
ret = tuple(x.rstrip() for x in result['stdout'])
return ret
def capacity(self):
"""
Returns the capacity of the online cache
:return: dict
"""
cmd = 'df -mhP {}'.format(self.orderpath)
result = self.execute_command(cmd)
line = result['stdout'][1].split(' ')
clean = [l for l in line if len(l) > 0]
results = {
'capacity': clean[1],
'used': clean[2],
'available': clean[3],
'percent_used': clean[4]
}
return results
def execute_command(self, cmd, silent=False):
"""
Execute the given command on the cache
:param cmd: cmd string to execute
:return: results of the command
"""
try:
result = self.client.execute(cmd)
except Exception, exception:
if not silent:
logger.critical('Error executing command: {} '
'Raised exception: {}'.format(cmd, exception))
raise OnlineCacheException(exception)
if 'stderr' in result and result['stderr']:
if not silent:
logger.critical('Error executing command: {} '
'stderror returned: {}'.format(
cmd, result['stderr']))
raise OnlineCacheException(result['stderr'])
logger.info('call to {} returned {}'.format(cmd, result))
return result
def get_system_config():
return ConfigurationProvider()._retrieve_config()
class AdministrationProvider(AdminProviderInterfaceV0):
config = ConfigurationProvider()
db = db_instance()
def orders(self, query=None, cancel=False):
pass
def system(self, key=None, disposition='on'):
pass
def products(self, query=None, resubmit=None):
pass
def access_configuration(self, key=None, value=None, delete=False):
if not key:
return self.config.configuration_keys
elif not delete and not value:
return self.config.get(key)
elif value and not delete:
return self.config.put(key, value)
elif delete and key:
return self.config.delete(key)
# def restore_configuration(self, filepath, clear=False):
def restore_configuration(self, filepath):
# self.config.load(filepath, clear=clear)
self.config.load(filepath)
def backup_configuration(self, path=None):
return self.config.dump(path)
def onlinecache(self,
list_orders=False,
orderid=None,
filename=None,
delete=False):
if delete and orderid and filename:
return OnlineCache().delete(orderid, filename)
elif delete and orderid:
return OnlineCache().delete(orderid)
elif list_orders:
return OnlineCache().list()
elif orderid:
return OnlineCache().list(orderid)
else:
return OnlineCache().capacity()
def jobs(self, jobid=None, stop=False):
if not jobid:
resp = HadoopHandler().list_jobs()
else:
resp = HadoopHandler().kill_job(jobid)
return resp
def error_to(self, orderid, state):
order = Order.find(orderid)
err_scenes = order.scenes({'status': 'error'})
try:
if state == 'submitted':
Scene.bulk_update(
[s.id for s in err_scenes], {
'status': state,
'orphaned': None,
'reported_orphan': None,
'log_file_contents': '',
'note': '',
'retry_count': 0
})
order.status = 'ordered'
order.completion_email_sent = None
order.save()
else:
Scene.bulk_update([s.id for s in err_scenes],
{'status': state})
return True
except SceneException as e:
logger.critical('ERR admin provider error_to\ntrace: {}'.format(
e.message))
raise AdministrationProviderException('ERR updating with error_to')
@staticmethod
def get_system_status():
sql = "select key, value from ordering_configuration where " \
"key in ('msg.system_message_body', 'msg.system_message_title', 'system.display_system_message');"
with db_instance() as db:
db.select(sql)
if db:
resp_dict = dict(db.fetcharr)
return {
'system_message_body':
resp_dict['msg.system_message_body'],
'system_message_title':
resp_dict['msg.system_message_title'],
'display_system_message':
resp_dict['system.display_system_message']
}
else:
return {'system_message_body': None, 'system_message_title': None}
@staticmethod
def update_system_status(params):
if set(params) != {
'system_message_title', 'system_message_body',
'display_system_message'
}:
return {
'msg':
'Only 3 params are valid, and they must be present:'
'system_message_title, system_message_body,'
'display_system_message'
}
sql = '''update ordering_configuration set value = %s where key = 'msg.system_message_title';
update ordering_configuration set value = %s where key = 'msg.system_message_body';
update ordering_configuration set value = %s where key = 'system.display_system_message'; '''
sql_vals = (params['system_message_title'],
params['system_message_body'],
params['display_system_message'])
try:
with db_instance() as db:
db.execute(sql, sql_vals)
db.commit()
except DBConnectException as e:
logger.critical("error updating system status: {}".format(e))
return {'msg': "error updating database: {}".format(e.message)}
return True
@staticmethod
def get_system_config():
return ConfigurationProvider()._retrieve_config()
@staticmethod
def admin_whitelist():
return api_cfg()['admin_whitelist']
@staticmethod
def stat_whitelist():
return api_cfg()['stat_whitelist']
import os
import logging
from logging import FileHandler
from logging import Formatter
from logging import Filter
from logging.handlers import SMTPHandler
from api.providers.configuration.configuration_provider import ConfigurationProvider
config = ConfigurationProvider()
LOG_FORMAT = ("%(asctime)s [%(levelname)s]: %(message)s in %(pathname)s:%(lineno)d")
class DbgFilter(Filter):
def filter(self, rec):
return rec.levelno == logging.DEBUG
ilogger = logging.getLogger("api")
ilogger.setLevel(logging.DEBUG)
ih = FileHandler("/var/log/uwsgi/espa-api-info.log")
dh = FileHandler("/var/log/uwsgi/espa-api-debug.log")
eh = SMTPHandler(mailhost='localhost', fromaddr=config.get('apiemailsender'), toaddrs=config.get('apiemailreceive').split(','), subject='ESPA API ERROR')
ih.setLevel(logging.INFO)
dh.setLevel(logging.DEBUG)
eh.setLevel(logging.DEBUG)
for handler in [ih, dh, eh]:
ilogger.addHandler(handler)
import os
import sys
import logging
from logging import StreamHandler, FileHandler
from logging import Formatter
from logging import Filter
from logging.handlers import SMTPHandler
from api.providers.configuration.configuration_provider import ConfigurationProvider
config = ConfigurationProvider()
if config.mode not in ('tst', 'dev'):
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("passlib.registry").setLevel(logging.WARNING)
else:
logging.getLogger('suds.client').setLevel(logging.DEBUG)
logging.getLogger("requests").setLevel(logging.DEBUG)
LOG_FORMAT = (
"%(asctime)s [%(levelname)s]: %(message)s in %(pathname)s:%(lineno)d")
ilogger = logging.getLogger("api")
ilogger.setLevel(logging.DEBUG)
espa_log_dir = os.getenv('ESPA_LOG_DIR')
if espa_log_dir and not os.getenv('ESPA_LOG_STDOUT'):
ih = FileHandler(os.path.join(espa_log_dir, 'espa-api-info.log'))
else:
ih = StreamHandler(stream=sys.stdout)
eh = SMTPHandler(mailhost='localhost',