示例#1
0
'''

import requests
import collections
import xml.etree.ElementTree as xml
from cStringIO import StringIO

from suds.client import Client as SoapClient
from suds.cache import ObjectCache

from api.domain import sensor
from api.providers.configuration.configuration_provider import ConfigurationProvider
from api.system.logger import ilogger as logger
from api import util as utils

config = ConfigurationProvider()


def check_lta_available():
    """
    Simple wrapper to check if lta is up
    :return: bool
    """
    url = config.url_for('earthexplorer')
    return utils.connections.is_reachable(url, timeout=1)


class LTAService(object):
    ''' Abstract service client for all of LTA services '''

    service_name = None
示例#2
0
class OnlineCache(object):
    """ Client code to interact with the LSRD online cache """

    config = ConfigurationProvider()

    __order_path_key = 'online_cache_orders_dir'
    __host_key = 'landsatds.host'
    __user_key = 'landsatds.username'
    __pw_key = 'landsatds.password'

    def __init__(self):
        self.orderpath = self.config.get(self.__order_path_key)

        if not self.orderpath:
            msg = '{} not defined in configurations'.format(
                self.__order_path_key)
            logger.critical(msg)
            raise OnlineCacheException(msg)

        host, user, pw = self.config.get(
            [self.__host_key, self.__user_key, self.__pw_key])

        self.client = sshcmd.RemoteHost(host, user, pw, timeout=5)

        try:
            self.client.execute('ls')
        except Exception as e:
            logger.critical('No connection to OnlineCache host: {}'.format(e))
            raise OnlineCacheException(e)

    def exists(self, orderid, filename=None):
        """ Check if an order [optional filename] exists on the onlinecache

        :param orderid:  associated order to check
        :param filename: file to check inside of an order
        :return: bool
        """
        if filename:
            path = os.path.join(self.orderpath, orderid, filename)
        else:
            path = os.path.join(self.orderpath, orderid)

        try:
            result = self.execute_command('ls -d {0}'.format(path),
                                          silent=True)
            ret = tuple(x.rstrip() for x in result['stdout'])
            return ret[-1] == path
        except OnlineCacheException as e:
            return False

    def delete(self, orderid, filename=None):
        """
        Removes an order from physical online cache disk

        :param filename: file to delete inside of an order
        :param orderid: associated order to delete
        """
        if not self.exists(orderid, filename):
            msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
            logger.critical(msg)
            return False

        if filename:
            path = os.path.join(self.orderpath, orderid, filename)
        else:
            path = os.path.join(self.orderpath, orderid)

        # this should be the dir where the order is held
        logger.info('Deleting {} from online cache'.format(path))
        try:
            self.execute_command(
                'sudo chattr -fR -i {0};rm -rf {0}'.format(path))
        except OnlineCacheException:
            # in the event /lustre is mounted to an NFS system
            logger.info(
                "onlinecache delete, chattr error, attempting chmod instead..."
            )
            self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))

        return True

    def list(self, orderid=None):
        """
        List the orders currently stored on cache, or files listed
        insed of a specific order

        :param orderid: order name to look inside of
        :return: list of folders/files
        """
        if orderid:
            path = os.path.join(self.orderpath, orderid)
        else:
            path = self.orderpath

        cmd = 'ls {}'.format(path)

        result = self.execute_command(cmd)
        ret = tuple(x.rstrip() for x in result['stdout'])

        return ret

    def capacity(self):
        """
        Returns the capacity of the online cache

        :return: dict
        """

        cmd = 'df -mhP {}'.format(self.orderpath)

        result = self.execute_command(cmd)

        line = result['stdout'][1].split(' ')

        clean = [l for l in line if len(l) > 0]

        results = {
            'capacity': clean[1],
            'used': clean[2],
            'available': clean[3],
            'percent_used': clean[4]
        }

        return results

    def execute_command(self, cmd, silent=False):
        """
        Execute the given command on the cache

        :param cmd: cmd string to execute
        :return: results of the command
        """
        try:
            result = self.client.execute(cmd)
        except Exception, exception:
            if not silent:
                logger.critical('Error executing command: {} '
                                'Raised exception: {}'.format(cmd, exception))
            raise OnlineCacheException(exception)

        if 'stderr' in result and result['stderr']:
            if not silent:
                logger.critical('Error executing command: {} '
                                'stderror returned: {}'.format(
                                    cmd, result['stderr']))

            raise OnlineCacheException(result['stderr'])

        logger.info('call to {} returned {}'.format(cmd, result))

        return result
示例#3
0
 def get_system_config():
     return ConfigurationProvider()._retrieve_config()
示例#4
0
class AdministrationProvider(AdminProviderInterfaceV0):
    config = ConfigurationProvider()
    db = db_instance()

    def orders(self, query=None, cancel=False):
        pass

    def system(self, key=None, disposition='on'):
        pass

    def products(self, query=None, resubmit=None):
        pass

    def access_configuration(self, key=None, value=None, delete=False):
        if not key:
            return self.config.configuration_keys
        elif not delete and not value:
            return self.config.get(key)
        elif value and not delete:
            return self.config.put(key, value)
        elif delete and key:
            return self.config.delete(key)

    # def restore_configuration(self, filepath, clear=False):
    def restore_configuration(self, filepath):
        # self.config.load(filepath, clear=clear)
        self.config.load(filepath)

    def backup_configuration(self, path=None):
        return self.config.dump(path)

    def onlinecache(self,
                    list_orders=False,
                    orderid=None,
                    filename=None,
                    delete=False):
        if delete and orderid and filename:
            return OnlineCache().delete(orderid, filename)
        elif delete and orderid:
            return OnlineCache().delete(orderid)
        elif list_orders:
            return OnlineCache().list()
        elif orderid:
            return OnlineCache().list(orderid)
        else:
            return OnlineCache().capacity()

    def jobs(self, jobid=None, stop=False):
        if not jobid:
            resp = HadoopHandler().list_jobs()
        else:
            resp = HadoopHandler().kill_job(jobid)

        return resp

    def error_to(self, orderid, state):
        order = Order.find(orderid)
        err_scenes = order.scenes({'status': 'error'})
        try:
            if state == 'submitted':
                Scene.bulk_update(
                    [s.id for s in err_scenes], {
                        'status': state,
                        'orphaned': None,
                        'reported_orphan': None,
                        'log_file_contents': '',
                        'note': '',
                        'retry_count': 0
                    })

                order.status = 'ordered'
                order.completion_email_sent = None
                order.save()

            else:
                Scene.bulk_update([s.id for s in err_scenes],
                                  {'status': state})

            return True
        except SceneException as e:
            logger.critical('ERR admin provider error_to\ntrace: {}'.format(
                e.message))
            raise AdministrationProviderException('ERR updating with error_to')

    @staticmethod
    def get_system_status():
        sql = "select key, value from ordering_configuration where " \
              "key in ('msg.system_message_body', 'msg.system_message_title', 'system.display_system_message');"
        with db_instance() as db:
            db.select(sql)

        if db:
            resp_dict = dict(db.fetcharr)
            return {
                'system_message_body':
                resp_dict['msg.system_message_body'],
                'system_message_title':
                resp_dict['msg.system_message_title'],
                'display_system_message':
                resp_dict['system.display_system_message']
            }
        else:
            return {'system_message_body': None, 'system_message_title': None}

    @staticmethod
    def update_system_status(params):

        if set(params) != {
                'system_message_title', 'system_message_body',
                'display_system_message'
        }:
            return {
                'msg':
                'Only 3 params are valid, and they must be present:'
                'system_message_title, system_message_body,'
                'display_system_message'
            }

        sql = '''update ordering_configuration set value = %s where key = 'msg.system_message_title';
                 update ordering_configuration set value = %s where key = 'msg.system_message_body';
                 update ordering_configuration set value = %s where key = 'system.display_system_message'; '''
        sql_vals = (params['system_message_title'],
                    params['system_message_body'],
                    params['display_system_message'])
        try:
            with db_instance() as db:
                db.execute(sql, sql_vals)
                db.commit()
        except DBConnectException as e:
            logger.critical("error updating system status: {}".format(e))
            return {'msg': "error updating database: {}".format(e.message)}

        return True

    @staticmethod
    def get_system_config():
        return ConfigurationProvider()._retrieve_config()

    @staticmethod
    def admin_whitelist():
        return api_cfg()['admin_whitelist']

    @staticmethod
    def stat_whitelist():
        return api_cfg()['stat_whitelist']
示例#5
0
import os
import logging

from logging import FileHandler
from logging import Formatter
from logging import Filter
from logging.handlers import SMTPHandler
from api.providers.configuration.configuration_provider import ConfigurationProvider

config = ConfigurationProvider()

LOG_FORMAT = ("%(asctime)s [%(levelname)s]: %(message)s in %(pathname)s:%(lineno)d")

class DbgFilter(Filter):
    def filter(self, rec):
        return rec.levelno == logging.DEBUG

ilogger = logging.getLogger("api")
ilogger.setLevel(logging.DEBUG)

ih = FileHandler("/var/log/uwsgi/espa-api-info.log")
dh = FileHandler("/var/log/uwsgi/espa-api-debug.log")
eh = SMTPHandler(mailhost='localhost', fromaddr=config.get('apiemailsender'), toaddrs=config.get('apiemailreceive').split(','), subject='ESPA API ERROR')

ih.setLevel(logging.INFO)
dh.setLevel(logging.DEBUG)
eh.setLevel(logging.DEBUG)

for handler in [ih, dh, eh]:
    ilogger.addHandler(handler)
示例#6
0
import os
import sys
import logging

from logging import StreamHandler, FileHandler
from logging import Formatter
from logging import Filter
from logging.handlers import SMTPHandler
from api.providers.configuration.configuration_provider import ConfigurationProvider

config = ConfigurationProvider()

if config.mode not in ('tst', 'dev'):
    logging.getLogger("requests").setLevel(logging.WARNING)
    logging.getLogger("passlib.registry").setLevel(logging.WARNING)
else:
    logging.getLogger('suds.client').setLevel(logging.DEBUG)
    logging.getLogger("requests").setLevel(logging.DEBUG)

LOG_FORMAT = (
    "%(asctime)s [%(levelname)s]: %(message)s in %(pathname)s:%(lineno)d")

ilogger = logging.getLogger("api")
ilogger.setLevel(logging.DEBUG)

espa_log_dir = os.getenv('ESPA_LOG_DIR')
if espa_log_dir and not os.getenv('ESPA_LOG_STDOUT'):
    ih = FileHandler(os.path.join(espa_log_dir, 'espa-api-info.log'))
else:
    ih = StreamHandler(stream=sys.stdout)
eh = SMTPHandler(mailhost='localhost',