Пример #1
0
 def __configure(self, config, update=True):
     """
     """
     msgs = []
 
     try:
         # create api manager
         params = {u'api_id':u'server-01',
                   u'api_name':config[u'api_system'],
                   u'api_subsystem':config[u'api_subsystem'],
                   u'database_uri':config[u'db_uri'],
                   u'api_module':[u'beehive.module.process.mod.ConfigModule'],
                   u'api_plugin':[]}
         manager = ApiManager(params)    
 
         # remove and create scchema
         if update is False:
             ConfigDbManager.remove_table(config[u'db_uri'])
         ConfigDbManager.create_table(config[u'db_uri'])
         self.logger.info(u'Create config DB %s' % (u''))
         msgs.append(u'Create config DB %s' % (u''))
 
         # create session
         operation.session = manager.get_session()
         #operation.perms = perms
         #operation.user = authuser
         
         # create config db manager
         db_manager = ConfigDbManager()
         
         # set configurations
         #
         # populate configs
         #
         for item in config[u'config']:
             value = item[u'value']
             if isinstance(value, dict):
                 value = json.dumps(value)
             res = db_manager.add(config[u'api_system'], 
                                  item[u'group'], 
                                  item[u'name'], 
                                  value)
             self.logger.info(u'Add configuration %s' % (res))
             msgs.append(u'Add configuration %s' % (res))
     except Exception as ex:
         self.logger.error(ex, exc_info=1)
         raise
     finally:
         # release session
         manager.release_session(operation.session)
         operation.session = None
         
     return msgs
Пример #2
0
def start_event_consumer(params):
    """Start event consumer

    :param params: configuration params
    """
    # internal logger
    logger = logging.getLogger('beehive.module.event.manager')

    logger_level = int(params.get('api_logging_level', logging.DEBUG))

    name = params['api_id'].decode('utf-8') + '.event.consumer'
    log_path = params.get('api_log', None)

    if log_path is None:
        log_path = '/var/log/%s/%s' % (params['api_package'],
                                       params['api_env'])
    else:
        log_path = log_path.decode('utf-8')

    file_name = log_path + name + '.log'
    loggers = [
        logger,
        logging.getLogger('beehive.common.event'),
        logging.getLogger('beehive.module.event.model')
    ]
    LoggerHelper.rotatingfile_handler(loggers, logger_level, file_name)

    # get event handlers
    event_handlers = params.pop('event_handler', [])

    # setup api manager
    api_manager = ApiManager(params)
    api_manager.configure()
    api_manager.register_modules()

    def terminate(*args):
        worker.should_stop = True

    for sig in (SIGHUP, SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM, SIGQUIT):
        signal(sig, terminate)

    with Connection(api_manager.redis_event_uri) as conn:
        try:
            worker = EventConsumerRedis(conn,
                                        api_manager,
                                        event_handlers=event_handlers)
            logger.info('Start event consumer')
            logger.debug('Event handlers: %s' % event_handlers)
            logger.debug('Active worker: %s' % worker)
            logger.debug('Use redis connection: %s' % conn)
            worker.run()
        except KeyboardInterrupt:
            logger.info('Stop event consumer')

    logger.info('Stop event consumer')
Пример #3
0
def start_scheduler(params):
    """start celery scheduler """
    name = ensure_text(params['api_id']) + '.scheduler'
    log_path = run_path = params.get('api_log', None)

    if log_path is not None:
        log_path = run_path = ensure_text(log_path)
    if log_path is None:
        log_path = '/var/log/%s/%s' % (params['api_package'],
                                       params['api_env'])
        run_path = '/var/run/%s/%s' % (params['api_package'],
                                       params['api_env'])

    file_name = log_path + name + '.log'
    pid_name = run_path + name + '.pid'

    logger_level = logging.INFO
    loggers = [
        logging.getLogger('beehive'),
        logging.getLogger('beecell'),
        logging.getLogger('beedrones'),
        logging.getLogger('celery'),
    ]
    LoggerHelper.rotatingfile_handler(loggers,
                                      logger_level,
                                      file_name,
                                      formatter=ExtTaskFormatter)

    api_manager = ApiManager(params)
    api_manager.configure()
    api_manager.register_modules()
    task_scheduler.api_manager = api_manager

    configure_task_scheduler(params['broker_url'],
                             params['result_backend'],
                             task_queue=params['broker_queue'])

    from beehive.module.scheduler.redis_scheduler import RedisScheduler

    beat = task_scheduler.Beat(
        loglevel=logging.getLevelName(internal_logger_level),
        logfile=file_name,
        pidfile=pid_name,
        scheduler_cls=RedisScheduler)

    def terminate(*args):
        pass

    for sig in (SIGHUP, SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM, SIGQUIT):
        signal(sig, terminate)

    beat.run()
Пример #4
0
def start_event_consumer(params, log_path=None):
    """Start event consumer
    """
    # setup kombu logger
    #setup_logging(loglevel=u'DEBUG', loggers=[u''])

    # internal logger
    logger = logging.getLogger(u'beehive.module.event.manager')

    logger_level = logging.DEBUG
    if log_path is None:
        log_path = u'/var/log/%s/%s' % (params[u'api_package'],
                                        params[u'api_env'])
    logname = u'%s/%s.event.consumer' % (log_path, params[u'api_id'])
    logger_file = u'%s.log' % logname
    #loggers = [logging.getLogger(), logger]
    loggers = [logger]
    LoggerHelper.rotatingfile_handler(loggers, logger_level, logger_file)

    # performance logging
    loggers = [logging.getLogger(u'beecell.perf')]
    logger_file = u'%s/%s.watch' % (log_path, params[u'api_id'])
    LoggerHelper.rotatingfile_handler(loggers,
                                      logging.DEBUG,
                                      logger_file,
                                      frmt=u'%(asctime)s - %(message)s')

    # setup api manager
    api_manager = ApiManager(params)
    api_manager.configure()
    api_manager.register_modules()

    def terminate(*args):
        worker.should_stop = True

    for sig in (SIGHUP, SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM, SIGQUIT):
        signal(sig, terminate)

    with Connection(api_manager.redis_event_uri) as conn:
        try:
            worker = EventConsumerRedis(conn, api_manager)
            logger.info(u'Start event consumer')
            worker.run()
        except KeyboardInterrupt:
            logger.info(u'Stop event consumer')

    logger.info(u'Stop event consumer')
Пример #5
0
def start_scheduler(params):
    """start celery scheduler """
    log_path = u'/var/log/%s/%s' % (params[u'api_package'], params[u'api_env'])
    run_path = u'/var/run/%s/%s' % (params[u'api_package'], params[u'api_env'])
    logger_file = u'%s/%s.scheduler.log' % (log_path, params[u'api_id'])
    loggers = [
        logging.getLogger(u'beehive'),
        logging.getLogger(u'beecell'),
        logging.getLogger(u'beedrones'),
        logging.getLogger(u'celery'),
    ]

    LoggerHelper.rotatingfile_handler(loggers,
                                      logger_level,
                                      logger_file,
                                      formatter=ExtTaskFormatter)

    api_manager = ApiManager(params)
    api_manager.configure()
    api_manager.register_modules()
    #worker = ProcessEventConsumerRedis(api_manager)
    #from beehive.module.tasks import task_manager
    task_scheduler.api_manager = api_manager

    configure_task_scheduler(params['broker_url'], params['result_backend'])

    #from beehive.module.scheduler.scheduler import RedisScheduler
    from beehive.module.scheduler.redis_scheduler import RedisScheduler

    beat = task_scheduler.Beat(
        loglevel=logging.getLevelName(logger_level),
        logfile='%s/%s.scheduler.log' % (log_path, params['api_id']),
        pidfile='%s/%s.scheduler.pid' % (run_path, params['api_id']),
        scheduler_cls=RedisScheduler)

    def terminate(*args):
        #run_command(['celery', 'multi', 'stopwait', 'worker1',
        #             '--pidfile="run/celery-%n.pid"'])
        #beat.Service.stop()
        pass

    for sig in (SIGHUP, SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM, SIGQUIT):
        signal(sig, terminate)

    beat.run()
Пример #6
0
def start_catalog_consumer(params):
    """Start catalog consumer
    
    :param params: configuration params    
    """
    # internal logger
    logger = getLogger('beehive')

    logger_level = int(params.get('api_logging_level', DEBUG))

    name = params['api_id'].decode('utf-8') + '.catalog'
    log_path = params.get('api_log', None)

    if log_path is None:
        log_path = '/var/log/%s/%s' % (params['api_package'],
                                       params['api_env'])
    else:
        log_path = log_path.decode('utf-8')

    file_name = log_path + name + '.log'
    loggers = [getLogger(), logger]
    LoggerHelper.rotatingfile_handler(loggers, logger_level, file_name)

    # setup api manager
    api_manager = ApiManager(params)
    api_manager.configure()
    api_manager.register_modules()

    def terminate(*args):
        worker.should_stop = True

    for sig in (SIGHUP, SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM, SIGQUIT):
        signal(sig, terminate)

    with Connection(api_manager.redis_catalog_uri) as conn:
        try:
            worker = CatalogConsumerRedis(conn, api_manager)
            logger.info('Start catalog consumer')
            worker.run()
        except KeyboardInterrupt:
            logger.info('Stop catalog consumer')

    logger.info('Stop catalog consumer')
Пример #7
0
def start_task_manager(params):
    """Start celery task manager
    """
    name = ensure_text(params['api_id']) + '.worker'
    log_path = run_path = params.get('api_log', None)

    if log_path is not None:
        log_path = run_path = ensure_text(log_path)
    if log_path is None:
        log_path = '/var/log/%s/%s' % (params['api_package'],
                                       params['api_env'])
        run_path = '/var/run/%s/%s' % (params['api_package'],
                                       params['api_env'])

    file_name = log_path + name + '.log'
    pid_name = run_path + name + '.pid'

    frmt = u'[%(asctime)s: %(levelname)s/%(task_name)s:%(task_id)s] %(name)s:%(funcName)s:%(lineno)d - %(message)s'

    logger_level = int(params.get('api_logging_level', logging.DEBUG))
    main_loggers = [
        logging.getLogger('beehive'),
        logging.getLogger('beehive.common.model'),
        logging.getLogger('beehive_service'),
        logging.getLogger('beehive_resource'),
        logging.getLogger('beehive.db'),
        logging.getLogger('beecell'),
        logging.getLogger('beedrones'),
        logging.getLogger('celery'),
        logging.getLogger('proxmoxer'),
        logging.getLogger('requests')
    ]
    LoggerHelper.rotatingfile_handler(main_loggers,
                                      logger_level,
                                      file_name,
                                      frmt=frmt,
                                      formatter=ExtTaskFormatter)

    # setup api manager
    api_manager = ApiManager(params, hostname=gethostname())
    api_manager.configure()
    api_manager.register_modules(register_api=False)
    task_manager.api_manager = api_manager

    # elk logger
    if api_manager.elasticsearch is not None:
        frmt = '{"timestamp":"%(asctime)s", "levelname":"%(levelname)s", "task_name":"%(task_name)s", ' \
               '"task_id":"%(task_id)s", "module":"%(name)s", "func":"%(funcName)s", "lineno":"%(lineno)d",' \
               '"message":"%(message)s"}'
        tags = []
        LoggerHelper.elastic_handler(main_loggers,
                                     logger_level,
                                     api_manager.elasticsearch,
                                     index='cmp',
                                     frmt=frmt,
                                     tags=tags,
                                     server=api_manager.server_name,
                                     app=api_manager.app_id,
                                     component='task')
    configure_task_manager(params['broker_url'],
                           params['result_backend'],
                           tasks=params['task_module'],
                           expire=params['expire'],
                           task_queue=params['broker_queue'],
                           time_limit=params['task_time_limit'])

    argv = [
        '',
        '--hostname=' + ensure_text(params['broker_queue']) + '@%h',
        '--loglevel=%s' % logging.getLevelName(internal_logger_level),
        '--purge',
        '--logfile=%s' % file_name,
        '--pidfile=%s' % pid_name,
    ]

    def terminate(*args):
        task_manager.stop()

    # for sig in (SIGHUP, SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM, SIGQUIT):
    #    signal(sig, terminate)

    task_manager.worker_main(argv)
Пример #8
0
def start_task_manager(params):
    """Start celery task manager
    """
    logname = "%s.task" % params['api_id']
    frmt = u'[%(asctime)s: %(levelname)s/%(processName)s] ' \
           u'%(name)s:%(funcName)s:%(lineno)d - %(message)s'

    frmt = u'[%(asctime)s: %(levelname)s/%(task_name)s:%(task_id)s] '\
           u'%(name)s:%(funcName)s:%(lineno)d - %(message)s'

    log_path = u'/var/log/%s/%s' % (params[u'api_package'], params[u'api_env'])
    run_path = u'/var/run/%s/%s' % (params[u'api_package'], params[u'api_env'])

    #loggers = [logging.getLogger('beehive.common.event')]
    #LoggerHelper.rotatingfile_handler(loggers, logger_level,
    #                                  '%s/%s.event.log' % (log_path, logname),
    #                                  frmt=frmt)

    # base logging
    loggers = [
        logging.getLogger(u'beehive'),
        logging.getLogger(u'beehive.db'),
        logging.getLogger(u'beecell'),
        logging.getLogger(u'beedrones'),
        logging.getLogger(u'celery'),
        logging.getLogger(u'proxmoxer'),
        logging.getLogger(u'requests')
    ]
    LoggerHelper.rotatingfile_handler(loggers,
                                      logger_level,
                                      u'%s/%s.log' % (log_path, logname),
                                      frmt=frmt,
                                      formatter=ExtTaskFormatter)

    # transaction and db logging
    loggers = [
        logging.getLogger('beehive.util.data'),
        logging.getLogger('sqlalchemy.engine'),
        logging.getLogger('sqlalchemy.pool')
    ]
    LoggerHelper.rotatingfile_handler(loggers, logger_level,
                                      '%s/%s.db.log' % (log_path, logname))

    # performance logging
    loggers = [logging.getLogger('beecell.perf')]
    LoggerHelper.rotatingfile_handler(loggers,
                                      logger_level,
                                      '%s/%s.watch' %
                                      (log_path, params[u'api_id']),
                                      frmt='%(asctime)s - %(message)s')

    api_manager = ApiManager(params, hostname=gethostname())
    api_manager.configure()
    api_manager.register_modules()
    #worker = ProcessEventConsumerRedis(api_manager)
    #from beehive.module.tasks import task_manager
    task_manager.api_manager = api_manager

    logger_file = '%s/%s.log' % (log_path, logname)

    configure_task_manager(params['broker_url'],
                           params['result_backend'],
                           tasks=params['task_module'],
                           expire=params['expire'],
                           logger_file=logger_file)

    argv = [
        u'',
        u'--loglevel=%s' % logging.getLevelName(logger_level),
        #u'--pool=prefork',
        u'--pool=gevent',
        u'--purge',
        #'--time-limit=600',
        #'--soft-time-limit=300',
        u'--concurrency=100',
        u'--maxtasksperchild=100',
        #u'--autoscale=100,10',
        u'--logfile=%s' % logger_file,
        u'--pidfile=%s/%s.task.pid' % (run_path, logname)
    ]

    def terminate(*args):
        #run_command(['celery', 'multi', 'stopwait', 'worker1',
        #             '--pidfile="run/celery-%n.pid"'])
        task_manager.stop()

    #for sig in (SIGHUP, SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM, SIGQUIT):
    #    signal(sig, terminate)

    task_manager.worker_main(argv)
Пример #9
0
    def __init__(self, *args, **kwargs):
        super(BeehiveApp, self).__init__(*args, **kwargs)

        # set debug mode
        self.debug = False

        # flask secret
        self.secret_key = urandom(48)

        self.http_socket = uwsgi_util.opt['http-socket']
        self.server_name = uwsgi_util.opt['api_host']
        self.server_fqdn = uwsgi_util.opt['api_fqdn']
        try:
            self.server_ip = gethostbyname(uwsgi_util.opt['api_fqdn'])
        except:
            self.server_ip = '127.0.0.1'

        self.app_name = uwsgi_util.opt['api_name']
        self.app_id = uwsgi_util.opt['api_id']

        # api instance static config
        self.params = uwsgi_util.opt

        # set logging path
        log_path = '/var/log/%s/%s' % (self.params['api_package'],
                                       self.params['api_env'])
        self.log_path = self.params.get('api_log', log_path)

        def error405(e):
            error = {
                'code': 405,
                'message': 'Method Not Allowed',
                'description': 'Method Not Allowed'
            }
            logger.error('Api response: %s' % error)
            return Response(response=json.dumps(error),
                            mimetype='application/json',
                            status=405)

        self._register_error_handler(None, 405, error405)

        def error404(e):
            error = {
                'code': 404,
                'message': 'Uri %s not found' % request.path,
                'description': 'Uri %s not found' % request.path
            }
            logger.error('Api response: %s' % error)
            return Response(response=json.dumps(error),
                            mimetype='application/json',
                            status=404)

        self._register_error_handler(None, 404, error404)

        # setup loggers
        loggin_level = int(self.params['api_logging_level'])
        self.setup_loggers(level=loggin_level)

        logger.info('########## SERVER STARTING ##########')
        start = time()

        # api manager reference
        self.api_manager = ApiManager(self.params,
                                      app=self,
                                      hostname=self.server_ip)

        # server configuration
        # self.api_manager.configure_logger()
        self.api_manager.configure()
        # self.get_configurations()

        # setup additional handler
        if self.api_manager.elasticsearch is not None:
            tags = []
            self.setup_additional_loggers(self.api_manager.elasticsearch,
                                          level=loggin_level,
                                          tags=tags,
                                          server=self.server_name,
                                          app=self.app_id,
                                          component='api')

        # load modules
        self.api_manager.register_modules()

        # register in catalog
        self.api_manager.register_catalog()

        # register in moitor
        # self.api_manager.register_monitor()

        logger.info('Setup server over: %s' % self.api_manager.app_uri)
        logger.info('Setup server over: %s' % self.api_manager.uwsgi_uri)

        logger.info('########## SERVER STARTED ########## - %s' %
                    round(time() - start, 2))
Пример #10
0
class BeehiveApp(Flask):
    """Custom Flask app used to read configuration and initialize security.

    :param args: positional args
    :param kwargs: key value args
    """
    def __init__(self, *args, **kwargs):
        super(BeehiveApp, self).__init__(*args, **kwargs)

        # set debug mode
        self.debug = False

        # flask secret
        self.secret_key = urandom(48)

        self.http_socket = uwsgi_util.opt['http-socket']
        self.server_name = uwsgi_util.opt['api_host']
        self.server_fqdn = uwsgi_util.opt['api_fqdn']
        try:
            self.server_ip = gethostbyname(uwsgi_util.opt['api_fqdn'])
        except:
            self.server_ip = '127.0.0.1'

        self.app_name = uwsgi_util.opt['api_name']
        self.app_id = uwsgi_util.opt['api_id']

        # api instance static config
        self.params = uwsgi_util.opt

        # set logging path
        log_path = '/var/log/%s/%s' % (self.params['api_package'],
                                       self.params['api_env'])
        self.log_path = self.params.get('api_log', log_path)

        def error405(e):
            error = {
                'code': 405,
                'message': 'Method Not Allowed',
                'description': 'Method Not Allowed'
            }
            logger.error('Api response: %s' % error)
            return Response(response=json.dumps(error),
                            mimetype='application/json',
                            status=405)

        self._register_error_handler(None, 405, error405)

        def error404(e):
            error = {
                'code': 404,
                'message': 'Uri %s not found' % request.path,
                'description': 'Uri %s not found' % request.path
            }
            logger.error('Api response: %s' % error)
            return Response(response=json.dumps(error),
                            mimetype='application/json',
                            status=404)

        self._register_error_handler(None, 404, error404)

        # setup loggers
        loggin_level = int(self.params['api_logging_level'])
        self.setup_loggers(level=loggin_level)

        logger.info('########## SERVER STARTING ##########')
        start = time()

        # api manager reference
        self.api_manager = ApiManager(self.params,
                                      app=self,
                                      hostname=self.server_ip)

        # server configuration
        # self.api_manager.configure_logger()
        self.api_manager.configure()
        # self.get_configurations()

        # setup additional handler
        if self.api_manager.elasticsearch is not None:
            tags = []
            self.setup_additional_loggers(self.api_manager.elasticsearch,
                                          level=loggin_level,
                                          tags=tags,
                                          server=self.server_name,
                                          app=self.app_id,
                                          component='api')

        # load modules
        self.api_manager.register_modules()

        # register in catalog
        self.api_manager.register_catalog()

        # register in moitor
        # self.api_manager.register_monitor()

        logger.info('Setup server over: %s' % self.api_manager.app_uri)
        logger.info('Setup server over: %s' % self.api_manager.uwsgi_uri)

        logger.info('########## SERVER STARTED ########## - %s' %
                    round(time() - start, 2))

    def del_configurations(self):
        del self.db_uri
        del self.tcp_proxy

    def setup_loggers(self, level=LoggerHelper.DEBUG):
        """Setup loggers

        :param level:
        :return:
        """
        logname = uwsgi_util.opt['api_id'].decode('utf-8')

        # base logging
        file_name = self.log_path.decode('utf-8') + logname + '.api.log'

        loggers = [
            logger,
            logging.getLogger('oauthlib'),
            logging.getLogger('beehive'),
            logging.getLogger('beecell'),
            logging.getLogger('beedrones'),
            logging.getLogger('beehive_oauth2'),
            logging.getLogger('beehive_service'),
            logging.getLogger('beehive_resource'),
            logging.getLogger('beehive_ssh'),
            # logging.getLogger('beehive.common.data')
        ]
        # LoggerHelper.DEBUG2
        LoggerHelper.rotatingfile_handler(loggers, level, file_name)

        # # transaction and db logging
        # file_name = '%s/%s.db.log' % (self.log_path, logname)
        # loggers = [
        #     logging.getLogger('sqlalchemy.engine'),
        #     logging.getLogger('sqlalchemy.pool')
        # ]
        # LoggerHelper.rotatingfile_handler(loggers, logging.DEBUG, file_name)

    def setup_additional_loggers(self,
                                 elasticsearch,
                                 level=LoggerHelper.DEBUG,
                                 tags=[],
                                 **custom_fields):
        """Setup loggers

        :param elasticsearch: elasticsearch.Elasticsearch class instance
        :param level:
        :return:
        """
        loggers = [
            logger,
            logging.getLogger('oauthlib'),
            logging.getLogger('beehive'),
            logging.getLogger('beehive.db'),
            logging.getLogger('beecell'),
            logging.getLogger('beedrones'),
            logging.getLogger('beehive_oauth2'),
            logging.getLogger('beehive_service'),
            logging.getLogger('beehive_resource'),
            logging.getLogger('beehive_ssh'),
            # logging.getLogger('beehive.common.data')
        ]
        # LoggerHelper.DEBUG2
        LoggerHelper.elastic_handler(loggers,
                                     level,
                                     elasticsearch,
                                     index='cmp',
                                     tags=tags,
                                     **custom_fields)

    def open_db_session(self):
        """Open database session.
        """
        operation.session = self.api_manager.get_session()
        # try:
        #     operation.session = self.api_manager.db_manager.get_session()
        #     return operation.session
        # except MysqlManagerError as e:
        #     logger.error(e)
        #     raise BeehiveAppError(e)

    def release_db_session(self):
        """Release database session.
        """
        self.api_manager.release_session()
Пример #11
0
 def __init_subsystem(self, config, update=True):
     """Init beehive subsystem
     
     :param dict config: subsystem configuration
     :param update: if update is True don't replace database schema
     :return: trace of execution
     """
     msgs = []
 
     try:
         # create api manager
         params = {u'api_id':u'server-01',
                   u'api_name':config[u'api_system'],
                   u'api_subsystem':config[u'api_subsystem'],
                   u'database_uri':config[u'db_uri'],
                   u'api_module':config[u'api_modules'],
                   u'api_plugin':config[u'api_plugins']}
         manager = ApiManager(params)
         manager.configure()
         manager.register_modules()
 
         # create config db manager
         config_db_manager = ConfigDbManager()
 
         for db_manager_class in config[u'db_managers']:
             db_manager = import_class(db_manager_class)
     
             # remove and create/update scchema
             if update is False:
                 db_manager.remove_table(config[u'db_uri'])
             db_manager.create_table(config[u'db_uri'])
             self.logger.info(u'Create DB %s' % (db_manager_class))
             msgs.append(u'Create DB %s' % (db_manager_class))
     except Exception as ex:
         self.logger.error(ex, exc_info=1)
         raise
 
     self.set_permissions(classes=self.classes)
 
     # create module
     for item in config[u'api_modules']:
         try:
             self.logger.info(u'Load module %s' % (item))
             module = manager.modules[item.split(u'.')[-1]]
             controller = module.get_controller()
             
             # create session
             operation.session = manager.get_session()
             
             # init module
             module.init_object()
             self.logger.info(u'Init module %s' % (module))
             msgs.append(u'Init module %s' % (module))
             
             # create system users and roles
             if module.name == u'AuthModule':
                 res = self.__create_main_users(controller, config, 
                                           config_db_manager)
                 controller.set_superadmin_permissions()
                 msgs.extend(res)
                 
             elif module.name == u'Oauth2Module':
                 controller.set_superadmin_permissions()
                 
             elif module.name == u'BasicModule':
                 controller.set_superadmin_permissions()  
                 
             elif module.name == u'CatalogModule':
                 res = self.__create_main_catalogs(controller, config, 
                                              config_db_manager)
                 controller.set_superadmin_permissions()
                 msgs.extend(res)
           
         except Exception as ex:
             self.logger.error(ex, exc_info=1)
             raise
         finally:
             # release session
             module.release_session(operation.session)
             operation.session = None
             
     self.logger.info(u'Init subsystem %s' % (config[u'api_subsystem']))
     msgs.append(u'Init subsystem %s' % (config[u'api_subsystem']))
     
     return msgs
Пример #12
0
    def __configure(self, config, update=True):
        """Main configuration steps

        :param config: subsystem configuration
        :param update: if update is True don't replace database schema
        :return:
        """
        msgs = []
        manager = None
        try:
            # create api manager
            params = {'api_id': 'server-01',
                      'api_name': config['api_system'],
                      'api_subsystem': config['api_subsystem'],
                      'database_uri': config['db_uri'],
                      'api_module': 1,
                      'api_module.1': 'beehive.module.process.mod.ConfigModule',
                      'api_plugin': 0}
            manager = ApiManager(params)    
    
            # remove and create scchema
            if update is False:
                ConfigDbManager.remove_table(config['db_uri'])
            ConfigDbManager.create_table(config['db_uri'])
            self.logger.info('Create config DB %s' % (''))
            msgs.append('Create config DB %s' % (''))
    
            # create session
            operation.session = manager.get_session()
            
            # create config db manager
            db_manager = ConfigDbManager()
            
            # set configurations
            #
            # populate configs
            #
            for item in config['config']:
                # check if config already exists
                value = item['value']
                if isinstance(value, dict):
                    value = json.dumps(value)
                try:
                    res = db_manager.get(app=config['api_system'], group=item['group'], name=item['name'])
                    self.logger.warning('Configuration %s %s %s already exist' %
                                        (config['api_system'], item['group'], item['name']))
                    msgs.append('Configuration %s %s %s already exist' %
                                (config['api_system'], item['group'], item['name']))
                except QueryError as ex:
                    res = db_manager.add(config['api_system'], item['group'], item['name'], value)
                    self.logger.info('Add configuration %s' % res)
                    msgs.append('Add configuration %s' % res)
        except Exception as ex:
            self.logger.error(ex, exc_info=True)
            raise
        finally:
            # release session
            if manager is not None:
                manager.release_session()
            
        return msgs
Пример #13
0
    def __init_subsystem(self, config, update=True):
        """Init beehive subsystem
        
        :param config: subsystem configuration
        :param update: if update is True don't replace database schema
        :return: trace of execution
        """
        msgs = []
    
        try:
            # create api manager
            params = {'api_id': 'server-01',
                      'api_name': config['api_system'],
                      'api_subsystem': config['api_subsystem'],
                      'database_uri': config['db_uri'],
                      'redis_identity_uri': config['redis_identity_uri'],
                      'api_module': config['api_module'],
                      'api_plugin': config['api_plugin'],
                      'api_endpoint': config['api_endpoint'],
                      'api_catalog': config['api_catalog']}
            for i in range(1, params['api_module']+1):
                params['api_module.%s' % i] = config['api_module.%s' % i]
            if config['api_plugin'] > 0:
                for i in range(1, params['api_plugin']+1):
                    params['api_plugin.%s' % i] = config['api_plugin.%s' % i]
            manager = ApiManager(params)
            manager.configure()
            manager.register_modules()
    
            # create config db manager
            config_db_manager = ConfigDbManager()
    
            for db_manager_class in config['db_managers']:
                db_manager = import_class(db_manager_class)
        
                # remove and create/update scchema
                if update is False:
                    db_manager.remove_table(config['db_uri'])
                db_manager.create_table(config['db_uri'])
                self.logger.info('Create DB %s' % db_manager_class)
                msgs.append('Create DB %s' % db_manager_class)
        except Exception as ex:
            self.logger.error(ex, exc_info=True)
            raise
    
        self.set_permissions(classes=self.classes)
    
        # create module
        for i in range(1, config['api_module']+1):
            item = config['api_module.%s' % i]
            try:
                self.logger.info('Load module %s' % item)
                module = manager.modules[item.split('.')[-1]]
                controller = module.get_controller()
                
                # create session
                operation.session = manager.get_session()
                
                # init module
                module.init_object()
                self.logger.info('Init module %s' % module)
                msgs.append('Init module %s' % module)
                
                # create system users and roles
                if module.name == 'AuthModule':
                    res = self.__create_main_users(controller, config, config_db_manager, update)
                    controller.set_superadmin_permissions()
                    msgs.extend(res)
                    
                elif module.name == 'Oauth2Module':
                    controller.set_superadmin_permissions()
                    
                elif module.name == 'BasicModule':
                    controller.set_superadmin_permissions()  
                    
                elif module.name == 'CatalogModule':
                    res = self.__create_main_catalogs(controller, config, config_db_manager)
                    controller.set_superadmin_permissions()
                    msgs.extend(res)

                elif module.name == 'ServiceModule':
                    controller.populate(config['db_uri'])
                    msgs.extend('Populate service database')
              
            except Exception as ex:
                self.logger.error(ex, exc_info=True)
                raise
            finally:
                # release session
                module.release_session()
                
        self.logger.info('Init subsystem %s' % (config['api_subsystem']))
        msgs.append('Init subsystem %s' % (config['api_subsystem']))
        
        return msgs
Пример #14
0
    def __init__(self, *args, **kwargs):
        """ """
        #self._config = kwargs.pop('config')

        super(BeehiveApp, self).__init__(*args, **kwargs)

        # set debug mode
        self.debug = False

        # flask secret
        self.secret_key = urandom(48)

        self.http_socket = uwsgi_util.opt[u'http-socket']
        self.server_name = gethostname()

        self.app_name = uwsgi_util.opt[u'api_name']
        self.app_id = uwsgi_util.opt[u'api_id']

        # api instance static config
        self.params = uwsgi_util.opt

        # set logging path
        log_path = u'/var/log/%s/%s' % (self.params[u'api_package'],
                                        self.params[u'api_env'])
        self.log_path = self.params.get(u'api_log', log_path)

        def error(e):
            error = {
                u'status': u'error',
                u'api': u'',
                u'operation': u'',
                u'data': u'',
                u'exception': u'',
                u'code': str(405),
                u'msg': u'Method Not Allowed'
            }
            return Response(response=json.dumps(error),
                            mimetype=u'application/json',
                            status=405)

        self._register_error_handler(None, 405, error)

        # setup loggers
        self.setup_loggers()

        self.logger.info("##### SERVER STARTING #####")
        start = time()

        # api manager reference
        self.api_manager = ApiManager(self.params,
                                      app=self,
                                      hostname=self.server_name)

        # server configuration
        #self.api_manager.configure_logger()
        self.api_manager.configure()
        #self.get_configurations()

        # load modules
        self.api_manager.register_modules()

        # register in catalog
        self.api_manager.register_catalog()

        # register in moitor
        self.api_manager.register_monitor()

        self.logger.info(u'Setup uwsgi over %s:%s' %
                         (self.server_name, self.http_socket))

        self.logger.info("##### SERVER STARTED ##### - %s" %
                         round(time() - start, 2))
Пример #15
0
class BeehiveApp(Flask):
    """Custom Flask app used to read configuration and initialize security.
    
    TODO: pooller that execcute some periodically task like verify orchestrators
          are active
    """
    def __init__(self, *args, **kwargs):
        """ """
        #self._config = kwargs.pop('config')

        super(BeehiveApp, self).__init__(*args, **kwargs)

        # set debug mode
        self.debug = False

        # flask secret
        self.secret_key = urandom(48)

        self.http_socket = uwsgi_util.opt[u'http-socket']
        self.server_name = gethostname()

        self.app_name = uwsgi_util.opt[u'api_name']
        self.app_id = uwsgi_util.opt[u'api_id']

        # api instance static config
        self.params = uwsgi_util.opt

        # set logging path
        log_path = u'/var/log/%s/%s' % (self.params[u'api_package'],
                                        self.params[u'api_env'])
        self.log_path = self.params.get(u'api_log', log_path)

        def error(e):
            error = {
                u'status': u'error',
                u'api': u'',
                u'operation': u'',
                u'data': u'',
                u'exception': u'',
                u'code': str(405),
                u'msg': u'Method Not Allowed'
            }
            return Response(response=json.dumps(error),
                            mimetype=u'application/json',
                            status=405)

        self._register_error_handler(None, 405, error)

        # setup loggers
        self.setup_loggers()

        self.logger.info("##### SERVER STARTING #####")
        start = time()

        # api manager reference
        self.api_manager = ApiManager(self.params,
                                      app=self,
                                      hostname=self.server_name)

        # server configuration
        #self.api_manager.configure_logger()
        self.api_manager.configure()
        #self.get_configurations()

        # load modules
        self.api_manager.register_modules()

        # register in catalog
        self.api_manager.register_catalog()

        # register in moitor
        self.api_manager.register_monitor()

        self.logger.info(u'Setup uwsgi over %s:%s' %
                         (self.server_name, self.http_socket))

        self.logger.info("##### SERVER STARTED ##### - %s" %
                         round(time() - start, 2))

    def del_configurations(self):
        del self.db_uri
        del self.tcp_proxy
        #del self.orchestrators
        #self.orchestrators = OrchestratorManager()

    def setup_loggers(self):
        """ """
        logname = uwsgi_util.opt[u'api_id']

        # base logging
        file_name = u'%s/%s.log' % (self.log_path, logname)
        loggers = [
            self.logger,
            logging.getLogger(u'oauthlib'),
            logging.getLogger(u'beehive'),
            logging.getLogger(u'beehive.db'),
            logging.getLogger(u'beecell'),
            logging.getLogger(u'beedrones'),
            logging.getLogger(u'beehive_oauth2'),
            logging.getLogger(u'beehive_monitor'),
            logging.getLogger(u'beehive_service'),
            logging.getLogger(u'beehive_resource')
        ]
        LoggerHelper.rotatingfile_handler(loggers,
                                          logging.DEBUG,
                                          file_name,
                                          formatter=ColorFormatter)

        # transaction and db logging
        file_name = u'%s/%s.db.log' % (self.log_path, logname)
        loggers = [
            logging.getLogger(u'beehive.util.data'),
            logging.getLogger(u'sqlalchemy.engine'),
            logging.getLogger(u'sqlalchemy.pool')
        ]
        LoggerHelper.rotatingfile_handler(loggers, logging.DEBUG, file_name)

        # performance logging
        file_name = u'%s/%s.watch' % (self.log_path, logname)
        file_name = u'%s/beehive.watch' % (self.log_path)
        loggers = [logging.getLogger(u'beecell.perf')]
        LoggerHelper.rotatingfile_handler(loggers,
                                          logging.DEBUG,
                                          file_name,
                                          frmt=u'%(asctime)s - %(message)s')

        #from openstack import utils
        #utils.enable_logging(debug=True)

    def open_db_session(self):
        """Open database session.
        """
        try:
            operation.session = self.api_manager.db_manager.get_session()
            return operation.session
        except MysqlManagerError, e:
            self.logger.error(e)
            raise BeehiveAppError(e)