def _configure_loggers(self): # TODO: configure loggers for Python, Matlab, and R modules level = map_logging_verbosity(self.verbosity) jtlib_logger = logging.getLogger('jtlib') jtlib_logger.setLevel(level) jtmodules_logger = logging.getLogger('jtmodules') jtmodules_logger.setLevel(level)
def __main__(cls): '''Main entry point for command line interface. Parsers the command line arguments and configures logging. Returns ------- int ``0`` when program completes successfully and ``1`` otherwise Raises ------ SystemExit exitcode ``1`` when the call raises an :class:`Exception` Warning ------- Don't do any other logging configuration anywhere else! ''' parser = cls._get_parser() args = parser.parse_args() configure_logging() level = map_logging_verbosity(args.verbosity) lib_logger = logging.getLogger('tmlib') lib_logger.setLevel(level) logger.debug('processing on node: %s', socket.gethostname()) # Use multiple database connection simultaneouls per job. # The major bottleneck for tools is I/O, i.e. reading feature data from # the database and writing label values back. # Since we use a distributed database, we can speed up I/O using # multiple connections. tm.utils.set_pool_size(10) manager = cls(args.experiment_id, args.name, args.verbosity) manager._print_logo() payload = manager.get_payload(args.submission_id) tool_cls = get_tool_class(args.name) tool = tool_cls(args.experiment_id) tool.process_request(args.submission_id, payload) logger.info('done')
def __main__(cls): '''Main entry point for command line interface. Parsers the command line arguments and configures logging. Returns ------- int ``0`` when program completes successfully and ``1`` otherwise Raises ------ SystemExit exitcode ``1`` when the call raises an :class:`Exception` Warning ------- Don't do any other logging configuration anywhere else! ''' parser = cls._get_parser() args = parser.parse_args() configure_logging() level = map_logging_verbosity(args.verbosity) lib_logger = logging.getLogger('tmlib') lib_logger.setLevel(level) logger.debug('processing on node: %s', socket.gethostname()) # Use multiple database connection simultaneouls per job. # The major bottleneck for tools is I/O, i.e. reading feature data from # the database and writing label values back. # Since we use a distributed database, we can speed up I/O using # multiple connections. tm.utils.set_pool_size(10) manager = cls(args.experiment_id, args.name, args.verbosity) manager._print_logo() payload = manager.get_payload(args.submission_id) tool_cls = get_tool_class(args.name) tool = tool_cls(args.experiment_id) tool.process_request(args.submission_id, payload) logger.info('done')
def __main__(cls): '''Main entry point for command line interface. Parsers the command line arguments and configures logging. Returns ------- int ``0`` when program completes successfully and ``1`` otherwise Raises ------ SystemExit exitcode ``1`` when the call raises an :class:`Exception` Warning ------- Don't do any other logging configuration anywhere else! ''' parser = cls._get_parser() args = parser.parse_args() configure_logging() logger = logging.getLogger('tmlib') level = map_logging_verbosity(args.verbosity) logger.setLevel(level) # Silence some chatty loggers gc3libs_logger = logging.getLogger('gc3.gc3libs') if args.verbosity < 3: gc3libs_logger.setLevel(logging.ERROR) elif args.verbosity < 4: gc3libs_logger.setLevel(logging.WARNING) elif args.verbosity < 5: gc3libs_logger.setLevel(logging.INFO) else: gc3libs_logger.setLevel(logging.DEBUG) cli_instance = cls(args.experiment_id, args.verbosity) cli_instance(args)
def create_app(verbosity=None): """Creates a Flask application object that registers all the blueprints on which the actual routes are defined. Parameters ---------- verbosity: int, optional logging verbosity to override the :attr:`logging_verbosity <tmserver.config.ServerConfig.logging_verbosity>` setting in the configuration file (default: ``None``) Returns ------- flask.Flask Flask application """ log_formatter = logging.Formatter( fmt='%(process)5d/%(threadName)-12s| %(levelname)-8s| %(message)s [[%(name)s @ %(pathname)s:%(lineno)d]]', datefmt='%Y-%m-%d %H:%M:%S' ) log_handler = logging.StreamHandler(stream=sys.stdout) log_handler.setFormatter(log_formatter) if verbosity is None: verbosity = cfg.logging_verbosity log_level = map_logging_verbosity(verbosity) app = Flask('wsgi') app.config['PROPAGATE_EXCEPTIONS'] = True app.logger.handlers = [] # remove standard handlers app.logger.setLevel(log_level) app.logger.addHandler(log_handler) tmserver_logger = logging.getLogger('tmserver') tmserver_logger.setLevel(log_level) tmserver_logger.addHandler(log_handler) tmlib_logger = logging.getLogger('tmlib') tmlib_logger.setLevel(log_level) tmlib_logger.addHandler(log_handler) flask_jwt_logger = logging.getLogger('flask_jwt') flask_jwt_logger.setLevel(log_level) flask_jwt_logger.addHandler(log_handler) gevent_logger = logging.getLogger('gevent') gevent_logger.addHandler(log_handler) gc3pie_logger = logging.getLogger('gc3.gc3libs') gc3pie_logger.addHandler(log_handler) sqlalchemy_logger = logging.getLogger('sqlalchemy.engine') sqlalchemy_logger.addHandler(log_handler) wsgi_logger = logging.getLogger('wsgi') wsgi_logger.addHandler(log_handler) if verbosity > 4: gevent_logger.setLevel(logging.DEBUG) gc3pie_logger.setLevel(logging.DEBUG) sqlalchemy_logger.setLevel(logging.DEBUG) wsgi_logger.setLevel(logging.DEBUG) elif verbosity > 3: gevent_logger.setLevel(logging.INFO) gc3pie_logger.setLevel(logging.INFO) sqlalchemy_logger.setLevel(logging.INFO) wsgi_logger.setLevel(logging.INFO) else: gevent_logger.setLevel(logging.ERROR) gc3pie_logger.setLevel(logging.ERROR) sqlalchemy_logger.setLevel(logging.ERROR) wsgi_logger.setLevel(logging.ERROR) app.json_encoder = TmJSONEncoder if cfg.secret_key == 'default_secret_key': app.logger.warn('The application will run with the default secret key!') elif not cfg.secret_key: app.logger.critical('Specify a secret key for this application!') sys.exit(1) app.config['SECRET_KEY'] = cfg.secret_key app.config['JWT_EXPIRATION_DELTA'] = cfg.jwt_expiration_delta ## Error handling # Register custom error classes register_http_error_classes(app) # Register SQLAlchemy error classes @app.errorhandler(NoResultFound) def _handle_no_result_found(error): response = jsonify(error={ 'message': error.message, 'status_code': 400, 'type': error.__class__.__name__ }) logger.error('no result found: ' + error.message) response.status_code = 400 return response @app.errorhandler(MultipleResultsFound) def _multiple_results_found(error): response = jsonify(error={ 'message': error.message, 'status_code': 409, 'type': error.__class__.__name__ }) logger.error('multiple results found: ' + error.message) response.status_code = 409 return response @app.errorhandler(IntegrityError) def _handle_integrity_error(error): response = jsonify(error={ 'error': True, 'message': error.message, 'status_code': 500, 'type': error.__class__.__name__ }) logger.error('database integrity error: ' + error.message) response.status_code = 500 return response ## Initialize Plugins jwt.init_app(app) # Create a session scope for interacting with the main database engine = create_db_engine(cfg.db_master_uri) create_db_tables(engine) session_factory = create_db_session_factory() session_factory.configure(bind=engine) session = flask_scoped_session(session_factory, app) from tmserver.extensions import gc3pie gc3pie.init_app(app) ## Import and register blueprints from tmserver.api import api app.register_blueprint(api, url_prefix='/api') from tmserver.jtui import jtui app.register_blueprint(jtui, url_prefix='/jtui') # For uWSGI fork() engine.dispose() return app
def __main__(cls): '''Main entry point for command line interfaces. Parses the command line arguments and configures logging. Returns ------- int ``0`` when program completes successfully and ``1`` otherwise Raises ------ SystemExit exitcode ``1`` when the call raises an :class:`Exception` Warning ------- Don't do any other logging configuration anywhere else! ''' # NOTE: This hack is required to ensure correct UTF8 encoding. import sys reload(sys) sys.setdefaultencoding('utf-8') arguments = cls._parser.parse_args() configure_logging() logger = logging.getLogger('tmlib') level = map_logging_verbosity(arguments.verbosity) logger.setLevel(level) # Adapt level of GC3Pie logger, which is very chatty. gc3libs_logger = logging.getLogger('gc3.gc3libs') if arguments.verbosity > 4: gc3libs_logger.setLevel(logging.DEBUG) elif arguments.verbosity == 4: gc3libs_logger.setLevel(logging.INFO) else: gc3libs_logger.setLevel(logging.ERROR) logger.debug('processing on node: %s', socket.gethostname()) logger.debug('running program: %s' % cls.__name__.lower()) # Use only a single database connection for the job. # This is necessary on a large cluster to prevent too many concurrent # connections by parallel running jobs, which would overload the # database server. # Using more than one connection also wouldn't be of any help, since # the job runs in a single process, such that the same connection can # be reused. tm.utils.set_pool_size(1) try: logger.debug('instantiate API class "%s"', cls._api_class.__name__) # Derived CLI classes may provide additional arguments for the main # parser, i.e. arguments for the constructor of the API class. kwargs = dict() valid_arg_names = inspect.getargspec(cls._api_class.__init__).args for arg_name, arg_value in vars(arguments).iteritems(): if arg_name in valid_arg_names: kwargs[arg_name] = arg_value api_instance = cls._api_class(**kwargs) logger.debug('instantiate CLI class "%s"', cls.__name__) cli_instance = cls(api_instance, arguments.verbosity) cli_instance(arguments) logger.info('JOB COMPLETED') sys.exit(0) return 0 except Exception as error: sys.stderr.write('\nJOB FAILED:\n%s\n' % str(error)) exc_type, exc_value, exc_traceback = sys.exc_info() for tb in traceback.format_tb(exc_traceback): sys.stderr.write(tb) sys.exit(1) return 1
def create_app(verbosity=None): """Creates a Flask application object that registers all the blueprints on which the actual routes are defined. Parameters ---------- verbosity: int, optional logging verbosity to override the :attr:`logging_verbosity <tmserver.config.ServerConfig.logging_verbosity>` setting in the configuration file (default: ``None``) Returns ------- flask.Flask Flask application """ log_formatter = logging.Formatter( fmt= '%(process)5d/%(threadName)-12s| %(levelname)-8s| %(message)s [[%(name)s @ %(pathname)s:%(lineno)d]]', datefmt='%Y-%m-%d %H:%M:%S') log_handler = logging.StreamHandler(stream=sys.stdout) log_handler.setFormatter(log_formatter) if verbosity is None: verbosity = cfg.logging_verbosity log_level = map_logging_verbosity(verbosity) app = Flask('wsgi') app.config['PROPAGATE_EXCEPTIONS'] = True app.logger.handlers = [] # remove standard handlers app.logger.setLevel(log_level) app.logger.addHandler(log_handler) tmserver_logger = logging.getLogger('tmserver') tmserver_logger.setLevel(log_level) tmserver_logger.addHandler(log_handler) tmlib_logger = logging.getLogger('tmlib') tmlib_logger.setLevel(log_level) tmlib_logger.addHandler(log_handler) flask_jwt_logger = logging.getLogger('flask_jwt') flask_jwt_logger.setLevel(log_level) flask_jwt_logger.addHandler(log_handler) gevent_logger = logging.getLogger('gevent') gevent_logger.addHandler(log_handler) gc3pie_logger = logging.getLogger('gc3.gc3libs') gc3pie_logger.addHandler(log_handler) sqlalchemy_logger = logging.getLogger('sqlalchemy.engine') sqlalchemy_logger.addHandler(log_handler) wsgi_logger = logging.getLogger('wsgi') wsgi_logger.addHandler(log_handler) if verbosity > 4: gevent_logger.setLevel(logging.DEBUG) gc3pie_logger.setLevel(logging.DEBUG) sqlalchemy_logger.setLevel(logging.DEBUG) wsgi_logger.setLevel(logging.DEBUG) elif verbosity > 3: gevent_logger.setLevel(logging.INFO) gc3pie_logger.setLevel(logging.INFO) sqlalchemy_logger.setLevel(logging.INFO) wsgi_logger.setLevel(logging.INFO) else: gevent_logger.setLevel(logging.ERROR) gc3pie_logger.setLevel(logging.ERROR) sqlalchemy_logger.setLevel(logging.ERROR) wsgi_logger.setLevel(logging.ERROR) app.json_encoder = TmJSONEncoder if cfg.secret_key == 'default_secret_key': app.logger.warn( 'The application will run with the default secret key!') elif not cfg.secret_key: app.logger.critical('Specify a secret key for this application!') sys.exit(1) app.config['SECRET_KEY'] = cfg.secret_key app.config['JWT_EXPIRATION_DELTA'] = cfg.jwt_expiration_delta ## Error handling # Register custom error classes register_http_error_classes(app) # Register SQLAlchemy error classes @app.errorhandler(NoResultFound) def _handle_no_result_found(error): response = jsonify( error={ 'message': error.message, 'status_code': 400, 'type': error.__class__.__name__ }) logger.error('no result found: ' + error.message) response.status_code = 400 return response @app.errorhandler(MultipleResultsFound) def _multiple_results_found(error): response = jsonify( error={ 'message': error.message, 'status_code': 409, 'type': error.__class__.__name__ }) logger.error('multiple results found: ' + error.message) response.status_code = 409 return response @app.errorhandler(IntegrityError) def _handle_integrity_error(error): response = jsonify( error={ 'error': True, 'message': error.message, 'status_code': 500, 'type': error.__class__.__name__ }) logger.error('database integrity error: ' + error.message) response.status_code = 500 return response ## Initialize Plugins jwt.init_app(app) # Create a session scope for interacting with the main database engine = create_db_engine(cfg.db_master_uri) create_db_tables(engine) session_factory = create_db_session_factory() session_factory.configure(bind=engine) session = flask_scoped_session(session_factory, app) from tmserver.extensions import gc3pie gc3pie.init_app(app) ## Import and register blueprints from tmserver.api import api app.register_blueprint(api, url_prefix='/api') from tmserver.jtui import jtui app.register_blueprint(jtui, url_prefix='/jtui') # For uWSGI fork() engine.dispose() return app