def _configure_loggers(self):
# TODO: configure loggers for Python, Matlab, and R modules
level = map_logging_verbosity(self.verbosity)
jtlib_logger = logging.getLogger('jtlib')
jtlib_logger.setLevel(level)
jtmodules_logger = logging.getLogger('jtmodules')
jtmodules_logger.setLevel(level)
def __main__(cls):
'''Main entry point for command line interface.
Parsers the command line arguments and configures logging.
Returns
-------
int
``0`` when program completes successfully and ``1`` otherwise
Raises
------
SystemExit
exitcode ``1`` when the call raises an :class:`Exception`
Warning
-------
Don't do any other logging configuration anywhere else!
'''
parser = cls._get_parser()
args = parser.parse_args()
configure_logging()
level = map_logging_verbosity(args.verbosity)
lib_logger = logging.getLogger('tmlib')
lib_logger.setLevel(level)
logger.debug('processing on node: %s', socket.gethostname())
# Use multiple database connection simultaneouls per job.
# The major bottleneck for tools is I/O, i.e. reading feature data from
# the database and writing label values back.
# Since we use a distributed database, we can speed up I/O using
# multiple connections.
tm.utils.set_pool_size(10)
manager = cls(args.experiment_id, args.name, args.verbosity)
manager._print_logo()
payload = manager.get_payload(args.submission_id)
tool_cls = get_tool_class(args.name)
tool = tool_cls(args.experiment_id)
tool.process_request(args.submission_id, payload)
logger.info('done')
def __main__(cls):
'''Main entry point for command line interface.
Parsers the command line arguments and configures logging.
Returns
-------
int
``0`` when program completes successfully and ``1`` otherwise
Raises
------
SystemExit
exitcode ``1`` when the call raises an :class:`Exception`
Warning
-------
Don't do any other logging configuration anywhere else!
'''
parser = cls._get_parser()
args = parser.parse_args()
configure_logging()
level = map_logging_verbosity(args.verbosity)
lib_logger = logging.getLogger('tmlib')
lib_logger.setLevel(level)
logger.debug('processing on node: %s', socket.gethostname())
# Use multiple database connection simultaneouls per job.
# The major bottleneck for tools is I/O, i.e. reading feature data from
# the database and writing label values back.
# Since we use a distributed database, we can speed up I/O using
# multiple connections.
tm.utils.set_pool_size(10)
manager = cls(args.experiment_id, args.name, args.verbosity)
manager._print_logo()
payload = manager.get_payload(args.submission_id)
tool_cls = get_tool_class(args.name)
tool = tool_cls(args.experiment_id)
tool.process_request(args.submission_id, payload)
logger.info('done')
def __main__(cls):
'''Main entry point for command line interface.
Parsers the command line arguments and configures logging.
Returns
-------
int
``0`` when program completes successfully and ``1`` otherwise
Raises
------
SystemExit
exitcode ``1`` when the call raises an :class:`Exception`
Warning
-------
Don't do any other logging configuration anywhere else!
'''
parser = cls._get_parser()
args = parser.parse_args()
configure_logging()
logger = logging.getLogger('tmlib')
level = map_logging_verbosity(args.verbosity)
logger.setLevel(level)
# Silence some chatty loggers
gc3libs_logger = logging.getLogger('gc3.gc3libs')
if args.verbosity < 3:
gc3libs_logger.setLevel(logging.ERROR)
elif args.verbosity < 4:
gc3libs_logger.setLevel(logging.WARNING)
elif args.verbosity < 5:
gc3libs_logger.setLevel(logging.INFO)
else:
gc3libs_logger.setLevel(logging.DEBUG)
cli_instance = cls(args.experiment_id, args.verbosity)
cli_instance(args)
def create_app(verbosity=None):
"""Creates a Flask application object that registers all the blueprints on
which the actual routes are defined.
Parameters
----------
verbosity: int, optional
logging verbosity to override the
:attr:`logging_verbosity <tmserver.config.ServerConfig.logging_verbosity>`
setting in the configuration file (default: ``None``)
Returns
-------
flask.Flask
Flask application
"""
log_formatter = logging.Formatter(
fmt='%(process)5d/%(threadName)-12s| %(levelname)-8s| %(message)s [[%(name)s @ %(pathname)s:%(lineno)d]]',
datefmt='%Y-%m-%d %H:%M:%S'
)
log_handler = logging.StreamHandler(stream=sys.stdout)
log_handler.setFormatter(log_formatter)
if verbosity is None:
verbosity = cfg.logging_verbosity
log_level = map_logging_verbosity(verbosity)
app = Flask('wsgi')
app.config['PROPAGATE_EXCEPTIONS'] = True
app.logger.handlers = [] # remove standard handlers
app.logger.setLevel(log_level)
app.logger.addHandler(log_handler)
tmserver_logger = logging.getLogger('tmserver')
tmserver_logger.setLevel(log_level)
tmserver_logger.addHandler(log_handler)
tmlib_logger = logging.getLogger('tmlib')
tmlib_logger.setLevel(log_level)
tmlib_logger.addHandler(log_handler)
flask_jwt_logger = logging.getLogger('flask_jwt')
flask_jwt_logger.setLevel(log_level)
flask_jwt_logger.addHandler(log_handler)
gevent_logger = logging.getLogger('gevent')
gevent_logger.addHandler(log_handler)
gc3pie_logger = logging.getLogger('gc3.gc3libs')
gc3pie_logger.addHandler(log_handler)
sqlalchemy_logger = logging.getLogger('sqlalchemy.engine')
sqlalchemy_logger.addHandler(log_handler)
wsgi_logger = logging.getLogger('wsgi')
wsgi_logger.addHandler(log_handler)
if verbosity > 4:
gevent_logger.setLevel(logging.DEBUG)
gc3pie_logger.setLevel(logging.DEBUG)
sqlalchemy_logger.setLevel(logging.DEBUG)
wsgi_logger.setLevel(logging.DEBUG)
elif verbosity > 3:
gevent_logger.setLevel(logging.INFO)
gc3pie_logger.setLevel(logging.INFO)
sqlalchemy_logger.setLevel(logging.INFO)
wsgi_logger.setLevel(logging.INFO)
else:
gevent_logger.setLevel(logging.ERROR)
gc3pie_logger.setLevel(logging.ERROR)
sqlalchemy_logger.setLevel(logging.ERROR)
wsgi_logger.setLevel(logging.ERROR)
app.json_encoder = TmJSONEncoder
if cfg.secret_key == 'default_secret_key':
app.logger.warn('The application will run with the default secret key!')
elif not cfg.secret_key:
app.logger.critical('Specify a secret key for this application!')
sys.exit(1)
app.config['SECRET_KEY'] = cfg.secret_key
app.config['JWT_EXPIRATION_DELTA'] = cfg.jwt_expiration_delta
## Error handling
# Register custom error classes
register_http_error_classes(app)
# Register SQLAlchemy error classes
@app.errorhandler(NoResultFound)
def _handle_no_result_found(error):
response = jsonify(error={
'message': error.message,
'status_code': 400,
'type': error.__class__.__name__
})
logger.error('no result found: ' + error.message)
response.status_code = 400
return response
@app.errorhandler(MultipleResultsFound)
def _multiple_results_found(error):
response = jsonify(error={
'message': error.message,
'status_code': 409,
'type': error.__class__.__name__
})
logger.error('multiple results found: ' + error.message)
response.status_code = 409
return response
@app.errorhandler(IntegrityError)
def _handle_integrity_error(error):
response = jsonify(error={
'error': True,
'message': error.message,
'status_code': 500,
'type': error.__class__.__name__
})
logger.error('database integrity error: ' + error.message)
response.status_code = 500
return response
## Initialize Plugins
jwt.init_app(app)
# Create a session scope for interacting with the main database
engine = create_db_engine(cfg.db_master_uri)
create_db_tables(engine)
session_factory = create_db_session_factory()
session_factory.configure(bind=engine)
session = flask_scoped_session(session_factory, app)
from tmserver.extensions import gc3pie
gc3pie.init_app(app)
## Import and register blueprints
from tmserver.api import api
app.register_blueprint(api, url_prefix='/api')
from tmserver.jtui import jtui
app.register_blueprint(jtui, url_prefix='/jtui')
# For uWSGI fork()
engine.dispose()
return app
def __main__(cls):
'''Main entry point for command line interfaces.
Parses the command line arguments and configures logging.
Returns
-------
int
``0`` when program completes successfully and ``1`` otherwise
Raises
------
SystemExit
exitcode ``1`` when the call raises an :class:`Exception`
Warning
-------
Don't do any other logging configuration anywhere else!
'''
# NOTE: This hack is required to ensure correct UTF8 encoding.
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
arguments = cls._parser.parse_args()
configure_logging()
logger = logging.getLogger('tmlib')
level = map_logging_verbosity(arguments.verbosity)
logger.setLevel(level)
# Adapt level of GC3Pie logger, which is very chatty.
gc3libs_logger = logging.getLogger('gc3.gc3libs')
if arguments.verbosity > 4:
gc3libs_logger.setLevel(logging.DEBUG)
elif arguments.verbosity == 4:
gc3libs_logger.setLevel(logging.INFO)
else:
gc3libs_logger.setLevel(logging.ERROR)
logger.debug('processing on node: %s', socket.gethostname())
logger.debug('running program: %s' % cls.__name__.lower())
# Use only a single database connection for the job.
# This is necessary on a large cluster to prevent too many concurrent
# connections by parallel running jobs, which would overload the
# database server.
# Using more than one connection also wouldn't be of any help, since
# the job runs in a single process, such that the same connection can
# be reused.
tm.utils.set_pool_size(1)
try:
logger.debug('instantiate API class "%s"', cls._api_class.__name__)
# Derived CLI classes may provide additional arguments for the main
# parser, i.e. arguments for the constructor of the API class.
kwargs = dict()
valid_arg_names = inspect.getargspec(cls._api_class.__init__).args
for arg_name, arg_value in vars(arguments).iteritems():
if arg_name in valid_arg_names:
kwargs[arg_name] = arg_value
api_instance = cls._api_class(**kwargs)
logger.debug('instantiate CLI class "%s"', cls.__name__)
cli_instance = cls(api_instance, arguments.verbosity)
cli_instance(arguments)
logger.info('JOB COMPLETED')
sys.exit(0)
return 0
except Exception as error:
sys.stderr.write('\nJOB FAILED:\n%s\n' % str(error))
exc_type, exc_value, exc_traceback = sys.exc_info()
for tb in traceback.format_tb(exc_traceback):
sys.stderr.write(tb)
sys.exit(1)
return 1
def create_app(verbosity=None):
"""Creates a Flask application object that registers all the blueprints on
which the actual routes are defined.
Parameters
----------
verbosity: int, optional
logging verbosity to override the
:attr:`logging_verbosity <tmserver.config.ServerConfig.logging_verbosity>`
setting in the configuration file (default: ``None``)
Returns
-------
flask.Flask
Flask application
"""
log_formatter = logging.Formatter(
fmt=
'%(process)5d/%(threadName)-12s| %(levelname)-8s| %(message)s [[%(name)s @ %(pathname)s:%(lineno)d]]',
datefmt='%Y-%m-%d %H:%M:%S')
log_handler = logging.StreamHandler(stream=sys.stdout)
log_handler.setFormatter(log_formatter)
if verbosity is None:
verbosity = cfg.logging_verbosity
log_level = map_logging_verbosity(verbosity)
app = Flask('wsgi')
app.config['PROPAGATE_EXCEPTIONS'] = True
app.logger.handlers = [] # remove standard handlers
app.logger.setLevel(log_level)
app.logger.addHandler(log_handler)
tmserver_logger = logging.getLogger('tmserver')
tmserver_logger.setLevel(log_level)
tmserver_logger.addHandler(log_handler)
tmlib_logger = logging.getLogger('tmlib')
tmlib_logger.setLevel(log_level)
tmlib_logger.addHandler(log_handler)
flask_jwt_logger = logging.getLogger('flask_jwt')
flask_jwt_logger.setLevel(log_level)
flask_jwt_logger.addHandler(log_handler)
gevent_logger = logging.getLogger('gevent')
gevent_logger.addHandler(log_handler)
gc3pie_logger = logging.getLogger('gc3.gc3libs')
gc3pie_logger.addHandler(log_handler)
sqlalchemy_logger = logging.getLogger('sqlalchemy.engine')
sqlalchemy_logger.addHandler(log_handler)
wsgi_logger = logging.getLogger('wsgi')
wsgi_logger.addHandler(log_handler)
if verbosity > 4:
gevent_logger.setLevel(logging.DEBUG)
gc3pie_logger.setLevel(logging.DEBUG)
sqlalchemy_logger.setLevel(logging.DEBUG)
wsgi_logger.setLevel(logging.DEBUG)
elif verbosity > 3:
gevent_logger.setLevel(logging.INFO)
gc3pie_logger.setLevel(logging.INFO)
sqlalchemy_logger.setLevel(logging.INFO)
wsgi_logger.setLevel(logging.INFO)
else:
gevent_logger.setLevel(logging.ERROR)
gc3pie_logger.setLevel(logging.ERROR)
sqlalchemy_logger.setLevel(logging.ERROR)
wsgi_logger.setLevel(logging.ERROR)
app.json_encoder = TmJSONEncoder
if cfg.secret_key == 'default_secret_key':
app.logger.warn(
'The application will run with the default secret key!')
elif not cfg.secret_key:
app.logger.critical('Specify a secret key for this application!')
sys.exit(1)
app.config['SECRET_KEY'] = cfg.secret_key
app.config['JWT_EXPIRATION_DELTA'] = cfg.jwt_expiration_delta
## Error handling
# Register custom error classes
register_http_error_classes(app)
# Register SQLAlchemy error classes
@app.errorhandler(NoResultFound)
def _handle_no_result_found(error):
response = jsonify(
error={
'message': error.message,
'status_code': 400,
'type': error.__class__.__name__
})
logger.error('no result found: ' + error.message)
response.status_code = 400
return response
@app.errorhandler(MultipleResultsFound)
def _multiple_results_found(error):
response = jsonify(
error={
'message': error.message,
'status_code': 409,
'type': error.__class__.__name__
})
logger.error('multiple results found: ' + error.message)
response.status_code = 409
return response
@app.errorhandler(IntegrityError)
def _handle_integrity_error(error):
response = jsonify(
error={
'error': True,
'message': error.message,
'status_code': 500,
'type': error.__class__.__name__
})
logger.error('database integrity error: ' + error.message)
response.status_code = 500
return response
## Initialize Plugins
jwt.init_app(app)
# Create a session scope for interacting with the main database
engine = create_db_engine(cfg.db_master_uri)
create_db_tables(engine)
session_factory = create_db_session_factory()
session_factory.configure(bind=engine)
session = flask_scoped_session(session_factory, app)
from tmserver.extensions import gc3pie
gc3pie.init_app(app)
## Import and register blueprints
from tmserver.api import api
app.register_blueprint(api, url_prefix='/api')
from tmserver.jtui import jtui
app.register_blueprint(jtui, url_prefix='/jtui')
# For uWSGI fork()
engine.dispose()
return app