def setup_logging(self):
"""Initialize common logging framework. Everything is logged to central
graylog server. Depending on setting messages of DEBUG or INFO and higher
go to console."""
logger = logging.getLogger()
logger.setLevel(logging.WARN)
# Enable logging to console
try:
from dlstbx.util.colorstreamhandler import ColorStreamHandler
self.console = ColorStreamHandler()
except ImportError:
self.console = logging.StreamHandler()
self.console.setLevel(logging.INFO)
logger.addHandler(self.console)
logging.getLogger("workflows").setLevel(logging.INFO)
logging.getLogger("zocalo").setLevel(logging.DEBUG)
self.log = logging.getLogger("zocalo.service")
self.log.setLevel(logging.DEBUG)
# Enable logging to graylog
zocalo.enable_graylog()
def config(verbosity=0, name=None, info=None, debug=None, logfile=None):
"""
Configure the logging.
:param verbosity: Verbosity level of log output. Possible values:
* 0: Info log output to stdout/logfile
* 1: Info & debug log output to stdout/logfile
:type verbosity: int
:param logfile: Filename for log output. If False, no log file is written.
:type logfile: str
"""
if info:
warnings.warn(
"info= parameter is deprecated, use logfile=",
DeprecationWarning,
stacklevel=2,
)
if debug:
warnings.warn(
"debug= parameter is deprecated, use logfile= and verbosity=",
DeprecationWarning,
stacklevel=2,
)
if name:
warnings.warn("name= parameter is deprecated",
DeprecationWarning,
stacklevel=2)
if os.getenv("COLOURLOG") and ColorStreamHandler:
console = ColorStreamHandler(sys.stdout)
else:
console = logging.StreamHandler(sys.stdout)
dials_logger = logging.getLogger("dials")
dials_logger.addHandler(console)
if verbosity:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
logfilename = logfile or info or debug
if logfilename:
fh = logging.FileHandler(filename=logfilename, mode="w")
fh.setLevel(loglevel)
dials_logger.addHandler(fh)
dials_logger.setLevel(loglevel)
# logging.getLogger("dxtbx").setLevel(logging.DEBUG)
console.setLevel(loglevel)
print_banner(use_logging=True)
def setup_logging(logfile=None, debugfile=None, verbose=False):
"""
Initialise logging for xia2
:param logfile: Filename for info/info+debug log output.
:type logfile: str
:param debugfile: Filename for debug log output.
:type debugfile: str
:param verbose: Enable debug output for logfile and console.
:type verbose: bool
"""
if verbose:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
if os.getenv("COLOURLOG") and ColorStreamHandler:
console = ColorStreamHandler(sys.stdout)
else:
console = logging.StreamHandler(sys.stdout)
console.setLevel(loglevel)
xia2_logger = logging.getLogger("xia2")
xia2_logger.addHandler(console)
xia2_logger.setLevel(loglevel)
other_loggers = [
logging.getLogger(package) for package in ("dials", "dxtbx")
]
if logfile:
fh = logging.FileHandler(filename=logfile, mode="w")
fh.setLevel(loglevel)
xia2_logger.addHandler(fh)
for logger in other_loggers:
logger.addHandler(fh)
logger.setLevel(loglevel)
if debugfile:
fh = logging.FileHandler(filename=debugfile, mode="w")
fh.setLevel(logging.DEBUG)
for logger in [xia2_logger] + other_loggers:
logger.addHandler(fh)
logger.setLevel(logging.DEBUG)
def setup_logging(self):
'''Initialize common logging framework. Everything is logged to central
graylog server. Depending on setting messages of DEBUG or INFO and higher
go to console.'''
logger = logging.getLogger()
logger.setLevel(logging.WARN)
# Enable logging to console
self.console = ColorStreamHandler()
self.console.setLevel(logging.INFO)
logger.addHandler(self.console)
# logging.getLogger('stomp.py').setLevel(logging.DEBUG)
logging.getLogger('workflows').setLevel(logging.INFO)
#FIXME: this is hard-coding don't think it's needed
self.log = logging.getLogger('dlstbx.service')
self.log.setLevel(logging.DEBUG)
# Enable logging to graylog
enable_graylog()
class DLSTBXServiceStarter(workflows.contrib.start_service.ServiceStarter):
__frontendref = None
use_live_infrastructure = False
def __init__(self):
# initialize logging
self.setup_logging()
self.log.debug('Loading dlstbx workflows plugins')
dlstbx = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
self.log.debug('Loading dlstbx credentials')
# change settings when in live mode
default_configuration = '/dls_sw/apps/zocalo/secrets/credentials-testing.cfg'
if '--live' in sys.argv:
self.use_live_infrastructure = True
default_configuration = '/dls_sw/apps/zocalo/secrets/credentials-live.cfg'
# override default stomp host
from workflows.transport.stomp_transport import StompTransport
try:
StompTransport.load_configuration_file(default_configuration)
except workflows.Error as e:
self.log.warning(e)
def setup_logging(self):
'''Initialize common logging framework. Everything is logged to central
graylog server. Depending on setting messages of DEBUG or INFO and higher
go to console.'''
logger = logging.getLogger()
logger.setLevel(logging.WARN)
# Enable logging to console
self.console = ColorStreamHandler()
self.console.setLevel(logging.INFO)
logger.addHandler(self.console)
# logging.getLogger('stomp.py').setLevel(logging.DEBUG)
logging.getLogger('workflows').setLevel(logging.INFO)
#FIXME: this is hard-coding don't think it's needed
self.log = logging.getLogger('dlstbx.service')
self.log.setLevel(logging.DEBUG)
# Enable logging to graylog
enable_graylog()
def on_parser_preparation(self, parser):
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Show debug output")
parser.add_option(
"--tag",
dest="tag",
metavar="TAG",
default=None,
help="Individual tag related to this service instance")
parser.add_option("-d",
"--debug",
dest="debug",
action="store_true",
default=False,
help="Set debug log level for workflows")
parser.add_option("-r",
"--restart",
dest="service_restart",
action="store_true",
default=False,
help="Restart service on failure")
parser.add_option(
"--test",
action="store_true",
dest="test",
help="Run in ActiveMQ testing namespace (zocdev, default)")
parser.add_option("--live",
action="store_true",
dest="test",
help="Run in ActiveMQ live namespace (zocalo)")
self.log.debug('Launching ' + str(sys.argv))
def on_parsing(self, options, args):
if options.verbose:
self.console.setLevel(logging.DEBUG)
logging.getLogger('dials').setLevel(logging.DEBUG)
logging.getLogger('dlstbx').setLevel(logging.DEBUG)
logging.getLogger('xia2').setLevel(logging.DEBUG)
if options.debug:
logging.getLogger('workflows').setLevel(logging.DEBUG)
self.options = options
def before_frontend_construction(self, kwargs):
kwargs['verbose_service'] = True
kwargs['environment'] = kwargs.get('environment', {})
kwargs['environment']['live'] = self.use_live_infrastructure
return kwargs
def on_frontend_preparation(self, frontend):
self.log.info('Attaching ActiveMQ logging to transport')
def logging_call(record):
if frontend._transport.is_connected():
try:
record = record.__dict__['records']
except:
record = record.__dict__
frontend._transport.broadcast('transient.log', record)
amq_handler = workflows.logging.CallbackHandler(logging_call)
if not self.options.verbose:
amq_handler.setLevel(logging.INFO)
logging.getLogger().addHandler(amq_handler)
if self.options.service_restart:
frontend.restart_service = True
extended_status = {}
if self.options.tag:
extended_status['tag'] = self.options.tag
for env in ('SGE_CELL', 'JOB_ID'):
if env in os.environ:
extended_status['cluster_' + env] = os.environ[env]
extended_status['dlstbx'] = dlstbx_version()
original_status_function = frontend.get_status
def extend_status_wrapper():
status = original_status_function()
status.update(extended_status)
status['mem-uss'] = dlstbx.util.get_process_uss()
return status
frontend.get_status = extend_status_wrapper
class ServiceStarter(workflows.contrib.start_service.ServiceStarter):
"""Starts a workflow service"""
__frontendref = None
def setup_logging(self):
"""Initialize common logging framework. Everything is logged to central
graylog server. Depending on setting messages of DEBUG or INFO and higher
go to console."""
logger = logging.getLogger()
logger.setLevel(logging.WARN)
# Enable logging to console
try:
from dlstbx.util.colorstreamhandler import ColorStreamHandler
self.console = ColorStreamHandler()
except ImportError:
self.console = logging.StreamHandler()
self.console.setLevel(logging.INFO)
logger.addHandler(self.console)
logging.getLogger("workflows").setLevel(logging.INFO)
logging.getLogger("zocalo").setLevel(logging.DEBUG)
self.log = logging.getLogger("zocalo.service")
self.log.setLevel(logging.DEBUG)
def __init__(self):
# load configuration and initialize logging
self._zc = zocalo.configuration.from_file()
envs = self._zc.activate()
self.use_live_infrastructure = ("live" in envs) or (
"default" in envs
) # deprecated
self.setup_logging()
if not hasattr(self._zc, "graylog") or not self._zc.graylog:
# Enable logging to graylog, deprecated
zocalo.enable_graylog()
if (
self._zc.storage
and self._zc.storage.get("zocalo.default_transport")
in workflows.transport.get_known_transports()
):
workflows.transport.default_transport = self._zc.storage[
"zocalo.default_transport"
]
def on_parser_preparation(self, parser):
parser.add_option(
"-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="Show debug output",
)
parser.add_option(
"--tag",
dest="tag",
metavar="TAG",
default=None,
help="Individual tag related to this service instance",
)
parser.add_option(
"-d",
"--debug",
action="store_true",
dest="debug",
default=False,
help="Set debug log level for workflows",
)
parser.add_option(
"-r",
"--restart",
action="store_true",
dest="service_restart",
default=False,
help="Restart service on failure",
)
self._zc.add_command_line_options(parser)
self.log.debug("Launching %r", sys.argv)
def on_parsing(self, options, args):
if options.verbose:
self.console.setLevel(logging.DEBUG)
if options.debug:
self.console.setLevel(logging.DEBUG)
logging.getLogger("pika").setLevel(logging.INFO)
logging.getLogger("stomp.py").setLevel(logging.DEBUG)
logging.getLogger("workflows").setLevel(logging.DEBUG)
self.options = options
def before_frontend_construction(self, kwargs):
kwargs["verbose_service"] = True
kwargs["environment"] = kwargs.get("environment", {})
kwargs["environment"]["live"] = self.use_live_infrastructure
kwargs["environment"]["config"] = self._zc
return kwargs
def on_frontend_preparation(self, frontend):
if self.options.service_restart:
frontend.restart_service = True
extended_status = zocalo.util.extended_status_dictionary()
if self.options.tag:
extended_status["tag"] = self.options.tag
original_status_function = frontend.get_status
def extend_status_wrapper():
status = original_status_function()
status.update(extended_status)
return status
frontend.get_status = extend_status_wrapper
class ServiceStarter(workflows.contrib.start_service.ServiceStarter):
"""Starts a workflow service"""
__frontendref = None
def setup_logging(self):
"""Initialize common logging framework. Everything is logged to central
graylog server. Depending on setting messages of DEBUG or INFO and higher
go to console."""
logger = logging.getLogger()
logger.setLevel(logging.WARN)
# Enable logging to console
try:
from dlstbx.util.colorstreamhandler import ColorStreamHandler
self.console = ColorStreamHandler()
except ImportError:
self.console = logging.StreamHandler()
self.console.setLevel(logging.INFO)
logger.addHandler(self.console)
logging.getLogger("workflows").setLevel(logging.INFO)
logging.getLogger("zocalo").setLevel(logging.DEBUG)
self.log = logging.getLogger("zocalo.service")
self.log.setLevel(logging.DEBUG)
# Enable logging to graylog
zocalo.enable_graylog()
def __init__(self):
# initialize logging
self.setup_logging()
# change settings when in live mode
default_configuration = "/dls_sw/apps/zocalo/secrets/credentials-testing.cfg"
if "--live" in sys.argv:
self.use_live_infrastructure = True
default_configuration = "/dls_sw/apps/zocalo/secrets/credentials-live.cfg"
else:
self.use_live_infrastructure = False
if os.path.exists(default_configuration):
StompTransport.load_configuration_file(default_configuration)
def on_parser_preparation(self, parser):
parser.add_option(
"-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="Show debug output",
)
parser.add_option(
"--tag",
dest="tag",
metavar="TAG",
default=None,
help="Individual tag related to this service instance",
)
parser.add_option(
"-d",
"--debug",
action="store_true",
dest="debug",
default=False,
help="Set debug log level for workflows",
)
parser.add_option(
"-r",
"--restart",
action="store_true",
dest="service_restart",
default=False,
help="Restart service on failure",
)
parser.add_option(
"--test",
action="store_true",
dest="test",
help="Run in ActiveMQ testing namespace (zocdev, default)",
)
parser.add_option(
"--live",
action="store_true",
dest="test",
help="Run in ActiveMQ live namespace (zocalo)",
)
self.log.debug("Launching " + str(sys.argv))
def on_parsing(self, options, args):
if options.verbose:
self.console.setLevel(logging.DEBUG)
if options.debug:
self.console.setLevel(logging.DEBUG)
logging.getLogger("stomp.py").setLevel(logging.DEBUG)
logging.getLogger("workflows").setLevel(logging.DEBUG)
self.options = options
def before_frontend_construction(self, kwargs):
kwargs["verbose_service"] = True
kwargs["environment"] = kwargs.get("environment", {})
kwargs["environment"]["live"] = self.use_live_infrastructure
return kwargs
def on_frontend_preparation(self, frontend):
if self.options.service_restart:
frontend.restart_service = True
extended_status = {"zocalo": zocalo.__version__}
if self.options.tag:
extended_status["tag"] = self.options.tag
for env in ("SGE_CELL", "JOB_ID"):
if env in os.environ:
extended_status["cluster_" + env] = os.environ[env]
original_status_function = frontend.get_status
def extend_status_wrapper():
status = original_status_function()
status.update(extended_status)
return status
frontend.get_status = extend_status_wrapper