def main():
if options.output == 'syslog':
logger.addHandler(
SysLogHandler(address=(options.sysloghostname,
options.syslogport)))
else:
sh = logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
#logger.debug(options)
try:
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
s = requests.Session()
s.headers.update({'Accept': 'application/json'})
s.headers.update({'Content-type': 'application/json'})
s.headers.update({'Authorization': 'SSWS {0}'.format(options.apikey)})
#capture the time we start running so next time we catch any events created while we run.
lastrun = toUTC(datetime.now()).isoformat()
#in case we don't archive files..only look at today and yesterday's files.
yesterday = date.strftime(datetime.utcnow() - timedelta(days=1),
'%Y/%m/%d')
today = date.strftime(datetime.utcnow(), '%Y/%m/%d')
r = s.get('https://{0}/api/v1/events?startDate={1}&limit={2}'.format(
options.oktadomain,
toUTC(options.lastrun).strftime('%Y-%m-%dT%H:%M:%S.000Z'),
options.recordlimit))
if r.status_code == 200:
oktaevents = json.loads(r.text)
for event in oktaevents:
if 'published' in event.keys():
if toUTC(event['published']) > options.lastrun:
try:
mozdefEvent = dict()
mozdefEvent['utctimestamp'] = toUTC(
event['published']).isoformat()
mozdefEvent['category'] = 'okta'
mozdefEvent['tags'] = ['okta']
if 'action' in event.keys(
) and 'message' in event['action'].keys():
mozdefEvent['summary'] = event['action'][
'message']
mozdefEvent['details'] = event
jbody = json.dumps(mozdefEvent)
res = es.index(index='events',
doc_type='okta',
doc=jbody)
logger.debug(res)
except Exception as e:
logger.error(
'Error handling log record {0} {1}'.format(
r, e))
continue
else:
logger.error(
'Okta event does not contain published date: {0}'.
format(event))
setConfig('lastrun', lastrun, options.configfile)
except Exception as e:
logger.error("Unhandled exception, terminating: %r" % e)
Content-Disposition: attachment;filename={call_id}.xml
'''.format(call_id=call_id)
yield dtmf_xml_fh
yield '\n'
yield '--CUT-HERE--\n'
@contextmanager
def noop_ctx(value=None):
yield value
if __name__ == '__main__':
log.setLevel(logging.INFO)
log.addHandler(SysLogHandler(address='/dev/log'))
try:
config = get_config(os.environ)
except Exception:
log.exception('Unable to determine desired behaviour')
raise
log.info(
'Preparing recording email for: %s',
format_recipient_details(config['recipient_details']))
try:
with tempfile_ctx() as temp_path:
dtmf_header_content = subprocess.check_output([
'dtmf2xml', '--text', '--infile={}'.format(config['wavpath']),
'--outfile={}'.format(temp_path)]
)
with open(config['wavpath'], 'rb') as wav:
chown(cfg.LOG_FILE,
getpwnam(cfg.USER).pw_uid,
getgrnam(cfg.GROUP).gr_gid)
log_handler = WatchedFileHandler(cfg.LOG_FILE)
log_handler.setFormatter(formatter)
log.addHandler(log_handler)
# std err console output
if cfg.LOG_CONSOLE:
log_handler = StreamHandler()
log_handler.setFormatter(formatter)
log.addHandler(log_handler)
if cfg.LOG_SYSLOG:
# time should be added by syslog daemon
hostname = gethostname().split('.')[0]
formatter = Formatter(hostname + ' {name} {levelname} {message}',
style='{')
# if /socket/file is given use this as address
if cfg.LOG_SYSLOG_DESTINATION.startswith('/'):
destination = cfg.LOG_SYSLOG_DESTINATION
# if host and port are defined use them...
elif cfg.LOG_SYSLOG_DESTINATION.count(':') == 1:
destination = tuple(cfg.LOG_SYSLOG_DESTINATION.split(':'))
# ...otherwise add port 514 to given host address
else:
destination = (cfg.LOG_SYSLOG_DESTINATION, 514)
log_handler = SysLogHandler(
address=destination,
facility=SysLogHandler.__dict__['LOG_' + cfg.LOG_SYSLOG_FACILITY])
log_handler.setFormatter(formatter)
log.addHandler(log_handler)
from logging.handlers import SysLogHandler
LOG_PATH = 'src/base/logs/log.txt'
truckID = -1
truckID = config.get_truck_id()
# papertrail logging start
class ContextFilter(logging.Filter):
hostname = socket.gethostname()
def filter(self, record):
record.hostname = ContextFilter.hostname
return True
syslog = SysLogHandler(address=('logs2.papertrailapp.com', ##papertrailID))
syslog.addFilter(ContextFilter())
format = '%(asctime)s %(hostname)s TRUCK{truckID}: %(message)s'.format(truckID=truckID)
formatter = logging.Formatter(format, datefmt='%b %d %H:%M:%S')
syslog.setFormatter(formatter)
logger_pt = logging.getLogger()
logger_pt.addHandler(syslog)
logger_pt.setLevel(logging.INFO)
# papertrail logging end
def log_start():
try:
logger_pt.info('---LOG START---')
if not os.path.exists(LOG_PATH):
def initialize_logging(logger_name):
try:
log_format = '%%(asctime)s | %%(levelname)s | %s | %%(name)s(%%(filename)s:%%(lineno)s) | %%(message)s' % logger_name
log_date_format = "%Y-%m-%d %H:%M:%S %Z"
config = configuration.Config()
logging_config = config.get_config(sections='Logging')
logging.basicConfig(
format=log_format,
level=logging_config['log_level'] or logging.INFO,
)
# set up file loggers
log_file = logging_config.get('%s_log_file' % logger_name)
if log_file is not None and not logging_config['disable_file_logging']:
# make sure the log directory is writeable
# NOTE: the entire directory needs to be writable so that rotation works
if os.access(os.path.dirname(log_file), os.R_OK | os.W_OK):
file_handler = logging.handlers.RotatingFileHandler(
log_file, maxBytes=LOGGING_MAX_BYTES, backupCount=1)
formatter = logging.Formatter(log_format, log_date_format)
file_handler.setFormatter(formatter)
root_log = logging.getLogger()
root_log.addHandler(file_handler)
else:
sys.stderr.write("Log file is unwritable: '%s'\n" % log_file)
# set up syslog
if logging_config['log_to_syslog']:
try:
syslog_format = '%s[%%(process)d]: %%(levelname)s (%%(filename)s:%%(lineno)s): %%(message)s' % logger_name
from logging.handlers import SysLogHandler
if logging_config['syslog_host'] is not None and logging_config[
'syslog_port'] is not None:
sys_log_addr = (logging_config['syslog_host'],
logging_config['syslog_port'])
else:
sys_log_addr = "/dev/log"
# Special-case macs
if sys.platform == 'darwin':
sys_log_addr = "/var/run/syslog"
handler = SysLogHandler(address=sys_log_addr,
facility=SysLogHandler.LOG_DAEMON)
handler.setFormatter(
logging.Formatter(syslog_format, log_date_format))
root_log = logging.getLogger()
root_log.addHandler(handler)
except Exception as e:
sys.stderr.write("Error setting up syslog: '%s'\n" % str(e))
traceback.print_exc()
except Exception as e:
sys.stderr.write("Couldn't initialize logging: %s\n" % str(e))
traceback.print_exc()
# if config fails entirely, enable basic stdout logging as a fallback
logging.basicConfig(
format=log_format,
level=logging.INFO,
)
# re-get the log after logging is initialized
global log
log = logging.getLogger(__name__)
ROOT_URLCONF = 'rfweb.urls'
# Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
TEMPLATE_DIRS = (
os.path.join(_BASEDIR, 'rfwebapp', 'templates').replace('\\', '/'),
)
INSTALLED_APPS = (
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.admin',
'dajaxice',
'dajax',
'rfweb.rfwebapp'
)
import logging
from logging.handlers import SysLogHandler
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
syslog = SysLogHandler(address='/dev/log', facility='local0')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s')
syslog.setFormatter(formatter)
logger.addHandler(syslog)
def add_syslog_handler(address):
"""Send events to a remote syslog."""
from logging.handlers import SysLogHandler
handler = SysLogHandler(address)
handler.setFormatter(logging.Formatter(Logger.SYSLOG_FORMAT))
Logger.logger.addHandler(handler)
'--verbosity',
action='store_true',
help=('increase verbosity of logged actions. check '
'/dev/log'))
args = parser.parse_args()
# set basic logging configuration
if args.verbosity:
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s : %(message)s')
else:
logging.basicConfig(level=logging.WARNING,
format='%(asctime)s - %(levelname)s : %(message)s')
# make sure logging is sent to /var/log/messages using local logger
logging.getLogger().addHandler(SysLogHandler(address='/dev/log'))
#run compresser as daemon service
if args.directory and args.email:
signalmap = {signal.SIGTERM: end_program, signal.SIGTSTP: end_program}
estimate = dry_run(args.directory)
prompt = ('Estimated memory savings is {} percent.'
'Continue?[y/n]'.format((1 - estimate) * 100))
proceed = input(prompt)
if proceed == 'y':
with daemon.DaemonContext(working_directory=os.getcwd(),
signal_map=signalmap):
main(args.directory,
target_email=args.email,
threshold=args.threshold)
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from .enum import Enum
from .enum import unique
from .utils import create_instance
from .utils import positive_int_value
from pyeclib_c import get_liberasurecode_version
import logging
from logging.handlers import SysLogHandler
logger = logging.getLogger('pyeclib')
syslog_handler = SysLogHandler()
logger.addHandler(syslog_handler)
def check_backend_available(backend_name):
try:
from pyeclib_c import check_backend_available
if backend_name.startswith('flat_xor_hd'):
int_type = PyECLib_EC_Types.get_by_name('flat_xor_hd')
else:
int_type = PyECLib_EC_Types.get_by_name(backend_name)
if not int_type:
return False
return check_backend_available(int_type.value)
except ImportError:
def cli(ctx, host, userid, password, output_format, transpose, error_format,
timestats, log, log_dest, syslog_facility):
"""
Command line interface for the IBM Z HMC.
The options shown in this help text are general options that can also
be specified on any of the (sub-)commands.
Parameters:
ctx (:class:`click.Context`): The click context object. Created by the
``@click.pass_context`` decorator.
: The remaining parameters are defined by the ``@click.option``
decorators.
"""
# Concept: In interactive mode, the global options specified in the command
# line are used as defaults for the commands that are issued interactively.
# The interactive commands may override these options.
# This requires being able to determine for each option whether it has been
# specified. This is the reason the options don't define defaults in the
# decorators that define them.
if ctx.obj is None:
# We are in command mode or are processing the command line options in
# interactive mode.
# We apply the documented option defaults.
if output_format is None:
output_format = DEFAULT_OUTPUT_FORMAT
if transpose is None:
transpose = False
if error_format is None:
error_format = DEFAULT_ERROR_FORMAT
if timestats is None:
timestats = DEFAULT_TIMESTATS
else:
# We are processing an interactive command.
# We apply the option defaults from the command line options.
if host is None:
host = ctx.obj.host
if userid is None:
userid = ctx.obj.userid
if password is None:
password = ctx.obj._password
if output_format is None:
output_format = ctx.obj.output_format
if transpose is None:
transpose = ctx.obj.transpose
if error_format is None:
error_format = ctx.obj.error_format
if timestats is None:
timestats = ctx.obj.timestats
if transpose and output_format == 'json':
raise_click_exception(
"Transposing output tables (-x / --transpose) conflicts with "
"non-table output format (-o / --output-format): {}".format(
output_format), error_format)
# TODO: Add context support for the following options:
if log is None:
log = DEFAULT_LOG
if log_dest is None:
log_dest = DEFAULT_LOG_DESTINATION
if syslog_facility is None:
syslog_facility = DEFAULT_SYSLOG_FACILITY
# Now we have the effective values for the options as they should be used
# by the current command, regardless of the mode.
# Set up logging
for lc in LOG_COMPONENTS:
reset_logger(lc)
if log_dest == 'syslog':
# The choices in SYSLOG_FACILITIES have been validated by click
# so we don't need to further check them.
facility = SysLogHandler.facility_names[syslog_facility]
system = platform.system()
if system.startswith('CYGWIN_NT'):
# Value is 'CYGWIN_NT-6.1'; strip off trailing version:
system = 'CYGWIN_NT'
try:
addresses = SYSLOG_ADDRESSES[system]
except KeyError:
raise NotImplementedError(
"Logging to syslog is not supported on this platform: {}".
format(system))
assert isinstance(addresses, list)
for address in addresses:
try:
handler = SysLogHandler(address=address, facility=facility)
except Exception:
continue
break
else:
exc = sys.exc_info()[1]
exc_name = exc.__class__.__name__ if exc else None
raise RuntimeError(
"Creating SysLogHandler with addresses {!r} failed. "
"Failure on last address {!r} was: {}: {}".format(
addresses, address, exc_name, exc))
fs = '%(levelname)s %(name)s: %(message)s'
handler.setFormatter(logging.Formatter(fs))
elif log_dest == 'stderr':
handler = StreamHandler(stream=sys.stderr)
fs = '%(levelname)s %(name)s: %(message)s'
handler.setFormatter(logging.Formatter(fs))
else:
# The choices in LOG_DESTINATIONS have been validated by click
assert log_dest == 'none'
handler = None
log_specs = log.split(',')
for log_spec in log_specs:
# ignore extra ',' at begin, end or in between
if log_spec == '':
continue
try:
log_comp, log_level = log_spec.split('=', 1)
except ValueError:
raise_click_exception(
"Missing '=' in COMP=LEVEL specification in "
"--log option: {ls}".format(ls=log_spec), error_format)
level = getattr(logging, log_level.upper(), None)
if level is None:
raise_click_exception(
"Invalid log level in COMP=LEVEL "
"specification in --log option: {ls}".format(ls=log_spec),
error_format)
if log_comp not in LOG_COMPONENTS:
raise_click_exception(
"Invalid log component in COMP=LEVEL "
"specification in --log option: {ls}".format(ls=log_spec),
error_format)
if handler:
setup_logger(log_comp, handler, level)
session_id = os.environ.get('ZHMC_SESSION_ID', None)
if session_id and session_id.startswith('faked_session:'):
# This should be used by the zhmc function tests only.
# A SyntaxError raised by an incorrect expression is considered
# an internal error in the function tests and is therefore not
# handled.
expr = session_id.split(':', 1)[1]
faked_session = eval(expr)
assert isinstance(faked_session, zhmcclient_mock.FakedSession)
session_id = faked_session
def get_password_via_prompt(host, userid):
"""
Password retrieval function that prompts for the password.
It follows the interface defined in
:func:`~zhmcclient.get_password_interface` and needs access to the
click context (ctx).
"""
if userid is not None and host is not None:
ctx.obj.spinner.stop()
password = click.prompt(
"Enter password (for user {userid} at HMC {host})".format(
userid=userid, host=host),
hide_input=True,
confirmation_prompt=False,
type=str,
err=True)
ctx.obj.spinner.start()
return password
else:
raise raise_click_exception(
"{cmd} command requires logon, but no "
"session-id or userid provided.".format(
cmd=ctx.invoked_subcommand), error_format)
# We create a command context for each command: An interactive command has
# its own command context different from the command context for the
# command line.
ctx.obj = CmdContext(host, userid, password, output_format, transpose,
error_format, timestats, session_id,
get_password_via_prompt)
# Invoke default command
if ctx.invoked_subcommand is None:
ctx.invoke(repl)
--- lshell/checkconfig.py.orig 2013-08-19 19:59:37 UTC
+++ lshell/checkconfig.py
@@ -307,7 +307,7 @@ class CheckConfig:
try:
if logfilename == "syslog":
from logging.handlers import SysLogHandler
- syslog = SysLogHandler(address='/dev/log')
+ syslog = SysLogHandler(address='/var/run/log')
syslog.setFormatter(syslogformatter)
syslog.setLevel(self.levels[self.conf['loglevel']])
logger.addHandler(syslog)
import logging
import os
from logging.handlers import SysLogHandler
###### Global variables
G_SYSLOG_SERVER = "logs3.papertrailapp.com:52775"
# Twitter username
TWITTER_USER = os.environ["TWITTER_USER"]
syslog_addr = G_SYSLOG_SERVER.split(':')
syslog = SysLogHandler(address=(syslog_addr[0], int(syslog_addr[1])))
formatter = logging.Formatter('%(asctime)s twitter.importer: ' + TWITTER_USER +
' %(message).60s',
datefmt='%b %d %H:%M:%S')
logger = logging.getLogger()
logger.setLevel(logging.INFO)
syslog.setFormatter(formatter)
logger.addHandler(syslog)
worker_def = keiconf.worker_def
# If the environment variable DEBUG is defined, specify DEBUG as the log level.
# DEBUG 環境変数が定義されていたらログレベルをDEBUGとする
if 'DEBUG' in os.environ:
debug = os.environ['DEBUG']
else:
debug = 0
if debug:
LOGLEVEL = logging.DEBUG
else:
LOGLEVEL = logging.INFO
# ロギングは syslogd の local3 ファシリティにておこなう
syshandler = SysLogHandler(facility=SysLogHandler.LOG_LOCAL3)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
syshandler.setFormatter(formatter)
# Setting the root logger（ルートロガーの設定）
logger = logging.getLogger('')
logger.setLevel(LOGLEVEL)
logger.addHandler(syshandler)
# Defining signal handlers（シグナルハンドラの定義）
# 1. End process（終了処理）
def exit_handler(signal, frame):
logger.info('Stopping all threads. Please wait ...')
# 動作中の woker にストップイベントをセット
def enableSyslog(name='xcat'):
h = SysLogHandler(address='/dev/log', facility=SysLogHandler.LOG_LOCAL4)
h.setFormatter(
logging.Formatter('%s: ' % name + '%(levelname)s %(message)s'))
logging.getLogger('xcatagent').addHandler(h)
# Pipelines
WAREHOUSE_URL = os.environ['WAREHOUSE_URL']
TEST_WAREHOUSE_URL = os.environ.get('TEST_WAREHOUSE_URL', None)
ITEM_PIPELINES = {
'collectors.base.pipelines.Warehouse': 100,
}
# Logging
logging.basicConfig(level=logging.DEBUG)
if os.environ.get('LOGGING_URL', None):
root_logger = logging.getLogger()
host, port = os.environ['LOGGING_URL'].split(':')
syslog_handler = SysLogHandler(address=(host, int(port)))
syslog_handler.setLevel(logging.INFO)
root_logger.addHandler(syslog_handler)
# ICTRP
ICTRP_USER = os.environ.get('ICTRP_USER', None)
ICTRP_PASS = os.environ.get('ICTRP_PASS', None)
# HRA
HRA_ENV = os.environ.get('HRA_ENV', None)
HRA_URL = os.environ.get('HRA_URL', None)
HRA_USER = os.environ.get('HRA_USER', None)
HRA_PASS = os.environ.get('HRA_PASS', None)
def websockify_init():
# Setup basic logging to stderr.
logger = logging.getLogger(WebSocketProxy.log_prefix)
logger.propagate = False
logger.setLevel(logging.INFO)
stderr_handler = logging.StreamHandler()
stderr_handler.setLevel(logging.DEBUG)
log_formatter = logging.Formatter("%(message)s")
stderr_handler.setFormatter(log_formatter)
logger.addHandler(stderr_handler)
# Setup optparse.
usage = "\n %prog [options]"
usage += " [source_addr:]source_port [target_addr:target_port]"
usage += "\n %prog [options]"
usage += " [source_addr:]source_port -- WRAP_COMMAND_LINE"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--verbose",
"-v",
action="store_true",
help="verbose messages")
parser.add_option("--traffic",
action="store_true",
help="per frame traffic")
parser.add_option("--record",
help="record sessions to FILE.[session_number]",
metavar="FILE")
parser.add_option("--daemon",
"-D",
dest="daemon",
action="store_true",
help="become a daemon (background process)")
parser.add_option("--run-once",
action="store_true",
help="handle a single WebSocket connection and exit")
parser.add_option("--timeout",
type=int,
default=0,
help="after TIMEOUT seconds exit when not connected")
parser.add_option(
"--idle-timeout",
type=int,
default=0,
help="server exits after TIMEOUT seconds if there are no "
"active connections")
parser.add_option("--cert",
default="self.pem",
help="SSL certificate file")
parser.add_option("--key",
default=None,
help="SSL key file (if separate from cert)")
parser.add_option("--ssl-only",
action="store_true",
help="disallow non-encrypted client connections")
parser.add_option("--ssl-target",
action="store_true",
help="connect to SSL target as SSL client")
parser.add_option(
"--verify-client",
action="store_true",
help="require encrypted client to present a valid certificate "
"(needs Python 2.7.9 or newer or Python 3.4 or newer)")
parser.add_option(
"--cafile",
metavar="FILE",
help="file of concatenated certificates of authorities trusted "
"for validating clients (only effective with --verify-client). "
"If omitted, system default list of CAs is used.")
parser.add_option(
"--ssl-version",
type="choice",
default="default",
choices=["default", "tlsv1_1", "tlsv1_2", "tlsv1_3"],
action="store",
help="minimum TLS version to use (default, tlsv1_1, tlsv1_2, tlsv1_3)")
parser.add_option(
"--ssl-ciphers",
action="store",
help="list of ciphers allowed for connection. For a list of "
"supported ciphers run `openssl ciphers`")
parser.add_option("--unix-target",
help="connect to unix socket target",
metavar="FILE")
parser.add_option("--inetd",
help="inetd mode, receive listening socket from stdin",
action="store_true")
parser.add_option("--web",
default=None,
metavar="DIR",
help="run webserver on same port. Serve files from DIR.")
parser.add_option("--web-auth",
action="store_true",
help="require authentication to access webserver.")
parser.add_option("--wrap-mode",
default="exit",
metavar="MODE",
choices=["exit", "ignore", "respawn"],
help="action to take when the wrapped program exits "
"or daemonizes: exit (default), ignore, respawn")
parser.add_option("--prefer-ipv6",
"-6",
action="store_true",
dest="source_is_ipv6",
help="prefer IPv6 when resolving source_addr")
parser.add_option("--libserver",
action="store_true",
help="use Python library SocketServer engine")
parser.add_option(
"--target-config",
metavar="FILE",
dest="target_cfg",
help="Configuration file containing valid targets "
"in the form 'token: host:port' or, alternatively, a "
"directory containing configuration files of this form "
"(DEPRECATED: use `--token-plugin TokenFile --token-source "
" path/to/token/file` instead)")
parser.add_option(
"--token-plugin",
default=None,
metavar="CLASS",
help="use a Python class, usually one from websockify.token_plugins, "
"such as TokenFile, to process tokens into host:port pairs")
parser.add_option("--token-source",
default=None,
metavar="ARG",
help="an argument to be passed to the token plugin "
"on instantiation")
parser.add_option("--host-token",
action="store_true",
help="use the host HTTP header as token instead of the "
"token URL query parameter")
parser.add_option(
"--auth-plugin",
default=None,
metavar="CLASS",
help="use a Python class, usually one from websockify.auth_plugins, "
"such as BasicHTTPAuth, to determine if a connection is allowed")
parser.add_option("--auth-source",
default=None,
metavar="ARG",
help="an argument to be passed to the auth plugin "
"on instantiation")
parser.add_option("--heartbeat",
type=int,
default=0,
metavar="INTERVAL",
help="send a ping to the client every INTERVAL seconds")
parser.add_option("--log-file",
metavar="FILE",
dest="log_file",
help="File where logs will be saved")
parser.add_option("--syslog",
default=None,
metavar="SERVER",
help="Log to syslog server. SERVER can be local socket, "
"such as /dev/log, or a UDP host:port pair.")
(opts, args) = parser.parse_args()
opts.ssl_options = select_ssl_version(opts.ssl_version)
del opts.ssl_version
if opts.log_file:
# Setup logging to user-specified file.
opts.log_file = os.path.abspath(opts.log_file)
log_file_handler = logging.FileHandler(opts.log_file)
log_file_handler.setLevel(logging.DEBUG)
log_file_handler.setFormatter(log_formatter)
logger.addHandler(log_file_handler)
del opts.log_file
if opts.syslog:
# Determine how to connect to syslog...
if opts.syslog.count(':'):
# User supplied a host:port pair.
syslog_host, syslog_port = opts.syslog.rsplit(':', 1)
try:
syslog_port = int(syslog_port)
except ValueError:
parser.error("Error parsing syslog port")
syslog_dest = (syslog_host, syslog_port)
else:
# User supplied a local socket file.
syslog_dest = os.path.abspath(opts.syslog)
from logging.handlers import SysLogHandler
# Determine syslog facility.
if opts.daemon:
syslog_facility = SysLogHandler.LOG_DAEMON
else:
syslog_facility = SysLogHandler.LOG_USER
# Start logging to syslog.
syslog_handler = SysLogHandler(address=syslog_dest,
facility=syslog_facility)
syslog_handler.setLevel(logging.DEBUG)
syslog_handler.setFormatter(log_formatter)
logger.addHandler(syslog_handler)
del opts.syslog
if opts.verbose:
logger.setLevel(logging.DEBUG)
# Validate options.
if opts.token_source and not opts.token_plugin:
parser.error("You must use --token-plugin to use --token-source")
if opts.host_token and not opts.token_plugin:
parser.error("You must use --token-plugin to use --host-token")
if opts.auth_source and not opts.auth_plugin:
parser.error("You must use --auth-plugin to use --auth-source")
if opts.web_auth and not opts.auth_plugin:
parser.error("You must use --auth-plugin to use --web-auth")
if opts.web_auth and not opts.web:
parser.error("You must use --web to use --web-auth")
# Transform to absolute path as daemon may chdir
if opts.target_cfg:
opts.target_cfg = os.path.abspath(opts.target_cfg)
if opts.target_cfg:
opts.token_plugin = 'TokenFile'
opts.token_source = opts.target_cfg
del opts.target_cfg
if sys.argv.count('--'):
opts.wrap_cmd = args[1:]
else:
opts.wrap_cmd = None
if not websockifyserver.ssl and opts.ssl_target:
parser.error("SSL target requested and Python SSL module not loaded.")
if opts.ssl_only and not os.path.exists(opts.cert):
parser.error("SSL only and %s not found" % opts.cert)
if opts.inetd:
opts.listen_fd = sys.stdin.fileno()
else:
if len(args) < 1:
parser.error("Too few arguments")
arg = args.pop(0)
# Parse host:port and convert ports to numbers
if arg.count(':') > 0:
opts.listen_host, opts.listen_port = arg.rsplit(':', 1)
opts.listen_host = opts.listen_host.strip('[]')
else:
opts.listen_host, opts.listen_port = '', arg
try:
opts.listen_port = int(opts.listen_port)
except ValueError:
parser.error("Error parsing listen port")
del opts.inetd
if opts.wrap_cmd or opts.unix_target or opts.token_plugin:
opts.target_host = None
opts.target_port = None
else:
if len(args) < 1:
parser.error("Too few arguments")
arg = args.pop(0)
if arg.count(':') > 0:
opts.target_host, opts.target_port = arg.rsplit(':', 1)
opts.target_host = opts.target_host.strip('[]')
else:
parser.error("Error parsing target")
try:
opts.target_port = int(opts.target_port)
except ValueError:
parser.error("Error parsing target port")
if len(args) > 0 and opts.wrap_cmd == None:
parser.error("Too many arguments")
if opts.token_plugin is not None:
if '.' not in opts.token_plugin:
opts.token_plugin = ('websockify.token_plugins.%s' %
opts.token_plugin)
token_plugin_module, token_plugin_cls = opts.token_plugin.rsplit(
'.', 1)
__import__(token_plugin_module)
token_plugin_cls = getattr(sys.modules[token_plugin_module],
token_plugin_cls)
opts.token_plugin = token_plugin_cls(opts.token_source)
del opts.token_source
if opts.auth_plugin is not None:
if '.' not in opts.auth_plugin:
opts.auth_plugin = 'websockify.auth_plugins.%s' % opts.auth_plugin
auth_plugin_module, auth_plugin_cls = opts.auth_plugin.rsplit('.', 1)
__import__(auth_plugin_module)
auth_plugin_cls = getattr(sys.modules[auth_plugin_module],
auth_plugin_cls)
opts.auth_plugin = auth_plugin_cls(opts.auth_source)
del opts.auth_source
# Create and start the WebSockets proxy
libserver = opts.libserver
del opts.libserver
if libserver:
# Use standard Python SocketServer framework
server = LibProxyServer(**opts.__dict__)
server.serve_forever()
else:
# Use internal service framework
server = WebSocketProxy(**opts.__dict__)
server.start_server()
def setup(logger_name: str = None,
destination: int = SYSLOG,
level: int = logging.WARNING,
propagate: bool = False) -> logging.Logger:
"""Configures a `logging.Logger`_ object
Once configured, a logger can also be retrieved via
`logging.getLogger`_
It is inefficient to call this function more than once for the same
logger name.
Args:
logger_name:
The name of the logger to configure. Use None (the default)
to configure the root logger.
level:
The `logging level`_ to use when filtering log entries.
Defaults to logging.WARNING.
propagate:
Whether to send log entries to ancestor loggers. Defaults to False.
destination:
- SYSLOG: (the default) Send messages to syslog.
- View with: ``tail -f /var/log/syslog | sed 's/#012/\n\t/g'``
- CONSOLE: Send message to stdout
Returns:
A `logging.Logger`_ object
.. _logging.Logger: https://docs.python.org/3/library/logging.html#logging.Logger
.. _logging level: https://docs.python.org/3/library/logging.html#levels
.. _logging.getLogger: https://docs.python.org/3/library/logging.html#logging.getLogger
"""
def _get_process_name():
# Example: ps -eaf | grep 5175 | grep -v grep | awk -F '--name=' '{print $2}'
pid = os.getpid()
cmd = "ps -eaf | grep {} | grep -v grep | awk -F '--name=' '{{print $2}}'| tr -d '\n'".format(
pid)
read_process_name = subprocess.Popen(
cmd, shell=True, stdout=subprocess.PIPE).stdout.readlines()
binary_to_string = [b.decode() for b in read_process_name]
pname = 'Fledge ' + binary_to_string[
0] if binary_to_string else 'Fledge'
return pname
logger = logging.getLogger(logger_name)
if destination == SYSLOG:
handler = SysLogHandler(address='/dev/log')
elif destination == CONSOLE:
handler = logging.StreamHandler(sys.stdout)
else:
raise ValueError("Invalid destination {}".format(destination))
# TODO: Consider using %r with message when using syslog .. \n looks better than #
process_name = _get_process_name()
fmt = '{}[%(process)d] %(levelname)s: %(module)s: %(name)s: %(message)s'.format(
process_name)
formatter = logging.Formatter(fmt=fmt)
handler.setFormatter(formatter)
logger.setLevel(level)
logger.propagate = propagate
logger.addHandler(handler)
return logger
# - ex: filename='mydisk.log',filemode='a'
# - https://docs.python.org/2/howto/logging.html#useful-handlers
# * Filters provide a finer grained facility for determining which log records to output.
# - ex: level=logging.DEBUG
# * Formatters specify the layout of log records in the final output.
# - ex: format='%(asctime)s - %(name)s - %(levelname)s - %(message)s '
# Logging setup
# create logger
logger = logging.getLogger('disk_monitor') # logger name/Application name
logger.setLevel(logging.DEBUG) # filter at logger level.
# create console handler and set level to debug
ch = SysLogHandler(address="/dev/log") # handlers
ch.setLevel(logging.DEBUG) # filter at handler level.
# create formatter
formatter = logging.Formatter(' : %(name)s - %(levelname)s - %(message)s') # Formatter
# add formatter to ch
ch.setFormatter(formatter) # handler to formatter
# add ch to logger
logger.addHandler(ch) # logger to handler
# disk_info=$(df -h /|tail -n 1|awk '{print $5}'|sed -e 's#%##g')
p1 = Popen(['df','-h','/'],stdout=PIPE)
p2 = Popen(['tail','-n','-1'],stdin=p1.stdout,stdout=PIPE)
output = p2.communicate()[0]
import logging
#import logging.handlers
from logging.handlers import SysLogHandler
import socket
class ContextFilter(logging.Filter):
hostname = socket.gethostname()
def filter(self, record):
record.hostname = ContextFilter.hostname
return True
## FIXME! no more control on verbosity of both script and modules (or comment those lines)
if w_remotesyslog == 1:
logger = logging.getLogger()
logger.setLevel(logging.INFO)
syslog = SysLogHandler(address='/dev/log')
#syslog = logging.handlers.SysLogHandler(address = ('IP', PORT))
#formatter = logging.Formatter('%(name)s: %(levelname)s %(message)s')
formatter = logging.Formatter('pdns-query: %(message)s')
syslog.setFormatter(formatter)
logger.addHandler(syslog)
#logging.basicConfig(format="%(filename)s:%(funcName)s:%(message)s", filename='debug.log',level=logging.DEBUG)
#logging.basicConfig(format="%(filename)s:%(funcName)s:%(message)s", level=logging.DEBUG, stream=sys.stderr)
logging.basicConfig(level=logging.ERROR)
## https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning
#logging.captureWarnings(True)
## https://github.com/shazow/urllib3/issues/497
import requests.packages.urllib3
requests.packages.urllib3.disable_warnings()
def __init__(self, log_name, log_handler=None, use_mwformat=True,
log_level='INFO', facility='local1', event_handler=None,
**kwargs):
'''
log_name: name of the logger
log_handler: It is a handler object or string object or None.
StreamHandler, SysLogHandler or FileHandler will be created
when log_handler is not logging.handler object.
Rule is as follow:
None: StreamHandler(sys.stderr)
string object:
{
'syslog': SysLogHandler(address="/dev/log",
facility=SysLogHandler.facility_names.get(facility, 1))
'other': logging.FileHandler(log_handler)
}
logging.Handler: use log_handler
use_mwformat: formatter is:
'%(threadName)s %(asctime)s %(name)s/%(levelname)s/
%(filename)s:%(lineno)d:%(funcName)s:%(process)d/%(thread)d:%(message)s'
if it is set to True, otherwise is '%(message)s' only
log_level: 'DEBUG/INFO/ERROR/WARN', must be capital, default is INFO
facility: default is user, all is follow:
{'ftp': 11, 'daemon': 3, 'uucp': 8, 'security': 4, 'local7': 23,
'local4': 20, 'lpr': 6, 'auth': 4, 'local0': 16, 'cron': 9,
'syslog': 5, 'user': 1, 'mail': 2, 'local5': 21,
'kern': 0, 'news': 7, 'local6': 22, 'local1': 17,
'authpriv': 10, 'local3': 19, 'local2': 18}
event_handler: a udp handler to monitor module, default is None
**kwargs: additional message, will be logged by json format
'''
super(ALogger, self).__init__(log_name)
self._args = {}
self._args['log_name'] = log_name
self._args['log_handler'] = log_handler
self._args['log_level'] = log_level
self._args['facility'] = facility
self._additional_msg = OrderedDict(**kwargs)
self._args.update(**kwargs)
self.setLevel(log_level)
# Format should be set in configures?
if use_mwformat:
self._format = '%(threadName)s %(asctime)s %(name)s/'\
'%(levelname)s/%(filename)s:%(lineno)d:'\
'%(funcName)s:%(process)d/%(thread)d:%(message)s'
self.formatter = MwFormatter(self._format, **kwargs)
else:
self._format = '%(message)s'
self.formatter = logging.Formatter(self._format)
if isinstance(log_handler, logging.Handler):
self.hdlr = log_handler
elif isinstance(log_handler, basestring):
if log_handler == 'syslog':
self.hdlr = SysLogHandler(
address="/dev/log",
facility=SysLogHandler.facility_names.get(facility, 1))
else:
self.hdlr = logging.FileHandler(log_handler)
elif log_handler is None:
self.hdlr = logging.StreamHandler(sys.stderr)
else:
raise Exception("Invalid Logging Handler")
self.hdlr.setFormatter(self.formatter)
self.addHandler(self.hdlr)
# create event logger
if event_handler and isinstance(event_handler, UDPHandler):
self._elogger = self.__class__(
log_name, log_handler=event_handler, **kwargs)
else:
self._elogger = None
session = web.config.get("_session")
# Set application name.
logger = logging.getLogger("iredadmin")
# Set log level.
_log_level = getattr(logging, str(settings.SYSLOG_LOG_LEVEL).upper())
logger.setLevel(_log_level)
# Log format
_formatter = logging.Formatter("%(name)s %(message)s (%(pathname)s, line %(lineno)d)")
_facility = getattr(SysLogHandler, "LOG_" + settings.SYSLOG_FACILITY.upper())
if settings.SYSLOG_SERVER.startswith("/"):
# Log to a local socket
_handler = SysLogHandler(address=settings.SYSLOG_SERVER, facility=_facility)
else:
# Log to a network address
_server = (settings.SYSLOG_SERVER, settings.SYSLOG_PORT)
_handler = SysLogHandler(address=_server, facility=_facility)
_handler.setFormatter(_formatter)
logger.addHandler(_handler)
def log_traceback():
exc_type, exc_value, exc_traceback = sys.exc_info()
msg = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.error(msg)
def __init__(self, *args, **kwargs):
""" Load Yaml configuration and Init logger """
super(PFUI_Firewall, self).__init__(*args, **kwargs)
self.threads = []
self.soc = None
self.db = None
# Load YAML Configuration
try:
self.cfg = safe_load(open(CONFIG_LOCATION))
if "LOGGING" not in self.cfg:
self.cfg['LOGGING'] = True
if "LOG_LEVEL" not in self.cfg:
self.cfg['LOG_LEVEL'] = "DEBUG"
if "SOCKET_LISTEN" not in self.cfg:
self.cfg['SOCKET_LISTEN'] = "0.0.0.0"
if "SOCKET_PORT" not in self.cfg:
self.cfg['SOCKET_PORT'] = 10001
if "SOCKET_TIMEOUT" not in self.cfg:
self.cfg['SOCKET_TIMEOUT'] = 2
if "SOCKET_BUFFER" not in self.cfg:
self.cfg['SOCKET_BUFFER'] = 1024
if "SOCKET_BACKLOG" not in self.cfg:
self.cfg['SOCKET_BACKLOG'] = 5
if "REDIS_HOST" not in self.cfg:
self.cfg['REDIS_HOST'] = "127.0.0.1"
if "REDIS_PORT" not in self.cfg:
self.cfg['REDIS_PORT'] = 6379
if "REDIS_DB" not in self.cfg:
self.cfg['REDIS_DB'] = 1024
if "SCAN_PERIOD" not in self.cfg:
self.cfg['SCAN_PERIOD'] = 60
if "TTL_MULTIPLIER" not in self.cfg:
self.cfg['TTL_MULTIPLIER'] = 1
if "CTL" not in self.cfg:
self.cfg['CTL'] = "IOCTL"
if "DEVPF" not in self.cfg:
self.cfg['DEVPF'] = "/dev/pf"
if "AF4_TABLE" not in self.cfg:
print("AF4_TABLE not found in YAML Config File. Exiting.")
sys.exit(2)
if "AF4_FILE" not in self.cfg:
print("AF4_FILE not found in YAML Config File. Exiting.")
sys.exit(2)
if "AF6_TABLE" not in self.cfg:
print("AF6_TABLE not found in YAML Config File. Exiting.")
sys.exit(2)
if "AF6_FILE" not in self.cfg:
print("AF6_FILE not found in YAML Config File. Exiting.")
sys.exit(2)
except Exception as e:
print("YAML Config File not found or cannot load. {}".format(e))
sys.exit(2)
# Init Logging
self.logger.addHandler(
SysLogHandler(address=find_syslog(),
facility=SysLogHandler.LOG_DAEMON))
if self.cfg['LOG_LEVEL'] == 'DEBUG' or self.cfg['LOG_LEVEL'] == 'INFO':
self.logger.setLevel(logging.DEBUG)
else:
self.logger.setLevel(logging.ERROR)
# Custom exception reporter to include some details
DEFAULT_EXCEPTION_REPORTER_FILTER = \
'weblate.trans.debug.WeblateExceptionReporterFilter'
# Default logging of Weblate messages
# - to syslog in production (if available)
# - otherwise to console
# - you can also choose 'logfile' to log into separate file
# after configuring it below
# Detect if we can connect to syslog
HAVE_SYSLOG = False
if platform.system() != 'Windows':
try:
handler = SysLogHandler(address='/dev/log',
facility=SysLogHandler.LOG_LOCAL2)
handler.close()
HAVE_SYSLOG = True
except IOError:
HAVE_SYSLOG = False
if DEBUG or not HAVE_SYSLOG:
DEFAULT_LOG = 'console'
else:
DEFAULT_LOG = 'syslog'
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/stable/topics/logging for
# more details on how to customize your logging configuration.
def main():
if options.output=='syslog':
logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport)))
else:
sh=logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
#logger.debug(options)
try:
es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
s = requests.Session()
s.headers.update({'Accept': 'application/json'})
s.headers.update({'Content-type': 'application/json'})
s.headers.update({'Authorization':'SSWS {0}'.format(options.apikey)})
#capture the time we start running so next time we catch any events created while we run.
state = State(options.state_file)
lastrun = toUTC(datetime.now()).isoformat()
#in case we don't archive files..only look at today and yesterday's files.
yesterday=date.strftime(datetime.utcnow()-timedelta(days=1),'%Y/%m/%d')
today = date.strftime(datetime.utcnow(),'%Y/%m/%d')
r = s.get('https://{0}/api/v1/events?startDate={1}&limit={2}'.format(
options.oktadomain,
toUTC(state.data['lastrun']).strftime('%Y-%m-%dT%H:%M:%S.000Z'),
options.recordlimit
))
if r.status_code == 200:
oktaevents = json.loads(r.text)
for event in oktaevents:
if 'published' in event.keys():
if toUTC(event['published']) > toUTC(state.data['lastrun']):
try:
mozdefEvent = dict()
mozdefEvent['utctimestamp']=toUTC(event['published']).isoformat()
mozdefEvent['receivedtimestamp']=toUTC(datetime.now()).isoformat()
mozdefEvent['category'] = 'okta'
mozdefEvent['tags'] = ['okta']
if 'action' in event.keys() and 'message' in event['action'].keys():
mozdefEvent['summary'] = event['action']['message']
mozdefEvent['details'] = event
# Actor parsing
# While there are various objectTypes attributes, we just take any attribute that matches
# in case Okta changes it's structure around a bit
# This means the last instance of each attribute in all actors will be recorded in mozdef
# while others will be discarded
# Which ends up working out well in Okta's case.
if 'actors' in event.keys():
for actor in event['actors']:
if 'ipAddress' in actor.keys():
if netaddr.valid_ipv4(actor['ipAddress']):
mozdefEvent['details']['sourceipaddress'] = actor['ipAddress']
if 'login' in actor.keys():
mozdefEvent['details']['username'] = actor['login']
if 'requestUri' in actor.keys():
mozdefEvent['details']['source_uri'] = actor['requestUri']
# We are renaming action to activity because there are
# currently mapping problems with the details.action field
mozdefEvent['details']['activity'] = mozdefEvent['details']['action']
mozdefEvent['details'].pop('action')
jbody=json.dumps(mozdefEvent)
res = es.save_event(doc_type='okta',body=jbody)
logger.debug(res)
except Exception as e:
logger.error('Error handling log record {0} {1}'.format(r, e))
continue
else:
logger.error('Okta event does not contain published date: {0}'.format(event))
state.data['lastrun'] = lastrun
state.write_state_file()
else:
logger.error('Could not get Okta events HTTP error code {} reason {}'.format(r.status_code, r.reason))
except Exception as e:
logger.error("Unhandled exception, terminating: %r"%e)
def __init__(self, mpf_path, machine_path, args):
"""Run mpf game."""
signal.signal(signal.SIGINT, self.exit)
parser = argparse.ArgumentParser(
description='Starts the MPF game engine')
parser.add_argument("-a",
action="store_true",
dest="no_load_cache",
help="Forces the config to be loaded from files "
"and not cache")
parser.add_argument("-A",
action="store_false",
dest="create_config_cache",
help="Does not create the cache config files")
parser.add_argument("-b",
action="store_false",
dest="bcp",
default=True,
help="Runs MPF without making a connection "
"attempt to a "
"BCP Server")
parser.add_argument("-c",
action="store",
dest="configfile",
default="config.yaml",
metavar='config_file',
help="The name of a config file to load. Default "
"is "
"config.yaml. Multiple files can be used "
"via a comma-"
"separated list (no spaces between)")
parser.add_argument("-C",
action="store",
dest="mpfconfigfile",
default=os.path.join(mpf_path, "mpfconfig.yaml"),
metavar='config_file',
help="The MPF framework default config file. "
"Default is "
"mpf/mpfconfig.yaml")
parser.add_argument("-f",
action="store_true",
dest="force_assets_load",
default=False,
help="Load all assets upon startup. Useful for "
"ensuring all assets are set up properly "
"during development.")
parser.add_argument("--json-logging",
action="store_true",
dest="jsonlogging",
default=False,
help="Enables json logging to file. ")
parser.add_argument(
"-l",
action="store",
dest="logfile",
metavar='file_name',
default=os.path.join(
"logs",
datetime.now().strftime("%Y-%m-%d-%H-%M-%S-mpf-" +
socket.gethostname() + ".log")),
help="The name (and path) of the log file")
parser.add_argument("-p",
action="store_true",
dest="pause",
default=False,
help="Pause the terminal window on exit. Useful "
"when launching in a separate window so you can "
"see any errors before the window closes.")
parser.add_argument(
"-P",
action="store_true",
dest="production",
default=False,
help=
"Production mode. Will suppress errors, wait for hardware on start and "
"try to exit when startup fails. Run this inside a loop.")
parser.add_argument("-t",
action="store_false",
dest='text_ui',
default=True,
help="Use the ASCII test-based UI")
parser.add_argument("-v",
action="store_const",
dest="loglevel",
const=logging.DEBUG,
default=15,
help="Enables verbose logging to the"
" log file")
parser.add_argument(
"-V",
action="store_const",
dest="consoleloglevel",
const=logging.DEBUG,
default=logging.INFO,
help="Enables verbose logging to the console. DO "
"NOTE: On Windows platforms you must also use -v for "
"this to work.")
parser.add_argument("-x",
action="store_const",
dest="force_platform",
const='virtual',
help="Forces the virtual platform to be "
"used for all devices")
parser.add_argument("--syslog_address",
action="store",
dest="syslog_address",
help="Log to the specified syslog address. This "
"can be a domain socket such as /dev/og on "
"Linux or /var/run/syslog on Mac. "
"Alternatively, you an specify host:port for "
"remote logging over UDP.")
parser.add_argument("-X",
action="store_const",
dest="force_platform",
const='smart_virtual',
help="Forces the smart virtual platform to be "
"used for all"
" devices")
# The following are just included for full compatibility with mc
# which is needed when using "mpf both".
parser.add_argument("-L",
action="store",
dest="mc_file_name",
metavar='mc_file_name',
default=None,
help=argparse.SUPPRESS)
parser.add_argument("--no-sound",
action="store_true",
dest="no_sound",
default=False)
self.args = parser.parse_args(args)
self.args.configfile = Util.string_to_list(self.args.configfile)
# Configure logging. Creates a logfile and logs to the console.
# Formatting options are documented here:
# https://docs.python.org/2.7/library/logging.html#logrecord-attributes
try:
os.makedirs(os.path.join(machine_path, 'logs'))
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
full_logfile_path = os.path.join(machine_path, self.args.logfile)
try:
os.remove(full_logfile_path)
except OSError:
pass
if self.args.text_ui:
console_log = logging.NullHandler()
console_log.setLevel(logging.ERROR)
else:
console_log = logging.StreamHandler()
console_log.setLevel(self.args.consoleloglevel)
# tell the handler to use this format
console_log.setFormatter(
logging.Formatter('%(levelname)s : %(name)s : %(message)s'))
# initialise async handler for console
console_log_queue = Queue()
console_queue_handler = QueueHandler(console_log_queue)
self.console_queue_listener = logging.handlers.QueueListener(
console_log_queue, console_log)
self.console_queue_listener.start()
# initialise file log
file_log = logging.FileHandler(full_logfile_path)
if self.args.jsonlogging:
formatter = JSONFormatter()
else:
formatter = logging.Formatter(
'%(asctime)s : %(levelname)s : %(name)s : %(message)s')
file_log.setFormatter(formatter)
# initialise async handler for file log
file_log_queue = Queue()
file_queue_handler = QueueHandler(file_log_queue)
self.file_queue_listener = logging.handlers.QueueListener(
file_log_queue, file_log)
self.file_queue_listener.start()
# add loggers
logger = logging.getLogger()
logger.addHandler(console_queue_handler)
logger.addHandler(file_queue_handler)
logger.setLevel(self.args.loglevel)
if self.args.syslog_address:
try:
host, port = self.args.syslog_address.split(":")
except ValueError:
syslog_logger = SysLogHandler(self.args.syslog_address)
else:
syslog_logger = SysLogHandler((host, int(port)))
logger.addHandler(syslog_logger)
try:
MachineController(mpf_path, machine_path, vars(self.args)).run()
logging.info("MPF run loop ended.")
self.exit()
# pylint: disable-msg=broad-except
except Exception as e:
self.exit(exception=e)
def setup_syslog(verbosity, quiet=False, logdest="syslog"):
'''
TBD
'''
try:
_fmsg = ""
_status = 100
_my_uuid, _oscp, _mscp, _lscp = get_stores_parms()
# HACK ALERT - A very crude "syslog facility selector"
_syslog_selector = {}
_syslog_selector["16"] = SysLogHandler.LOG_LOCAL0
_syslog_selector["17"] = SysLogHandler.LOG_LOCAL1
_syslog_selector["18"] = SysLogHandler.LOG_LOCAL2
_syslog_selector["19"] = SysLogHandler.LOG_LOCAL3
_syslog_selector["20"] = SysLogHandler.LOG_LOCAL4
_syslog_selector["21"] = SysLogHandler.LOG_LOCAL5
_syslog_selector["22"] = SysLogHandler.LOG_LOCAL6
_syslog_selector["23"] = SysLogHandler.LOG_LOCAL7
_verbosity = int(verbosity)
logger = getLogger()
# Reset the logging handlers
while len(logger.handlers) != 0:
logger.removeHandler(logger.handlers[0])
if logdest == "console" or (not _lscp["hostname"]
or not _lscp["port"]):
hdlr = StreamHandler(stdout)
else:
_facility = int(21)
if _facility > 23 or _facility < 16:
_facility = 23
hdlr = SysLogHandler(address = (_lscp["hostname"], \
int(_lscp["port"])), \
facility=_syslog_selector[str(_facility)])
formatter = Formatter("[%(asctime)s] [%(levelname)s] %(message)s")
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
if _verbosity:
if int(_verbosity) >= 6:
logger.setLevel(DEBUG)
elif int(_verbosity) >= 5:
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
logger.setLevel(DEBUG)
elif int(_verbosity) >= 4:
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
logger.setLevel(DEBUG)
elif int(_verbosity) >= 3:
# Filter out gmetad logging statements
hdlr.addFilter(VerbosityFilter("gmetad"))
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from the "remote"
# subdirectory.
hdlr.addFilter(VerbosityFilter("remote"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
hdlr.addFilter(MsgFilter("Exit point"))
hdlr.addFilter(MsgFilter("Entry point"))
logger.setLevel(DEBUG)
elif int(_verbosity) >= 2:
# Filter out gmetad logging statements
hdlr.addFilter(VerbosityFilter("gmetad"))
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from all modules in the
# "collectors" subdirectory.
hdlr.addFilter(VerbosityFilter("collectors"))
# Used to filter out all function calls from the "remote"
# subdirectory.
hdlr.addFilter(VerbosityFilter("remote"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
logger.setLevel(DEBUG)
elif int(_verbosity) == 1:
# Filter out gmetad logging statements
hdlr.addFilter(VerbosityFilter("gmetad"))
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
# Used to filter out all function calls from all modules in the
# "collectors" subdirectory.
hdlr.addFilter(VerbosityFilter("collectors"))
# Used to filter out all function calls from the "remote"
# subdirectory.
hdlr.addFilter(VerbosityFilter("remote"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("clouds"))
logger.setLevel(DEBUG)
else:
logger.setLevel(INFO)
if quiet:
logger.setLevel(ERROR)
_status = 0
except Exception, e:
_status = 23
_fmsg = str(e)
def initialize_logging(logger_name):
global windows_file_handler_added
try:
logging_config = get_logging_config()
logging.basicConfig(
format=get_log_format(logger_name),
level=logging_config['log_level'] or logging.INFO,
)
# set up file loggers
if get_os() == 'windows' and not windows_file_handler_added:
logger_name = 'agent'
windows_file_handler_added = True
log_file = logging_config.get('%s_log_file' % logger_name)
if log_file is not None and not logging_config['disable_file_logging']:
# make sure the log directory is writeable
# NOTE: the entire directory needs to be writable so that rotation works
if os.access(os.path.dirname(log_file), os.R_OK | os.W_OK):
file_handler = logging.handlers.RotatingFileHandler(
log_file, maxBytes=LOGGING_MAX_BYTES, backupCount=1)
formatter = logging.Formatter(get_log_format(logger_name), get_log_date_format())
file_handler.setFormatter(formatter)
root_log = logging.getLogger()
root_log.addHandler(file_handler)
else:
sys.stderr.write("Log file is unwritable: '%s'\n" % log_file)
# set up syslog
if logging_config['log_to_syslog']:
try:
from logging.handlers import SysLogHandler
if logging_config['syslog_host'] is not None and logging_config[
'syslog_port'] is not None:
sys_log_addr = (logging_config['syslog_host'], logging_config['syslog_port'])
else:
sys_log_addr = "/dev/log"
# Special-case macs
if sys.platform == 'darwin':
sys_log_addr = "/var/run/syslog"
handler = SysLogHandler(address=sys_log_addr, facility=SysLogHandler.LOG_DAEMON)
handler.setFormatter(
logging.Formatter(get_syslog_format(logger_name), get_log_date_format()))
root_log = logging.getLogger()
root_log.addHandler(handler)
except Exception as e:
sys.stderr.write("Error setting up syslog: '%s'\n" % str(e))
traceback.print_exc()
# Setting up logging in the event viewer for windows
if get_os() == 'windows' and logging_config['log_to_event_viewer']:
try:
from logging.handlers import NTEventLogHandler
nt_event_handler = NTEventLogHandler(
logger_name,
get_win32service_file(
'windows',
'win32service.pyd'),
'Application')
nt_event_handler.setFormatter(
logging.Formatter(get_syslog_format(logger_name), get_log_date_format()))
nt_event_handler.setLevel(logging.ERROR)
app_log = logging.getLogger(logger_name)
app_log.addHandler(nt_event_handler)
except Exception as e:
sys.stderr.write("Error setting up Event viewer logging: '%s'\n" % str(e))
traceback.print_exc()
except Exception as e:
sys.stderr.write("Couldn't initialize logging: %s\n" % str(e))
traceback.print_exc()
# if config fails entirely, enable basic stdout logging as a fallback
logging.basicConfig(
format=get_log_format(logger_name),
level=logging.INFO,
)
# re-get the log after logging is initialized
global log
log = logging.getLogger(__name__)
'''
Manages backups for etcd.
Saves the backup in a specific directory and rotates old backups, keeping 7
of them.
'''
import argparse
import glob
import logging
from logging.handlers import SysLogHandler
import os
import shutil
import subprocess
log = logging.getLogger()
log.addHandler(SysLogHandler('/dev/log'))
log.setLevel(logging.INFO)
etcdctl = '/usr/bin/etcdctl'
def rotate_logs(cluster, backup_root, num_logs):
backup_dir = os.path.join(
backup_root,
'etcd-{0}-backup'.format(cluster)
)
backupdir_content = glob.glob(backup_dir + '*')
backupdirs = []
for filepath in backupdir_content:
if not os.path.isdir(filepath):
continue
# Note: RLIMIT_NPROC doesn't really work
# Note: RLIMIT_AS is more of a suggestion than a hard limit in Mac OS X
# Note: All other resource limits seem to be working, but besides RLIMIT_CPU and
# RLIMIT_AS they don't actually kill off offending processes
"resource_limits": {"RLIMIT_CPU": 120, # CPU time in seconds
"RLIMIT_AS": 3*(2**30), #Maximum address space in bytes; this sets 3 GB
},
# The log file will be in the home directory of the untrusted account
"log_file": "sagecell.log",
"max_kernels": 30,
"preforked_kernels": 5,
# These set paramaters for a heartbeat channel checking whether a given kernel is alive.
# Setting first_beat lower than 1.0 may cause javascript errors.
"beat_interval": 0.5,
"first_beat": 1.0}
for i in range(4):
computers.append(_default_config)
import logging
from logging.handlers import SysLogHandler
h = SysLogHandler(address='/dev/log', facility=SysLogHandler.LOG_LOCAL3)
h.setFormatter(logging.Formatter('%(asctime)s %(name)s %(process)d: %(message)s'))
logging.getLogger('sagecell.stats').addHandler(h)
systemlog_handler = SysLogHandler(address='/dev/log', facility=SysLogHandler.LOG_LOCAL4)
systemlog_handler.setFormatter(logging.Formatter('%(asctime)s %(name)s %(process)d: %(message)r'))
logging.getLogger("tornado.application").addHandler(systemlog_handler)
logging.getLogger('sagecell.system').addHandler(systemlog_handler)
# -*- coding: utf-8 -*-
import argparse
import os
import logging
from logging.handlers import SysLogHandler
from sync import Sync
from local import Local
from remote import Remote
# TODO: get version from setup.cfg
version = '0.1.0'
logging.basicConfig()
logger = logging.getLogger("flickrsmartsync_oauth")
hdlr = SysLogHandler()
formatter = logging.Formatter('flickrsmartsync_oauth %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
# flickrsmartsync_oauth.main()
def main():
parser = argparse.ArgumentParser(
description='Sync current folder to your flickr account.')
parser.add_argument('--monitor',
action='store_true',
help='Start monitoring daemon.')
parser.add_argument(
'--starts-with',
