def init_logger(log_name='corelibrary', log_path=None, log_level=10, stdout_level=30, file_level=10):
"""inits logger"""
_short_format = '%(asctime)s %(levelname)-5s %(message)s'
_long_format = '%(asctime)s %(module)-14s %(levelname)-8s %(message)-120s'
if log_path is None:
log_path = os.getcwd() + os.path.sep + 'pace.log'
elif os.path.isdir(log_path):
log_path += os.path.sep + 'pace.log'
core_logger = getLogger(log_name)
if stdout_level is not None:
stdout_handler = StreamHandler()
stdout_handler.setFormatter(Formatter(_short_format, '%Y%m%d %H%M%S'))
stdout_handler.setLevel(stdout_level)
core_logger.addHandler(stdout_handler)
if file_level is not None:
file_handler = RotatingFileHandler(log_path, backupCount=10)
file_handler.doRollover()
file_handler.setFormatter(Formatter(_long_format, '%Y-%m-%d %H:%M:%S'))
file_handler.setLevel(file_level)
core_logger.addHandler(file_handler)
core_logger.setLevel(log_level)
return core_logger
def _file_handler(loglevel, log_file, log_format, command):
""" Add a rotating file handler for the current Faceswap session. 1 backup is always kept.
Parameters
----------
loglevel: str
The requested log level that messages should be logged at.
log_file: str
The location of the log file to write Faceswap's log to
log_format: :class:`FaceswapFormatter:
The formatting to store log messages as
command: str
The Faceswap command that is being run. Used to dictate whether the log file should
have "_gui" appended to the filename or not.
Returns
-------
:class:`logging.RotatingFileHandler`
The logging file handler
"""
if log_file is not None:
filename = log_file
else:
filename = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "faceswap")
# Windows has issues sharing the log file with sub-processes, so log GUI separately
filename += "_gui.log" if command == "gui" else ".log"
should_rotate = os.path.isfile(filename)
log_file = RotatingFileHandler(filename, backupCount=1)
if should_rotate:
log_file.doRollover()
log_file.setFormatter(log_format)
log_file.setLevel(loglevel)
return log_file
def setup_logging_on_disk(log_dir, backup_count=5):
from logging.handlers import RotatingFileHandler
root = logging.getLogger()
webcontent = logging.getLogger("webcontent")
class Formatter(logging.Formatter):
def formatMessage(self, record):
fmt = ("%(levelname)s %(name)s: [%(url)s] %(message)s"
if record.name == "webcontent" else
"%(levelname)s: %(message)s")
return fmt % record.__dict__
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
handler = RotatingFileHandler(os.path.join(log_dir, "log"),
backupCount=backup_count,
delay=True)
handler.setFormatter(Formatter())
handler.doRollover()
handler.setLevel(logging.DEBUG)
for logger in (root, webcontent):
logger.addHandler(handler)
def make_logger(name: str,
stream_level=logging.WARN,
file_level=logging.DEBUG):
"""Create a logger for a given name
The log file on disk will rotate every 65536 bytes or when there is another execution.
There should always be a historic log file for each of the last two runs. The stream
handler is conventionally set to a higher level than the file handler, the defaults are
WARN and DEBUG. This is to reduce chattyness on cli.
:param name: name of the logger
:param stream_level: One of logging level [default: logging.WARN]
:param file_level: One of logging level [default: logging.DEBUG]
:returns: logging.Logger
"""
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setLevel(stream_level)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
file_handler = RotatingFileHandler(f"{name}.log",
backupCount=1,
maxBytes=65536)
file_handler.setLevel(file_level)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
file_handler.doRollover()
return logger
def __init__(self, ind_logs, logpath, stdout, curCmd):
self.individual_logs = ind_logs
self.path = logpath
self.stdout = stdout
self.cmd = curCmd
if ind_logs:
sys.logger.debug(
"Starting SensorNode-Output Log in individual logfile mode. Writing logfiles to: %s "
% self.path
) #@UndefinedVariable (this is to ignore the error in eclipse)
self.loggerdict = dict()
else:
sys.logger.debug(
"Starting SensorNode-Output Log in cumulative logfile mode. Writing logfile to: %s "
% self.path
) #@UndefinedVariable (this is to ignore the error in eclipse)
#initialize the sensor node logger
logfile = logpath.rstrip('/') + '/parallel-wsn-sensornodes.log'
self.snlogger = logging.getLogger('sensornode')
self.snlogger.setLevel(logging.DEBUG)
lfh = RotatingFileHandler(logfile,
mode='a',
maxBytes=1000000,
backupCount=5)
self.snlogger.addHandler(
lfh
) #@UndefinedVariable (this is to ignore the error in eclipse)
formatter = logging.Formatter('%(asctime)s %(message)s')
lfh.setFormatter(formatter)
#don't log to console
self.snlogger.propagate = False
#on init start a new file if existing one is not empty
if os.path.getsize(logfile) > 0:
lfh.doRollover()
def get_batch_logger(logdir,
project_name,
loglevel,
backupcount,
name=__name__):
"""
Get rotating file logger for storing logs of mirroring of given project.
:param logdir: log directory
:param project_name: name of the project
:param loglevel: logging level
:param backupcount count of log files to keep around
:param name name of the logger
:return logger
"""
logger = logging.getLogger(name)
logfile = os.path.join(logdir, project_name + ".log")
handler = RotatingFileHandler(logfile,
maxBytes=0,
mode='a',
backupCount=backupcount)
formatter = logging.Formatter(
"%(asctime)s - %(levelname)s: "
"%(message)s", '%m/%d/%Y %I:%M:%S %p')
handler.setFormatter(formatter)
handler.doRollover()
logger.setLevel(loglevel)
logger.propagate = False
logger.handlers = []
logger.addHandler(handler)
return logger
def main():
import argparse
config = Configuration()
log_fn = pathlib.Path(config.log_path, "benten-ls.log")
roll_over = log_fn.exists()
handler = RotatingFileHandler(log_fn, backupCount=5)
formatter = logging.Formatter(
fmt='[%(levelname)-7s] %(asctime)s (%(name)s) %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
handler.setFormatter(formatter)
if roll_over:
handler.doRollover()
# logging.basicConfig(filename=log_fn, filemode="w", level=logging.INFO)
# logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser(description="")
parser.add_argument("--mode",
default="stdio",
help="communication (stdio|tcp)")
parser.add_argument("--addr",
default=4389,
help="server listen (tcp)",
type=int)
parser.add_argument("--debug", action="store_true")
args = parser.parse_args()
logging.basicConfig(
level=(logging.DEBUG if args.debug else logging.WARNING))
logger.addHandler(handler)
logger.info(
f"Benten {__version__}: CWL Language Server from Rabix (Seven Bridges)"
)
logger.info(f"ruamel.yaml: {__ruamel_version__}")
logger.info(f"cwl-format: {__cwl_fmt_version__}")
config.initialize()
if args.mode == "stdio":
logger.info("Reading on stdin, writing on stdout")
s = LangServer(conn=JSONRPC2Connection(
ReadWriter(sys.stdin.buffer, sys.stdout.buffer)),
config=config)
s.run()
elif args.mode == "tcp":
host, addr = "0.0.0.0", args.addr
logger.info("Accepting TCP connections on %s:%s", host, addr)
ForkingTCPServer.allow_reuse_address = True
ForkingTCPServer.daemon_threads = True
LangserverTCPTransport.config = config
s = ForkingTCPServer((host, addr), LangserverTCPTransport)
try:
s.serve_forever()
finally:
s.shutdown()
def get_batch_logger(logdir, project_name, loglevel, backupcount,
name=__name__):
"""
Get rotating file logger for storing logs of mirroring of given project.
:param logdir: log directory
:param project_name: name of the project
:param loglevel: logging level
:param backupcount count of log files to keep around
:param name name of the logger
:return logger
"""
logger = logging.getLogger(name)
logfile = os.path.join(logdir, project_name + ".log")
handler = RotatingFileHandler(logfile, maxBytes=0, mode='a',
backupCount=backupcount)
formatter = logging.Formatter("%(asctime)s - %(levelname)s: "
"%(message)s", '%m/%d/%Y %I:%M:%S %p')
handler.setFormatter(formatter)
handler.doRollover()
logger.setLevel(loglevel)
logger.propagate = False
logger.handlers = []
logger.addHandler(handler)
return logger
def configure_logger():
log_dir = Path(os.path.dirname(__file__)) / 'logs'
log_file = log_dir / 'build.log'
if not log_dir.exists():
log_dir.mkdir(parents=True)
if log_file.exists():
do_rollover = True
else:
do_rollover = False
stream_handler = logging.StreamHandler()
file_handler = RotatingFileHandler(str(log_file),
backupCount=5,
encoding=None,
delay=0)
if do_rollover:
file_handler.doRollover()
formatter = logging.Formatter(fmt=('%(asctime)s.%(msecs)03d '
'%(name)-18s '
'%(levelname)-8s '
'%(threadName)-10s '
'%(message)s'),
datefmt='%H:%M:%S')
stream_handler.setFormatter(formatter)
file_handler.setFormatter(formatter)
logger.setLevel(logging.DEBUG)
logger.addHandler(stream_handler)
logger.addHandler(file_handler)
def init_logger(log_level, log_location, app_name):
# Getting log level
log_level = _define_log_level(log_level)
# Modify logger log level
logger = logging.getLogger()
logger.setLevel(log_level)
# Set file formatter
formatter = logging.Formatter("%(asctime)s :: %(levelname)s :: " + app_name + " :: %(message)s")
log_filename = "%s_log.txt" % app_name
log_location = expand_var_and_user(log_location)
if os.path.isdir(log_location):
log_file_path = os.path.join(log_location, log_filename)
need_roll = os.path.isfile(log_file_path)
# Redirect logs into a log file
file_handler = RotatingFileHandler(log_file_path, backupCount=10, maxBytes=2*1024*1024)
file_handler.setLevel(log_level)
file_handler.setFormatter(formatter)
if need_roll:
file_handler.doRollover()
logger.addHandler(file_handler)
# Redirect logs into the user console
stream_handler = logging.StreamHandler()
stream_handler.setLevel(log_level)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
else:
raise BotsicoteException("Log directory: %s does not exist, create it before relaunching the program" %
log_location)
def logsetup(self, logbasepath, level="INFO"):
logname="sofa-display"
log_formatter = logging.Formatter('%(asctime)-6s.%(msecs).03d %(levelname).1s%(lineno)4d: %(message)s','%m/%d %H:%M:%S')
logpath=os.path.join(logbasepath, logname)
logfile=os.path.join(logpath,"%s.log" % logname)
loglink=os.path.join(logbasepath,"%s.log" % logname)
if not os.path.exists(logpath):
os.makedirs(logpath)
#check if a log file already exists and if so rotate it
needRoll = os.path.isfile(logfile)
log_handler = RotatingFileHandler(logfile, mode='a', maxBytes=1024*1024, backupCount=5)
log_handler.setFormatter(log_formatter)
log_handler.setLevel(getattr(logging,level))
if needRoll:
log_handler.doRollover()
console = logging.StreamHandler()
console.setFormatter(log_handler)
console.setLevel(logging.INFO)
logging.getLogger(logname).addHandler(console)
self.log = logging.getLogger(logname)
self.log.setLevel(logging.INFO)
self.log.addHandler(log_handler)
if not os.path.exists(loglink):
os.symlink(logfile, loglink)
self.log.info('-- -----------------------------------------------')
def set_logging(component, logging_level=logging.DEBUG):
""" Configure logging
:param component: name of a component
:type component: str
:param logging_level: loglevel
:type logging_level: int
"""
if not os.path.isdir(LOG_DIR) or not os.access(LOG_DIR, os.W_OK):
log_handler = logging.NullHandler()
log_file = ""
else:
log_file = "/var/log/blivet-gui/%s.log" % component
rotate = os.path.isfile(log_file)
log_handler = RotatingFileHandler(log_file, backupCount=5)
log_handler.setLevel(logging_level)
formatter = logging.Formatter('%(levelname)s:%(name)s: %(message)s')
log_handler.setFormatter(formatter)
if rotate:
log_handler.doRollover()
logger = logging.getLogger(component)
logger.addHandler(log_handler)
logger.setLevel(logging_level)
return log_file, logger
def __init__(self, version):
# disable old log process
logging.shutdown()
# Directories
self.addonDir = join(mw.pm.addonFolder(), "1129289384")
self.mediaDir = mw.col.media.dir()
os.makedirs(self.mediaDir, exist_ok=True)
# Paths
self.iconPath = join(self.addonDir, r'resources/anki.png')
self.ankiCsvPath = join(self.addonDir, Constant.ANKI_DECK)
# Config Logging (Rotate Every 10MB)
os.makedirs(join(self.addonDir, r'logs'), exist_ok=True)
self.ankiFlashLog = join(self.addonDir, r'logs/ankiflash.log')
rfh = RotatingFileHandler(
filename=self.ankiFlashLog, maxBytes=50000000, backupCount=3, encoding='utf-8')
should_roll_over = os.path.isfile(self.ankiFlashLog)
if should_roll_over:
rfh.doRollover()
logging.basicConfig(level=logging.INFO,
format=u"%(asctime)s - %(threadName)s [%(thread)d] - %(message)s",
datefmt="%d-%b-%y %H:%M:%S",
handlers=[rfh])
# Create Generator Dialog
self.generator = GeneratorDialog(
version, self.iconPath, self.addonDir, self.mediaDir)
self.generator.show()
def __init__(self,
name="logger",
level=logging.DEBUG,
console_level=logging.INFO,
mode='a',
config=None):
self.logger = logging.getLogger(name)
#防止同一个实例加载好几次的handler
if len(self.logger.handlers) > 0:
return None
# 设置logger的等级
#super().__init__(name)
# 注意这各会设置最低的等级，后续的设置只能比这个高
self.logger.setLevel(level)
# 组织一个带时间戳的字符串作为日志文件的名字,实现每天记录一个日志文件
date_time = time.strftime("%Y%m%d", time.localtime(time.time()))
if config == None or config.noconf:
log_path_str = os.path.join(
os.path.abspath(os.path.join(os.getcwd(), "")), "ipbehaviour")
logmaxBytes = 256 * 1024 * 1024
logbackupCount = 4
logrotatestarted = 1
else:
log_path_str = config.logFile
logmaxBytes = config.logmaxBytes
logbackupCount = config.logbackupCount
logrotatestarted = config.logrotatestarted
# python 在创建filehandler时路径不存在会报FileNotFoundError，这里要新建下路径（而具体文件存不存在都时可以的，python会自动创建文件）
if not os.path.exists(log_path_str):
os.makedirs(log_path_str)
logFile = os.path.join(log_path_str, date_time + '.log')
# 创建一个logging输出到文件的handler并设置等级和输出格式
# mode属性用于控制写文件的模式，w模式每次程序运行都会覆盖之前的logger，而默认的是a则每次在文件末尾追加
print(logFile)
formatter = logging.Formatter(
'%(asctime)-.19s %(levelname)-.1s %(name)-5.5s %(funcName)-15.15s "%(message)s"'
)
fh = RotatingFileHandler(logFile,
'a',
maxBytes=int(logmaxBytes),
backupCount=int(logbackupCount))
#fh = logging.FileHandler(logFile, mode)
if logrotatestarted == 1:
fh.doRollover()
fh.setLevel(level)
fh.setFormatter(formatter)
self.logger.addHandler(fh)
self.logger.w_filehandler = fh
# 控制台句柄
ch = logging.StreamHandler()
ch.setFormatter(formatter)
ch.setLevel(console_level)
self.logger.addHandler(ch)
self.logger.w_consolehandle = ch
def doRollover(self, callback=None):
newfile = self.get_old_filename() + '.1'
logger.debug('doing rollover')
RotatingFileHandler.doRollover(self)
if newfile is not None:
if callback is None:
self.callback(newfile)
else:
callback(newfile)
def rotateLogs(path, logs_count, owner):
bounce_log_rotator = RotatingFileHandler(path,
maxBytes=0,
backupCount=logs_count)
bounce_log_rotator.doRollover()
uid = pwd.getpwnam(owner).pw_uid
gid = grp.getgrnam(owner).gr_gid
os.chown(path, uid, gid)
os.chmod(path, 0o600)
def get_logging_handler(self, log_level=None):
logfile = os.path.join(self.application_home, 'hydropick.log')
# make handler
from logging.handlers import RotatingFileHandler
handler = RotatingFileHandler(logfile, backupCount=5)
handler.doRollover()
handler.setFormatter(self.get_formatter(log_level))
handler.addFilter(self.get_filter(log_level))
return handler
def shouldRollover(self, record):
if RotatingFileHandler.shouldRollover(self , record):
RotatingFileHandler.doRollover(self)
t = time.strftime("%Y%m%d" , time.localtime(time.time()))
if (cmp(self.current , t) < 0) :
return 1
return 0
def log_setup(self, name=None, level="INFO", errorOnly=[], log_path=None):
if not log_path:
if hasattr(self, 'log_directory'):
log_path = self.log_directory
if not name:
name = __file__.rsplit('.', 1)[0]
log_formatter = logging.Formatter(
'%(asctime)-6s.%(msecs).03d %(filename).8s %(levelname).1s%(lineno)4d: %(message)s',
'%m/%d %H:%M:%S')
if not os.path.isdir(log_path):
os.makedirs(log_path)
log_file = os.path.join(log_path, "%s.log" % name)
if not os.path.exists(log_path):
os.makedirs(log_path)
if os.path.isfile(log_file):
roll = True
else:
roll = False
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=1024 * 1024,
backupCount=5)
log_handler.setFormatter(log_formatter)
log_handler.setLevel(getattr(logging, level))
if roll:
log_handler.doRollover()
console = logging.StreamHandler()
console.setFormatter(log_handler)
console.setLevel(getattr(logging, level))
logging.getLogger(name).addHandler(console)
log = logging.getLogger(name)
log.setLevel(getattr(logging, level))
log.addHandler(log_handler)
log.info('-- %s -----------------------------------------------' %
name)
for lg in logging.Logger.manager.loggerDict:
#self.log.info('.. Active logger: %s' % lg)
for item in errorOnly:
if lg.startswith(item):
log.debug('.. Logger set to error and above: %s' % lg)
logging.getLogger(lg).setLevel(logging.ERROR)
self.log = log
return log
def doRollover(self):
"""
Override base class method to make the new log file group writable.
"""
# Rotate the file first.
RotatingFileHandler.doRollover(self)
# Add group write to the current permissions.
currMode = os.stat(self.baseFilename).st_mode
os.chmod(self.baseFilename, 0777)
def get_rot_file_logger(name, path):
"""Returns a logger with a rotating file handler"""
logger = logging.getLogger(name)
handler = RotatingFileHandler(path, backupCount=5, encoding="utf-8")
handler.doRollover() # rollover existing log files
handler.terminator = "" # click.echo already adds a newline
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
return logger
def _get_handler(self, log_file, count=3):
_handler = RotatingFileHandler(log_file,
delay=True,
mode="w",
backupCount=count,
encoding="utf-8")
self._formatter(_handler)
if os.path.isfile(log_file):
_handler.doRollover()
return _handler
class LoggerFactory:
initialized = False
def __init__(self,
folder: str = '.',
filename: str = 'application.log',
console_level: str = "INFO",
file_level: str = "INFO"):
if not self.initialized:
self.initialized = True
self._log_folder = folder
self._log_filename = folder + os.sep + filename
if not os.path.exists(self._log_folder):
os.mkdir(self._log_folder)
should_roll_over = os.path.isfile(self._log_filename)
self._formatter = logging.Formatter(LOG_FORMAT)
self._console_handler = logging.StreamHandler()
self._console_handler.setLevel(
LoggerFactory._translate_level(console_level))
self._console_handler.setFormatter(self._formatter)
self._file_handler = RotatingFileHandler(self._log_filename,
mode='a',
backupCount=50)
self._file_handler.setLevel(
LoggerFactory._translate_level(file_level))
self._file_handler.setFormatter(self._formatter)
if should_roll_over:
self._file_handler.doRollover()
def __new__(cls, *args, **kwargs):
if not hasattr(cls, '_inst'):
cls._inst = super(LoggerFactory, cls).__new__(cls)
return cls._inst
@staticmethod
def _translate_level(level):
lv = logging.INFO
if level == "DEBUG":
lv = logging.DEBUG
elif level == "WARNING":
lv = logging.WARNING
elif level == "ERROR":
lv = logging.ERROR
elif level == "CRITICAL":
lv = logging.CRITICAL
return lv
def get_new_logger(self, name: str):
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
logger.addHandler(self._console_handler)
logger.addHandler(self._file_handler)
return logger
def _logging_handler_default(self):
from logging.handlers import RotatingFileHandler
logfile = os.path.join(self.application_home, 'hydropick.log')
print logfile, self.application_home
handler = RotatingFileHandler(logfile, backupCount=5)
handler.doRollover()
# format output
datefmt = '%Y%m%d:%H%M%S'
line_fmt = '%(asctime)s :: %(name)s : %(levelname)s : %(message)s'
formatter = logging.Formatter(line_fmt, datefmt=datefmt)
handler.setFormatter(formatter)
return handler
def initLogger():
logname = getProjectRelativePath("app.log")
handler = RotatingFileHandler(logname,
mode="w",
maxBytes=100000,
backupCount=1)
handler.suffix = "%Y%m%d"
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO,
handlers=[handler])
if os.path.isfile(logname): # log already exists, roll over!
handler.doRollover()
def doRollover(self):
# If you shut down Miro then start it up again immediately
# afterwards, then we get in this squirrely situation where
# the log is opened by another process. We ignore the
# exception, but make sure we have an open file. (bug #11228)
try:
RotatingFileHandler.doRollover(self)
except WindowsError:
if not self.stream or self.stream.closed:
self.stream = open(self.baseFilename, "a")
try:
RotatingFileHandler.doRollover(self)
except WindowsError:
pass
def setup_custom_logger(logfile=None, quiet=False):
global logfilename
maxBytes = config.settings.main.logging.logMaxSize * 1024
rotateAfterDays = config.settings.main.logging.logRotateAfterDays
backupCount = config.settings.main.logging.keepLogFiles
logfileMessage = None
if logfile is None:
logfilename = config.settings.main.logging.logfilename
else:
logfilename = logfile
logfileMessage = "Logging to file %s as defined in the command line" % logfilename
console_log_level = config.settings.main.logging.consolelevel.upper()
file_log_level = config.settings.main.logging.logfilelevel.upper()
# set console log level from config file
if not quiet:
console_logger.setLevel(console_log_level)
logger.setLevel(console_log_level)
# add log file handler
if rotateAfterDays is None:
file_logger = RotatingFileHandler(filename=logfilename,
maxBytes=maxBytes,
backupCount=backupCount)
else:
file_logger = TimedRotatingFileHandler(filename=logfilename,
when="d",
interval=rotateAfterDays,
backupCount=backupCount)
file_logger.setFormatter(logging.Formatter(LOGGER_DEFAULT_FORMAT))
file_logger.setLevel(file_log_level)
logger.addHandler(file_logger)
# we need to set the global log level to the lowest defined
if getattr(logging, file_log_level) < getattr(logging, console_log_level):
logger.setLevel(file_log_level)
# add sensitive data filter
logger.addFilter(SensitiveDataFilter())
logging.getLogger("requests").setLevel(logging.CRITICAL)
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
logging.getLogger('werkzeug').setLevel(logging.CRITICAL)
logger.info(logfileMessage)
if not config.settings.main.isFirstStart and config.settings.main.logging.rolloverAtStart:
logger.info("Starting new log file as configured")
file_logger.doRollover()
if config.settings.main.logging.logfileUmask and platform.system().lower(
) == "linux":
umask = config.settings.main.logging.logfileUmask
if not umask.startswith("0"):
umask = "0" + umask
logger.info("Setting umask of log file %s to %s", logfilename, umask)
os.chmod(logfilename, int(umask, 8))
return logger
def doRollover(self):
# If you shut down Miro then start it up again immediately
# afterwards, then we get in this squirrely situation where
# the log is opened by another process. We ignore the
# exception, but make sure we have an open file. (bug #11228)
try:
RotatingFileHandler.doRollover(self)
except WindowsError:
if not self.stream or self.stream.closed:
self.stream = open(self.baseFilename, "a")
try:
RotatingFileHandler.doRollover(self)
except WindowsError:
pass
def setupLogging(self):
"""
setup logging file
"""
try:
hdlr = RotatingFileHandler(self.configDir / 'exe.log', 'a', 500000,
10)
hdlr.doRollover()
except OSError:
# ignore the error we get if the log file is logged
hdlr = logging.FileHandler(self.configDir / 'exe.log')
format = "%(asctime)s %(name)s %(levelname)s %(message)s"
log = logging.getLogger()
hdlr.setFormatter(logging.Formatter(format))
log.addHandler(hdlr)
loggingLevels = {
"DEBUG": logging.DEBUG,
"INFO": logging.INFO,
"WARNING": logging.WARNING,
"ERROR": logging.ERROR,
"CRITICAL": logging.CRITICAL
}
if self.configParser.has_section('logging'):
for logger, level in self.configParser._sections["logging"].items(
):
if logger == "root":
logging.getLogger().setLevel(loggingLevels[level])
else:
logging.getLogger(logger).setLevel(loggingLevels[level])
if not G.application.portable:
log.info("************** eXe logging started **************")
log.info("version = %s" % version.version)
log.info("configPath = %s" % self.configPath)
log.info("exePath = %s" % self.exePath)
log.info("libPath = %s" %
Path(twisted.__path__[0]).splitpath()[0])
log.info("browser = %s" % self.browser)
log.info("webDir = %s" % self.webDir)
log.info("jsDir = %s" % self.jsDir)
log.info("localeDir = %s" % self.localeDir)
log.info("port = %d" % self.port)
log.info("dataDir = %s" % self.dataDir)
log.info("configDir = %s" % self.configDir)
log.info("locale = %s" % self.locale)
log.info("internalAnchors = %s" % self.internalAnchors)
log.info("License = %s" % self.defaultLicense)
def init(base_path, file_name):
log = logging.getLogger(NAME)
log.setLevel(logging.DEBUG)
log_formatter = logging.Formatter(
"%(asctime)s - %(levelname)s :: %(message)s")
log_file = os.path.join(base_path, file_name)
handler = RotatingFileHandler(log_file, backupCount=3)
handler.doRollover()
handler.setFormatter(log_formatter)
log.addHandler(handler)
def doRollover(self):
RotatingFileHandler.doRollover(self)
try:
dirPath = os.path.dirname(self.baseFilename)
baseName = os.path.basename(self.baseFilename)
fileNameList = os.listdir(dirPath)
for fileName in fileNameList:
if not re.match('%s.\\\\w+$' % baseName, fileName):
continue
rollingFilePath = os.path.join(dirPath, fileName)
zipCompressFile(rollingFilePath, baseName)
removeExceededBackupFiles(dirPath, baseName, self.backupCount)
except Exception:
getDefaultLogger().exception('backup trace catch an exception.')
def setup_logger(name, log_file, level=logging.INFO):
"""
To setup as many loggers as you want
"""
handler = logging.FileHandler(log_file, mode='a')
# rotate into 10 files when file reaches 50MB
handler = RotatingFileHandler(log_file, maxBytes=50*1024*1024, backupCount=10)
handler.setFormatter(formatter)
handler.doRollover()
logger = logging.getLogger(name)
logger.setLevel(level)
logger.addHandler(handler)
return logger
def _init_logging(self):
# add a file logger for our run
log_file_handler = RotatingFileHandler(self.project_workspace /
'log.txt',
backupCount=5,
delay=True)
log_file_handler.doRollover()
logging.getLogger('fab').addHandler(log_file_handler)
logger.info(f"{datetime.now()}")
if self.multiprocessing:
logger.info(f'machine cores: {cpu_count()}')
logger.info(f'available cores: {len(os.sched_getaffinity(0))}')
logger.info(f'using n_procs = {self.n_procs}')
logger.info(f"workspace is {self.project_workspace}")
def setup_logging(logdir=NW_DEFAULT_LOG_FILE_PATH,
filelog=True,
rotate=True,
maxBytes=NW_DEFAULT_ROTATING_FILES_MAX_BYTE,
backupCount=NW_DEFAULT_ROTATING_BACKUP_NUM,
loglevel=NW_DEFAULT_LOG_LEVEL,
console=False
):
# Make sure this function is only called once
global _nwLogger_Configured
if _nwLogger_Configured:
return
# Drop any previous logging setup
root = logging.getLogger()
if root.handlers:
for handler in root.handlers:
root.removeHandler(handler)
# Setup the root logger
log = logging.getLogger()
log.setLevel(loglevel)
# Setup the logger format
log_formatter = logging.Formatter(NW_DEFAULT_LOG_MSG_FORMAT)
if filelog:
if not rotate: backupCount = 0
logdir = os.path.abspath(logdir)
if not os.path.exists(logdir): os.mkdir(logdir)
logfile = _nwBuildFullPath(logdir)
file_hdlr = RotatingFileHandler(filename=logfile,
maxBytes=maxBytes,
backupCount=backupCount,
mode="a"
)
file_hdlr.doRollover()
file_hdlr.setFormatter(log_formatter)
log.addHandler(file_hdlr)
log.info("Logger Initialized!")
if console:
console_hdlr = logging.StreamHandler()
console_hdlr.setFormatter(log_formatter)
log.addHandler(console_hdlr)
_nwLogger_Configured = True
def setup_local(name=__name__,
logdir='log',
console=True,
text=True,
loglevel=logging.DEBUG,
logfile=None):
"""
function:
prepares a logger that can print to both the console and a logfile
parameters:
name - the logger's name (for the call logging.getLogger(name))
logdir - location for the output logfile (default is current dir)
console - print to console
text - print to logfile
loglevel - level for logger (logging.DEBUG)
logfile - name of logfile (default=name + '.log')
use:
mylogger.setup(name, etc.)
logger = mylogger.getLogger(name)
:param name:
:param logdir:
:param console:
:param text:
:param loglevel:
:param logfile:
"""
logdir = os.path.abspath(logdir)
os.makedirs(logdir, exist_ok=True)
logger = logging.getLogger(name)
logger.setLevel(loglevel)
formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s')
if not logfile:
logfile = name + '.log'
if text:
txt_handler = RotatingFileHandler(os.path.join(logdir, logfile),
backupCount=5)
txt_handler.doRollover()
txt_handler.setFormatter(formatter)
logger.addHandler(txt_handler)
if console:
scrn_handler = logging.StreamHandler()
scrn_handler.setFormatter(formatter)
logger.addHandler(scrn_handler)
return logging.getLogger(name)
def configure(stdout: bool = True,
rotating=False,
loglevel: str = 'INFO') -> None:
'''configure logging'''
dir = 'log'
filename = 'client_gps.log'
if not os.path.isdir(dir):
try:
os.makedirs(dir)
except OSError:
raise Exception(f'Creation of the log directory "{dir}" failed')
if not filename.endswith('.log'):
filename += '.log'
log_exists = os.path.isfile(os.path.join(dir, filename))
format = '%(asctime)s: %(levelname)s [%(name)s] %(message)s'
formatter = logging.Formatter(format)
if rotating:
handler = RotatingFileHandler(filename=os.path.join(dir, filename),
mode='a',
maxBytes=5 * 1024 * 1024,
backupCount=2)
handler.setFormatter(formatter)
if log_exists:
handler.doRollover()
logging.getLogger().addHandler(handler)
else:
logging.basicConfig(filename=os.path.join(dir, filename),
level=logging.INFO,
format=format)
if stdout:
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
# set log level
numeric_level = getattr(logging, loglevel.upper(), None)
if not isinstance(numeric_level, int):
raise Exception(loglevel, 'invalid log level')
logging.getLogger().setLevel(numeric_level)
# always log version and command line arguments
logging.getLogger().info(f'arguments: {sys.argv[1:]}')
def setupLogging(self):
"""
setup logging file
"""
try:
hdlr = RotatingFileHandler(self.configDir / 'exe.log', 'a', 500000,
10)
hdlr.doRollover()
except OSError:
# ignore the error we get if the log file is logged
hdlr = logging.FileHandler(self.configDir / 'exe.log')
format = "%(asctime)s %(name)s %(levelname)s %(message)s"
log = logging.getLogger()
hdlr.setFormatter(logging.Formatter(format))
log.addHandler(hdlr)
loggingLevels = {
"DEBUG": logging.DEBUG,
"INFO": logging.INFO,
"WARNING": logging.WARNING,
"ERROR": logging.ERROR,
"CRITICAL": logging.CRITICAL
}
if self.configParser.has_section('logging'):
for logger, level in self.configParser._sections["logging"].items(
):
if logger == "root":
logging.getLogger().setLevel(loggingLevels[level])
else:
logging.getLogger(logger).setLevel(loggingLevels[level])
if not G.application.portable:
log.info("************** eXe logging started **************")
log.info("version = %s" % version.version)
log.info("configPath = %s" % self.configPath)
log.info("exePath = %s" % self.exePath)
log.info(
"libPath = %s" % Path(twisted.__path__[0]).splitpath()[0])
log.info("browser = %s" % self.browser)
log.info("webDir = %s" % self.webDir)
log.info("jsDir = %s" % self.jsDir)
log.info("localeDir = %s" % self.localeDir)
log.info("port = %d" % self.port)
log.info("dataDir = %s" % self.dataDir)
log.info("configDir = %s" % self.configDir)
log.info("locale = %s" % self.locale)
log.info("internalAnchors = %s" % self.internalAnchors)
def configure_logging(level=logging.INFO, logfile=None): # pragma: no cover
"""
Configure the root logger for our purposes.
"""
logging.root.setLevel(level)
formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s')
ch = logging.StreamHandler()
ch.setFormatter(formatter)
logging.root.addHandler(ch)
if logfile:
ch2 = RotatingFileHandler(logfile, backupCount=10)
ch2.doRollover()
ch2.setFormatter(formatter)
logging.root.addHandler(ch2)
logging.getLogger('paramiko.transport').setLevel(logging.ERROR)
def create_ind_logfile(self, node_name):
#initialize this sensor node's logger
logfile = self.path.rstrip('/')+'/parallel-wsn-'+node_name+'.log'
snlogger = logging.getLogger(node_name)
snlogger.setLevel(logging.DEBUG)
lfh = RotatingFileHandler(logfile, mode='a', maxBytes=1000000, backupCount=5)
snlogger.addHandler(lfh) #@UndefinedVariable (this is to ignore the error in eclipse)
formatter = logging.Formatter('%(asctime)s %(message)s')
lfh.setFormatter(formatter)
#don't log to console
sys.logger.propagate = False
#on init start a new file if existing one is not empty
if os.path.getsize(logfile) > 0:
lfh.doRollover()
return snlogger
def setup_custom_logger(logfile=None, quiet=False):
global logfilename
maxBytes = config.settings.main.logging.logMaxSize * 1024
rotateAfterDays = config.settings.main.logging.logRotateAfterDays
backupCount = config.settings.main.logging.keepLogFiles
logfileMessage = None
if logfile is None:
logfilename = config.settings.main.logging.logfilename
else:
logfilename = logfile
logfileMessage = "Logging to file %s as defined in the command line" % logfilename
console_log_level = config.settings.main.logging.consolelevel.upper()
file_log_level = config.settings.main.logging.logfilelevel.upper()
# set console log level from config file
if not quiet:
console_logger.setLevel(console_log_level)
logger.setLevel(console_log_level)
# add log file handler
if rotateAfterDays is None:
file_logger = RotatingFileHandler(filename=logfilename, maxBytes=maxBytes, backupCount=backupCount)
else:
file_logger = TimedRotatingFileHandler(filename=logfilename, when="d", interval=rotateAfterDays, backupCount=backupCount)
file_logger.setFormatter(logging.Formatter(LOGGER_DEFAULT_FORMAT))
file_logger.setLevel(file_log_level)
logger.addHandler(file_logger)
# we need to set the global log level to the lowest defined
if getattr(logging, file_log_level) < getattr(logging, console_log_level):
logger.setLevel(file_log_level)
# add sensitive data filter
logger.addFilter(SensitiveDataFilter())
logging.getLogger("requests").setLevel(logging.CRITICAL)
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
logging.getLogger('werkzeug').setLevel(logging.CRITICAL)
logger.info(logfileMessage)
if not config.settings.main.isFirstStart and config.settings.main.logging.rolloverAtStart:
logger.info("Starting new log file as configured")
file_logger.doRollover()
if config.settings.main.logging.logfileUmask and platform.system().lower() == "linux":
umask = config.settings.main.logging.logfileUmask
if not umask.startswith("0"):
umask = "0" + umask
logger.info("Setting umask of log file %s to %s", logfilename, umask)
os.chmod(logfilename, int(umask, 8))
return logger
def setupLogging(logdir=None, scrnlog=True, txtlog=True, loglevel=logging.DEBUG):
logdir = os.path.abspath(logdir)
if not os.path.exists(logdir):
os.mkdir(logdir)
log = logging.getLogger('scrapper')
log.setLevel(loglevel)
log_formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s :: %(message)s")
if txtlog:
txt_handler = RotatingFileHandler(os.path.join(logdir, "scrapper.log"), backupCount=5)
txt_handler.doRollover()
txt_handler.setFormatter(log_formatter)
log.addHandler(txt_handler)
log.info("Logger initialised.")
if scrnlog:
console_handler = logging.StreamHandler()
console_handler.setFormatter(log_formatter)
log.addHandler(console_handler)
def get_config_parser(filepath="/etc/cinder/cinder.conf", write_on_exit=False):
from ConfigParser import RawConfigParser
from logging.handlers import RotatingFileHandler
try:
from collections import OrderedDict
except ImportError:
from .cinder.collections import OrderedDict
parser = RawConfigParser(dict_type=OrderedDict)
parser.optionxform = str # make options case-sensitive
parser.read(filepath)
try:
yield parser
finally:
if write_on_exit:
handler = RotatingFileHandler(filepath, mode='a', maxBytes=0, backupCount=10)
handler.doRollover()
with open(filepath, 'w') as fd:
parser.write(fd)
def GetLogger(logfile=None, debug=False):
"""Initialise the root logger
Option Parameters:
logfile: the full path to the logfile to write, if not supplied logging will be to console only
debug: enable the output of debug log messages
Returns: The root logger class
"""
# setup a root logger - this will capture output from all loggers
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(module)-16s %(funcName)-20s %(message)s')
if logfile != None:
# create file handler which logs even debug messages
fh = RotatingFileHandler(logfile, maxBytes=0, backupCount=14)
fh.doRollover()
fh.setFormatter(formatter)
if debug:
fh.setLevel(logging.DEBUG)
else:
fh.setLevel(logging.INFO)
logger.addHandler(fh)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setFormatter(formatter)
if debug:
ch.setLevel(logging.DEBUG)
else:
ch.setLevel(logging.INFO)
logger.addHandler(ch)
return logger
def initRotatingLogger(logName, fileName, logDir=None, toScreen=True, toText=True, logLevel=logging.WARNING):
if not os.path.exists(logDir): os.makedirs(os.path.abspath(logDir))
logger = logging.getLogger(logName)
logger.setLevel(logLevel)
log_formatter = logging.Formatter("%(asctime)s - %(levelname)s :: %(message)s")
if toText:
txt_handler = RotatingFileHandler(os.path.join(logDir, fileName), backupCount=5)
txt_handler.doRollover()
txt_handler.setFormatter(log_formatter)
logger.addHandler(txt_handler)
logger.info("Logger initialized.")
if toScreen:
console_handler = logging.StreamHandler()
console_handler.setFormatter(log_formatter)
logger.addHandler(console_handler)
return logger
def _configure_logger(l, config=None, level=None):
verbose = 0
log_file = None
quiet = False
if config:
verbose = config.verbose
log_file = config.log_file
quiet = config.quiet
if level:
log_level = level
else:
if verbose == 0:
log_level = logging.ERROR
if verbose == 1:
log_level = logging.WARN
if verbose == 2:
log_level = logging.INFO
if verbose > 2:
log_level = logging.DEBUG
if log_file:
max_size = 1 * 1024 * 1024 # 1MB
filehandler = RotatingFileHandler(log_file, maxBytes=max_size,
backupCount=5)
filehandler.addFilter(context_filter)
l.addHandler(filehandler)
if os.path.getsize(log_file) > max_size/2:
filehandler.doRollover()
if quiet:
l.removeHandler(consolehandler)
l.setLevel(log_level)
logger.info('Finished setting up logger %s %s %s', l.name, log_level,
log_file)
def __init__(self, ind_logs, logpath, stdout, curCmd):
self.individual_logs = ind_logs
self.path = logpath
self.stdout = stdout
self.cmd = curCmd
if ind_logs:
sys.logger.debug("Starting SensorNode-Output Log in individual logfile mode. Writing logfiles to: %s " %self.path) #@UndefinedVariable (this is to ignore the error in eclipse)
self.loggerdict = dict()
else:
sys.logger.debug("Starting SensorNode-Output Log in cumulative logfile mode. Writing logfile to: %s " %self.path) #@UndefinedVariable (this is to ignore the error in eclipse)
#initialize the sensor node logger
logfile = logpath.rstrip('/')+'/parallel-wsn-sensornodes.log'
self.snlogger = logging.getLogger('sensornode')
self.snlogger.setLevel(logging.DEBUG)
lfh = RotatingFileHandler(logfile, mode='a', maxBytes=1000000, backupCount=5)
self.snlogger.addHandler(lfh) #@UndefinedVariable (this is to ignore the error in eclipse)
formatter = logging.Formatter('%(asctime)s %(message)s')
lfh.setFormatter(formatter)
#don't log to console
self.snlogger.propagate = False
#on init start a new file if existing one is not empty
if os.path.getsize(logfile) > 0:
lfh.doRollover()
def doRollover(self):
RotatingFileHandler.doRollover(self)
# Compress the old log.
for i in range(self.backupCount - 1, 0, -1):
sfn = "%s.%d.z" % (self.baseFilename, i)
dfn = "%s.%d.z" % (self.baseFilename, i + 1)
if os.path.exists(sfn):
if os.path.exists(dfn):
os.remove(dfn)
os.rename(sfn, dfn)
old_log = self.baseFilename + ".1"
if self.compress_cls == gzip:
output = self.compress_cls.open(old_log + '.z', 'w')
with open(old_log) as log:
output.writelines(log)
output.close()
if self.compress_cls == zipfile:
output = self.compress_cls.ZipFile(old_log + '.z', 'w', zipfile.ZIP_DEFLATED)
output.write(old_log, os.path.split(self.baseFilename)[1])
output.close()
os.remove(old_log)
def setupLogging(self):
"""
setup logging file
"""
try:
hdlr = RotatingFileHandler(self.configDir / "exe.log", "a", 500000, 10)
hdlr.doRollover()
except OSError:
hdlr = logging.FileHandler(self.configDir / "exe.log")
format = "%(asctime)s %(name)s %(levelname)s %(message)s"
log = logging.getLogger()
hdlr.setFormatter(logging.Formatter(format))
log.addHandler(hdlr)
loggingLevels = {
"DEBUG": logging.DEBUG,
"INFO": logging.INFO,
"WARNING": logging.WARNING,
"ERROR": logging.ERROR,
"CRITICAL": logging.CRITICAL,
}
if self.configParser.has_section("logging"):
for logger, level in self.configParser._sections["logging"].items():
if logger == "root":
logging.getLogger().setLevel(loggingLevels[level])
else:
logging.getLogger(logger).setLevel(loggingLevels[level])
log.info("************** eXe logging started **************")
log.info("configPath = %s" % self.configPath)
log.info("exePath = %s" % self.exePath)
log.info("browserPath = %s" % self.browserPath)
log.info("webDir = %s" % self.webDir)
log.info("xulDir = %s" % self.xulDir)
log.info("localeDir = %s" % self.localeDir)
log.info("port = %d" % self.port)
log.info("dataDir = %s" % self.dataDir)
log.info("configDir = %s" % self.configDir)
log.info("locale = %s" % self.locale)
def _initLogging(self, verbose=False, logfile=None, logfilecount=0):
log = logging.getLogger()
if logfile == "console":
h = logging.StreamHandler()
elif logfile is not None:
from logging.handlers import RotatingFileHandler
doRotation = True if os.path.exists(logfile) else False
h = RotatingFileHandler(logfile, backupCount=logfilecount)
if doRotation:
h.doRollover()
else:
return
if verbose:
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
formatter = logging.Formatter("%(asctime)s - %(name)s "
"- %(levelname)s - %(message)s")
h.setFormatter(formatter)
log.addHandler(h)
# http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
# http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
# set logger
import logging
from logging.handlers import RotatingFileHandler
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh = RotatingFileHandler('debug.log', backupCount=30)
fh.setLevel( logging.DEBUG)
fh.setFormatter( formatter )
fh.doRollover()
logger.addHandler(fh)
BOT_NAME = 'crawlers'
SPIDER_MODULES = ['crawlers.spiders']
NEWSPIDER_MODULE = 'crawlers.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
def doRollover(self):
"""
Rename every backups so the current log can be promoted to backup number one.
The new log file is empty. If this process fails due to any I/O error, rollover is
deactivated for this handler and logs will be appended to the current log file indefinitely.
"""
temp_backup_name = "%s.%s" % (self.baseFilename, uuid.uuid4())
# We need to close the file before renaming it (windows!)
if self.stream:
self.stream.close()
self.stream = None
# Before doing the rollover, check if the first file will fail at all.
# If it does, then it is a good thing that we checked otherwise the last
# backup would have been blown away before encountering the error.
# Take the scenario where there's only one backup. This means that
# doRollover would first delete the backup (.1) file so it can make
# room for the main file to be renamed to .1. However, if the main file
# can't be renamed, we've effectively lost 50% of the logs we had, which
# is not cool. Since most of the time only the first file will be locked,
# we will try to rename it first. If that fails right away as expected,
# we don't try any rollover and append to the current log file.
# and raise the _disable_rollover flag.
try:
os.rename(self.baseFilename, temp_backup_name)
except:
# It failed, so we'll simply append from now on.
log.debug(
"Cannot rotate log file '%s'. Logging will continue to this file, "
"exceeding the specified maximum size", self.baseFilename, exc_info=True
)
self._handle_rename_failure("a", disable_rollover=True)
return
# Everything went well, so now simply move the log file back into place
# so doRollover can do its work.
try:
os.rename(temp_backup_name, self.baseFilename)
except:
# For some reason we couldn't move the backup in its place.
log.debug(
"Unexpected issue while rotating log file '%s'. Logging will continue to this file, "
"exceeding the specified maximum size", self.baseFilename, exc_info=True
)
# The main log file doesn't exist anymore, so create a new file.
# Don't disable the rollover, this has nothing to do with rollover
# failing.
self._handle_rename_failure("w")
return
# Python 2.6 expects the file to be opened during rollover.
if not self.stream and sys.version_info[:2] < (2, 7):
self.mode = "a"
self.stream = self._open()
# Now, that we are back in the original state we were in,
# were pretty confident that the rollover will work. However, due to
# any number of reasons it could still fail. If it does, simply
# disable rollover and append to the current log.
try:
RotatingFileHandler.doRollover(self)
except:
# Something probably failed trying to rollover the backups,
# since the code above proved that in theory the main log file
# should be renamable. In any case, we didn't succeed in renaming,
# so disable rollover and reopen the main log file in append mode.
log.debug(
"Cannot rotate log file '%s'. Logging will continue to this file, "
"exceeding the specified maximum size", self.baseFilename, exc_info=True
)
self._handle_rename_failure("a", disable_rollover=True)
'.json':serialize(send_host_obj, human=True),
'-distro.html':self.get_distro_specific_html(),
'.rst':'\n'.join(map(to_ascii, self.getProfile())),
}
logdir = os.path.expanduser('~/.smolt/')
try:
if not os.path.exists(logdir):
os.mkdir(logdir, 1700)
for k, v in log_matrix.items():
filename = os.path.expanduser(os.path.join(
logdir, 'submission%s' % k))
r = RotatingFileHandler(filename, \
maxBytes=1000000, backupCount=9)
r.stream.write(v)
r.doRollover()
r.close()
os.remove(filename)
except:
pass
del logdir
del log_matrix
debug('sendHostStr: %s' % serialized_host_obj_machine)
debug('Sending Host')
if batch:
entry_point = "/client/batch_add_json"
logging.debug('Submitting in asynchronous mode')
else:
if using_engage:
root_logger.setLevel(min(root_logger.level, logging.DEBUG))
if running_from_command_line:
# we only create the master.log logfile if we are running directly in the command
# line. If we are called as a library, we let the caller setup the logfile
log_dir = engage_file_locator.get_log_directory()
if not os.path.exists(log_dir):
os.makedirs(log_dir)
log_file = os.path.join(log_dir, "master.log")
do_log_rollover = os.path.exists(log_file)
handler = RotatingFileHandler(log_file, backupCount=5)
formatter = logging.Formatter(defs.DATABLOX_LOG_FORMAT,
defs.DATABLOX_LOG_DATEFMT)
handler.setFormatter(formatter)
if do_log_rollover: # we do a rollover each time the master is run
handler.doRollover()
handler.setLevel(logging.DEBUG)
root_logger.addHandler(handler)
else:
root_logger.setLevel(min(root_logger.level, log_levels[options.log_level]))
Master(bloxpath, args[0], args[1:], using_engage,
_log_level=log_levels[options.log_level],
_debug_block_list=debug_block_list,
reuse_existing_installs=reuse_existing_installs,
poll_interval=options.poll_interval,
stats_multiple=options.stats_multiple,
block_args=block_args,
loads_file=options.loads_file,
log_stats_hist=options.log_stats_hist,
time_limit=options.time_limit,
def run(self):
optparser = optparse.OptionParser(usage="usage: %prog [-chHl] start|stop|restart|status", conflict_handler='resolve')
optparser.add_option('-c', '--configfile', dest='cf', type='string', action='store', metavar='config-file', help='specify the path to the config file (default is ./parallel-wsn.config)', default=(str(os.getcwd())+'/parallel-wsn.config'))
optparser.add_option('-l', '--logpath', dest='logpath', type='string', action='store', help='define the path for the logfiles (default is ./ (CWD))')
optparser.add_option('-h', '--hosts', dest='host_files', action='append', metavar='HOST_FILE', help='hosts file (each line "[user@]host[:port]")')
optparser.add_option('-H', '--host', dest='host_strings', action='append',metavar='HOST_STRING', help='additional host entries ("[user@]host[:port]")')
optparser.add_option('-u', '--user', dest='user', help='username (OPTIONAL)')
optparser.add_option('-t', '--waiting-time', dest='waiting_time', help='define the time in seconds for parallel-wsn to wait for answers from the sensor nodes (OPTIONAL)')
optparser.add_option('-v', '--verbose', dest='verbose', action='store_true', help='turn on warning and diagnostic messages (OPTIONAL)')
(options, args) = optparser.parse_args(sys.argv)
print options
print args
#check if verbose option was set
if options.verbose:
sys.log_lvl = logging.DEBUG
#init parser to parse the config-file
cp = ConfigParser.RawConfigParser()
print ("reading configuration file from %s" %options.cf) #@UndefinedVariable (this is to ignore the error in eclipse) #TODO: if verbose...
res = cp.read(options.cf)
if len(res) == 0:
print "Could not read config-file at %s!\n" % options.cf
sys.exit(-1)
#initialize the logger
if options.logpath:
path = options.logpath.rstrip('/')
ensure_dir(path)
else:
path = cp.get("main", "logpath")
if path[0] == "$":
envvar = os.getenv(path.split()[0][1:])
if len(path.split()) > 1:
path = envvar + path.split()[1]
else:
path = envvar
print path
ensure_dir(path)
logfile = path.rstrip('/') + '/parallel-wsn-'+socket.gethostname()+'.log'
if options.verbose:
print "logging to: %s" %logfile
sys.logger = logging.getLogger('parallel-wsn')
sys.logger.setLevel(sys.log_lvl) #@UndefinedVariable (this is to ignore the error in eclipse)
lfh = RotatingFileHandler(logfile, mode='a', maxBytes=1000000, backupCount=5)
sys.logger.addHandler(lfh) #@UndefinedVariable (this is to ignore the error in eclipse)
formatter = logging.Formatter('%(asctime)s %(message)s')
lfh.setFormatter(formatter)
#don't log to console
sys.logger.propagate = False
#on init start a new file if existing one is not empty
if os.path.getsize(logfile) > 0:
lfh.doRollover()
global comm_server
comm_server = communications.ClientCommHandler
global comm_client
comm_client = communications.CommClient()
if not args[1]:
print "No command specified!"
sys.logger.error("No command specified") #@UndefinedVariable (this is to ignore the error in eclipse)
sys.exit(-1)
if cp.get("main", "individual_logfiles") == "yes":
handler = tcp_handler(True, path.rstrip('/'), cp.get("main", "stdout"), args[1])
else:
handler = tcp_handler(False, path.rstrip('/'), cp.get("main", "stdout"), args[1])
if os.uname()[1] == "uhu":
#bind server only to the VLAN ip-address so that nobody outside can access the server
ip_addr = communications.ip_address().get_ip_address("br-meshnet")
else:
#not running on huhu - bind server to all valid ip-addresses of this machine
ip_addr = "0"
#server in own thread
sys.logger.debug("starting TCP-Server on ip %s port %s and putting it in own thread" %(ip_addr, int(cp.get("main", "client_port")))) #@UndefinedVariable (this is to ignore the error in eclipse)
ch = communications.myTCPServer((ip_addr, int(cp.get("main", "client_port"))), comm_server, handler)
ch.allow_reuse_address = True
ch_thread = threading.Thread(target=ch.serve_forever)
ch_thread.setDaemon(1)
ch_thread.start()
#parse the passed hosts
sys.logger.debug("parsing the hosts_file at %s" %options.host_files) #@UndefinedVariable (this is to ignore the error in eclipse)
hosts = hostsParser.read_host_files(options.host_files)
#if additional host_strings were defined add them too
if options.host_strings:
sys.logger.debug("parsing the additional hosts string %s" %options.host_strings) #@UndefinedVariable (this is to ignore the error in eclipse)
for host_string in options.host_strings:
res = hostsParser.parse_host_string(host_string)
if res:
hosts.extend(res)
if not hosts:
print "No Hosts specified!"
sys.logger.error("No hosts were specified in either a file or the string (-h or -H option)") #@UndefinedVariable (this is to ignore the error in eclipse)
sys.exit(-1)
if options.verbose:
print hosts
daemon_port = cp.get("main", "daemon_port")
sys.logger.debug("sending command to the host(s) now") #@UndefinedVariable (this is to ignore the error in eclipse)
for host in hosts:
if options.verbose:
print "sending %s, %s, %s" %(args[1], str(host[0]), (host[1] if host[1] else daemon_port))
sys.logger.debug("sending %s, %s, %s" %(args[1], str(host[0]), (host[1] if host[1] else daemon_port))) #@UndefinedVariable (this is to ignore the error in eclipse)
try:
comm_client.send(args[1], str(host[0]), (host[1] if host[1] else daemon_port))
except:
sys.logger.error("Couldn't send %s to %s" %(args[1], str(host[0]))) #@UndefinedVariable (this is to ignore the error in eclipse)
print ("Couldn't send %s to %s" %(args[1], str(host[0])))
if options.verbose:
print "waiting for answers"
if options.waiting_time:
sys.logger.debug("waiting %s seconds for answers" %options.waiting_time) #@UndefinedVariable (this is to ignore the error in eclipse)
time.sleep(float(options.waiting_time))
else:
sys.logger.debug("waiting %s seconds for answers" %cp.get("main", "waiting_time")) #@UndefinedVariable (this is to ignore the error in eclipse)
time.sleep(float(cp.get("main", "waiting_time")))
sys.logger.debug("shutting down TCP-Server and exiting system now") #@UndefinedVariable (this is to ignore the error in eclipse)
ch.server_close()
ch.shutdown()
ensure_dir(path)
logfile = path.rstrip('/') + '/parallel-wsn-daemon-'+socket.gethostname()+'.log'
if options.verbose:
print "logging to: %s" %logfile
lfh = RotatingFileHandler(logfile, mode='a', maxBytes=1000000, backupCount=5)
sys.logger.addHandler(lfh) #@UndefinedVariable (this is to ignore the error in eclipse)
formatter = logging.Formatter('%(asctime)s %(message)s')
lfh.setFormatter(formatter)
#don't log to console
sys.logger.propagate = False
#if starting or restarting - (re-)read the configuration and write new logfile
if (args[1] == 'start' or args[1] == 'restart'):
#on init start a new file if existing one is not empty
if os.path.getsize(logfile) > 0:
lfh.doRollover()
#check the daemon commands
if options.verbose:
daemon = parallel_wsn_daemon(pidfile, sys.stdin, sys.stdout, sys.stderr)
else:
daemon = parallel_wsn_daemon(pidfile)
if len(args) == 2:
if 'start' == args[1]:
sys.logger.debug("Received Signal to start the daemon") #@UndefinedVariable (this is to ignore the error in eclipse)
daemon.start()
elif 'stop' == args[1]:
sys.logger.debug("Received Signal to stop the daemon") #@UndefinedVariable (this is to ignore the error in eclipse)
daemon.stop()
elif 'restart' == args[1]:
sys.logger.debug("Received Signal to restart the daemon") #@UndefinedVariable (this is to ignore the error in eclipse)
def doRollover(self):
RotatingFileHandler.doRollover(self)
# Try to fix up permissions on new file
set_log_owner(self.baseFilename)
class Job(object):
"""Holds the state of a job in memory"""
def __init__(self, job_id, job_dir, parsed_json, previous_stats = None):
self.state = {}
self.state["id"] = job_id
self.state["job_directory"] = job_dir
self.state["json_config"] = parsed_json
self.state["next_run"] = const.DATETIME_NEVER
self.state["last_run"] = const.DATETIME_NEVER
self.state["last_status"] = ""
self.state["name"] = parsed_json["name"]
if previous_stats:
if 'last_run' in previous_stats:
self.state["last_run"] = previous_stats['last_run']
if 'last_run_status' in previous_stats:
self.state["last_status"] = previous_stats['last_run_status']
self.state["run_now"] = False
self.state["is_running"] = False
self.job_logger = logging.getLogger('jobs.' + job_id)
log_file_handler = logging.FileHandler(os.path.join(self.state["job_directory"], 'job.log'))
log_file_handler.setLevel(logging.DEBUG)
self.job_logger.addHandler(log_file_handler)
self.job_run_logger = logging.getLogger('jobs.' + self.state["id"] + '.last_run')
self.job_run_file_handler = RotatingFileHandler(
os.path.join(self.state["job_directory"],
'job.last_run.log'),
backupCount = 3)
self.job_run_file_handler.setLevel(logging.DEBUG)
self.job_run_logger.addHandler(self.job_run_file_handler)
self.update_schedule()
def serialize(self):
"""create a serializable dict from the job's fields"""
next_run_str = self.state['next_run']
if self.state['next_run'] is const.DATETIME_NEVER:
next_run_str = 'NEVER'
last_run_str = self.state['last_run']
if self.state['last_run'] is const.DATETIME_NEVER:
last_run_str = 'NEVER'
return {
'next_run' : next_run_str,
'last_run' : last_run_str,
'last_run_status' : self.state['last_status'],
'name' : self.state['name'],
'id' : self.state['id']
}
def __repr__(self):
return self.state['name']
@property
def job_id(self):
"""accessor for human readable job id"""
return self.state['id']
@property
def name(self):
"""accessor for human readable job name"""
return self.state['name']
@property
def next_scheduled_run(self):
"""accessor for next scheduled run as a datetime"""
return self.state['next_run']
@property
def is_running(self):
"""accessor for is_running boolean"""
return self.state['is_running']
def run_now(self):
"""schedules the job to run immediately"""
self.state['next_run'] = datetime.datetime.now()
self.state['run_now'] = True
def update_schedule(self):
""" polls all of the job's triggers to see if anything changes
require an update of the schedule
"""
if self.state['run_now']:
self.state['next_run'] = datetime.datetime.now()
return
next_run = const.DATETIME_NEVER
self.job_logger.debug('evaluating triggers...')
for trigger_data in self.state['json_config']['triggers']:
trigger_class = dynamic_import_trigger(trigger_data['className'])
trigger = trigger_class(self.state['id'], trigger_data, self.job_logger)
temp_next_run = trigger.next_run()
next_run = min(next_run, temp_next_run)
self.state['next_run'] = next_run
def _resolve_workspace_dir(self):
if not 'workspace' in self.state['json_config']:
working_dir = os.path.join(self.state['job_directory'], 'workspace')
else:
working_dir = self.state['json_config']['workspace']['workspace_path']
if not os.path.isabs(working_dir):
working_dir = os.path.join(self.state['job_directory'], working_dir)
return working_dir
def run(self):
"""runs the actions of a job"""
self.state['run_now'] = False
self.state['is_running'] = True
self.update_schedule()
working_dir = self._resolve_workspace_dir()
if not os.path.isdir(working_dir):
os.mkdir(working_dir)
logging.debug(self.state['json_config']['actions'])
self.state['last_run'] = datetime.datetime.now()
job_status = const.SUCCESS
self.job_logger.info('executing job...')
self.job_run_file_handler.doRollover()
self.job_run_logger.handlers[0].doRollover()
for action_data in self.state['json_config']['actions']:
action_class = dynamic_import_action(action_data['className'])
action = action_class(action_data, working_dir, self.job_run_logger)
result = action.run()
if ((result is not const.SUCCESS) and
(result is not const.UNSTABLE) and
(result is not const.FAILURE)):
logging.error('job action returned an invalid result: ' + result)
# TODO: throw/error job here
job_status = result
self.job_run_logger.info(job_status)
self.state['last_status'] = job_status
with open(os.path.join(self.state['job_directory'], 'job.stats'), 'w') as stats_file:
json.dump({
"last_run": self.state['last_run'].isoformat(),
"last_run_status": self.state['last_status']},
stats_file)
self.state['is_running'] = False
# -*- coding: utf-8 -*- # # Author: Ingelrest François ([email protected]) # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import consts, logging from logging.handlers import RotatingFileHandler __logHandler = RotatingFileHandler(consts.fileLog, maxBytes=0, backupCount=2) __logHandler.doRollover() __logHandler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)-7s %(message)s', datefmt='%y%m%d%H%M%S')) logger = logging.getLogger(consts.appNameShort) logger.setLevel(logging.INFO) logger.addHandler(__logHandler)
