def set_log(verbose,
debug,
log_dir,
name='train',
job_number=0,
tmp_dir='/tmp'):
log = logging.getLogger('')
log.setLevel(0)
if (log.hasHandlers()):
log.handlers.clear()
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if not os.path.isdir(log_dir):
log_dir = '/tmp'
h_formatter = logging.Formatter(
'%(asctime)s [%(levelname).1s] %(message)s')
formatter = logging.Formatter('[%(levelname).1s] %(message)s')
stream_handler = logging.StreamHandler()
if job_number:
fn = os.path.join(log_dir, f'{name}.log.{job_number}')
file_handler = FileHandler(fn)
else:
fn = os.path.join(log_dir, f'{name}.log')
file_handler = RotatingFileHandler(fn,
maxBytes=5000000,
backupCount=10)
file_handler.doRollover()
fn = os.path.join(log_dir, 'dump.log')
dump_file_handler = RotatingFileHandler(fn, maxBytes=1, backupCount=20)
log_level = logging.ERROR
if verbose == 1:
log_level = logging.WARNING
if verbose > 1:
log_level = logging.INFO
if debug:
log_level = logging.DEBUG
stream_handler.setFormatter(formatter)
stream_handler.setLevel(log_level)
log.addHandler(stream_handler)
file_handler.setFormatter(h_formatter)
file_handler.setLevel(logging.DEBUG)
log.addHandler(file_handler)
dump_file_handler.setFormatter(h_formatter)
dump_file_handler.setLevel(logging.DEBUG)
log.addHandler(dump_file_handler)
if log_dir == tmp_dir:
logging.warning('will log in %s', tmp_dir)
def dump_filter(record):
r = record.getMessage().startswith('DUMPED')
# print('dump filter', r)
# if r:
# logging.error('An error has been dumped somewhere')
return r
def no_dump_filter(record):
r = record.getMessage().startswith('DUMPED')
# print('no dump filter', not r)
return not r
for h in (stream_handler, file_handler):
h.addFilter(no_dump_filter)
pass #
dump_file_handler.addFilter(dump_filter)
# log.error('not an error, just showing what logs look like')
# log.info('Verbose is on')
# log.debug(f'Debug is on')
return log
