def __init__(self, path=""):
if path:
self.__path = path
pathos = Path(path)
pathos.parent.mkdir(parents=True, exist_ok=True)
self.__logger = logging.getLogger("ePiframe")
self.__logger.setLevel(logging.INFO)
handler = TimedRotatingFileHandler(path,
when="midnight",
interval=1,
backupCount=6)
def namer(name):
return name + ".gz"
def rotator(source, dest):
with open(source, "rb") as sf:
data = sf.read()
with gzip.open(dest, "wb") as df:
df.write(data)
os.remove(source)
handler.rotator = rotator
handler.namer = namer
self.__logger.addHandler(handler)
else:
self.__path = ""
def add_file_handler(filename):
handler = TimedRotatingFileHandler(filename=filename, when='midnight')
handler.setFormatter(logging.Formatter(fmt=log_format))
handler.namer = file_namer
handler.rotator = file_rotator
log.addHandler(handler)
log.info('Writing log to %s', filename)
def __init__(self):
if not os.path.exists('logs'):
os.makedirs('logs')
fName = '{}/{}'.format(os.path.dirname(os.getcwd()), 'logs/dk_log.log')
if not os.path.isfile(fName):
fName = 'logs/dk_log.log'
self.logger = logging.getLogger()
conHandler = logging.StreamHandler()
fileHandler = TimedRotatingFileHandler(filename=fName,
when='midnight',
backupCount=90,
interval=1)
conHandler.setLevel(logging.WARNING)
fileHandler.setLevel(logging.INFO)
logformatCon = logging.Formatter(
'%(asctime)s %(levelname)s %(message)s',
datefmt='%d-%b-%y %H:%M:%S')
logformatfile = logging.Formatter(
'%(asctime)s %(name)s %(levelname)s %(message)s',
datefmt='%d-%b-%y %H:%M:%S')
conHandler.setFormatter(logformatCon)
fileHandler.setFormatter(logformatfile)
fileHandler.rotator = GZipRotator()
self.logger.addHandler(conHandler)
self.logger.addHandler(fileHandler)
self.logger.setLevel(logging.INFO)
def create_logger(a_path: str) -> Logger:
"""
Creates a rotating log
"""
ensure_exists(a_path)
log_name: str = a_path.split("/")[-1]
if log_name in LOGGERS:
return LOGGERS[log_name]
logger = logging.getLogger(log_name.split(".")[0])
logger.setLevel(logging.INFO)
log_handler = TimedRotatingFileHandler(filename=a_path, when="h")
log_formatter = logging.Formatter(
"%(asctime)s [%(levelname)s] [%(name)s] %(message)s")
log_handler.setFormatter(log_formatter)
log_handler.setLevel(logging.INFO)
log_handler.rotator = GZipRotator()
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
stream_handler.setFormatter(log_formatter)
logger.addHandler(stream_handler)
logger.addHandler(log_handler)
LOGGERS[log_name] = logger
return logger
def logToFile(filename=None):
if not filename:
filename = '/var/log/myDevices/cayenne.log'
handler = TimedRotatingFileHandler(filename, when="midnight", interval=1, backupCount=7)
handler.setFormatter(LOG_FORMATTER)
handler.rotator=rotator
handler.namer=namer
LOGGER.addHandler(handler)
def setup_log_file(log_file_directory: str, log_level=logging.DEBUG, log_backup_count=30):
root_logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s')
file_handler = TimedRotatingFileHandler(filename=f'{log_file_directory}/nctl_logs',
when='d', interval=1, backupCount=log_backup_count)
file_handler.rotator = nauta_log_rotator
file_handler.setFormatter(formatter)
file_handler.setLevel(log_level)
root_logger.addHandler(file_handler)
return file_handler
def getFileLogger(name: str, fPath: str, backupCount: int, numRollingHrs: int) -> LoggerAdapter:
logger = logging.getLogger(name)
logger.setLevel(logging.INFO)
# streamHandler = logging.StreamHandler()
fileHandler = TimedRotatingFileHandler(
fPath, backupCount=backupCount, when='h', interval=numRollingHrs)
fileHandler.namer = lambda name: name.replace(".log", "") + ".zip"
fileHandler.rotator = rotator
streamFormatter = logging.Formatter("%(message)s")
# streamHandler.setFormatter(streamFormatter)
fileHandler.setFormatter(streamFormatter)
# logger.addHandler(streamHandler)
logger.addHandler(fileHandler)
loggerAdapter = logging.LoggerAdapter(logger, extra={})
return loggerAdapter
def pure_api_record_logger(name='pure_api_record',
dirname=dirname,
type='pure-api-record-type'):
path = dirname + '/' + type + '.log'
logger = logging.getLogger('experts_etl.' + name)
logger.setLevel(logging.INFO)
handler = TimedRotatingFileHandler(
path,
when='S',
interval=86400, # seconds/day
backupCount=365)
handler.setFormatter(PureApiRecordFormatter())
handler.rotator = rotator
handler.namer = namer
logger.addHandler(handler)
return logger
def command_line_interface():
parser = argparse.ArgumentParser(description='Scrape remote weblisting')
parser.add_argument('--url', type=str, help='url to scrape')
parser.add_argument('--output', default='./')
parser.add_argument('--config', type=str)
parser.add_argument('--service', type=bool, default=False)
parser.add_argument('--logfile', type=str, default=None)
parser.add_argument('--debug', type=bool, default=False)
args = parser.parse_args()
FORMAT = '%(levelname)-8s | %(asctime)-15s | %(pathname)-15s +%(lineno)-4d | %(message)s'
logFormatter = logging.Formatter(FORMAT)
logging.basicConfig(format=FORMAT)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
if args.logfile is not None:
logHandler = TimedRotatingFileHandler(args.logfile,
when="d",
interval=5,
backupCount=5)
logHandler.rotator = log_rotator
logHandler.namer = log_namer
logHandler.setFormatter( logFormatter )
logHandler.setLevel(logging.INFO)
logger.addHandler(logHandler)
if args.debug or args.debug == "True":
logger.setLevel(logging.DEBUG)
if (args.url is None) and (args.config is None):
raise RuntimeError('Requires either --url or --config ')
if args.config is not None:
with open(args.config) as f:
config = yaml.safe_load(f)
else:
config = {'scraped_site': {'url': args.url,
'download_location': args.output},
}
config['service'] = args.service
logging.info(f'Running with config {config}')
return config
def experts_etl_logger(name='experts_etl', dirname=dirname):
path = dirname + '/' + name + '.log'
logger = logging.getLogger('experts_etl.' + name)
logger.setLevel(logging.INFO)
handler = TimedRotatingFileHandler(
path,
when='S',
interval=86400, # seconds/day
backupCount=365)
handler.setFormatter(
ExpertsEtlFormatter(
'%(timestamp)s %(levelname)s %(name)s %(message)s %(pathname)s %(funcName)s %(lineno)i'
))
handler.rotator = rotator
handler.namer = namer
logger.addHandler(handler)
return logger
'SECRET_KEY',
b'3OVQ\xb6\x12\xe1w\xf8z\x8eK4Y\x8c\x0f\x078\x08\xe1\x18\x07\x00\x11')
SQLALCHEMY_POOL_SIZE = os.getenv('SQLALCHEMY_POOL_SIZE', 20)
SQLALCHEMY_MAX_OVERFLOW = os.getenv('SQLALCHEMY_MAX_OVERFLOW', 100)
class GZipRotator:
def __call__(self, source, dest):
os.rename(source, dest)
f_in = open(dest, 'rb')
f_out = gzip.open("%s.gz" % dest, 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.remove(dest)
GzipRotatorHandler = TimedRotatingFileHandler(
filename=os.getenv('SERVICE_LOG', 'vpn-monitor.log'),
when="midnight",
backupCount=os.getenv('MAX_LOGFILE_CNT', 30))
GzipRotatorHandler.rotator = GZipRotator()
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
handlers=[GzipRotatorHandler])
