def cli(context, pkg_path, configfiles, debug, confvars):
pkg_path = list(pkg_path)
if 'REMAND_PKG_PATH' in os.environ:
pkg_path.extend(os.environ['REMAND_PKG_PATH'].split(os.pathsep))
# add contrib to pkg path
import remand.contrib as contrib
pkg_path.append(os.path.abspath(os.path.dirname(contrib.__file__)))
# pluginbase is imported here because just importing it breaks Crypto
# (and with it paramiko)
import pluginbase
plugin_base = pluginbase.PluginBase(package='remand.ext')
obj = context.obj = {}
handler = ColorizedStderrHandler(
level=logbook.DEBUG if debug else logbook.INFO)
# setup logging
logbook.compat.redirect_logging()
handler.push_application()
# read configuration and host registry
obj['config'] = load_configuration(APP_NAME, configfiles)
# set configuration values
for k, v in confvars:
obj['config']['Match:.*'][k] = v
log.debug('Set Match.*:[{!r}] = {!r}'.format(k, v))
obj['hosts'] = HostRegistry(obj['config'])
plugin_source = plugin_base.make_plugin_source(
searchpath=list(pkg_path) + ['.'])
obj['plugin_source'] = plugin_source
def cli(ctx, githome, loglevel):
ctx.obj = {}
if loglevel is None:
loglevel = logbook.INFO
# setup sqlalchemy loglevel
if loglevel is logbook.DEBUG:
redirect_logging()
logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)
# setup console logging
NullHandler().push_application()
handler = ColorizedStderrHandler(level=loglevel)
handler.format_string = '{record.channel}: {record.message}'
handler.push_application()
ctx.obj['githome_path'] = pathlib.Path(githome)
# if we're just calling init, do not initialize githome
if ctx.invoked_subcommand == 'init':
return
# check if the home is valid
if not GitHome.check(ctx.obj['githome_path']):
log.critical('Not a valid githome: "{}"; use {} init to initialize it '
'first.'.format(githome, 'githome'))
abort(1)
# create and add to context
gh = GitHome(ctx.obj['githome_path'])
ctx.obj['githome'] = gh
def log_handler(args, thread_wrapping=True):
"""
Return log handler with given config
"""
if not isinstance(args, dict):
args = vars(args)
if args.get("quiet"):
stderr_handler = ColorizedStderrHandler(level="ERROR")
elif args.get("verbose"):
stderr_handler = ColorizedStderrHandler(level="DEBUG")
else:
stderr_handler = ColorizedStderrHandler(level=args.get(
"stderr_level", "NOTICE").upper(),
bubble=True)
if args.get("log_file"):
file_handler = FileHandler(args.get("log_file"),
level=args.get("log_file_level",
"DEBUG").upper(),
bubble=True)
else:
file_handler = NullHandler()
if thread_wrapping:
file_handler = ThreadedWrapperHandler(file_handler)
stderr_handler = ThreadedWrapperHandler(stderr_handler)
return NestedSetup([
NullHandler(), # catch everything else
file_handler,
stderr_handler
])
def get_handlers(self, log_name):
logger_dir = LOGGER_CONF["path"]
logger_fmt = LOGGER_CONF["format"]
# logger_size = int(LOGGER_CONF["size"])
logger_level = LOGGER_CONF["level"].upper()
if not os.path.exists(logger_dir):
os.makedirs(logger_dir)
def log_type(record, handler):
log = logger_fmt.format(
date = record.time, # 日志时间
level = record.level_name, # 日志等级
filename = os.path.split(record.filename)[-1], # 文件名
func_name = record.func_name, # 函数名
lineno = record.lineno, # 行号
msg = record.message, # 日志内容
channel = record.channel, # 通道
pid = self._pid,
ppid = self._ppid
)
return log
# 日志打印到屏幕
log_std = ColorizedStderrHandler(bubble=True, level=logger_level)
log_std.formatter = log_type
# 日志打印到文件
log_file = TimedRotatingFileHandler(os.path.join(logger_dir, '{}.log'.format(log_name)), date_format='%Y-%m-%d', rollover_format='{basename}_{timestamp}{ext}', bubble=True, level=logger_level, encoding='utf-8')
log_file.formatter = log_type
logbook.set_datetime_format("local")
return [log_std, log_file]
def print_handler():
# 日志路径，在主工程下生成log目录
Log_Dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'log')
if not os.path.exists(Log_Dir):
os.makedirs(Log_Dir)
# 打印到屏幕句柄
user_std_handler = ColorizedStderrHandler(bubble=True)
user_std_handler.formatter = log_formatter
# 打印到文件句柄
file_handler = TimedRotatingFileHandler(os.path.join(Log_Dir, "%s.log" % "test_log"), date_format="%Y%m%d", bubble=True)
file_handler.formatter = log_formatter
return user_std_handler, file_handler
def initialize_log_handler(log_level):
from logbook.more import ColorizedStderrHandler
debug_format_str = (
"[{record.level_name}] {record.channel} {record.func_name} "
"({record.lineno}): {record.message}")
if log_level == logbook.DEBUG:
info_format_str = debug_format_str
else:
info_format_str = "[{record.level_name}] {record.channel}: {record.message}"
ColorizedStderrHandler(level=logbook.DEBUG,
format_string=debug_format_str).push_application()
ColorizedStderrHandler(level=logbook.INFO,
format_string=info_format_str).push_application()
def __init__(self, log_name, custom_path=False, debug=False):
self.name = log_name
self.logger = Logger(log_name)
self.levels = []
self.log_path = os.path.dirname(os.path.realpath(__file__))
if custom_path:
self.log_dir = os.path.join(custom_path, "logs")
else:
self.log_dir = os.path.join(self.log_path, "logs")
if debug:
debug_handler = ColorizedStderrHandler(bubble=True)
debug_handler.formatter = self.user_handler_log_formatter
self.logger.handlers.append(debug_handler)
def get_logger(perform_rollover=False):
"""
Push to the app stack the needed handlers and return a Logger object.
:rtype: logbook.Logger
"""
# NOTE: make sure that the folder exists, the logger is created before
# saving settings on the first run.
_base = os.path.join(get_path_prefix(), "leap")
mkdir_p(_base)
bitmask_log_file = os.path.join(_base, 'bitmask.log')
level = logbook.WARNING
if flags.DEBUG:
level = logbook.NOTSET
# This handler consumes logs not handled by the others
null_handler = logbook.NullHandler()
null_handler.push_application()
silencer = SelectiveSilencerFilter()
zmq_handler = SafeZMQHandler('tcp://127.0.0.1:5000', multi=True,
level=level, filter=silencer.filter)
zmq_handler.push_application()
file_handler = logbook.RotatingFileHandler(
bitmask_log_file, format_string=LOG_FORMAT, bubble=True,
filter=silencer.filter, max_size=sys.maxint)
if perform_rollover:
file_handler.perform_rollover()
file_handler.push_application()
# don't use simple stream, go for colored log handler instead
# stream_handler = logbook.StreamHandler(sys.stdout,
# format_string=LOG_FORMAT,
# bubble=True)
# stream_handler.push_application()
stream_handler = ColorizedStderrHandler(
level=level, format_string=LOG_FORMAT, bubble=True,
filter=silencer.filter)
stream_handler.push_application()
logger = logbook.Logger('leap')
return logger
def get_logger(name=APP_NAME, file_log=False):
logbook.set_datetime_format('local')
## 是否输出到sys.stdout
ColorizedStderrHandler(bubble=False).push_application()
if file_log:
logbook.TimedRotatingFileHandler(LOG_PATH, date_format='%Y%m%d', bubble=True).push_application()
return logbook.Logger(name)
def __init__(self,
log_folder,
log_name=None,
max_size=100,
backup_count=10):
"""
log_folder: 日志文件夹
log_name: 日志文件名称
max_size: 单个日志文件的大小，单位 MB
backup_count: 总备份数量，默认为 5
log_path: 日志文件全路径
注意：所有 handler 中的 bubble 表示记录是否给下个 handler 用。
"""
# 设置日志信息时间的格式
set_datetime_format('local')
self.log_folder = log_folder
self.log_name = str(log_name) if log_name else 'pms'
self.log_path = self.__file_path()
# 检查存放日志的文件夹是否存在，不存在则创建
self.__check_path()
self.log_ = Logger(self.log_name.split('.')[0])
self.log_.handlers.append(
RotatingFileHandler(filename=self.log_path,
mode='a',
level='INFO',
max_size=max_size * 1024**2,
backup_count=backup_count,
bubble=True))
self.log_.handlers.append(ColorizedStderrHandler(bubble=False))
def __init__(self):
# 脚本日志
self.run_log = Logger("script_log")
# 日志存放路径
self.LOG_DIR = log_path
if not os.path.exists(self.LOG_DIR):
os.makedirs(self.LOG_DIR)
def log_type(record, handler):
log = "[{date}] [{level}] [{filename}] [{func_name}] [{lineno}] {msg}".format(
date=record.time, # 日志时间
level=record.level_name, # 日志等级
filename=os.path.split(record.filename)[-1], # 文件名
func_name=record.func_name, # 函数名
lineno=record.lineno, # 行号
msg=record.message) # 日志内容
return log
# 日志打印到屏幕
self.log_std = ColorizedStderrHandler(bubble=False)
self.log_std.formatter = log_type
# 日志打印到文件
self.log_file = TimedRotatingFileHandler(os.path.join(
self.LOG_DIR, '%s.log' % 'log'),
date_format='%Y-%m-%d',
bubble=True,
encoding='utf-8')
self.log_file.formatter = log_type
def log_init(name=__name__,
level='NOTICE',
show_level=None,
format_string=FORMAT_STRING,
logfile=None):
"""Initialize a new Logger to file and colorized stderr stream"""
logfile = log_file_init(log_name=name, logfile=logfile)
file_handler = FileHandler(logfile,
level=level,
format_string=format_string,
bubble=True)
show_level = show_level if show_level else level
cstd_handler = ColorizedStderrHandler(level=show_level,
format_string=format_string,
bubble=False)
level = logbook.lookup_level(level)
logger = Logger(name, level=level)
logger.handlers.append(file_handler)
logger.handlers.append(cstd_handler)
logger.filename = logfile
return logger
def get_logger(name='appium', file_log=file_stream, level=''):
logbook.set_datetime_format('local')
ColorizedStderrHandler(bubble=False, level=level).push_thread()
logbook.TimedRotatingFileHandler(
os.path.join(LOG_DIR, '%s.log' % name),
date_format='%Y-%m-%d-%H', bubble=True, encoding='utf-8').push_thread()
return logbook.Logger(name)
def get_logger(name='jiekou', file_log=file_stream, level=''):
""" get logger Factory function """
logbook.set_datetime_format('local')
ColorizedStderrHandler(bubble=False, level=level).push_thread()
logbook.TimedRotatingFileHandler(
os.path.join(log_path, '%s.log' % name),
date_format='%Y-%m-%d-%H', bubble=True, encoding='utf-8').push_thread()
return logbook.Logger(name)
def get_logger(name="interface_Test", file_log=file_stream, level=''):
""" get logger Factory function """
logbook.set_datetime_format("local")
ColorizedStderrHandler(bubble=False, level=level).push_thread()
logbook.TimedRotatingFileHandler(
os.path.join(LOG_DIR, "%s.log" %"Log"),
date_format="%Y-%m-%d-%H", bubble=True, encoding="utf-8").push_thread()
return logbook.Logger(name)
def get_logger(name='LOGBOOK', log_path='', file_log=False):
logbook.set_datetime_format('local')
ColorizedStderrHandler(bubble=True).push_application()
log_dir = os.path.join('log') if not log_path else log_path
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if file_log:
TimedRotatingFileHandler(os.path.join(log_dir, '%s.log' % name.lower()), date_format='%Y-%m-%d', bubble=True).push_application()
return Logger(name)
def get_logger(name='log', level='INFO'):
logbook.set_datetime_format('local')
# 输出到控制台
ColorizedStderrHandler(bubble=False, level=level).push_thread()
# 写入log文件
# logbook.TimedRotatingFileHandler(
# os.path.join(log_dir, '%s.log' % name),
# date_format='%Y-%m-%driver-%H', bubble=True, encoding='utf-8').push_thread()
return logbook.Logger(name)
def get_logger(name='system', level=''):
""" get logger Factory function """
logbook.set_datetime_format('local')
ColorizedStderrHandler(bubble=False, level=level).push_thread()
logbook.StreamHandler(sys.stdout,
bubble=False,
encoding='utf-8',
level=level).push_thread()
return logbook.Logger(name)
def get_logger(name='接口', level=''):
#设置时间格式
logbook.set_datetime_format('local')
ColorizedStderrHandler(bubble=False, level=level).push_thread()
logbook.TimedRotatingFileHandler(os.path.join(path, '%s.log' % name),
date_format='%Y-%m-%d-%H',
bubble=True,
encoding='utf-8').push_thread()
#print('join_path =' + os.path.join(path,'%s.log' % name))
return logbook.Logger(name)
def initialize_cli(options):
debug_format_str = (
"[{record.level_name}] {record.channel} {record.func_name} "
"({record.lineno}): {record.message}")
if options.log_level == logbook.DEBUG:
info_format_str = debug_format_str
else:
info_format_str = "[{record.level_name}] {record.channel}: {record.message}"
ColorizedStderrHandler(level=logbook.DEBUG,
format_string=debug_format_str).push_application()
ColorizedStderrHandler(level=logbook.INFO,
format_string=info_format_str).push_application()
set_log_level(options.log_level)
SubprocessRunner.is_save_history = True
if options.is_output_stacktrace:
SubprocessRunner.is_output_stacktrace = options.is_output_stacktrace
def get_logger(perform_rollover=False):
"""
Push to the app stack the needed handlers and return a Logger object.
:rtype: logbook.Logger
"""
level = logbook.WARNING
if flags.DEBUG:
level = logbook.NOTSET
# This handler consumes logs not handled by the others
null_handler = logbook.NullHandler()
null_handler.push_application()
silencer = SelectiveSilencerFilter()
zmq_handler = SafeZMQHandler('tcp://127.0.0.1:5000', multi=True,
level=level, filter=silencer.filter)
zmq_handler.push_application()
file_handler = logbook.RotatingFileHandler(
BITMASK_LOG_FILE, format_string=LOG_FORMAT, bubble=True,
filter=silencer.filter, max_size=sys.maxint)
if perform_rollover:
file_handler.perform_rollover()
file_handler.push_application()
# don't use simple stream, go for colored log handler instead
# stream_handler = logbook.StreamHandler(sys.stdout,
# format_string=LOG_FORMAT,
# bubble=True)
# stream_handler.push_application()
stream_handler = ColorizedStderrHandler(
level=level, format_string=LOG_FORMAT, bubble=True,
filter=silencer.filter)
stream_handler.push_application()
logger = logbook.Logger('leap')
return logger
def generate_logger():
# 日志名称
logger = Logger(PROJECT_NAME_EN, level=LOG_LEVEL)
logbook.set_datetime_format("local")
logger.handlers = []
# 日志打印到文件
log_file = TimedRotatingFileHandler(os.path.join(
LOG_DIR, '%s.log' % PROJECT_NAME_EN),
date_format=DAY_FORMAT,
bubble=True,
encoding='utf-8')
log_file.formatter = log_type
logger.handlers.append(log_file)
if DEBUG:
# 日志打印到屏幕
log_std = ColorizedStderrHandler(bubble=True)
log_std.formatter = log_type
logger.handlers.append(log_std)
return logger
def get_logger(name='interface', level=''):
""" get logger Factory function """
logbook.set_datetime_format('local')
# 打印到屏幕句柄
ColorizedStderrHandler(bubble=False, level=level).push_thread()
# 打印到文件句柄
logbook.TimedRotatingFileHandler(os.path.join(LOG_DIR, '%s.log' % name),
date_format='%Y-%m-%d-%H',
bubble=True,
encoding='utf-8').push_thread()
return logbook.Logger(name)
def cli(context, pkg_path, configfiles, debug, confvars, debug_ssh):
pkg_path = list(pkg_path)
if 'REMAND_PKG_PATH' in os.environ:
pkg_path.extend(os.environ['REMAND_PKG_PATH'].split(os.pathsep))
# add contrib to pkg path
import remand.contrib as contrib
pkg_path.append(os.path.abspath(os.path.dirname(contrib.__file__)))
# pluginbase is imported here because just importing it breaks Crypto
# (and with it paramiko)
import pluginbase
plugin_base = pluginbase.PluginBase(package='remand.ext')
obj = context.obj = {}
handler = ColorizedStderrHandler(
level=logbook.DEBUG if debug else logbook.INFO)
# setup logging
logbook.compat.redirect_logging()
handler.push_application()
if not debug_ssh:
logbook.NullHandler(filter=lambda r, h: r.channel.startswith(
'paramiko')).push_application()
# read configuration and host registry
obj['config'] = load_configuration(APP_NAME, configfiles)
# set configuration values
for k, v in confvars:
obj['config']['Match:.*'][k] = v
log.debug('Set Match.*:[{!r}] = {!r}'.format(k, v))
obj['hosts'] = HostRegistry(obj['config'])
plugin_source = plugin_base.make_plugin_source(searchpath=list(pkg_path) +
['.'])
obj['plugin_source'] = plugin_source
def __init__(self, log_dir=BASE_DIR, log_name='log.log', backup_count=10, log_type=log_type, stdOutFlag=False):
if not os.path.exists(log_dir):
os.mkdir(log_dir)
self.log_dir = log_dir
self.backup_count = backup_count
handler = TimedRotatingFileHandler(filename= os.path.join(self.log_dir, log_name),
date_format='%Y-%m-%d',
backup_count=self.backup_count)
self.handler = handler
if log_type is not None:
handler.formatter = log_type
handler.push_application()
if not stdOutFlag:
return
handler_std = ColorizedStderrHandler(bubble=True)
if log_type is not None:
handler_std.formatter = log_type
handler_std.push_application()
def get_logger(logname):
user_log = logbook.Logger('log')
logbook.set_datetime_format("local") # 格式化时间
# 日志打印到屏幕
log_std = ColorizedStderrHandler(bubble=True)
log_std.formatter = log_type
# 日志打印到文件
log_file = TimedRotatingFileHandler(os.path.join(full_path,
'%s.log' % logname),
date_format='%Y-%m-%d',
bubble=True,
encoding='utf-8') # 日期分割显示文件(带日期的)
log_file.formatter = log_type
user_log.handlers = []
user_log.handlers.append(log_file)
user_log.handlers.append(log_std)
return user_log
def get_logger(perform_rollover=False):
"""
Push to the app stack the needed handlers and return a Logger object.
:rtype: logbook.Logger
"""
# NOTE: make sure that the folder exists, the logger is created before
# saving settings on the first run.
_base = os.path.join(get_path_prefix(), "leap")
mkdir_p(_base)
bitmask_log_file = os.path.join(_base, 'bitmask.log')
# level = logbook.WARNING
# if flags.DEBUG:
# level = logbook.NOTSET
level = logbook.NOTSET
# This handler consumes logs not handled by the others
null_handler = logbook.NullHandler()
null_handler.push_application()
file_handler = logbook.RotatingFileHandler(bitmask_log_file,
format_string=LOG_FORMAT,
bubble=True,
max_size=sys.maxint)
if perform_rollover:
file_handler.perform_rollover()
file_handler.push_application()
stream_handler = ColorizedStderrHandler(level=level,
format_string=LOG_FORMAT,
bubble=True)
stream_handler.push_application()
logger = logbook.Logger('leap')
return logger
def get_logger(name='monkey小工具各流程日志输出', file_log=file_stream, level=''):
'''获取日志工厂功能'''
'''
py:attr:`LogRecord.time` will be a datetime in local time zone
(but not time zone aware)
'''
logbook.set_datetime_format('local')
ColorizedStderrHandler(bubble=False, level=level).push_thread()
# # 日志打印到文件中
logbook.TimedRotatingFileHandler(os.path.join(LOG_DIR, '%s.log' % name),
date_format='%Y-%m-%d-%H',
bubble=True,
encoding='utf-8').push_thread()
return logbook.Logger(name)
def __init__(self, clazz):
logbook.set_datetime_format("local")
self.serverName = clazz.__name__[clazz.__name__.rfind('.') + 1:]
if not os.path.exists(log_dir):
os.makedirs(log_dir)
self.log_file = TimedRotatingFileHandler(os.path.join(
log_dir, '%s.log' % self.serverName),
date_format='%Y-%m-%d',
bubble=True,
encoding='utf-8')
self.log_std = ColorizedStderrHandler(bubble=True)
self.log = Logger(self.serverName)
self.__init_logger()
self.__setting()
def get_logger(perform_rollover=False):
"""
Push to the app stack the needed handlers and return a Logger object.
:rtype: logbook.Logger
"""
# NOTE: make sure that the folder exists, the logger is created before
# saving settings on the first run.
_base = os.path.join(get_path_prefix(), "leap")
mkdir_p(_base)
bitmask_log_file = os.path.join(_base, "bitmask.log")
# level = logbook.WARNING
# if flags.DEBUG:
# level = logbook.NOTSET
level = logbook.NOTSET
# This handler consumes logs not handled by the others
null_handler = logbook.NullHandler()
null_handler.push_application()
file_handler = logbook.RotatingFileHandler(
bitmask_log_file, format_string=LOG_FORMAT, bubble=True, max_size=sys.maxint
)
if perform_rollover:
file_handler.perform_rollover()
file_handler.push_application()
stream_handler = ColorizedStderrHandler(level=level, format_string=LOG_FORMAT, bubble=True)
stream_handler.push_application()
logger = logbook.Logger("leap")
return logger
def get_logs_dispatcher(uri=None, debug=False):
handlers = []
if not debug:
handlers.append(NullHandler(level=DEBUG))
handlers.append(ColorizedStderrHandler(level=INFO))
if not uri:
# Find an open port.
# This is a race condition as the port could be used between
# the check and its binding. However, this is probably one of the
# best solution without patching Logbook.
tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tmpsock.bind(('localhost', 0))
uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
tmpsock.close()
subscriber = ZeroMQSubscriber(uri, multi=True)
return uri, subscriber.dispatch_in_background(setup=NestedSetup(handlers))
def __init__(self, logname, toscreen=False):
# 设置日志名称
self.logname = logname
self.toscreen = toscreen
# 设置日志目录
self.LOG_DIR = self.setpath()
# 设置本地时间
logbook.set_datetime_format("local")
# 设置终端输出格式
self.log_standard = ColorizedStderrHandler(bubble=True)
self.log_standard.formatter = self.logformat
# 设置文件输出格式
self.log_file = TimedRotatingFileHandler(os.path.join(
self.LOG_DIR, '{}.log'.format(self.logname)),
date_format='%Y-%m-%d',
bubble=True,
encoding='utf-8')
self.log_file.formatter = self.logformat
# 执行log记录
self.log = Logger("SatanLogging")
self.logrun()
def user_std_handler_log_formatter(record, handler):
from ..environment import Environment
try:
dt = Environment.get_instance().calendar_dt.strftime(DATETIME_FORMAT)
except Exception:
dt = datetime.now().strftime(DATETIME_FORMAT)
log = "{dt} {level} {msg}".format(
dt=dt,
level=record.level_name,
msg=to_utf8(record.message),
)
return log
user_std_handler = ColorizedStderrHandler(bubble=True)
user_std_handler.formatter = user_std_handler_log_formatter
def formatter_builder(tag):
def formatter(record, handler):
log = "[{formatter_tag}] [{time}] {level}: {msg}".format(
formatter_tag=tag,
level=record.level_name,
msg=to_utf8(record.message),
time=record.time,
)
if record.formatted_exception:
log += "\n" + record.formatted_exception
self.timer.stop()
def connect_failed(self, error):
logger.error('client {} connecting failed, reason {}'.format(id, error))
def timeout(self, watcher, events):
logger.error('timeout')
def main():
loop = pyev.default_loop()
signal_handler = whizzer.signal_handler(loop)
factory = whizzer.ProtocolFactory()
factory.protocol = EchoClientProtocol
clients = []
# number of parallel clients
for x in range(0, 2):
clients.append(EchoClient(x, loop, factory))
signal_handler.start()
loop.loop()
if __name__ == "__main__":
stderr_handler = ColorizedStderrHandler(level='INFO')
null_handler = NullHandler()
with null_handler.applicationbound():
with stderr_handler.applicationbound():
main()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--smtp-ip', default='127.0.0.1', metavar='IP', help='SMTP ip (default: 127.0.0.1)')
parser.add_argument('--smtp-port', default=1025, type=int, metavar='PORT', help='SMTP port (deault: 1025)')
parser.add_argument('--http-ip', default='127.0.0.1', metavar='IP', help='HTTP ip (default: 127.0.0.1)')
parser.add_argument('--http-port', default=1080, type=int, metavar='PORT', help='HTTP port (deault: 1080)')
parser.add_argument('--db', metavar='PATH', help='SQLite database - in-memory if missing')
parser.add_argument('--htpasswd', metavar='HTPASSWD', help='Apache-style htpasswd file')
parser.add_argument('-v', '--version', help='Display the version and exit', action='store_true')
parser.add_argument('-f', '--foreground', help='Run in the foreground (default if no pid file is specified)',
action='store_true')
parser.add_argument('-d', '--debug', help='Run the web app in debug mode', action='store_true')
parser.add_argument('-a', '--autobuild-assets', help='Automatically rebuild assets if necessary',
action='store_true')
parser.add_argument('-n', '--no-quit', help='Do not allow clients to terminate the application',
action='store_true')
parser.add_argument('-p', '--pidfile', help='Use a PID file')
parser.add_argument('--stop', help='Sends SIGTERM to the running daemon (needs --pidfile)', action='store_true')
args = parser.parse_args()
if args.version:
from maildump.util import get_version
print('MailDump {0}'.format(get_version()))
sys.exit(0)
# Do we just want to stop a runnign daemon?
if args.stop:
if not args.pidfile or not os.path.exists(args.pidfile):
print('PID file not specified or not found')
sys.exit(1)
try:
pid = read_pidfile(args.pidfile)
except ValueError as e:
print('Could not read PID file: {0}'.format(e))
sys.exit(1)
try:
os.kill(pid, signal.SIGTERM)
except OSError as e:
print('Could not send SIGTERM: {0}'.format(e))
sys.exit(1)
sys.exit(0)
# Default to foreground mode if no pid file is specified
if not args.pidfile and not args.foreground:
print('No PID file specified; runnning in foreground')
args.foreground = True
# Warn about relative paths and absolutize them
if args.db and not os.path.isabs(args.db):
args.db = os.path.abspath(args.db)
print('Database path is relative, using {0}'.format(args.db))
if args.htpasswd and not os.path.isabs(args.htpasswd):
args.htpasswd = os.path.abspath(args.htpasswd)
print('Htpasswd path is relative, using {0}'.format(args.htpasswd))
# Check if the password file is valid
if args.htpasswd and not os.path.isfile(args.htpasswd):
print('Htpasswd file does not exist')
sys.exit(1)
# Check if the static folder is writable
asset_folder = os.path.join(pkgutil.get_loader('maildump').filename, 'static')
if args.autobuild_assets and not os.access(asset_folder, os.W_OK):
print('Autobuilding assets requires write access to {0}'.format(asset_folder))
sys.exit(1)
daemon_kw = {'monkey_greenlet_report': False,
'signal_map': {signal.SIGTERM: terminate_server,
signal.SIGINT: terminate_server}}
if args.foreground:
# Do not detach and keep std streams open
daemon_kw.update({'detach_process': False,
'stdin': sys.stdin,
'stdout': sys.stdout,
'stderr': sys.stderr})
pidfile = None
if args.pidfile:
pidfile = os.path.abspath(args.pidfile) if not os.path.isabs(args.pidfile) else args.pidfile
if os.path.exists(pidfile):
pid = read_pidfile(pidfile)
if not os.path.exists(os.path.join('/proc', str(pid))):
print('Deleting obsolete PID file (process {0} does not exist)'.format(pid))
os.unlink(pidfile)
daemon_kw['pidfile'] = TimeoutPIDLockFile(pidfile, 5)
# Unload threading module to avoid error on exit (it's loaded by lockfile)
if 'threading' in sys.modules:
del sys.modules['threading']
context = GeventDaemonContext(**daemon_kw)
try:
context.open()
except lockfile.LockTimeout:
print('Could not acquire lock on pid file {0}'.format(pidfile))
print('Check if the daemon is already running.')
sys.exit(1)
except KeyboardInterrupt:
print()
sys.exit(1)
with context:
# Imports are here to avoid importing anything before monkeypatching
from maildump import app, start
from maildump.web import assets
assets.debug = app.debug = args.debug
assets.auto_build = args.autobuild_assets
app.config['MAILDUMP_HTPASSWD'] = HtpasswdFile(args.htpasswd) if args.htpasswd else None
app.config['MAILDUMP_NO_QUIT'] = args.no_quit
level = logbook.DEBUG if args.debug else logbook.INFO
format_string = (
u'[{record.time:%Y-%m-%d %H:%M:%S}] {record.level_name:<8} {record.channel}: {record.message}'
)
stderr_handler = ColorizedStderrHandler(level=level, format_string=format_string)
with NullHandler().applicationbound():
with stderr_handler.applicationbound():
start(args.http_ip, args.http_port, args.smtp_ip, args.smtp_port, args.db)
from common.HTML import Html
from common.SendMail import MailClient
from utils import generate_link
from utils import register_to_db
# 禁止from xx import *
__all__ = []
# 追加路径依赖
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(BASE_DIR)
# 定义logger
logger = Logger(os.path.split(__file__)[-1].split('.')[0])
set_datetime_format('local')
# logger.handlers.append(TimedRotatingFileHandler(log_name, level=log_level, bubble=True, date_format='%Y-%m-%d',))
logger.handlers.append(ColorizedStderrHandler(level='INFO', bubble=True))
meta_dict_list = []
def main():
try:
for link in generate_link():
logger.info(link)
web = Html(link) # 实例化网页抓取器
article_title = web.title # 文章标题
html_file = web.archive_html(archive_dir=OtherMisc.archive_dir)
mail_client = MailClient()
mail_client.subject_body(subject=article_title, body=web.foreword+'\n'*2+link)
mail_client.attach_file(file_type='string', target=web.body_md)
mail_client.attach_file(file_type='file', target=html_file)
lineno=record.lineno, # 行号
msg=record.message # 日志内容
)
log_dir = os.getcwd()
print('123')
if os.path.exists(log_dir):
print('文件存在')
else:
print('文件不存在')
# print(path)
#日志打印到屏幕
log_led = ColorizedStderrHandler(bubble=True) #获取打印在屏幕上的句柄
log_led.formatter = log_type #log打印的格式设置
# 日志打印到文件
log_file = TimedRotatingFileHandler(os.path.join(log_dir,'%s.log' % 'log'),date_format='%Y-%m-%d',bubble=True,encoding='utf-8')
log_file.formatter = log_type
#脚本日志
run_log = Logger("script_log")
def init_logger():
logbook.set_datetime_format("local")
run_log.handlers = []
run_log.handlers.append(log_file)
run_log.handlers.append(log_led)
logger = init_logger()
__all__ = [
"user_log",
]
def user_log_formatter(record, handler):
return "[{dt}] {level}: {msg}".format(
dt=ExecutionContext.get_current_dt(),
level=record.level_name,
msg=record.message,
)
# handler = StreamHandler(sys.stdout)
handler = ColorizedStderrHandler()
handler.formatter = user_log_formatter
handler.push_application()
user_log = Logger("user_log")
def user_print(*args, **kwargs):
sep = kwargs.get("sep", " ")
end = kwargs.get("end", "")
message = sep.join(map(str, args)) + end
user_log.info(message)
