def main():
warnings.simplefilter('error', UserWarning)
warnings.simplefilter('error', Warning)
level='WARNING'
from logbook import FileHandler, StreamHandler
log_handler1 = FileHandler('application.log')
log_handler1.push_application()
test4()
pylab.show()
def setup_logging(config):
log_file = os.path.join(config['daemon']['app_path'],
config['daemon']['log']['file'])
# if running in debug mode, disable log rotation because it makes
# things confusing
if config['daemon']['debug']:
log_handler = FileHandler(log_file)
else:
max_size = config['daemon']['log']['rotate_size']
backup_count = config['daemon']['log']['rotate_count']
log_handler = RotatingFileHandler(log_file, max_size=max_size,
backup_count=backup_count)
log_handler.push_application()
log = Logger('edgy_crits')
return log
def __init__(self, settings={}, prefix=""):
"""initialize the Application with a settings dictionary and an optional
``prefix`` if this is a sub application"""
self.settings = settings
self.mapper = routes.Mapper()
self.setup_handlers(self.mapper)
self.loghandler = FileHandler(self.logfilename)
def start(self):
"""Initialize workdir, logging, etc. in preparation for running jobs.
"""
# make a working directory for each job
for job in self.jobs:
job["workdir"] = os.path.join(self.workdir, job["description"])
fs.maybe_mkdir(job["workdir"])
# temporary ipython profile directory
self.ipythondir = os.path.join(self.workdir, ".ipython")
fs.maybe_mkdir(self.ipythondir)
# log dir
self.logdir = os.path.join(self.workdir, "log")
fs.maybe_mkdir(self.logdir)
# determine which IP we are going to listen on for logging
try:
self.listen_ip = localinterfaces.public_ips()[0]
except:
raise ValueError("This machine appears not to have"
" any publicly visible IP addresses")
# setup ZMQ logging
self.handler = FileHandler(os.path.join(self.logdir, "dish.log"))
self.listen_port = str(randint(5000, 10000))
self.subscriber = ZeroMQPullSubscriber("tcp://" + self.listen_ip +
":" + self.listen_port)
self.controller = self.subscriber.dispatch_in_background(self.handler)
self.logger = Logger("dish_master")
def __init__(self, filament):
self.logger = Logger(Fibratus.__name__)
self.file_handler = FileHandler(os.path.join(os.path.abspath(__file__), '..', '..', '..', 'fibratus.log'),
mode='w+')
self.kevt_streamc = KEventStreamCollector(etw.KERNEL_LOGGER_NAME.encode())
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags()
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of kernel trace
with self.file_handler.applicationbound():
self.logger.info('Starting fibratus...')
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
self.thread_registry = ThreadRegistry(self.handle_repository, self._handles)
self.kevent = KEvent(self.thread_registry)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.requires_render = {}
self.filters_count = 0
def main():
"""
The main routine which kicks everything off
:return:
"""
# Setup the command line arguments
flags = argparse.ArgumentParser(description="Tool to validate and fix errors in CSV files for TADC imports")
flags.add_argument('csv_file', type=str, help="Path to a CSV file to validate")
flags.add_argument('header_rows', type=str, help="Number of header rows")
flags.add_argument('--fix-missing', '-f', action='store_true', help="Fix missing fields by inserting the value 'unknown'")
flags.add_argument('--output-dir', '-o', type=str, help='Where to put output files', default=os.getcwd())
flags.add_argument('--log-dir', '-l', type=str, help='Where to put log files', default='/tmp')
flags.add_argument('--log-level', type=str, help='Choose a log level', default='INFO')
flags.add_argument('--old-date-format', type=str, help="the format of dates that will be fixed", default='%d/%m/%Y')
args = flags.parse_args()
log_filename = os.path.join(
args.log_dir,
'tadc_import_validator_{}.log'.format(os.path.basename(time.strftime('%Y%m%d-%H%M%S')))
)
# register some logging handlers
log_handler = FileHandler(
log_filename,
mode='w',
level=args.log_level,
bubble=True
)
stdout_handler = StreamHandler(sys.stdout, level=args.log_level, bubble=True)
with stdout_handler.applicationbound():
with log_handler.applicationbound():
log.info("Arguments: {}".format(args))
start = time.time()
log.info("starting at {}".format(time.strftime('%l:%M%p %Z on %b %d, %Y')))
with CSVFileValidator(
csv_file=args.csv_file,
header_rows=args.header_rows,
output_dir=args.output_dir,
old_date_format=args.old_date_format,
fix_missing=args.fix_missing) as validator:
validator.validate_file()
log.info("Running time: {}".format(str(datetime.timedelta(seconds=(round(time.time() - start, 3))))))
log.info("Log written to {}:".format(log_filename))
log.info("Fixed data is in: {}".format(validator.get_fixed_filename()))
def __init__(self, config=None, merge=False, **kwargs):
"""
See qdb.config for more information about the configuration of
qdb.
merge denotes how config and kwargs should be merged.
QdbConfig.kwargs_first says config will trample kwargs,
QdbConfig.config_first says kwargs will trample config.
Otherwise, kwargs and config cannot both be passed.
"""
super(Qdb, self).__init__()
if config and kwargs:
if merge == QdbConfig.kwargs_first:
first = kwargs
second = config
elif merge == QdbConfig.config_first:
first = config
second = kwargs
else:
raise TypeError('Cannot pass config and kwargs')
config = first.merge(second)
else:
config = QdbConfig.get_config(config or kwargs)
self.address = config.host, config.port
self.set_default_file(config.default_file)
self.default_namespace = config.default_namespace or {}
self.exception_serializer = config.exception_serializer or \
default_exception_serializer
self.eval_fn = config.eval_fn or default_eval_fn
self.green = config.green
self._file_cache = {}
self.redirect_output = config.redirect_output
self.retry_attepts = config.retry_attepts
self.repr_fn = config.repr_fn
self.skip_fn = config.skip_fn or (lambda _: False)
self.pause_signal = config.pause_signal \
if config.pause_signal else signal.SIGUSR2
self.uuid = str(config.uuid or uuid4())
self.watchlist = {}
self.execution_timeout = config.execution_timeout
# We need to be able to send stdout back to the user debugging the
# program. We hold a handle to this in case the program resets stdout.
if self.redirect_output:
self._old_stdout = sys.stdout
self._old_stderr = sys.stderr
self.stdout = StringIO()
self.stderr = StringIO()
sys.stdout = self.stdout
sys.stderr = self.stderr
self.forget()
self.log_handler = None
if config.log_file:
self.log_handler = FileHandler(config.log_file)
self.log_handler.push_application()
self.cmd_manager = (config.cmd_manager or RemoteCommandManager)(self)
self.cmd_manager.start(config.auth_msg)
def main():
"""watch a specific directory, logging changes and
running python scripts when they are written to disk"""
home_dir = Path(environ.get('HOME'))
run_logfile = home_dir / 'pyrun.log'
watchdog_logfile = home_dir / 'pydir.log'
run_log = FileHandler(str(run_logfile), level='NOTICE', bubble=True, mode='w', delay=True)
file_log = FileHandler(str(watchdog_logfile), level='INFO', bubble=True)
with run_log.applicationbound():
with file_log.applicationbound():
watched_dir = home_dir / 'code' / 'pyrep' / 'coderunner' / 'snippets'
handler = MyEventHandler(run_logfile, run_log)
obs = InotifyObserver()
obs.schedule(handler, str(watched_dir), False)
obs.start()
try:
while True:
sleep(1)
except: # pylint: disable=bare-except
obs.stop()
obs.join()
def log(self, message):
try:
if self._logger is None:
FileHandler(self.args.log,
format_string=
"{record.time:%Y-%m-%d %H:%M:%S} {record.message}"
).push_application()
if self.args.verbose > 1:
StderrHandler(bubble=True).push_application()
self._logger = Logger("tupa")
self._logger.warn(message)
except OSError:
pass
def logging_context(path=None, level=None):
from logbook import StderrHandler, FileHandler
from logbook.compat import redirected_logging
with StderrHandler(level=level or 'INFO').applicationbound():
if path:
if not os.path.isdir(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
with FileHandler(path, bubble=True).applicationbound():
with redirected_logging():
yield
else:
with redirected_logging():
yield
def test_formatter_with_file_handler(monkeypatch):
r_fixture = {
'channel': 'formatter.test',
'level': 2,
'msg': 'My test log message',
'args': None,
'kwargs': None,
'exc_info': None,
'extra': None,
'frame': None,
'dispatcher': None
}
fh_fixture = {
'filename': '/tmp/bogus.log',
'mode': 'a',
'encoding': 'utf-8',
'level': 0,
'format_string': None,
'delay': False,
'filter': None,
'bubble': False
}
# This date fixture translates to '2014-12-05T14:12:49.303830Z'
timestamp_fixture = FROZEN_DATETIME.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
monkeypatch.setattr(datetime, 'datetime', MockedDate)
lf = LogstashFormatter()
json_msg = lf(
record=LogRecord(**r_fixture),
handler=FileHandler(**fh_fixture)
)
expected_json = {
'@fields': {
'extra': {}, 'level': r_fixture['level'], 'process': None,
'frame': None, 'args': [], 'kwargs': {},
'_dispatcher': r_fixture['dispatcher'],
'channel': r_fixture['channel']
},
'@handler': {
'level': fh_fixture['level'],
'_filename': fh_fixture['filename'], '_mode': fh_fixture['mode'],
'filter': fh_fixture['filter'], 'bubble': fh_fixture['bubble'],
'encoding': fh_fixture['encoding']
},
'@timestamp': timestamp_fixture,
'@source_host': 'localhost',
'@message': r_fixture['msg']
}
assert json.loads(json_msg) == expected_json
def __init__(self, filament):
self.logger = Logger(Fibratus.__name__)
self.file_handler = FileHandler(os.path.join(os.path.abspath(__file__),
'..', '..', '..',
'fibratus.log'),
mode='w+')
self.kevt_streamc = KEventStreamCollector(
etw.KERNEL_LOGGER_NAME.encode())
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags()
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of kernel trace
with self.file_handler.applicationbound():
self.logger.info('Starting fibratus...')
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
self.thread_registry = ThreadRegistry(self.handle_repository,
self._handles)
self.kevent = KEvent(self.thread_registry)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.requires_render = {}
self.filters_count = 0
def logger(name: str,
stream_level=DEBUG,
file_level=DEBUG,
folder: str = default_path('logs')) -> Logger:
set_datetime_format('local')
StreamHandler(sys.stdout, level=stream_level,
bubble=True).push_application()
filename = __file__.split('/')[-1][:-3]
FileHandler(
f'{folder}/{name}_{datetime.today().strftime("%Y-%m-%d_%H-%M")}.log',
bubble=True,
level=file_level,
delay=True).push_application()
return Logger(name)
def __init():
driver = config.get('app.log.driver', 'stderr')
level = config.get('app.log.level', 'DEBUG').upper()
global __handler
global __loggers
if driver == 'stderr':
__handler = StreamHandler(sys.stderr, level=level)
elif driver == 'stdout':
__handler = StreamHandler(sys.stdout, level=level)
elif driver == 'file':
__handler = FileHandler(filename=__get_log_file(), level=level)
else:
raise Exception('Invalid driver for log')
__handler.push_application()
__loggers['core'] = Logger('Core')
container.register('logger', __loggers['core'])
def setup_logging():
try:
print("setting up log file")
#get current folder
current_dir_path = os.path.dirname(os.path.realpath(__file__))
#join log folder to current path, if you \log then it will be created one level up
log_path=os.path.join(current_dir_path,"log")
log.info('log path:' + log_path)
#check if log folder exist
if not os.path.exists(log_path):
try:
print("creating log folder")
os.makedirs(log_path)
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
else:
log.info("Log folder exist")
log.info("create log file inside log folder")
log_filename = os.path.join(
log_path,
'google-drive-to-s3-{}.log'.format(os.path.basename(time.strftime('%Y%m%d-%H%M%S')))
)
print(log_filename)
"""#test to create log file on disk
if not os.path.exists(log_filename):
print("create log file")
try:
# double check against OS race
filehandle = open(log_filename, 'r')
except IOError:
# if file does not exist, create it
filehandle = open(log_filename, "w")
"""
# register some logging handlers
log_handler = FileHandler(
log_filename,
mode='w',
level=args.loglevel,
bubble=True
)
return log_handler
except (Exception) as e:
log.error('Error in creating log handler' + str(e))
def main(config_file, **kwargs):
with open(config_file) as fh:
config = yaml.load(fh)
try:
rmq_settings = config["rabbitmq_logging"]
except KeyError:
print("RabbitMQ logging not configured in {}".format(config_file))
sys.exit()
handlers = [NullHandler()]
if not kwargs["quiet"]:
handlers.append(StderrHandler(bubble=True))
if kwargs["filename"]:
handlers.append(FileHandler(kwargs["filename"], bubble=True))
if kwargs["log_db"]:
try:
cdb_settings = config["couchdb_logging"]
except KeyError:
print("CouchDB logging not configured in {}".format(config_file))
sys.exit()
db_handler = DatabaseHandler(cdb_settings["couchdb_url"],
backend=CouchDBBackend,
db=cdb_settings["database"],
bubble=True)
handlers.append(db_handler)
setup = NestedSetup(handlers)
print("Now waiting for log messages")
with setup:
subscriber = RabbitMQSubscriber(rmq_settings["url"],
queue=rmq_settings["log_queue"])
try:
subscriber.dispatch_forever()
except KeyboardInterrupt:
print("\nLog subscriber shutting down")
subscriber.close()
except Exception:
print("Log subscriber quit (unexpectedly)")
def __init__(self,
name='default',
log_type='stdout',
filepath='default.log',
loglevel='DEBUG'):
"""
:param name:
:param log_type:stdout 输出到屏幕 file 输出到文件
:param filepath:
:param loglevel:设定log等级 ['CRITICAL', 'ERROR', 'WARNING', 'NOTICE', 'INFO', 'DEBUG', 'TRACE', 'NOTSET']
"""
self.log = Logger(name)
if log_type == 'stdout':
StreamHandler(sys.stdout, level=loglevel).push_application()
elif log_type == 'file':
if os.path.isdir(filepath) and not os.path.exists(filepath):
os.makedirs(os.path.dirname(filepath))
file_hander = FileHandler(filepath, level=loglevel)
self.log.handlers.append(file_hander)
def __init__(self,
name='default',
log_type='stdout',
filepath='default.log',
loglevel='DEBUG'):
"""Log对象
:param name: log 名字
:param :logtype: 'stdout' 输出到屏幕, 'file' 输出到指定文件
:param :filename: log 文件名
:param :loglevel: 设定log等级 ['CRITICAL', 'ERROR', 'WARNING', 'NOTICE', 'INFO', 'DEBUG', 'TRACE', 'NOTSET']
:return log handler object
"""
self.log = Logger(name)
if log_type == 'stdout':
StreamHandler(sys.stdout, level=loglevel).push_application()
if log_type == 'file':
if os.path.isdir(filepath) and not os.path.exists(filepath):
os.makedirs(os.path.dirname(filepath))
file_handler = FileHandler(filepath, level=loglevel)
self.log.handlers.append(file_handler)
def __init__(self, name='default', log_type='stdout', file_path=default_log_path, loglevel='DEBUG'):
"""Log对象
:param name: log 名字
:param :logtype: 'stdout' 输出到屏幕, 'file' 输出到指定文件
:param :filename: log 文件名
:param :loglevel: 设定log等级 ['CRITICAL', 'ERROR', 'WARNING', 'NOTICE', 'INFO', 'DEBUG', 'TRACE', 'NOTSET']
:param :dir_path: 文件夹路径
:return log handler object
"""
self.log = Logger(name)
if log_type == 'stdout':
StreamHandler(sys.stdout, level=loglevel).push_application()
if log_type == 'file':
date = time.strftime("%Y%m%d", time.localtime())
# 一天对应一个文件夹
file_path = "%s%s/" % (file_path, date)
file_path_final = "%s%s-%s.log" % (file_path, date, name)
print(file_path)
if file_path[-1] == '/' and not os.path.exists(file_path):
os.makedirs(os.path.dirname(file_path))
file_handler = FileHandler(file_path_final, level=loglevel)
self.log.handlers.append(file_handler)
class Application(object):
"""a base class for dispatching WSGI requests"""
def __init__(self, settings={}, prefix=""):
"""initialize the Application with a settings dictionary and an optional
``prefix`` if this is a sub application"""
self.settings = settings
self.mapper = routes.Mapper()
self.setup_handlers(self.mapper)
self.loghandler = FileHandler(self.logfilename)
def __call__(self, environ, start_response):
with self.loghandler.threadbound():
request = werkzeug.Request(environ)
m = self.mapper.match(environ = environ)
if m is not None:
handler = m['handler'](app=self, request=request, settings=self.settings)
try:
return handler.handle(**m)(environ, start_response)
except werkzeug.exceptions.HTTPException, e:
return e(environ, start_response)
# no view found => 404
return werkzeug.exceptions.NotFound()(environ, start_response)
p.oplossing = data
elif soort == 'event':
p.events.append((dt.datetime.today().isoformat(' ')[:19], data))
elif soort == 'statuscode':
p.status = data
elif soort == 'arch':
p.set_arch(data)
list(p)
if update:
p.write()
return p
if __name__ == "__main__":
fnm = "afrift"
log_handler = FileHandler('get_acties_sql_1.log', mode='w')
with log_handler.applicationbound():
test_get_acties(fnm, {}, "")
test_get_acties(fnm, {"idlt": "2010"}, "")
test_get_acties(fnm, {
"idlt": "2010",
"id": "and",
"idgt": "2007-0003"
}, "")
test_get_acties(fnm, {
"idgt": "2010",
"id": "or",
"idlt": "2007-0003"
}, "")
test_get_acties(fnm, {"idgt": "2007-0003"}, "")
test_get_acties(fnm, {"status": ["1"]}, "")
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import platform
import shutil
from uuid import uuid4
from time import strftime
import psutil
from types import NoneType
from logbook import FileHandler, Logger, CRITICAL
log = Logger("Browser")
if os.path.exists("application.log"):
log_handler = FileHandler('application.log')
log_handler.push_application()
else:
log.level = CRITICAL
class Browser(object):
def _expand(self, path):
log.debug("Expanding: {}".format(path))
if self.os == "linux" or self.os == "darwin":
return os.path.expanduser(path)
elif self.os == "windows" and platform.release() != "XP":
return os.path.expandvars(path)
else:
log.error("Unsupported OS: {} - expanding failed.".format(self.os))
if not os.path.exists(restore_path):
print "Backup file at {} could not be found.".format(restore_path)
return
backup_browser = os.path.basename(restore_path)[0].lower()
if backup_browser == browser.name:
browser.restore_from_self(restore_path)
else:
restore_browser = self.all_browsers[backup_browser]
restore_data = restore_browser.get_data(restore_path)
browser.set_data(restore_data)
if __name__ == '__main__':
if os.path.exists("application.log"):
log_handler = FileHandler('application.log')
log_handler.push_application()
else:
log.level = CRITICAL
parser = argparse.ArgumentParser(prog='RESTool Command Line Interface. '
'See github for usage examples.')
parser.add_argument(
'-l',
'--list',
action='store_true',
help=
"List all available browsers and profiles to use for other commands.")
parser.add_argument(
'-b',
# -*- coding: utf-8 -*-
from logbook import Logger, FileHandler
import logbook
import sys
handler = FileHandler('app.log')
handler.push_application()
log = Logger('test')
def main():
log.info('something logging')
if __name__ == '__main__':
main()
# ptvsd.wait_for_attach()
# I use cspell plugin for spell checking, and I want it to ignore few words
# cspell:ignore reican lzma isdir
# max file size in lines (10M by default)
MAX_LINES_TO_READ = 10000000
# max file size in megabytes
MAX_FILE_SIZE = "50M"
# where to write application log
LOG_FILE_NAME = "reican.log"
TIMESTAMP_FORMAT = "YYYY-MM-DD HH:mm:ss"
log_handler = FileHandler(LOG_FILE_NAME)
log_handler.push_application()
log = Logger("Reican")
log.info("Logging started")
def func_log(function_name):
"""Decorator for logging and timing function execution."""
def log_it(*args, **kwargs):
"""Log function and its args, execute the function and return the result."""
t_start = time.time()
result = function_name(*args, **kwargs)
t_end = time.time() - t_start
msg = "Function call: {}".format(function_name.__name__)
if args:
def main():
parser = argparse.ArgumentParser(description='Extract features')
parser.add_argument(
'-i',
'--input',
required=True,
help='Raw data input dir'
)
parser.add_argument(
'-o',
'--output',
required=True,
help='Output dir'
)
parser.add_argument(
'--filter',
default='lowpass',
help='Filtering Type'
)
parser.add_argument(
'--window',
type=int,
required=True,
help='Window length'
)
parser.add_argument(
'--stride',
type=int,
required=True,
help='Stride length'
)
parser.add_argument(
'-f',
'--featurelist',
nargs='+',
help='Features to extact',
required=True
)
parser.add_argument(
'--downsample',
type=int,
default=1,
help='Downsample step, default takes no downsample'
)
parser.add_argument(
'--log',
default='info',
choices=['debug', 'warning', 'info', 'error'],
help='Logging level, default info'
)
parser.add_argument(
'--dataset',
choices=['ninapro-db1', 'ninapro-db2', 'ninapro-db3', 'ninapro-db4',
'ninapro-db5', 'ninapro-db6', 'ninapro-db7', 'biopatrec-db1',
'biopatrec-db2', 'biopatrec-db3', 'biopatrec-db4'],
help='Dataset choices',
required=True
)
args = parser.parse_args()
with NullHandler().applicationbound():
with StderrHandler(level=args.log.upper()).applicationbound():
with FileHandler(
os.path.join(ensure_dir(args.output), 'log'),
level=args.log.upper(),
bubble=True
).applicationbound():
try:
return run(args)
except:
log.exception('Failed')
if __name__ == '__main__':
'''Usage: python tk_maintain.py --log hehe.log
'''
from logbook import FileHandler
from logbook import Logger
from argparse import ArgumentParser
import sys
parser = ArgumentParser()
logpath = './log/'
parser.add_argument('--log', nargs=1, help='log path')
parser.add_argument('--version', nargs=1, help='maintain version')
args = parser.parse_args(sys.argv[1:])
logfilepath = logpath + args.log[0]
maintain_version = args.version[0]
log_handler = FileHandler(logfilepath)
logbk = Logger('Token Maintain')
with log_handler.applicationbound():
logbk.info('maintain prepare')
at_least = AT_LEAST_TOKEN_COUNT
max_tokens_redis_limit = MAX_TOKENS_IN_REDIS
logbk.info('maintain begin')
# 认证新用户，并将access_token加入mongodb，redis从mongodb导入新token，不重置已有token 的 req_count
if maintain_version == 'addatoken':
print 'generate new token, write to mongo, push to redis without reset request count'
generate_api_access_token(logbk)
add_without_reset_req_count(max_tokens_redis_limit, logbk)
import requests
import time
import json
import zmq
from multiprocessing import Process
from logbook import FileHandler, catch_exceptions
from sachintweets.models import connect, MongoException
from twitter import username, password
from os import remove, getpid
####################################### Log book setup #########################
log_handler = FileHandler('recv.log')
log_handler.push_application()
###################################### Constants ###############################
LOCK_FILE = 'recv.lock'
###################################### Mongodb connection ######################
try:
db = connect()
if db:
tweet = db.tweet
except Exception as e:
log_handler.write(e.message)
#################################### ZERO MQ PULLER ############################
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect("tcp://*:6789")
socket.setsockopt(zmq.SUBSCRIBE, "")
# - * - coding: utf-8 - * -
import logbook
import sys
from logbook import Logger, FileHandler, StreamHandler
# 日志文件
logFile = "rk-bd.log"
# 设置UTC时间格式为本地时间格式
logbook.set_datetime_format("local")
# 日志输出到标准输出设备，应用整个程序范围
StreamHandler(
sys.stdout,
level=logbook.DEBUG,
format_string=
"[{record.time:%Y-%m-%d %H:%M:%S}] {record.level_name}: {record.filename}:{record.lineno} {record.message}",
encoding="utf-8").push_application()
# 日志输出到文件，应用整个程序范围
FileHandler(
logFile,
level=logbook.INFO,
format_string=
"[{record.time:%Y-%m-%d %H:%M:%S}] {record.level_name}: {record.filename}:{record.lineno} {record.message}",
encoding="utf-8",
bubble=True).push_application()
log = Logger(__name__)
file_handle = kwargs.get("file_handle")
logger.debug(
f"Unlocking the file {file_name} on host {host}, owner={owner}, client={client_name},"
f" kwargs = {kwargs}")
file_handle = self._get_file_handle(
host, export, file_name) if not file_handle else file_handle
nlm_client = NLMClient(host)
unlock_arguments = get_packer_arguments("UNLOCK",
caller_name=client_name,
owner=owner,
fh=file_handle,
l_offset=offset,
l_len=length)
status = nlm_client.unlock(unlock_arguments)
return NLM4_Stats(status).name
def _get_file_handle(self, host, export, file_name):
file_handle = self.exposed_lookup_file(host, export, file_name)
if not file_handle:
raise FileNotFound(
f"{file_name} cannot be found and file_handle was not specified"
)
return file_handle
if __name__ == "__main__":
FileHandler("nfs_client.log").push_application()
t = ThreadedServer(NFSClientWrapper, port=9999)
t.daemon = True
logger.notice("Starting server on port 9999")
t.start()
servers = [Server(i) for i in range(1, NUM_SERVERS + 1)]
start_servers(servers)
time.sleep(10)
try:
return test(servers)
except Exception, e:
logger.exception('Test failed: %s' % e)
return 1
finally:
logger.info('Stopping')
stop(servers)
if __name__ == '__main__':
format = '[{record.time}] {record.level_name:>5} [{record.extra[worker_id]}] {record.message}'
logging_setup = NestedSetup([
NullHandler(),
FileHandler(
filename=os.path.join(os.path.dirname(__file__), 'log/client.log'),
format_string=format,
bubble=True,
),
StderrHandler(level=logbook.INFO, format_string=format, bubble=True),
])
with logging_setup.applicationbound():
sys.exit(main())
for k,v in headers.iteritems():
self.req.add_header(k,v)
def set_req(self):
self.req = urllib2.Request(self.url, urllib.urlencode(self.data))
#self.req = urllib2.Request(self.url)
def send(self):
self.set_req()
return urllib2.urlopen(self.req)
if __name__ == "__main__":
logger = Logger("TicketchangeToInfluxdb")
logfile = "ticketchangetoinfluxdb.log"
fh = FileHandler(logfile,"a")
fh.applicationbound()
fh.push_application()
client = Client()
client.test()
adapter = Adapter()
client.set_adapter(adapter)
a =client.get_adapter()
a.test()
print("This is just a test.")
logger.info("Testing logging.")
def init(log_path):
global logger
log_hander = FileHandler(log_path)
log_hander.formatter = log_type
log_hander.push_application()
logger = Logger('simulate_tcm')
collection = Collection(url_api_collection)
except Exception, e:
msg = "Exception in Collection {}, init {}".format(url_api_collection, str(e))
logbook.error(msg)
raise e
if not (collection["harvest_type"] in HARVEST_TYPES):
msg = "Collection {} wrong type {} for harvesting. Harvest type {} \
is not in {}".format(
url_api_collection, collection["harvest_type"], collection["harvest_type"], HARVEST_TYPES.keys()
)
logbook.error(msg)
raise ValueError(msg)
mail_handler.subject = "Error during harvest of " + collection.url
my_log_handler = None
if not log_handler: # can't init until have collection
my_log_handler = FileHandler(get_log_file_path(collection.slug))
my_log_handler.push_application()
logger = logbook.Logger("HarvestMain")
msg = "Init harvester next. Collection:{}".format(collection.url)
logger.info(msg)
# email directly
mimetext = create_mimetext_msg(
EMAIL_RETURN_ADDRESS, user_email, " ".join(("Starting harvest for ", collection.slug)), msg
)
try: # TODO: request more emails from AWS
mail_handler.deliver(mimetext, "*****@*****.**")
except:
pass
logger.info("Create DPLA profile document")
if not profile_path:
profile_path = os.path.abspath(os.path.join(dir_profile, collection.id + ".pjs"))
import requests
import time
import json
import zmq
from logbook import FileHandler, catch_exceptions
from sachintweets.models import connect, MongoException
####################################### Log book setup #########################
log_handler = FileHandler('fetch_user_details.log')
log_handler.push_application()
###################################### Constants ###############################
LOCK_FILE = 'recv.lock'
###################################### Mongodb connection ######################
try:
db = connect()
if db:
tweet = db.tweet
except Exception as e:
log_handler.write(e.message)
def fetch_user_details():
tweets = tweet.find({})
for t in tweets:
if not 'screen_name' in t:
r = requests.get("http://api.twitter.com/1/statuses/show/%d.json"\
%(t['tid']))
d = json.loads(r.content)
if 'user' in d:
dh = User(usernaam, password, flag)
if not flag:
if dh.paswdok:
logger.info(' {}'.format(dh.__dict__))
else:
logger.info(" password not ok")
else:
if dh.session_ok:
logger.info(' {}'.format(dh.__dict__))
else:
logger.info(" session not ok")
return dh
if __name__ == '__main__':
log_handler = FileHandler('test_user.log', mode="w")
with log_handler.applicationbound():
test_userlijst()
test_user('woefdram')
test_user('vader', "begin")
test_user('vader', "begin", False)
test_user('vader', "20015533280430", True)
usernaam, passwd = 'leerkracht', 'begin'
test_user(usernaam)
dh = test_user(usernaam, passwd)
login = dh.login
test_user(usernaam, passwd, False)
test_user(usernaam, "20015533280430", True)
test_user(usernaam, dh.login, True)
dh = test_user(usernaam, passwd)
class Fibratus():
"""Fibratus entrypoint.
Setup the core components including the kernel
event stream collector and the tracing controller.
At this point the system handles are also being
enumerated.
"""
def __init__(self, filament):
self.logger = Logger(Fibratus.__name__)
self.file_handler = FileHandler(os.path.join(os.path.abspath(__file__), '..', '..', '..', 'fibratus.log'),
mode='w+')
self.kevt_streamc = KEventStreamCollector(etw.KERNEL_LOGGER_NAME.encode())
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags()
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of kernel trace
with self.file_handler.applicationbound():
self.logger.info('Starting fibratus...')
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
self.thread_registry = ThreadRegistry(self.handle_repository, self._handles)
self.kevent = KEvent(self.thread_registry)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.requires_render = {}
self.filters_count = 0
def run(self):
@atexit.register
def _exit():
self.stop_ktrace()
self.kcontroller.start_ktrace(etw.KERNEL_LOGGER_NAME, self.ktrace_props)
def on_kstream_open():
if self._filament is None:
IO.write_console('Done! ')
self.kevt_streamc.set_kstream_open_callback(on_kstream_open)
self._open_kstream()
def _open_kstream(self):
try:
self.kevt_streamc.open_kstream(self._on_next_kevent)
except Exception as e:
with self.file_handler.applicationbound():
self.logger.error(e)
except KeyboardInterrupt:
self.stop_ktrace()
def stop_ktrace(self):
IO.write_console('Stopping fibratus...')
if self._filament:
self._filament.close()
self.kcontroller.stop_ktrace(self.ktrace_props)
self.kevt_streamc.close_kstream()
def add_filters(self, kevent_filters):
if len(kevent_filters) > 0:
self.filters_count = len(kevent_filters)
# include the basic filters
# that are essential to the
# rest of kernel events
self.kevt_streamc.add_kevent_filter(ENUM_PROCESS)
self.kevt_streamc.add_kevent_filter(ENUM_THREAD)
self.kevt_streamc.add_kevent_filter(ENUM_IMAGE)
self.kevt_streamc.add_kevent_filter(REG_CREATE_KCB)
self.kevt_streamc.add_kevent_filter(REG_DELETE_KCB)
# these kevents are necessary for consistent state
# of the trace. If the user doesn't include them
# in a filter list, then we do the job but set the
# kernel event type as not eligible for rendering
if not KEvents.CREATE_PROCESS in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_PROCESS)
self.requires_render[CREATE_PROCESS] = False
else:
self.requires_render[CREATE_PROCESS] = True
if not KEvents.CREATE_THREAD in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_THREAD)
self.requires_render[CREATE_THREAD] = False
else:
self.requires_render[CREATE_THREAD] = True
if not KEvents.CREATE_FILE in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_FILE)
self.requires_render[CREATE_FILE] = False
else:
self.requires_render[CREATE_FILE] = True
for kevent_filter in kevent_filters:
ktuple = kname_to_tuple(kevent_filter)
if isinstance(ktuple, list):
for kt in ktuple:
self.kevt_streamc.add_kevent_filter(kt)
if not kt in self.requires_render:
self.requires_render[kt] = True
else:
self.kevt_streamc.add_kevent_filter(ktuple)
if not ktuple in self.requires_render:
self.requires_render[ktuple] = True
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS,
CREATE_THREAD,
ENUM_PROCESS,
ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
self._render(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
self._render(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [CREATE_FILE,
DELETE_FILE,
CLOSE_FILE,
READ_FILE,
WRITE_FILE]:
self.fsio.parse_fsio(ktype, kparams)
self._render(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._render(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._render(ktype)
# registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [REG_CREATE_KEY,
REG_DELETE_KEY,
REG_OPEN_KEY,
REG_QUERY_KEY,
REG_SET_VALUE,
REG_DELETE_VALUE,
REG_QUERY_VALUE]:
self.hive_parser.parse_hive(ktype, kparams)
self._render(ktype)
# network kernel events
elif ktype in [SEND_SOCKET_TCPV4,
SEND_SOCKET_UDPV4,
RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4,
ACCEPT_SOCKET_TCPV4,
CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4,
RECONNECT_SOCKET_TCPV4]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._render(ktype)
if self._filament:
# call filament method
# to process the next
# kernel event from the stream
if ktype not in [ENUM_PROCESS,
ENUM_THREAD, ENUM_IMAGE]:
if self.kevent.name:
self._filament.process(self.kevent)
def _render(self, ktype):
"""Renders the kevent to the standard output stream.
Parameters
----------
ktype: tuple
Identifier of the kernel event
"""
if not self._filament:
if ktype in self.requires_render:
rr = self.requires_render[ktype]
if rr:
self.kevent.render()
elif self.filters_count == 0:
self.kevent.render()
class Qdb(Bdb, object):
"""
The Quantopian Remote Debugger.
"""
_instance = None
def __new__(cls, *args, **kwargs):
"""
Qdb objects are singletons that persist until their disable method is
called.
"""
if not cls._instance:
cls._instance = super(Qdb, cls).__new__(cls)
cls._instance._init(*args, **kwargs)
return cls._instance
def __init__(self, *args, **kwargs):
pass
def _init(self, config=None, merge=False, **kwargs):
"""
See qdb.config for more information about the configuration of
qdb.
merge denotes how config and kwargs should be merged.
QdbConfig.kwargs_first says config will trample kwargs,
QdbConfig.config_first says kwargs will trample config.
Otherwise, kwargs and config cannot both be passed.
"""
self.super_ = super(Qdb, self)
self.super_.__init__()
self.reset()
if config and kwargs:
if merge == QdbConfig.kwargs_first:
first = kwargs
second = config
elif merge == QdbConfig.config_first:
first = config
second = kwargs
else:
raise TypeError('Cannot pass config and kwargs')
config = first.merge(second)
else:
config = QdbConfig.get_config(config or kwargs)
self.address = config.host, config.port
self.set_default_file(config.default_file)
self.default_namespace = config.default_namespace or {}
self.exception_serializer = config.exception_serializer or \
default_exception_serializer
self.eval_fn = config.eval_fn or default_eval_fn
self.green = config.green
self._file_cache = {}
self.redirect_output = config.redirect_output
self.retry_attepts = config.retry_attepts
self.repr_fn = config.repr_fn
self._skip_fn = config.skip_fn or (lambda _: False)
self.pause_signal = config.pause_signal \
if config.pause_signal else signal.SIGUSR2
self.uuid = str(config.uuid or uuid4())
self.watchlist = {}
self.execution_timeout = config.execution_timeout
self.reset()
self.log_handler = None
if config.log_file:
self.log_handler = FileHandler(config.log_file)
self.log_handler.push_application()
# The timing between these lines might matter depending on the
# cmd_manager. Don't seperate them.
self.cmd_manager = (config.cmd_manager or RemoteCommandManager)(self)
self.cmd_manager.start(config.auth_msg)
# We need to be able to send stdout back to the user debugging the
# program. We hold a handle to this in case the program resets stdout.
if self.redirect_output:
self._old_stdout = sys.stdout
self._old_stderr = sys.stderr
sys.stdout = OutputTee(
sys.stdout,
RemoteOutput(self.cmd_manager, '<stdout>'),
)
sys.stderr = OutputTee(
sys.stderr,
RemoteOutput(self.cmd_manager, '<stderr>'),
)
def skip_fn(self, path):
return self._skip_fn(self.canonic(path))
def restore_output_streams(self):
"""
Restores the original output streams.
"""
if self.redirect_output:
sys.stdout = self._old_stdout
sys.stderr = self._old_stderr
def _new_execution_timeout(self, src):
"""
Return a new execution timeout context manager.
If not execution timeout is in place, returns ExitStack()
"""
# We use green=False because this could be cpu bound. This will
# still throw to the proper greenlet if this is gevented.
return (
Timeout(
self.execution_timeout,
QdbExecutionTimeout(src, self.execution_timeout),
green=False
) if self.execution_timeout else ExitStack()
)
def set_default_file(self, filename):
"""
Safely sets the new default file.
"""
self.default_file = self.canonic(filename) if filename else None
def get_line(self, filename, line):
"""
Checks for any user cached files before deferring to the linecache.
"""
# The line - 1 is so that querying line 1 gives us the first line in
# the file.
try:
return self._get_file_lines(filename)[line - 1]
except IndexError:
return 'No source available for this line.'
def get_file(self, filename):
"""
Retrieves a file out of cache or opens and caches it.
"""
return '\n'.join(self._get_file_lines(filename))
def _get_file_lines(self, filename):
"""
Retrieves the file from the file cache as a list of lines.
If the file does not exist in the cache, it is cached from
disk.
"""
canonic_name = self.canonic(filename)
try:
return self._file_cache[canonic_name]
except KeyError:
if not self.cache_file(canonic_name):
return []
return self._file_cache.get(canonic_name)
def cache_file(self, filename, contents=None):
"""
Caches filename from disk into memory.
This overrides whatever was cached for filename previously.
If contents is provided, it allows the user to cache a filename to a
string.
Returns True if the file caching succeeded, otherwise returns false.
"""
canonic_name = self.canonic(filename)
if contents:
self._file_cache[canonic_name] = contents.splitlines()
return True
try:
with open(canonic_name, 'r') as f:
self._file_cache[canonic_name] = map(
lambda l: l[:-1] if l.endswith('\n') else l,
f.readlines()
)
return True
except IOError:
# The caching operation failed.
return False
def set_break(self, filename, lineno, temporary=False, cond=None,
funcname=None, **kwargs):
"""
Sets a breakpoint. This is overridden to account for the filecache
and for unreachable lines.
**kwargs are ignored. This is to work with payloads that pass extra
fields to the set_break payload.
"""
filename = self.canonic(filename) if filename else self.default_file
try:
self.get_line(filename, lineno)
except IndexError:
raise QdbUnreachableBreakpoint({
'file': filename,
'line': lineno,
'temp': temporary,
'cond': cond,
'func': funcname,
})
blist = self.breaks.setdefault(filename, [])
if lineno not in blist:
blist.append(lineno)
Breakpoint(filename, lineno, temporary, cond, funcname)
def clear_break(self, filename, lineno, *args, **kwargs):
"""
Wrapper to make the breakpoint json standardized for setting
and removing of breakpoints.
This means that the same json data that was used to set a break point
may be fed into this function with the extra values ignored.
"""
self.super_.clear_break(filename, lineno)
def canonic(self, filename):
canonic_filename = self.super_.canonic(filename)
if canonic_filename.endswith('pyc'):
return canonic_filename[:-1]
return canonic_filename
def reset(self):
self.botframe = None
self._set_stopinfo(None, None)
self.forget()
def forget(self):
self.lineno = None
self.stack = []
self.curindex = 0
self.curframe = None
def setup_stack(self, stackframe, traceback):
"""
Sets up the state of the debugger object for this frame.
"""
self.forget()
self.stack, self.curindex = self.get_stack(stackframe, traceback)
self.curframe = self.stack[self.curindex][0]
self.curframe_locals = self.curframe.f_locals
self.update_watchlist()
def extend_watchlist(self, *args):
"""
Adds every arg to the watchlist and updates.
"""
for expr in args:
self.watchlist[expr] = (False, '')
self.update_watchlist()
def update_watchlist(self):
"""
Updates the watchlist by evaluating all the watched expressions in
our current frame.
"""
id_ = lambda n: n # Why is this NOT a builtin?
for expr in self.watchlist:
try:
with self._new_execution_timeout(expr), \
self.inject_default_namespace() as stackframe:
self.watchlist[expr] = (
None,
(self.repr_fn or id_)(
self.eval_fn(expr, stackframe)
)
)
except Exception as e:
self.watchlist[expr] = (
type(e).__name__,
self.exception_serializer(e)
)
def effective(self, file, line, stackframe):
"""
Finds the effective breakpoint for this line; called only
when we know that there is a breakpoint here.
returns the breakpoint paired with a flag denoting if we should
remove this breakpoint or not.
"""
for breakpoint in Breakpoint.bplist[file, line]:
if breakpoint.enabled == 0:
continue
if not checkfuncname(breakpoint, stackframe):
continue
# Count every hit when breakpoint is enabled
breakpoint.hits = breakpoint.hits + 1
if not breakpoint.cond:
# If unconditional, and ignoring go on to next, else break
if breakpoint.ignore > 0:
breakpoint.ignore = breakpoint.ignore - 1
continue
else:
return breakpoint, True
else:
# Conditional breakpoint
# Ignore count applies only to those bpt hits where the
# condition evaluates to true.
try:
with self._new_execution_timeout(breakpoint.cond), \
self.inject_default_namespace(stackframe) as frame:
val = self.eval_fn(
breakpoint.cond,
frame,
'eval'
)
except Exception as e:
# Send back a message to let the user know there was an
# issue with their breakpoint.
self.cmd_manager.send_error(
'condition', {
'cond': breakpoint.cond,
'line': line,
'exc': type(e).__name__,
'output': self.exception_serializer(e),
}
)
# Return this breakpoint to be safe. The user will be
# stopped here so that they can fix the breakpoint.
return breakpoint, False
if val:
if breakpoint.ignore > 0:
breakpoint.ignore = breakpoint.ignore - 1
else:
return breakpoint, True
return None, False
def break_here(self, stackframe):
"""
Checks if we should break execution in this stackframe.
This function handles the cleanup and ignore counts for breakpoints.
Returns True iff we should stop in the stackframe, False otherwise.
"""
filename = self.canonic(stackframe.f_code.co_filename)
if filename not in self.breaks:
return False
lineno = stackframe.f_lineno
if lineno not in self.breaks[filename]:
# The line itself has no breakpoint, but maybe the line is the
# first line of a function with breakpoint set by function name.
lineno = stackframe.f_code.co_firstlineno
if lineno not in self.breaks[filename]:
return False
# flag says ok to delete temporary breakpoints.
breakpoint, flag = self.effective(filename, lineno, stackframe)
if breakpoint:
self.currentbp = breakpoint.number
if flag and breakpoint.temporary:
self.do_clear(breakpoint.number)
return True
else:
return False
def trace_dispatch(self, stackframe, event, arg):
"""
Trace function that does some preliminary checks and then defers to
the event handler for each type of event.
"""
if self.quitting:
# We were told to quit by the user, bubble this up to their code.
return
if self.skip_fn(stackframe.f_code.co_filename):
# We want to skip this, don't stop but keep tracing.
return self.trace_dispatch
try:
return self.super_.trace_dispatch(stackframe, event, arg)
except BdbQuit:
raise QdbQuit() # Rewrap as a QdbError object.
def user_call(self, stackframe, arg):
if self.break_here(stackframe):
self.user_line(stackframe)
def user_line(self, stackframe):
self.setup_stack(stackframe, None)
self.cmd_manager.send_watchlist()
self.cmd_manager.send_stack()
self.cmd_manager.next_command()
def user_return(self, stackframe, return_value):
stackframe.f_locals['__return__'] = return_value
self.setup_stack(stackframe, None)
self.cmd_manager.send_watchlist()
self.cmd_manager.send_stack()
msg = fmt_msg('return', str(return_value), serial=pickle.dumps)
self.cmd_manager.next_command(msg)
def user_exception(self, stackframe, exc_info):
exc_type, exc_value, exc_traceback = exc_info
stackframe.f_locals['__exception__'] = exc_type, exc_value
self.setup_stack(stackframe, exc_traceback)
self.cmd_manager.send_watchlist()
self.cmd_manager.send_stack()
msg = fmt_msg(
'exception', {
'type': exc_type.__name__,
'value': str(exc_value),
'traceback': traceback.format_tb(exc_traceback)
},
serial=pickle.dumps,
)
self.cmd_manager.next_command(msg)
def do_clear(self, bpnum):
"""
Handles deletion of temporary breakpoints.
"""
if not (0 <= bpnum < len(Breakpoint.bpbynumber)):
return
self.clear_bpbynumber(bpnum)
def set_quit(self):
"""
Sets the quitting state and restores the program state.
"""
self.quitting = True
def disable(self, mode='soft'):
"""
Stops tracing.
"""
try:
if mode == 'soft':
self.clear_all_breaks()
self.set_continue()
# Remove this instance so that new ones may be created.
self.__class__._instance = None
elif mode == 'hard':
sys.exit(1)
else:
raise ValueError("mode must be 'hard' or 'soft'")
finally:
self.restore_output_streams()
if self.log_handler:
self.log_handler.pop_application()
self.cmd_manager.stop()
if sys.gettrace() is self.trace_dispatch:
sys.settrace(None)
def __enter__(self):
self.set_trace(sys._getframe().f_back, stop=False)
return self
def __exit__(self, type, value, traceback):
self.disable('soft')
def set_trace(self, stackframe=None, stop=True):
"""
Starts debugging in stackframe or in the callers frame.
If stop is True, begin stepping from here, otherwise, wait for
the first breakpoint or exception.
"""
# We need to look back 1 frame to get our caller.
stackframe = stackframe or sys._getframe().f_back
self.reset()
while stackframe:
stackframe.f_trace = self.trace_dispatch
self.botframe = stackframe
stackframe = stackframe.f_back
if stop:
self.set_step()
else:
self.set_continue()
sys.settrace(self.trace_dispatch)
@contextmanager
def inject_default_namespace(self, stackframe=None):
"""
Adds the default namespace to the frame, or if no frame is provided,
self.curframe is used.
"""
stackframe = stackframe or self.curframe
to_remove = set()
for k, v in self.default_namespace.iteritems():
if k not in stackframe.f_globals:
# Only add the default things if the name is unbound.
stackframe.f_globals[k] = v
to_remove.add(k)
try:
yield stackframe
finally:
for k in to_remove:
try:
del stackframe.f_globals[k]
except IndexError:
# The body of this manager might have del'd this.
pass
# Prevent exceptions from generating ref cycles.
del stackframe
class Pipeline(object):
"""Represents the abstraction of a pipeline of jobs to be run
distributed over machines
"""
def __init__(self, workdir, jobs, total_cores, scheduler=None, queue=None,
local=False, retries=None):
"""Initialize a pipeline.
:param workdir: Name of a directory to use for scratch space
and results. This needs to be visible to all nodes over NFS or
similar.
:param jobs: A list of jobs, which are just dicts. The only
required key for now is "description", which will be used for
the directory that holds all this job's output.
:param total_cores: The total number of cores you want to use
for processing.
:returns: A Pipeline object, which has methods that invoke
various kinds of distributed work.
"""
# validate things
for job in jobs:
if type(job) is not dict:
raise ValueError("job is not a dict: {}".format(job))
if not job.get("description"):
raise ValueError("job {} has not description".format(job))
workdir = os.path.abspath(os.path.expanduser(workdir))
if not os.path.exists(workdir):
raise ValueError(
"workdir: {} appears not to exist".format(workdir))
self.workdir = workdir
self.jobs = jobs
self.total_cores = total_cores
self.scheduler = scheduler
self.queue = queue
self.local = local
self.retries = retries
# setup default cluster_view
self._cluster_view = cluster_view
def start(self):
"""Initialize workdir, logging, etc. in preparation for running jobs.
"""
# make a working directory for each job
for job in self.jobs:
job["workdir"] = os.path.join(self.workdir, job["description"])
fs.maybe_mkdir(job["workdir"])
# temporary ipython profile directory
self.ipythondir = os.path.join(self.workdir, ".ipython")
fs.maybe_mkdir(self.ipythondir)
# log dir
self.logdir = os.path.join(self.workdir, "log")
fs.maybe_mkdir(self.logdir)
# determine which IP we are going to listen on for logging
try:
self.listen_ip = localinterfaces.public_ips()[0]
except:
raise ValueError("This machine appears not to have"
" any publicly visible IP addresses")
# setup ZMQ logging
self.handler = FileHandler(os.path.join(self.logdir, "dish.log"))
self.listen_port = str(randint(5000, 10000))
self.subscriber = ZeroMQPullSubscriber("tcp://" + self.listen_ip +
":" + self.listen_port)
self.controller = self.subscriber.dispatch_in_background(self.handler)
self.logger = Logger("dish_master")
def stop(self):
"""Gracefully shutdown the Pipeline, cleaning up threads, sockets,
etc. Leaves working directory intact so everything can in
principle be picked up again where we left off.
"""
self.controller.stop()
self.subscriber.close()
def _compute_resources(self, cores_per_engine, mem_per_engine,
max_engines):
if cores_per_engine > self.total_cores:
raise ValueError("A job requested {0} but only {1}"
" are available.".format(cores_per_engine,
self.total_cores))
num_engines = self.total_cores // cores_per_engine
if len(self.jobs) < num_engines:
# we don't even need this many engines
num_engines = self.jobs
if max_engines:
num_engines = min(num_engines, max_engines)
# TODO in the future, should maybe validate that requested
# cores and memory are actually going to be availible. This
# would unfortunately have to be specialized for each
# scheduler probably.
return num_engines, cores_per_engine, mem_per_engine
@contextmanager
def group(self, cores=1, mem="0.1", max=None):
"""Context manager for "grouping" a set of pipeline operations. A
group of operations is run on the same ipython cluster and has
it's resources specified in the group as opposed to in each
individual job. This is useful if there is some small amount
of setup work that isn't worth spinning up a new cluster for
but which needs to be done before a resource intensive task.
For example::
with p.group(cores=8, mem=12):
p.run("setup.sh . . .") # do some data munging or other setup
p.run("main_work -n 8 . . .") # call an expensive program
"""
# TODO this duplicates some code from p.map and is a bit
# clunky, there is probably a better abstraction here
engines, cores, mem = self._compute_resources(cores, mem, max)
extra_params = {"run_local": self.local,
"mem": mem}
old_view_factory = self._cluster_view
cm = self._cluster_view(self.scheduler, self.queue,
engines, profile=self.ipythondir,
cores_per_job=cores,
extra_params=extra_params,
retries=self.retries)
view = cm.gen.next()
@contextmanager
def reuse_view(*args, **kwargs):
yield view
# everything done in the block will use the view we just made
self._cluster_view = reuse_view
try:
yield
finally:
# restore the normal cluster_view context manager on exit
self._cluster_view = old_view_factory
try:
cm.gen.next() # clean up the view we've been using
except StopIteration:
pass
def _transaction_filter(self, targets):
"""Filter the `jobs` appropriately based on whether `targets` is a
function, str, or list of str"""
# TODO there has got to be a better way to do this -____-
to_run = []
dont_run = []
if callable(targets):
f = targets
for job in self.jobs:
if f(job):
dont_run.append(job)
else:
to_run.append(job)
return to_run, dont_run
elif isinstance(targets, str):
targets = [targets]
elif not isinstance(targets, list):
TypeError("transaction targets must be list, str, or callable")
for job in self.jobs:
canonical_targets = fs.canonicalize(job, targets)
if all((os.path.exists(target)
for target in canonical_targets)):
info = ("Skipping transaction for job {} targets {} "
"already present")
with self.handler.applicationbound():
self.logger.info(info.format(job["description"],
canonical_targets))
dont_run.append(job)
else:
# targets not present for this job
to_run.append(job)
return to_run, dont_run
@contextmanager
def transaction(self, targets):
"""Do some work "transacationally", in the sense that nothing done
inside a ``transaction`` block will be "commited" to the
workdir unless it all succeeds without error. The work done
inside a transaction is also idempotent in that you must
specify a ``target`` file or files for the tranasaction and it
will not be run if the target exists already. This is perhaps
best illustrated by a simple example::
with p.transaction("{workdir}/example.txt"):
p.run("{tmpdir}/touch example.txt")
This will result in a file ``B.txt`` in each job's
``workdir``. The creation of this file will be skipped if the
code is run again and the file already exists. This is
obviously a silly example, but the code inside the `with`
block can be any arbitrarily complex series of operations
which produces a set of target output files at the end. This
is a powerful feature in that it allows pipelines to be
restratable: if a pipeline crashes for some reason but you
have it's major sections wrapped in ``transaction`` blocks,
you can simple run it again and pick up where you left off
without redoing any work. The transaction blocks guarentee
that the ``workdir`` for each job is never in an inconsistent
state and that work that's already been completed isn't
redone.
Inside a transaction, each job has a special ``tmpdir`` key,
whose value is the path to a unique temporary directory for
the job. You can do work that produces files inside the
``tmpdir`` and expect everything in it to be moved to the
job's ``workdir`` if the transaction compeltes without error.
The ``tmpdir`` will be removed at the end of the transaction
regardless of whether or not it succeeds. We change
directories to the ``tmpdir`` before doing anything else and
implicitly consider targets to be relative to a job's
``workdir`` so the above example could also be written
written::
with p.transaction("example.txt"):
p.run("touch example.txt")
which sacrifices explicitness for brevity.
:param targets: a string or list of strings descsribing files
that must exist in order for the transaction to be skipped.
"""
to_run, dont_run = self._transaction_filter(targets)
for job in to_run:
job["tmpdir"] = tempfile.mkdtemp(dir=job["workdir"])
self.jobs = to_run
try:
yield
finally:
for job in self.jobs:
if not os.path.exists(os.path.join(job["tmpdir"], ".error")):
fs.liftdir(job["tmpdir"], job["workdir"])
shutil.rmtree(job["tmpdir"])
del job["tmpdir"]
self.jobs = dont_run + self.jobs
def localmap(self, f):
"""Just like ``map``, but work locally rather than launching an ipython
cluster. This is useful for tasks where the cluster launch
overhead would swamp the cost of the actual work to be done.
:params f: function of ``(job, logger)`` to be mapped over all jobs.
"""
self.jobs = map(logging_wrapper, self.jobs,
(f for j in self.jobs),
(self.listen_ip for j in self.jobs),
(self.listen_port for j in self.jobs))
def map(self, f, cores=1, mem="0.1", max=None):
"""Map the function ``f`` over all of the ``jobs`` in this
pipeline. ``f`` must be a function of two arguments, the job
and a logger. It should modify the job it is passed, which
will then be returned over the wire. A silly example::
def f(job, logger):
job["capitalized_description"] = job["description"].toupper()
p.map(f)
Will give each ``job`` in the pipeline a ``capitalized_description``
attribute, which can then be used in future pipline operations.
``cores`` and ``mem`` are used to specify the cores and memory
required by this step; they will be passed to the underlying
scheduler. ``max`` can be used as a hard limit on the number of
jobs to run. This is useful if, for example, a particular task
puts pressure on some sort of storage system (a distributed
file system, object store, etc.) that you know will fail under
too much load.
:param f: function of ``(job, logger)`` to be mapped over all jobs.
:param cores: cores required by this call.
:param mem: memory required by this call.
:param max: maximum number of jobs to submit.
"""
if not self.jobs:
# this looks very odd, it's necessary because sometimes
# being a transaction causes self.jobs to be empty, and
# IPython throws errors if you try to make over the empty
# list. It might be cleaner to catch the error after
# letting IPython do the map; will have to think about it.
return
engines, cores, mem = self._compute_resources(cores, mem, max)
extra_params = {"run_local": self.local,
"mem": mem}
with self._cluster_view(self.scheduler, self.queue,
engines, profile=self.ipythondir,
cores_per_job=cores,
extra_params=extra_params,
retries=self.retries) as view:
# using cloudpickle allows us to serialize all sorts of things
# we wouldn't otherwise be able to
dview = view.client.direct_view()
use_cloudpickle()
dview.apply(use_cloudpickle)
self.jobs = view.map_sync(logging_wrapper, self.jobs,
(f for j in self.jobs),
(self.listen_ip for j in self.jobs),
(self.listen_port for j in self.jobs))
def run(self, template, capture_in=None, **kwargs):
"""Run the ``template`` formatted with the contents of each
job. Example::
p.run("touch {workdir}/example.txt")
will make an example.txt file in each job's workdir.
``cores`` and ``mem`` mean the same thing they do in the
``map`` method.
If a string is passed for ``capture_in``, the stdout of the command
will be captured in ``job[capture_in]`` for each job.
"""
runner = cmdrunner(template, capture_in)
self.map(runner, **kwargs)
import sys
import time
import calendar as cal
from io import BytesIO
from datetime import datetime
import requests
from requests_html import HTMLSession
import pandas as pd
from logbook import Logger, FileHandler, StreamHandler
stream_handler = StreamHandler(
sys.stdout,
format_string=" | {record.message}", bubble=True)
file_handler = FileHandler(
'expiration_downloader_errors_{}.log'.format(datetime.today().strftime("%Y-%m-%d_%H-%M")),
format_string=" | {record.message}", bubble=True, delay=True)
log = Logger(__name__)
#stream_handler.push_application()
#file_handler.push_application()
class ExpirationDownloader:
"""
Download contract expiry dates from CME website.
Use urls from quandl provided meta data.
Parameters:
df: dataframe read from csv file downloaded from quandl
download: False - use file from disk, True - download file form CME
show_progress: zipline variable to be passed by caller (or not)
def start_logging(filename):
this_file = os.path.basename(filename)
log_file = '/var/log/' + remove_extn(this_file) + '.log'
log_handler = FileHandler(log_file, bubble=True)
log_handler.push_application()
def main(user_email,
url_api_collection,
log_handler=None,
mail_handler=None,
dir_profile='profiles',
profile_path=None,
config_file=None,
**kwargs):
'''Executes a harvest with given parameters.
Returns the ingest_doc_id, directory harvest saved to and number of
records.
'''
if not config_file:
config_file = os.environ.get('DPLA_CONFIG_FILE', 'akara.ini')
num_recs = -1
my_mail_handler = None
if not mail_handler:
my_mail_handler = logbook.MailHandler(
EMAIL_RETURN_ADDRESS, user_email, level='ERROR', bubble=True)
my_mail_handler.push_application()
mail_handler = my_mail_handler
try:
collection = Collection(url_api_collection)
except Exception as e:
msg = 'Exception in Collection {}, init {}'.format(url_api_collection,
str(e))
logbook.error(msg)
raise e
if not (collection['harvest_type'] in HARVEST_TYPES):
msg = 'Collection {} wrong type {} for harvesting. Harvest type {} \
is not in {}'.format(url_api_collection,
collection['harvest_type'],
collection['harvest_type'],
HARVEST_TYPES.keys())
logbook.error(msg)
raise ValueError(msg)
mail_handler.subject = "Error during harvest of " + collection.url
my_log_handler = None
if not log_handler: # can't init until have collection
my_log_handler = FileHandler(get_log_file_path(collection.slug))
my_log_handler.push_application()
logger = logbook.Logger('HarvestMain')
msg = 'Init harvester next. Collection:{}'.format(collection.url)
logger.info(msg)
# email directly
mimetext = create_mimetext_msg(EMAIL_RETURN_ADDRESS, user_email, ' '.join(
('Starting harvest for ', collection.slug)), msg)
try: # TODO: request more emails from AWS
mail_handler.deliver(mimetext, '*****@*****.**')
except:
pass
logger.info('Create DPLA profile document')
if not profile_path:
profile_path = os.path.abspath(
os.path.join(dir_profile, collection.id + '.pjs'))
with codecs.open(profile_path, 'w', 'utf8') as pfoo:
pfoo.write(collection.dpla_profile)
logger.info('DPLA profile document : ' + profile_path)
harvester = None
try:
harvester = HarvestController(
user_email,
collection,
profile_path=profile_path,
config_file=config_file,
**kwargs)
except Exception as e:
import traceback
msg = 'Exception in harvester init: type: {} TRACE:\n{}'.format(
type(e), traceback.format_exc())
logger.error(msg)
raise e
logger.info('Create ingest doc in couch')
ingest_doc_id = harvester.create_ingest_doc()
logger.info('Ingest DOC ID: ' + ingest_doc_id)
logger.info('Start harvesting next')
try:
num_recs = harvester.harvest()
msg = ''.join(('Finished harvest of ', collection.slug, '. ',
str(num_recs), ' records harvested.'))
harvester.update_ingest_doc('complete', items=num_recs, num_coll=1)
logger.info(msg)
# email directly
mimetext = create_mimetext_msg(
EMAIL_RETURN_ADDRESS, user_email, ' '.join(
('Finished harvest of raw records '
'for ', collection.slug, ' enriching next')), msg)
try:
mail_handler.deliver(mimetext, '*****@*****.**')
except:
pass
except Exception as e:
import traceback
error_msg = ''.join(("Error while harvesting: type-> ", str(type(e)),
" TRACE:\n" + str(traceback.format_exc())))
logger.error(error_msg)
harvester.update_ingest_doc(
'error', error_msg=error_msg, items=num_recs)
raise e
if my_log_handler:
my_log_handler.pop_application()
if my_mail_handler:
my_mail_handler.pop_application()
return ingest_doc_id, num_recs, harvester.dir_save, harvester
def main():
"""Shows basic usage of the Google Drive API.
Creates a Google Drive API service object and outputs the names and IDs
for up to 10 files.
"""
log_filename = os.path.join(
args.log_dir,
'google-drive-to-s3-{}.log'.format(os.path.basename(time.strftime('%Y%m%d-%H%M%S')))
)
# register some logging handlers
log_handler = FileHandler(
log_filename,
mode='w',
level=args.log_level,
bubble=True
)
stdout_handler = StreamHandler(sys.stdout, level=args.log_level, bubble=True)
with stdout_handler.applicationbound():
with log_handler.applicationbound():
log.info("Arguments: {}".format(args))
start = time.time()
log.info("starting at {}".format(time.strftime('%l:%M%p %Z on %b %d, %Y')))
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
drive_service = discovery.build('drive', 'v3', http=http)
s3 = boto3.resource('s3')
# load up a match file if we have one.
if args.match_file:
with open(args.match_file, 'r') as f:
match_filenames = f.read().splitlines()
else:
match_filenames = None
# get the files in the specified folder.
files = drive_service.files()
request = files.list(
pageSize=args.page_size,
q="'{}' in parents".format(args.folder_id),
fields="nextPageToken, files(id, name)"
)
# make sure our S3 Key prefix has a trailing slash
key_prefix = ensure_trailing_slash(args.key_prefix)
page_counter = 0
file_counter = 0
while request is not None:
file_page = request.execute(http=http)
page_counter += 1
page_file_counter = 0 # reset the paging file counter
# determine the page at which to start processing.
if page_counter >= args.start_page:
log.info(u"######## Page {} ########".format(page_counter))
for this_file in file_page['files']:
file_counter += 1
page_file_counter += 1
if we_should_process_this_file(this_file['name'], match_filenames):
log.info(u"#== Processing {} file number {} on page {}. {} files processed.".format(
this_file['name'],
page_file_counter,
page_counter,
file_counter
))
# download the file
download_request = drive_service.files().get_media(fileId=this_file['id'])
fh = io.BytesIO() # Using an in memory stream location
downloader = MediaIoBaseDownload(fh, download_request)
done = False
pbar = InitBar(this_file['name'])
while done is False:
status, done = downloader.next_chunk()
pbar(int(status.progress()*100))
# print("\rDownload {}%".format(int(status.progress() * 100)))
del pbar
# upload to bucket
log.info(u"Uploading to S3")
s3.Bucket(args.bucket).put_object(
Key="{}{}".format(key_prefix, this_file['name']),
Body=fh.getvalue(),
ACL='public-read'
)
log.info(u"Uploaded to S3")
fh.close() # close the file handle to release memory
else:
log.info(u"Do not need to process {}".format(this_file['name']))
# stop if we have come to the last user specified page
if args.end_page and page_counter == args.end_page:
log.info(u"Finished paging at page {}".format(page_counter))
break
# request the next page of files
request = files.list_next(request, file_page)
log.info("Running time: {}".format(str(datetime.timedelta(seconds=(round(time.time() - start, 3))))))
log.info("Log written to {}:".format(log_filename))
def __init__(self, filament, **kwargs):
self._start = datetime.now()
try:
log_path = os.path.join(os.path.expanduser('~'), '.fibratus',
'fibratus.log')
FileHandler(log_path, mode='w+').push_application()
StreamHandler(sys.stdout, bubble=True).push_application()
except PermissionError:
panic(
"ERROR - Unable to open log file for writing due to permission error"
)
self.logger = Logger(Fibratus.__name__)
self._config = YamlConfig()
self.logger.info('Starting Fibratus...')
enable_cswitch = kwargs.pop('cswitch', False)
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags(cswitch=enable_cswitch)
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
enum_handles = kwargs.pop('enum_handles', True)
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of the kernel trace
if enum_handles:
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
image_meta_config = self._config.image_meta
self.image_meta_registry = ImageMetaRegistry(
image_meta_config.enabled, image_meta_config.imports,
image_meta_config.file_info)
self.thread_registry = ThreadRegistry(self.handle_repository,
self._handles,
self.image_meta_registry)
self.kevt_streamc = KEventStreamCollector(
etw.KERNEL_LOGGER_NAME.encode())
skips = self._config.skips
image_skips = skips.images if 'images' in skips else []
if len(image_skips) > 0:
self.logger.info("Adding skips for images %s" % image_skips)
for skip in image_skips:
self.kevt_streamc.add_skip(skip)
self.kevent = KEvent(self.thread_registry)
self._output_classes = dict(console=ConsoleOutput,
amqp=AmqpOutput,
smtp=SmtpOutput,
elasticsearch=ElasticsearchOutput)
self._outputs = self._construct_outputs()
self.output_aggregator = OutputAggregator(self._outputs)
self._binding_classes = dict(yara=YaraBinding)
self._bindings = self._construct_bindings()
if filament:
filament.logger = self.logger
filament.do_output_accessors(self._outputs)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.context_switch_registry = ContextSwitchRegistry(
self.thread_registry, self.kevent)
self.output_kevents = {}
self.filters_count = 0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
from spider163.utils import config
from logbook import FileHandler, Logger
from terminaltables import AsciiTable
from colorama import Fore
from colorama import init
path = config.get_path()
log_handler = FileHandler(filename=path + '/spider163.log')
log_handler.push_application()
log = Logger("")
init(autoreset=True)
def Log(msg):
log.warn(msg)
def Table(tb):
print(AsciiTable(tb).table)
def Blue(msg):
return Fore.BLUE + msg
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, nested_scopes
import os
import logbook
from logbook import FileHandler
from logbook import Logger
log = Logger('scraper')
# Create a logs direcory if not exist
if not os.path.exists('logs'):
os.makedirs('logs')
file_handler = FileHandler('logs/app.log', level=logbook.DEBUG)
file_handler.push_application()
import itertools
from collections import defaultdict
import glob
import math
capdir = os.getcwd()
directory = os.path.join(capdir, "songs")
echonestkey = os.environ.get('ECHO_NEST_API_KEY')
config.ECHO_NEST_API_KEY=echonestkey
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
log = Logger('Logbook')
file_handler = FileHandler("keymixlog.log")
#error_handler = SyslogHandler('logbook example', level='ERROR')
#this seems to work though :)
null_handler = NullHandler()
harmonic_mixing_dict = {80:[10,80,30,101],
30:[80,30,110,61],
110:[30,110,50,11],
50:[110,50,0,81],
0:[50,0,70,31],
70:[0,70,20,91],
20:[70,20,90,51],
90:[20,90,40,1],
40:[90,40,100,71],
100:[40,100,60,21],
def setup_logger(test, path='test.log'):
test.log_handler = FileHandler(path)
test.log_handler.push_application()
"""
@version: 1.0
@ __author__: longbai
@ __file__: processortest.py
@ __mtime__: 2016/8/12 18:38
日志处理器
"""
import os
import sys
from logbook import Processor, StreamHandler, DEBUG, Logger, FileHandler
my_handler = FileHandler("test.log", encoding="utf-8", level=DEBUG)
# my_handler = StreamHandler(sys.stdout, level=DEBUG)
def log_other_info(record):
"""
a) 通过 with.processor可以让在其中的日志拥有共同的逻辑，相当于一个切面注入
比如这里的例子是 在每条日志中记录一些额外的信息（额外的信息是通过在日志对象（logRecord）的extra（字典对象）属性中添加
一些其他的信息），这样每条日志都会有这里添加的额外的信息。
b) 有个疑问就是，这些额外的信息怎么运用呢，比如这些信息如何能和日志一块记录在文件中呢
c) 关于日志的属性，见 logrecord.py
"""
record.extra['myname'] = 'kute'
record.extra['mycwd'] = os.getcwd()
# update myname propertiy
record.extra.update(myname="lisa")
"""Logger for wxPython."""
from logbook import FileHandler, Logger
logger = Logger()
file_handler = FileHandler("wx.log",
encoding="utf-8",
level="DEBUG",
delay=True)
file_handler.format_string = (
"{record.time:%Y-%m-%d %H:%M:%S.%f%z} [{record.level_name}] "
"{record.message}")
# Uncomment this to log to wx.log
#logger.handlers.append(file_handler)
def main():
"""
Copy a folder from Source to Target
"""
log_filename = os.path.join(
args.log_dir,
'copy-google-drive-folder-{}.log'.format(os.path.basename(time.strftime('%Y%m%d-%H%M%S')))
)
# register some logging handlers
log_handler = FileHandler(
log_filename,
mode='w',
level=args.log_level,
bubble=True
)
stdout_handler = StreamHandler(sys.stdout, level=args.log_level, bubble=True)
with stdout_handler.applicationbound():
with log_handler.applicationbound():
log.info("Arguments: {}".format(args))
start = time.time()
log.info("starting at {}".format(time.strftime('%l:%M%p %Z on %b %d, %Y')))
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
drive_service = discovery.build('drive', 'v3', http=http)
# get the files in the specified folder.
files = drive_service.files()
request = files.list(
pageSize=args.page_size,
q="'{}' in parents".format(args.source_folder_id),
fields="nextPageToken, files(id, name, mimeType)"
)
page_counter = 0
file_counter = 0
while request is not None:
file_page = request.execute(http=http)
page_counter += 1
page_file_counter = 0 # reset the paging file counter
# determine the page at which to start processing.
if page_counter >= args.start_page:
log.info(u"######## Page {} ########".format(page_counter))
for this_file in file_page['files']:
file_counter += 1
page_file_counter += 1
log.info(u"#== Processing {} {} file number {} on page {}. {} files processed.".format(
this_file['mimeType'],
this_file['name'],
page_file_counter,
page_counter,
file_counter
))
# if not a folder
if this_file['mimeType'] != 'application/vnd.google-apps.folder':
# Copy the file
new_file = {'title': this_file['name']}
copied_file = drive_service.files().copy(fileId=this_file['id'], body=new_file).execute()
# move it to it's new location
drive_service.files().update(
fileId=copied_file['id'],
addParents=args.target_folder_id,
removeParents=args.source_folder_id
).execute()
else:
log.info(u"Skipped Folder")
else:
log.info(u"Skipping Page {}".format(page_counter))
# stop if we have come to the last user specified page
if args.end_page and page_counter == args.end_page:
log.info(u"Finished paging at page {}".format(page_counter))
break
# request the next page of files
request = files.list_next(request, file_page)
log.info("Running time: {}".format(str(datetime.timedelta(seconds=(round(time.time() - start, 3))))))
log.info("Log written to {}:".format(log_filename))
class Fibratus():
"""Fibratus entrypoint.
Setup the core components including the kernel
event stream collector and the tracing controller.
At this point the system handles are also being
enumerated.
"""
def __init__(self, filament):
self.logger = Logger(Fibratus.__name__)
self.file_handler = FileHandler(os.path.join(os.path.abspath(__file__),
'..', '..', '..',
'fibratus.log'),
mode='w+')
self.kevt_streamc = KEventStreamCollector(
etw.KERNEL_LOGGER_NAME.encode())
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags()
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of kernel trace
with self.file_handler.applicationbound():
self.logger.info('Starting fibratus...')
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
self.thread_registry = ThreadRegistry(self.handle_repository,
self._handles)
self.kevent = KEvent(self.thread_registry)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.requires_render = {}
self.filters_count = 0
def run(self):
@atexit.register
def _exit():
self.stop_ktrace()
self.kcontroller.start_ktrace(etw.KERNEL_LOGGER_NAME,
self.ktrace_props)
def on_kstream_open():
if self._filament is None:
IO.write_console('Done! ')
self.kevt_streamc.set_kstream_open_callback(on_kstream_open)
self._open_kstream()
def _open_kstream(self):
try:
self.kevt_streamc.open_kstream(self._on_next_kevent)
except Exception as e:
with self.file_handler.applicationbound():
self.logger.error(e)
except KeyboardInterrupt:
self.stop_ktrace()
def stop_ktrace(self):
IO.write_console('Stopping fibratus...')
if self._filament:
self._filament.close()
self.kcontroller.stop_ktrace(self.ktrace_props)
self.kevt_streamc.close_kstream()
def add_filters(self, kevent_filters):
if len(kevent_filters) > 0:
self.filters_count = len(kevent_filters)
# include the basic filters
# that are essential to the
# rest of kernel events
self.kevt_streamc.add_kevent_filter(ENUM_PROCESS)
self.kevt_streamc.add_kevent_filter(ENUM_THREAD)
self.kevt_streamc.add_kevent_filter(ENUM_IMAGE)
self.kevt_streamc.add_kevent_filter(REG_CREATE_KCB)
self.kevt_streamc.add_kevent_filter(REG_DELETE_KCB)
# these kevents are necessary for consistent state
# of the trace. If the user doesn't include them
# in a filter list, then we do the job but set the
# kernel event type as not eligible for rendering
if not KEvents.CREATE_PROCESS in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_PROCESS)
self.requires_render[CREATE_PROCESS] = False
else:
self.requires_render[CREATE_PROCESS] = True
if not KEvents.CREATE_THREAD in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_THREAD)
self.requires_render[CREATE_THREAD] = False
else:
self.requires_render[CREATE_THREAD] = True
if not KEvents.CREATE_FILE in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_FILE)
self.requires_render[CREATE_FILE] = False
else:
self.requires_render[CREATE_FILE] = True
for kevent_filter in kevent_filters:
ktuple = kname_to_tuple(kevent_filter)
if isinstance(ktuple, list):
for kt in ktuple:
self.kevt_streamc.add_kevent_filter(kt)
if not kt in self.requires_render:
self.requires_render[kt] = True
else:
self.kevt_streamc.add_kevent_filter(ktuple)
if not ktuple in self.requires_render:
self.requires_render[ktuple] = True
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS, CREATE_THREAD, ENUM_PROCESS, ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(
self.kevent, ktype, kparams)
self._render(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent, ktype,
kparams)
self._render(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [
CREATE_FILE, DELETE_FILE, CLOSE_FILE, READ_FILE, WRITE_FILE
]:
self.fsio.parse_fsio(ktype, kparams)
self._render(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._render(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._render(ktype)
# registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [
REG_CREATE_KEY, REG_DELETE_KEY, REG_OPEN_KEY, REG_QUERY_KEY,
REG_SET_VALUE, REG_DELETE_VALUE, REG_QUERY_VALUE
]:
self.hive_parser.parse_hive(ktype, kparams)
self._render(ktype)
# network kernel events
elif ktype in [
SEND_SOCKET_TCPV4, SEND_SOCKET_UDPV4, RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4, ACCEPT_SOCKET_TCPV4, CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4, RECONNECT_SOCKET_TCPV4
]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._render(ktype)
if self._filament:
# call filament method
# to process the next
# kernel event from the stream
if ktype not in [ENUM_PROCESS, ENUM_THREAD, ENUM_IMAGE]:
if self.kevent.name:
self._filament.process(self.kevent)
def _render(self, ktype):
"""Renders the kevent to the standard output stream.
Parameters
----------
ktype: tuple
Identifier of the kernel event
"""
if not self._filament:
if ktype in self.requires_render:
rr = self.requires_render[ktype]
if rr:
self.kevent.render()
elif self.filters_count == 0:
self.kevent.render()
def __init__(self,
host='localhost',
port=8001,
auth_msg='',
default_file=None,
default_namespace=None,
eval_fn=None,
exception_serializer=None,
skip_fn=None,
pause_signal=None,
redirect_output=True,
retry_attepts=10,
uuid=None,
cmd_manager=None,
green=False,
repr_fn=None,
log_file=None,
execution_timeout=None):
"""
Host and port define the address to connect to.
The auth_msg is a message that will be sent with the start event to the
server. This can be used to do server/tracer authentication.
The default_file is a file to use if the file field is ommited from
payloads.
eval_fn is the function to eval code where the user may provide it,
for example in a conditional breakpoint, or in the repl.
skip_fn is simmilar to the skip list feature of Bdb, except that
it should be a function that takes a filename and returns True iff
the debugger should skip this file. These files will be suppressed from
stack traces.
The pause_signal is signal to raise in this program to trigger a pause
command. If this is none, this will default to SIGUSR2.
retry_attempts is the number of times to attempt to connect to the
server before raising a QdbFailedToConnect error.
The repr_fn is a function to use to convert objects to strings to send
then back to the server. By default, this wraps repr by catching
exceptions and reporting them to the user.
The uuid is the identifier on the server for this session. If none is
provided, it will generate a uuid4.
cmd_manager should be a callable that takes a Qdb instance and manages
commands by implementing a next_command method. If none, a new, default
manager will be created that reads commands from the server at
(host, port).
If green is True, this will use gevent safe timeouts, otherwise this
will use signal based timeouts.
repr_fn is the repr function to use when displaying results. If None,
use the builtin repr.
execution_timeout is the amount of time user code has to execute before
being cut short. This is applied to the repl, watchlist and conditional
breakpoints. If None, no timeout is applied.
"""
super(Qdb, self).__init__()
self.address = host, port
self.set_default_file(default_file)
self.default_namespace = default_namespace or {}
self.exception_serializer = exception_serializer or \
default_exception_serializer
self.eval_fn = eval_fn or default_eval_fn
self.green = green
self._file_cache = {}
self.redirect_output = redirect_output
self.retry_attepts = retry_attepts
self.repr_fn = repr_fn
self.skip_fn = skip_fn or (lambda _: False)
self.pause_signal = pause_signal if pause_signal else signal.SIGUSR2
self.uuid = str(uuid or uuid4())
self.watchlist = {}
self.execution_timeout = execution_timeout
# We need to be able to send stdout back to the user debugging the
# program. We hold a handle to this in case the program resets stdout.
if self.redirect_output:
self._old_stdout = sys.stdout
self._old_stderr = sys.stderr
self.stdout = StringIO()
self.stderr = StringIO()
sys.stdout = self.stdout
sys.stderr = self.stderr
self.forget()
self.log_handler = None
if log_file:
self.log_handler = FileHandler(log_file)
self.log_handler.push_application()
if not cmd_manager:
cmd_manager = RemoteCommandManager
self.cmd_manager = cmd_manager(self)
self.cmd_manager.start(auth_msg)
