#!/bin/env python import os, sys, urllib, urllib2, time from logbook import Logger, FileHandler user = "******" token = "password" message = bytes(user).encode('utf-8') secret = bytes(token).encode('utf-8') logger = Logger("Cache Purge") logfile = "cache-purge.log" fh = FileHandler(logfile, "a") fh.applicationbound() fh.push_application() api_root = "https://api.ccu.akamai.com" get_call = "/ccu/v2/queues/default" #data = {} try: req = None url = api_root + get_call mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() mgr.add_password(None, api_root, user, token) handler = urllib2.HTTPBasicAuthHandler(mgr) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) req = urllib2.Request(url) #req = urllib2.Request(api_root,urllib.urlencode(data))
# - * - coding: utf-8 - * - import logbook import sys from logbook import Logger, FileHandler, StreamHandler # 日志文件 logFile = "rk-bd.log" # 设置UTC时间格式为本地时间格式 logbook.set_datetime_format("local") # 日志输出到标准输出设备,应用整个程序范围 StreamHandler( sys.stdout, level=logbook.DEBUG, format_string= "[{record.time:%Y-%m-%d %H:%M:%S}] {record.level_name}: {record.filename}:{record.lineno} {record.message}", encoding="utf-8").push_application() # 日志输出到文件,应用整个程序范围 FileHandler( logFile, level=logbook.INFO, format_string= "[{record.time:%Y-%m-%d %H:%M:%S}] {record.level_name}: {record.filename}:{record.lineno} {record.message}", encoding="utf-8", bubble=True).push_application() log = Logger(__name__)
#!/usr/bin/env python # -*- coding: utf-8 -*- import logging from spider163.utils import config from logbook import FileHandler, Logger from terminaltables import AsciiTable from colorama import Fore from colorama import init path = config.get_path() log_handler = FileHandler(filename=path + '/spider163.log') log_handler.push_application() log = Logger("") init(autoreset=True) def Log(msg): log.warn(msg) def Table(tb): print(AsciiTable(tb).table) def Blue(msg): return Fore.BLUE + msg
# limitations under the License. import json import os import platform import shutil import sqlite3 from glob import glob from logbook import FileHandler, Logger, CRITICAL from browser import Browser log = Logger("Chromium") if os.path.exists("application.log"): log_handler = FileHandler('application.log') log_handler.push_application() else: log.level = CRITICAL class Chromium(Browser): def __init__(self): log.debug("Starting initialization") self.name = "chromium" self.process_names = ["chromium-browser", "chrome.exe"] self.os = platform.system().lower() self.path = None self.profile = None self.available_profiles = self._get_profiles()
servers = [Server(i) for i in range(1, NUM_SERVERS + 1)] start_servers(servers) time.sleep(10) try: return test(servers) except Exception, e: logger.exception('Test failed: %s' % e) return 1 finally: logger.info('Stopping') stop(servers) if __name__ == '__main__': format = '[{record.time}] {record.level_name:>5} [{record.extra[worker_id]}] {record.message}' logging_setup = NestedSetup([ NullHandler(), FileHandler( filename=os.path.join(os.path.dirname(__file__), 'log/client.log'), format_string=format, bubble=True, ), StderrHandler(level=logbook.INFO, format_string=format, bubble=True), ]) with logging_setup.applicationbound(): sys.exit(main())
if not token: return None elif not tk_alive.isalive(token): req_count.delete(token) tk_alive.drop_tk(token) continue return token, used if __name__ == '__main__': from logbook import FileHandler from logbook import Logger from argparse import ArgumentParser import sys parser = ArgumentParser() parser.add_argument('--log', nargs=1, help='log path') args = parser.parse_args(sys.argv[1:]) log_handler = FileHandler(args.log[0]) logbk = Logger('Token Maintain') with log_handler.applicationbound(): logbk.info('maintain prepare') at_least = AT_LEAST_TOKEN_COUNT logbk.info('maintain begin') maintain(at_least=at_least, hourly=True, logbk=logbk) logbk.info('maintain end')
# and/or other materials provided with the distribution. # 3. Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, # OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED # OF THE POSSIBILITY OF SUCH DAMAGE. """Log configuration for contexts.""" from logbook import FileHandler, Logger logger = Logger() file_handler = FileHandler("logs/contexts.log", encoding="utf-8", level="DEBUG", delay=True) file_handler.format_string = ( "{record.time:%Y-%m-%d %H:%M:%S.%f%z} [{record.level_name}] " "{record.message}") logger.handlers.append(file_handler)
def _init(self, config=None, merge=False, **kwargs): """ See qdb.config for more information about the configuration of qdb. merge denotes how config and kwargs should be merged. QdbConfig.kwargs_first says config will trample kwargs, QdbConfig.config_first says kwargs will trample config. Otherwise, kwargs and config cannot both be passed. """ self.super_ = super(Qdb, self) self.super_.__init__() self.reset() if config and kwargs: if merge == QdbConfig.kwargs_first: first = kwargs second = config elif merge == QdbConfig.config_first: first = config second = kwargs else: raise TypeError('Cannot pass config and kwargs') config = first.merge(second) else: config = QdbConfig.get_config(config or kwargs) self.address = config.host, config.port self.set_default_file(config.default_file) self.default_namespace = config.default_namespace or {} self.exception_serializer = config.exception_serializer or \ default_exception_serializer self.eval_fn = config.eval_fn or default_eval_fn self._file_cache = {} self.retry_attepts = config.retry_attepts self.repr_fn = config.repr_fn self._skip_fn = config.skip_fn or (lambda _: False) self.pause_signal = config.pause_signal \ if config.pause_signal else signal.SIGUSR2 self.uuid = str(config.uuid or uuid4()) self.watchlist = {} self.execution_timeout = config.execution_timeout self.reset() self.log_handler = None if config.log_file: self.log_handler = FileHandler(config.log_file) self.log_handler.push_application() self.bound_cmd_manager = config.cmd_manager or TerminalCommandManager() self.bound_cmd_manager.start(config.auth_msg) # We need to be able to send stdout back to the user debugging the # program. We hold a handle to this in case the program resets stdout. self._old_stdout = sys.stdout self._old_stderr = sys.stderr self.redirect_output = ( config.redirect_output and not isinstance(self.cmd_manager, TerminalCommandManager) ) if self.redirect_output: sys.stdout = OutputTee( sys.stdout, RemoteOutput(self.cmd_manager, '<stdout>'), ) sys.stderr = OutputTee( sys.stderr, RemoteOutput(self.cmd_manager, '<stderr>'), )
import asyncio import configparser import inspect import sys import time import os import discord from discord.ext import commands from logbook import Logger, StreamHandler, FileHandler logger = Logger("Discord Music") logger.handlers.append(StreamHandler(sys.stdout, bubble=True)) logger.handlers.append(FileHandler("last-run.log", bubble=True, mode="w")) logger.debug("Loading config files") default_config = "[Config]\ntoken = \nsnip = " config = configparser.ConfigParser() token = "" snip = "" if os.path.exists("config.ini"): config.read("config.ini") try: token = config['Config']['token'] except KeyError: logger.critical(
import asyncio from pathlib import Path from logbook import FileHandler, Logger from pony.orm import commit, core, db_session, set_sql_debug from data.base import db from data.search import search_permission from service.base import BaseService import settings from tools.delay import Delay # Logger logger = Logger() file_handler = FileHandler("logs/database.log", encoding="utf-8", level="DEBUG", delay=True) file_handler.format_string = ( "{record.time:%Y-%m-%d %H:%M:%S.%f%z} [{record.level_name}] " "{record.message}" ) logger.handlers.append(file_handler) class Service(BaseService): """Data service.""" name = "data" @property def has_admin(self): """
h = Actie(xmlfile, "2006-0001") logger.info(h.meld) if h.exists: h.list() h.setArch(True) h.write() h.read() h.list() if __name__ == "__main__": ## log_handler = FileHandler('get_acties_xml_1.log', mode='w') ## with log_handler.applicationbound(): ## test_acties(arch='') ## test_acties(select={"idgt": "2006-0010"}) ## test_acties(select={"idlt": "2005-0019"}) ## test_acties(select={"idgt": "2005-0019" , "idlt": "2006-0010"}) ## test_acties(select={"idgt": "2005-0019" , "idlt": "2006-0010", "id": "and" }) ## test_acties(select={"idgt": "2006-0010" , "idlt": "2005-0019", "id": "or" }) ## test_acties(select={"status": ("0", "1", "3")}) ## test_acties(select={"soort": ("W", "P")}) ## test_acties(select={"titel": ("tekst")}) ## log_handler = FileHandler('settings_xml_1.log', mode='w') ## with log_handler.applicationbound(): ## test_settings() log_handler = FileHandler('actie_xml.log', mode='w') with log_handler.applicationbound(): test_laatste() test_actie("2007-0001") test_actie("1") ## test_archiveren()
import numpy as np from logbook import FileHandler, Logger from sqlalchemy import create_engine from statsmodels.regression.linear_model import OLS from imp import reload import mysql_table import get_alphas import utils reload(mysql_table) reload(get_alphas) reload(utils) log_file_name = r'price_vol.log' log_handler = FileHandler(log_file_name).push_application() def check_log(date, tdays): ''' if the date next day successfully wrote to database, then ignore the date because there four py file need to be executed Params: date: str, like '%Y%m%d' tdays: trade day series ''' with open(log_file_name) as f: log_text = f.read()
import sys from logbook import Logger, NestedSetup, StreamHandler, FileHandler, StringFormatterHandlerMixin, NullHandler format_string = '[{record.time:%y%m%d %H:%M}] {record.level_name}: snakepot {record.channel}: {record.message}' NestedSetup([ FileHandler('logfile.log', format_string=format_string, level='DEBUG'), StreamHandler(sys.stderr, format_string=format_string, bubble=True) ]).push_application()
from logbook import Logger, FileHandler import six from . import helpers if six.PY2: import sys reload(sys) sys.setdefaultencoding('utf8') log = Logger(__file__) log_file = ".".join(__file__.split(os.sep)[-1].split(".")[:-1]) log_file = os.getcwd() + os.sep + log_file + ".log" file_handler = FileHandler(log_file, level="DEBUG") log.handlers.append(file_handler) class NotLoginError(Exception): def __init__(self, result=None): super(NotLoginError, self).__init__() self.result = result class WebTrader(object): global_config_path = os.path.dirname(__file__) + '/config/global.json' config_path = '' def __init__(self): self.__read_config()
def __init__(self, filament, **kwargs): self._start = datetime.now() try: log_path = os.path.join(os.path.expanduser('~'), '.fibratus', 'fibratus.log') FileHandler(log_path, mode='w+').push_application() StreamHandler(sys.stdout).push_application() except PermissionError: panic( "ERROR - Unable to open log file for writing due to permission error" ) self.logger = Logger(Fibratus.__name__) self._config = YamlConfig() self.logger.info('Starting...') enable_cswitch = kwargs.pop('cswitch', False) self.kcontroller = KTraceController() self.ktrace_props = KTraceProps() self.ktrace_props.enable_kflags(cswitch=enable_cswitch) self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME enum_handles = kwargs.pop('enum_handles', True) self.handle_repository = HandleRepository() self._handles = [] # query for handles on the # start of the kernel trace if enum_handles: self.logger.info('Enumerating system handles...') self._handles = self.handle_repository.query_handles() self.logger.info('%s handles found' % len(self._handles)) self.handle_repository.free_buffers() self.thread_registry = ThreadRegistry(self.handle_repository, self._handles) self.kevt_streamc = KEventStreamCollector( etw.KERNEL_LOGGER_NAME.encode()) image_skips = self._config.image_skips if len(image_skips) > 0: self.logger.info("Adding skips for images %s" % image_skips) for skip in image_skips: self.kevt_streamc.add_skip(skip) self.kevent = KEvent(self.thread_registry) self.keventq = Queue() self._output_classes = dict(console=ConsoleOutput, amqp=AmqpOutput, smtp=SmtpOutput, elasticsearch=ElasticsearchOutput) self._outputs = self._construct_outputs() if filament: filament.keventq = self.keventq filament.logger = log_path filament.setup_adapters(self._outputs) self._filament = filament self.fsio = FsIO(self.kevent, self._handles) self.hive_parser = HiveParser(self.kevent, self.thread_registry) self.tcpip_parser = TcpIpParser(self.kevent) self.dll_repository = DllRepository(self.kevent) self.context_switch_registry = ContextSwitchRegistry( self.thread_registry, self.kevent) self.output_kevents = {} self.filters_count = 0
line = line.strip().split('=', 1) config_dict[line[0]] = line[1] ################################################### # Adding parse arguements parser = argparse.ArgumentParser(description='An NGS pipeline') parser.add_argument('--input_dir', nargs='?') parser.add_argument('--config', nargs='?') args = parser.parse_args() #################################################### # Create a logger FileHandler('{input_dir}ngs_pipeline.log'.format(input_dir=args.input_dir)).push_application() log = Logger('pipeline_logger') log.info('Starting the pipeline as user ' + getpass.getuser()) git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() log.info('pipeline git hash: '+ git_hash) log.info('input_dir: '+ args.input_dir) log.info('config: '+ args.config) log.info('using python version: '+ sys.version.strip()) log.info('fastqc version: '+ subprocess.check_output([config_dict['fastqc'], '--version']).strip()) #################################################### # Set current time i = datetime.datetime.now() date_time_iso = '%s' %i.isoformat()
# # * 商业用途(商业用途指个人出于任何商业目的使用本软件,或者法人或其他组织出于任何目的使用本软件): # 未经米筐科技授权,任何个人不得出于任何商业目的使用本软件(包括但不限于向第三方提供、销售、出租、出借、转让本软件、本软件的衍生产品、引用或借鉴了本软件功能或源代码的产品或服务),任何法人或其他组织不得出于任何目的使用本软件,否则米筐科技有权追究相应的知识产权侵权责任。 # 在此前提下,对本软件的使用同样需要遵守 Apache 2.0 许可,Apache 2.0 许可与本许可冲突之处,以本许可为准。 # 详细的授权流程,请联系 [email protected] 获取。 import os import sys from datetime import date, timedelta from logbook import StreamHandler, Logger, FileHandler, set_datetime_format set_datetime_format("local") LOG_DIR = os.path.join(os.path.expanduser("~"), ".rqams-helper") if not os.path.exists(LOG_DIR): os.mkdir(LOG_DIR) LOG_PATH = os.path.join(LOG_DIR, "{}.log".format(date.today().strftime("%Y%m%d"))) # StreamHandler(sys.stdout, bubble=True).push_application() FileHandler(LOG_PATH, bubble=True).push_application() logger = Logger("RQAMS_HELPER") LOG_THRESHOLD = (date.today() - timedelta(30)).strftime("%Y%m%d") for file in os.listdir(LOG_DIR): if file.endswith(".log") and file.split(".")[0] < LOG_THRESHOLD: logger.info(f"old log file {file} has been removed") os.remove(os.path.join(LOG_DIR, file))
parser.add_argument( "--cul-path", default="/dev/ttyACM0", help="Path to usbmodem path of CUL, defaults to /dev/ttyACM0") parser.add_argument("--cul-baud", default="38400", help="Baudrate of the cul serial connection.") args = parser.parse_args() db.create_all() if args.detach: # init logger from logbook import FileHandler log_handler = FileHandler('server.log') log_handler.push_application() import detach with detach.Detach(daemonize=True) as d: if d.pid: print( "started process {} in background with log to server.log". format(d.pid)) else: main(args) else: # init logger from logbook.more import ColorizedStderrHandler log_handler = ColorizedStderrHandler() log_handler.push_application()
def main(): """ Copy a folder from Source to Target """ log_filename = os.path.join( args.log_dir, 'copy-google-drive-folder-{}.log'.format( os.path.basename(time.strftime('%Y%m%d-%H%M%S')))) # register some logging handlers log_handler = FileHandler(log_filename, mode='w', level=args.log_level, bubble=True) stdout_handler = StreamHandler(sys.stdout, level=args.log_level, bubble=True) with stdout_handler.applicationbound(): with log_handler.applicationbound(): log.info("Arguments: {}".format(args)) start = time.time() log.info("starting at {}".format( time.strftime('%l:%M%p %Z on %b %d, %Y'))) credentials = get_credentials() http = credentials.authorize(httplib2.Http()) drive_service = discovery.build('drive', 'v3', http=http) # get the files in the specified folder. files = drive_service.files() request = files.list( pageSize=args.page_size, q="'{}' in parents".format(args.source_folder_id), fields="nextPageToken, files(id, name, mimeType)") page_counter = 0 file_counter = 0 while request is not None: file_page = request.execute(http=http) page_counter += 1 page_file_counter = 0 # reset the paging file counter # determine the page at which to start processing. if page_counter >= args.start_page: log.info(u"######## Page {} ########".format(page_counter)) for this_file in file_page['files']: file_counter += 1 page_file_counter += 1 log.info( u"#== Processing {} {} file number {} on page {}. {} files processed." .format(this_file['mimeType'], this_file['name'], page_file_counter, page_counter, file_counter)) # if not a folder if this_file[ 'mimeType'] != 'application/vnd.google-apps.folder': # Copy the file new_file = {'title': this_file['name']} copied_file = drive_service.files().copy( fileId=this_file['id'], body=new_file).execute() # move it to it's new location drive_service.files().update( fileId=copied_file['id'], addParents=args.target_folder_id, removeParents=args.source_folder_id).execute() else: log.info(u"Skipped Folder") else: log.info(u"Skipping Page {}".format(page_counter)) # stop if we have come to the last user specified page if args.end_page and page_counter == args.end_page: log.info( u"Finished paging at page {}".format(page_counter)) break # request the next page of files request = files.list_next(request, file_page) log.info("Running time: {}".format( str(datetime.timedelta(seconds=(round(time.time() - start, 3)))))) log.info("Log written to {}:".format(log_filename))
level = env.LOGGING_LEVEL log = Logger(env.LOGGING_FILE_PREFIX, level=level) if env.LOGGING_ENABLED: if env.LOGGING_STDOUT: sh = StreamHandler(sys.stdout, bubble=True) sh.formatter = formatter # sh.format_string += ' (rank={})'.format(MPI_RANK) log.handlers.append(sh) # sh.push_application() if env.LOGGING_TOFILE: fh_directory = env.LOGGING_DIR fh_file_prefix = env.LOGGING_FILE_PREFIX fh = FileHandler(os.path.join( fh_directory, '{}-rank-{}.log'.format(fh_file_prefix, MPI_RANK)), bubble=True, mode=env.LOGGING_FILEMODE.lower()) fh.formatter = formatter # fh.format_string += ' (rank={})'.format(MPI_RANK) log.handlers.append(fh) log.notice('Initialized (Version {})'.format(utools.__version__)) # fh.push_application() class log_entry_exit(object): def __init__(self, f): self.f = f def __call__(self, *args, **kwargs): log.debug("entering callable {0}".format(self.f.__name__)) try:
#! /usr/bin/env python # -*- coding: utf-8 -*- # __author__ = 'kute' # __mtime__ = '2016/11/19 12:05' """ """ from logbook import Logger, FileHandler, INFO, Processor import pandas as pd formatstr = "{record.time}:{record.message}" handler = FileHandler(filename="test.log", mode="a", encoding="utf-8", level=INFO, format_string=formatstr)\ .push_application() mylog = Logger("processor") usercommentfile = "/Users/kute/work/docs/netease/data/UserComment" usercommentary = ["aa", "ab", "ac", "ad"] userinfofile = "/Users/kute/work/docs/netease/data/UserInfoa{}" userinfoary = [chr(i) for i in range(ord('a'), ord('v') + 1)] delfile = "/Users/kute/work/docs/netease/data/del20161008/{}_del20161008.txt" delary = ["yeah", "126", "163"] # ['aa', 'ab', ..., 'be'] del163ary = [('a' if i <= ord('z') else 'b') + chr(i if i <= ord('z') else i - ord('z') + ord('a') - 1) for i in range(ord('a'), ord('z') + 6)] del163file = "/Users/kute/work/docs/netease/data/del20161008/163_del20161008{}"
import os, sys from os import environ as env from logbook import Logger, FileHandler, DEBUG, INFO, NOTSET, StreamHandler, set_datetime_format from zipline.api import get_datetime import datetime import linecache import os import tracemalloc # log in local time instead of UTC set_datetime_format("local") LOG_ENTRY_FMT = '[{record.time:%Y-%m-%d %H:%M:%S}] {record.level_name}: {record.message}' logfilename = os.path.join(env["HOME"], "log", "sharadar-zipline.log") log = Logger('sharadar_db_bundle') log_file_handler = FileHandler(logfilename, level=DEBUG, bubble=True) log_file_handler.format_string = LOG_ENTRY_FMT log.handlers.append(log_file_handler) log_std_handler = StreamHandler(sys.stdout, level=INFO) log_std_handler.format_string = LOG_ENTRY_FMT log.handlers.append(log_std_handler) def log_top_mem_usage(logger, snapshot, key_type='lineno', limit=10): snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) logger.info("Top %s lines" % limit)
import matplotlib.pyplot as plt import numpy as np import pandas as pd from logbook import Logger, FileHandler import datetime from catalyst import run_algorithm from catalyst.api import (record, symbol, order, order_target_percent) from catalyst.exchange.utils.stats_utils import extract_transactions from catalyst.exchange.utils.exchange_utils import get_exchange_symbols NAMESPACE = 'OLMAR' log = Logger(NAMESPACE) file_handler = FileHandler("olmar.log") # file_handler.push_application() def initialize(context): context.ASSET_NAMES = ['btc_usdt'] #, 'eth_usdt', 'xrp_usdt'] # import pdb; pdb.set_trace() context.assets = [symbol(asset_name) for asset_name in context.ASSET_NAMES] context.m = len(context.assets) + 1 # +1 for cash context.b_t = np.ones(context.m) / context.m context.eps = 1.4 #change epsilon here context.init = True context.counter = 0 context.window = 4 # in minutes #set_slippage(slippage.VolumeShareSlippage(volume_limit=0.25, price_impact=0, delay=datetime.timedelta(minutes=0))) #set_commission(commission.PerShare(cost=0))
def run(): f = NamedTemporaryFile() with FileHandler(f.name) as handler: for x in xrange(500): log.warning(u'this is handled \x6f')
file_handle = kwargs.get("file_handle") logger.debug( f"Unlocking the file {file_name} on host {host}, owner={owner}, client={client_name}," f" kwargs = {kwargs}") file_handle = self._get_file_handle( host, export, file_name) if not file_handle else file_handle nlm_client = NLMClient(host) unlock_arguments = get_packer_arguments("UNLOCK", caller_name=client_name, owner=owner, fh=file_handle, l_offset=offset, l_len=length) status = nlm_client.unlock(unlock_arguments) return NLM4_Stats(status).name def _get_file_handle(self, host, export, file_name): file_handle = self.exposed_lookup_file(host, export, file_name) if not file_handle: raise FileNotFound( f"{file_name} cannot be found and file_handle was not specified" ) return file_handle if __name__ == "__main__": FileHandler("nfs_client.log").push_application() t = ThreadedServer(NFSClientWrapper, port=9999) t.daemon = True logger.notice("Starting server on port 9999") t.start()
""" from logbook import FileHandler, Logger, StreamHandler import sys # Two handlers are created. Notice that other parts of the application # will create handlers as well, these two are general handlers (one stream # handler configured on `sys.stdout`, one file handler set on "bui.log" # although you can change the file name). stream = StreamHandler(sys.stdout, encoding="utf-8", level="INFO", bubble=True) stream.format_string = ( "[{record.level_name}] {record.channel}: {record.message}") file = FileHandler("bui.log", encoding="utf-8", level="INFO", delay=True, bubble=True) file.format_string = ( "{record.time:%Y-%m-%d %H:%M:%S.%f%z} [{record.level_name}] " "{record.channel}: {record.message}") # At this point, neither handler is used, if we create the logger and # write in it, nothing will be logged unless `push_application` # is called on the handlers. logger = Logger("bui") def start_logging(): """Start logging, push the logger.""" stream.push_application()
def main(): """Shows basic usage of the Google Drive API. Creates a Google Drive API service object and outputs the names and IDs for up to 10 files. """ log_filename = os.path.join( args.log_dir, 'google-drive-to-s3-{}.log'.format(os.path.basename(time.strftime('%Y%m%d-%H%M%S'))) ) # register some logging handlers log_handler = FileHandler( log_filename, mode='w', level=args.log_level, bubble=True ) stdout_handler = StreamHandler(sys.stdout, level=args.log_level, bubble=True) with stdout_handler.applicationbound(): with log_handler.applicationbound(): log.info("Arguments: {}".format(args)) start = time.time() log.info("starting at {}".format(time.strftime('%l:%M%p %Z on %b %d, %Y'))) credentials = get_credentials() http = credentials.authorize(httplib2.Http()) drive_service = discovery.build('drive', 'v3', http=http) s3 = boto3.resource('s3') # load up a match file if we have one. if args.match_file: with open(args.match_file, 'r') as f: match_filenames = f.read().splitlines() else: match_filenames = None # get the files in the specified folder. files = drive_service.files() request = files.list( pageSize=args.page_size, q="'{}' in parents".format(args.folder_id), fields="nextPageToken, files(id, name)" ) # make sure our S3 Key prefix has a trailing slash key_prefix = ensure_trailing_slash(args.key_prefix) page_counter = 0 file_counter = 0 while request is not None: file_page = request.execute(http=http) page_counter += 1 page_file_counter = 0 # reset the paging file counter # determine the page at which to start processing. if page_counter >= args.start_page: log.info(u"######## Page {} ########".format(page_counter)) for this_file in file_page['files']: file_counter += 1 page_file_counter += 1 if we_should_process_this_file(this_file['name'], match_filenames): log.info(u"#== Processing {} file number {} on page {}. {} files processed.".format( this_file['name'], page_file_counter, page_counter, file_counter )) # download the file download_request = drive_service.files().get_media(fileId=this_file['id']) fh = io.BytesIO() # Using an in memory stream location downloader = MediaIoBaseDownload(fh, download_request) done = False pbar = InitBar(this_file['name']) while done is False: status, done = downloader.next_chunk() pbar(int(status.progress()*100)) # print("\rDownload {}%".format(int(status.progress() * 100))) del pbar # upload to bucket log.info(u"Uploading to S3") s3.Bucket(args.bucket).put_object( Key="{}{}".format(key_prefix, this_file['name']), Body=fh.getvalue(), ACL='public-read' ) log.info(u"Uploaded to S3") fh.close() # close the file handle to release memory else: log.info(u"Do not need to process {}".format(this_file['name'])) # stop if we have come to the last user specified page if args.end_page and page_counter == args.end_page: log.info(u"Finished paging at page {}".format(page_counter)) break # request the next page of files request = files.list_next(request, file_page) log.info("Running time: {}".format(str(datetime.timedelta(seconds=(round(time.time() - start, 3)))))) log.info("Log written to {}:".format(log_filename))
def logger(self, log_path): self._log_path = log_path FileHandler(self._log_path).push_application() self._logger = Logger(Filament.__name__)
def setup_logger(test, path='test.log'): test.log_handler = FileHandler(path) test.log_handler.push_application()
def push_file_handler(file: Path, level: int = NOTICE, encoding: str = 'utf-8') -> FileHandler: handler = FileHandler(file.expanduser().absolute().__str__(), level=level, encoding=encoding) handler.push_application() return handler