async def task(task_name: str): def inject_extra(record): record.extra['task_name'] = task_name record.extra['task_id'] = id(asyncio.current_task()) with Handler(bubble=True).contextbound(): with Processor(inject_extra).contextbound(): logger.info('I am the task') await asyncio.sleep(0) await util() logger.info('I am still the task')
def __init__(self, algo, sim_params, data_portal, clock, benchmark_source, restrictions, universe_func, instant_fill=False): # ============== # Simulation # Param Setup # ============== self.sim_params = sim_params self.env = algo.trading_environment self.data_portal = data_portal self.restrictions = restrictions self.instant_fill = instant_fill # ============== # Algo Setup # ============== self.algo = algo # ============== # Snapshot Setup # ============== # This object is the way that user algorithms interact with OHLCV data, # fetcher data, and some API methods like `data.can_trade`. self.current_data = self._create_bar_data(universe_func) # We don't have a datetime for the current snapshot until we # receive a message. self.simulation_dt = None self.clock = clock self.benchmark_source = benchmark_source # ============= # Logging Setup # ============= # Processor function for injecting the algo_dt into # user prints/logs. def inject_algo_dt(record): if 'algo_dt' not in record.extra: record.extra['algo_dt'] = self.simulation_dt self.processor = Processor(inject_algo_dt)
def test_mdc_works(): def inject_extra(record): record.extra['ip'] = '127.0.0.1' record.extra['username'] = '******' with TestHandler() as handler: handler.formatter = text_formatter logger = logging.getLogger('Dummy') with redirected_logging(): with Processor(inject_extra): logger.info('hello world') assert len(handler.formatted_records) == 1 assert 'INFO: Dummy: hello world <ip=127.0.0.1, username=Andrey>' in handler.formatted_records[0]
def __init__(self, blotter, perf_tracker, algo, algo_start): # ========== # Algo Setup # ========== # We extract the order book from the txn client so that # the algo can place new orders. self.blotter = blotter self.perf_tracker = perf_tracker self.perf_key = self.EMISSION_TO_PERF_KEY_MAP[ perf_tracker.emission_rate] self.algo = algo self.algo_start = algo_start.replace(hour=0, minute=0, second=0, microsecond=0) # Monkey patch the user algorithm to place orders in the # TransactionSimulator's order book and use our logger. self.algo.set_order(self.order) # ============== # Snapshot Setup # ============== # The algorithm's universe as of our most recent event. # We want an ndict that will have empty objects as default # values on missing keys. self.universe = ndict(internal=defaultdict(SIDData)) # We don't have a datetime for the current snapshot until we # receive a message. self.simulation_dt = None self.snapshot_dt = None # ============= # Logging Setup # ============= # Processor function for injecting the algo_dt into # user prints/logs. def inject_algo_dt(record): if not 'algo_dt' in record.extra: record.extra['algo_dt'] = self.snapshot_dt self.processor = Processor(inject_algo_dt)
def transact_stub(slippage, commission, event, open_orders): """ This is intended to be wrapped in a partial, so that the slippage and commission models can be enclosed. """ def inject_algo_dt(record): if not 'algo_dt' in record.extra: record.extra['algo_dt'] = event['dt'] with Processor(inject_algo_dt).threadbound(): transaction = slippage.simulate(event, open_orders) if transaction and transaction.amount != 0: direction = abs(transaction.amount) / transaction.amount per_share, total_commission = commission.calculate(transaction) transaction.price = transaction.price + (per_share * direction) transaction.commission = total_commission return transaction
def test_json_formatting_works(): def inject_extra(record): record.extra['ip'] = '127.0.0.1' record.extra['username'] = '******' with TestHandler() as handler: handler.formatter = json_formatter logger = logging.getLogger('Dummy') with redirected_logging(): with Processor(inject_extra): logger.info('hello world') assert len(handler.formatted_records) == 1 record = json.loads(handler.formatted_records[0]) assert record['level'] == 'INFO' assert record['name'] == 'Dummy' assert record['message'] == 'hello world' assert record['ip'] == '127.0.0.1' assert record['username'] == 'Andrey'
def __init__(self, algo, sim_params): # ============== # Simulation # Param Setup # ============== self.sim_params = sim_params # ============== # Algo Setup # ============== self.algo = algo self.algo_start = self.sim_params.first_open self.algo_start = self.algo_start.replace(hour=0, minute=0, second=0, microsecond=0) # ============== # Snapshot Setup # ============== # The algorithm's data as of our most recent event. # We want an object that will have empty objects as default # values on missing keys. self.current_data = BarData() # We don't have a datetime for the current snapshot until we # receive a message. self.simulation_dt = None # ============= # Logging Setup # ============= # Processor function for injecting the algo_dt into # user prints/logs. def inject_algo_dt(record): if not 'algo_dt' in record.extra: record.extra['algo_dt'] = self.simulation_dt self.processor = Processor(inject_algo_dt)
def _bootstrap(): """Get the Inyoka version and store it.""" global INYOKA_REVISION # the path to the contents of the Inyoka module conts = os.environ.setdefault('INYOKA_MODULE', realpath(join(dirname(__file__)))) # the path to the Inyoka instance folder os.environ['INYOKA_INSTANCE'] = realpath(join(conts, pardir)) os.environ['CELERY_LOADER'] = 'inyoka.core.celery_support.CeleryLoader' # get the `INYOKA_REVISION` using the mercurial python api try: ui = hgui.ui() repository = localrepository(ui, join(conts, '..')) ctx = repository['tip'] INYOKA_REVISION = ('%(num)s:%(id)s' % { 'num': ctx.rev(), 'id': shorthex(ctx.node()) }) except TypeError: # fail silently pass # This value defines the timeout for sockets in seconds. Per default python # sockets do never timeout and as such we have blocking workers. # Socket timeouts are set globally within the whole application. # The value *must* be a floating point value. socket.setdefaulttimeout(10.0) #: bind the context ctx = ApplicationContext() ctx.bind() # setup components ctx.load_packages(ctx.cfg['activated_components']) if ctx.cfg['testing']: logger.level_name = 'ERROR' # makes INYOKA_REVISION visible in the extra dict of every log record proc = Processor(lambda x: x.extra.update(INYOKA_REVISION=INYOKA_REVISION)) proc.push_application()
def transact_stub(slippage, commission, event, open_orders): """ This is intended to be wrapped in a partial, so that the slippage and commission models can be enclosed. """ def inject_algo_dt(record): if not 'algo_dt' in record.extra: record.extra['algo_dt'] = event['dt'] with Processor(inject_algo_dt).threadbound(): transactions = slippage.simulate(event, open_orders) for transaction in transactions: if (transaction and not zp_math.tolerant_equals(transaction.amount, 0)): direction = math.copysign(1, transaction.amount) per_share, total_commission = commission.calculate(transaction) transaction.price = transaction.price + (per_share * direction) transaction.commission = total_commission return transactions
def run(self): def inject_worker_id(record): record.extra['worker_id'] = self.id with Processor(inject_worker_id).threadbound(): iterations_left = NUM_ITERATIONS num_fails_in_sequence = 0 while iterations_left > 0 and not self.exit_event.is_set(): # acquire the locks locked = [] for data_id in range(NUM_DATA_SLOTS): key = 'key-%d' % data_id if self.lock(key): locked.append(key) else: break if len(locked) == NUM_DATA_SLOTS: num_fails_in_sequence = 0 for data_id in range(NUM_DATA_SLOTS): self.d[data_id].append(self.id) iterations_left -= 1 logger.info('decrement: locked=%r' % (locked, )) else: num_fails_in_sequence += 1 if num_fails_in_sequence > NUM_ITERATIONS * NUM_DATA_SLOTS * NUM_WORKERS: # seems that algorithm stuck somewhere, # exit from worker logger.warning('seems that algorithm stuck somewhere') break # release the locks for key in reversed(locked): self.unlock(key) time.sleep(random.random()) logger.debug('exit from the worker, locals=%r' % (locals(), ))
def __init__(self, algo, sim_params): # ============== # Simulation # Param Setup # ============== self.sim_params = sim_params # ============== # Algo Setup # ============== self.algo = algo self.algo_start = normalize_date(self.sim_params.first_open) self.env = algo.trading_environment # ============== # Snapshot Setup # ============== # The algorithm's data as of our most recent event. # We want an object that will have empty objects as default # values on missing keys. self.current_data = BarData() # We don't have a datetime for the current snapshot until we # receive a message. self.simulation_dt = None self.previous_dt = self.algo_start # ============= # Logging Setup # ============= # Processor function for injecting the algo_dt into # user prints/logs. def inject_algo_dt(record): if 'algo_dt' not in record.extra: record.extra['algo_dt'] = self.simulation_dt self.processor = Processor(inject_algo_dt)
def g(*args, **kargs): with Processor(partial(inject_func, f.__name__, args, kargs)).threadbound(): return f(*args, **kargs)
""" import os import sys from logbook import Processor, StreamHandler, DEBUG, Logger, FileHandler my_handler = FileHandler("test.log", encoding="utf-8", level=DEBUG) # my_handler = StreamHandler(sys.stdout, level=DEBUG) def log_other_info(record): """ a) 通过 with.processor可以让在其中的日志拥有共同的逻辑,相当于一个切面注入 比如这里的例子是 在每条日志中记录一些额外的信息(额外的信息是通过在日志对象(logRecord)的extra(字典对象)属性中添加 一些其他的信息),这样每条日志都会有这里添加的额外的信息。 b) 有个疑问就是,这些额外的信息怎么运用呢,比如这些信息如何能和日志一块记录在文件中呢 c) 关于日志的属性,见 logrecord.py """ record.extra['myname'] = 'kute' record.extra['mycwd'] = os.getcwd() # update myname propertiy record.extra.update(myname="lisa") print(record.to_dict()) if __name__ == "__main__": with my_handler.applicationbound(): with Processor(log_other_info).applicationbound(): mylog = Logger("processor") mylog.notice("notice msg.")
' : ', '{record.message}', ]), } def inject_extra(record): record.extra['basename'] = os.path.basename(record.filename) record.extra['level_color'] = get_log_color(record.level) record.extra['clear_color'] = color.ENDC logger = Logger('root') # extra info processor = Processor(inject_extra) processor.push_application() # for screen log screen_level = INFO stream_handler = StreamHandler(sys.stdout, level=screen_level, bubble=True) stream_handler.format_string = formatter['screen'] stream_handler.push_application() # for rolling file log p = os.environ['FBPATH'] if not os.path.isdir(p): os.system('mkdir -p {}'.format(p)) file_path = os.path.expanduser(os.path.join(p, 'logs')) if os.path.isdir(file_path): backup_count = 7
def __init__(self, algo, sim_params): # ============== # Simulation # Param Setup # ============== self.sim_params = sim_params # ============== # Algo Setup # ============== self.algo = algo self.algo_start = normalize_date(self.sim_params.first_open) self.env = algo.trading_environment # ============== # Snapshot Setup # ============== _day = timedelta(days=1) def _get_removal_date(sid, finder=self.env.asset_finder, default=self.sim_params.last_close + _day): """ Get the date of the morning on which we should remove an asset from data. If we don't have an auto_close_date, this is just the end of the simulation. If we have an auto_close_date, then we remove assets from data on max(asset.auto_close_date, asset.end_date + timedelta(days=1)) We hold assets at least until auto_close_date because up until that date the user might still hold positions or have open orders in an expired asset. We hold assets at least until end_date + 1, because an asset continues trading until the **end** of its end_date. Even if an asset auto-closed before the end_date (say, because Interactive Brokers clears futures positions prior the actual notice or expiration), there may still be trades arriving that represent signals for other assets that are still tradeable. (Particularly in the futures case, trading in the final days of a contract are likely relevant for trading the next contract on the same future chain.) 获取上午的日期,我们应该从数据中删除资产。 #如果我们没有auto_close_date,那只是结束了 #模拟。 #如果我们有auto_close_date,那么我们从数据中删除资产 #max(asset.auto_close_date,asset.end_date + timedelta(days = 1)) #我们至少持有资产直到auto_close_date,因为直到那之前 #用户可能仍然持有头寸或在一个开仓订单的日期 #过期资产。 #我们持有资产至少到end_date + 1,因为资产 #继续交易直到其end_date的**结束**。即使是 #end_date之前的资产自动关闭(比如因为Interactive #经纪人在实际通知之前清算期货头寸或 #到期),可能仍有交易到达代表 #其他资产仍可交易的信号。 (特别是在 #期货案例,在合约的最后几天进行交易 #可能与在同一个未来交易下一份合约有关 #链。) """ try: asset = finder.retrieve_asset(sid) except ValueError: # Handle sid not an int, such as from a custom source. # So that they don't compare equal to other sids, and we'd # blow up comparing strings to ints, let's give them unique # close dates. return default + timedelta(microseconds=id(sid)) except SidsNotFound: return default auto_close_date = asset.auto_close_date if auto_close_date is None: # If we don't have an auto_close_date, we never remove an asset # from the user's portfolio. return default end_date = asset.end_date if end_date is None: # If we have an auto_close_date but not an end_date, clear the # asset from data when we clear positions/orders. return auto_close_date # If we have both, make close once we're on or after the # auto_close_date, and strictly after the end_date. # See docstring above for an explanation of this logic. return max(auto_close_date, end_date + _day) self._get_removal_date = _get_removal_date # The algorithm's data as of our most recent event. # Maintain sids in order by asset close date, so that we can more # efficiently remove them when their times come...算法的数据是我们最近的事件。 # 按资产截止日期按顺序维护sid,以便我们可以在时间到来时更有效地删除它们... self.current_data = BarData(SortedDict(self._get_removal_date)) # We don't have a datetime for the current snapshot until we # receive a message.在收到消息之前,我们没有当前快照的日期时间 self.simulation_dt = None # ============= # Logging Setup # ============= # Processor function for injecting the algo_dt into # user prints/logs. def inject_algo_dt(record): if 'algo_dt' not in record.extra: record.extra['algo_dt'] = self.simulation_dt self.processor = Processor(inject_algo_dt)
from utils import get_ip def inject_information(record): record.extra['ip'] = get_ip() log_format = u'[{record.time:%Y-%m-%d %H:%M}] {record.channel} - {record.level_name}: {record.message} \t({record.extra[ip]})' # a nested handler setup can be used to configure more complex setups setup = NestedSetup([ #StderrHandler(format_string=u'[{record.time:%Y-%m-%d %H:%M}] {record.channel} - {record.level_name}: {record.message} \t({record.extra[ip]})'), StreamHandler(sys.stdout, format_string=log_format), # then write messages that are at least warnings to to a logfile FileHandler(os.environ['QTRADE_LOG'], level='WARNING'), Processor(inject_information) ]) color_setup = NestedSetup([ StreamHandler(sys.stdout, format_string=log_format), ColorizedStderrHandler(format_string=log_format, level='NOTICE'), Processor(inject_information) ]) remote_setup = NestedSetup([ ZeroMQHandler('tcp://127.0.0.1:5540'), Processor(inject_information) ]) log = Logger('Trade Labo')
def __init__(self, algo, sim_params): # ============== # Simulation # Param Setup # ============== self.sim_params = sim_params # ============== # Algo Setup # ============== self.algo = algo self.algo_start = normalize_date(self.sim_params.first_open) self.env = algo.trading_environment # ============== # Snapshot Setup # ============== _day = timedelta(days=1) def _get_removal_date(sid, finder=self.env.asset_finder, default=self.sim_params.last_close + _day): """ Get the date of the morning on which we should remove an asset from data. If we don't have an auto_close_date, this is just the end of the simulation. If we have an auto_close_date, then we remove assets from data on max(asset.auto_close_date, asset.end_date + timedelta(days=1)) We hold assets at least until auto_close_date because up until that date the user might still hold positions or have open orders in an expired asset. We hold assets at least until end_date + 1, because an asset continues trading until the **end** of its end_date. Even if an asset auto-closed before the end_date (say, because Interactive Brokers clears futures positions prior the actual notice or expiration), there may still be trades arriving that represent signals for other assets that are still tradeable. (Particularly in the futures case, trading in the final days of a contract are likely relevant for trading the next contract on the same future chain.) """ try: asset = finder.retrieve_asset(sid) except ValueError: # Handle sid not an int, such as from a custom source. # So that they don't compare equal to other sids, and we'd # blow up comparing strings to ints, let's give them unique # close dates. return default + timedelta(microseconds=id(sid)) except SidsNotFound: return default auto_close_date = asset.auto_close_date if auto_close_date is None: # If we don't have an auto_close_date, we never remove an asset # from the user's portfolio. return default end_date = asset.end_date if end_date is None: # If we have an auto_close_date but not an end_date, clear the # asset from data when we clear positions/orders. return auto_close_date # If we have both, make close once we're on or after the # auto_close_date, and strictly after the end_date. # See docstring above for an explanation of this logic. return max(auto_close_date, end_date + _day) self._get_removal_date = _get_removal_date # The algorithm's data as of our most recent event. # Maintain sids in order by asset close date, so that we can more # efficiently remove them when their times come... self.current_data = BarData(SortedDict(self._get_removal_date)) # We don't have a datetime for the current snapshot until we # receive a message. self.simulation_dt = None # ============= # Logging Setup # ============= # Processor function for injecting the algo_dt into # user prints/logs. def inject_algo_dt(record): if 'algo_dt' not in record.extra: record.extra['algo_dt'] = self.simulation_dt self.processor = Processor(inject_algo_dt)
# GMail 邮箱 ghandler = GMailHandler( account_id="*****@*****.**", password="******", recipients=["*****@*****.**"], format_string=formatstr ) def main(): mylog = Logger("MailHandler-APP") mylog.info("mailhandler message") def inject_other_info(record): record.extra['myscret'] = "do not tell you" record.extra.update( # some other info pp="pp info", ip="127.0.0.1", url="http://logbook.readthedocs.io/en/stable/", method="GET", myscret="do not tell you yet" ) if __name__ == "__main__": with ghandler.threadbound(): with Processor(callback=inject_other_info).threadbound(): main()
from logbook import warn, StreamHandler import sys from termcc.cc import cc my_handler = StreamHandler(sys.stdout) my_handler.push_application() warn(cc(':red: :yin_yang: This is a warning :reset:')) import os from logbook import Processor def inject_cwd(record): record.extra['cwd'] = os.getcwd() with my_handler.applicationbound(): with Processor(inject_cwd).applicationbound(): warn(cc(':blue: :yin_yang: This is a warning'))
def __init__(self, algo, sim_params): # ============== # Simulation # Param Setup # ============== self.sim_params = sim_params # ============== # Algo Setup # ============== self.algo = algo self.algo_start = normalize_date(self.sim_params.first_open) self.env = algo.trading_environment # ============== # Snapshot Setup # ============== def _get_asset_close_date(sid, finder=self.env.asset_finder, default=self.sim_params.last_close + timedelta(days=1)): try: asset = finder.retrieve_asset(sid) except ValueError: # Handle sid not an int, such as from a custom source. # So that they don't compare equal to other sids, and we'd # blow up comparing strings to ints, let's give them unique # close dates. return default + timedelta(microseconds=id(sid)) except SidsNotFound: return default # Default is used when the asset has no auto close date, # and is set to a time after the simulation ends, so that the # relevant asset isn't removed from the universe at all # (at least not for this reason). return asset.auto_close_date or default self._get_asset_close = _get_asset_close_date # The algorithm's data as of our most recent event. # Maintain sids in order by asset close date, so that we can more # efficiently remove them when their times come... self.current_data = BarData(SortedDict(self._get_asset_close)) # We don't have a datetime for the current snapshot until we # receive a message. self.simulation_dt = None # ============= # Logging Setup # ============= # Processor function for injecting the algo_dt into # user prints/logs. def inject_algo_dt(record): if 'algo_dt' not in record.extra: record.extra['algo_dt'] = self.simulation_dt self.processor = Processor(inject_algo_dt)
import sys from kute.easylog.easylog import geteasylog from logbook import StreamHandler, DEBUG, Processor, Logger easylog = geteasylog() StreamHandler(sys.stdout, level=DEBUG).push_application() def printrecorddetail(record): easylog.info(record.channel) # name of the logger easylog.info(record.dispatcher.name) easylog.info(record.exception_message) easylog.info(record.exception_name) easylog.info(record.extra) easylog.info(record.filename) easylog.info(record.level) easylog.info(record.level_name) easylog.info(record.message) easylog.info(record.msg) easylog.info(record.thread_name) easylog.info(record.time) easylog.info(record.to_dict(True)) # 纵览全部日志属性 if __name__ == "__main__": with Processor(printrecorddetail).applicationbound(): mylog = Logger("log-record-app-name") mylog.info("log record detail")