import os
import sys
import click
from elasticsearch import Elasticsearch
from logbook import Logger, StreamHandler
import sqlalchemy
from app import config
from app.analysis.model import db, Phrase, Sentence
es_host = os.environ.get('ES_HOST', 'localhost')
es_port = int(os.environ.get('ES_PORT', 9200))
es = Elasticsearch([{'host': es_host, 'port': es_port}])
StreamHandler(sys.stderr, level='INFO').push_application()
log = Logger('')
def str2datetime(s):
"""Parse a datetime string (with milliseconds)."""
parts = s.split('.')
dt = datetime.strptime(parts[0], "%Y-%m-%d %H:%M:%S")
return dt.replace(microsecond=int(parts[1]))
def unix_time(dt):
epoch = datetime.utcfromtimestamp(0)
delta = dt - epoch
return delta.total_seconds()
def __init__(self, **kwargs):
"""
Keyword Args:
filename=None (str, list): csv data file.
**datafeed_args (any): any datafeed-related args, passed through to
default btgym.datafeed class.
dataset=None (btgym.datafeed): BTgymDataDomain instance,
overrides `filename` or any other datafeed-related args.
strategy=None (btgym.startegy): strategy to be used by `engine`, any subclass of
btgym.strategy.base.BTgymBaseStrateg
engine=None (bt.Cerebro): environment simulation engine, any bt.Cerebro subclass,
overrides `strategy` arg.
network_address=`tcp://127.0.0.1:` (str): BTGym_server address.
port=5500 (int): network port to use for server - API_shell communication.
data_master=True (bool): let this environment control over data_server;
data_network_address=`tcp://127.0.0.1:` (str): data_server address.
data_port=4999 (int): network port to use for server -- data_server communication.
connect_timeout=60 (int): server connection timeout in seconds.
render_enabled=True (bool): enable rendering for this environment;
render_modes=['human', 'episode'] (list): `episode` - plotted episode results;
`human` - raw_state observation.
**render_args (any): any render-related args, passed through to renderer class.
verbose=0 (int): verbosity mode, {0 - WARNING, 1 - INFO, 2 - DEBUG}
log_level=None (int): logbook level {DEBUG=10, INFO=11, NOTICE=12, WARNING=13},
overrides `verbose` arg;
log=None (logbook.Logger): external logbook logger,
overrides `log_level` and `verbose` args.
task=0 (int): environment id
Environment kwargs applying logic::
if <engine> kwarg is given:
do not use default engine and strategy parameters;
ignore <strategy> kwarg and all strategy and engine-related kwargs.
else (no <engine>):
use default engine parameters;
if any engine-related kwarg is given:
override corresponding default parameter;
if <strategy> is given:
do not use default strategy parameters;
if any strategy related kwarg is given:
override corresponding strategy parameter;
else (no <strategy>):
use default strategy parameters;
if any strategy related kwarg is given:
override corresponding strategy parameter;
if <dataset> kwarg is given:
do not use default dataset parameters;
ignore dataset related kwargs;
else (no <dataset>):
use default dataset parameters;
if any dataset related kwarg is given:
override corresponding dataset parameter;
If any <other> kwarg is given:
override corresponding default parameter.
"""
# Parameters and default values:
self.params = dict(
# Backtrader engine mandatory parameters:
engine=dict(
start_cash=10.0, # initial trading capital.
broker_commission=
0.001, # trade execution commission, default is 0.1% of operation value.
fixed_stake=10, # single trade stake is fixed type by def.
),
# Dataset mandatory parameters:
dataset=dict(filename=None, ),
strategy=dict(state_shape=dict(), ),
render=dict(),
)
p2 = dict( # IS HERE FOR REFERENCE ONLY
# Strategy related parameters:
# Observation state shape is dictionary of Gym spaces,
# at least should contain `raw_state` field.
# By convention first dimension of every Gym Box space is time embedding one;
# one can define any shape; should match env.observation_space.shape.
# observation space state min/max values,
# For `raw_state' - absolute min/max values from BTgymDataset will be used.
state_shape=dict(raw_state=spaces.Box(
shape=(10, 4), low=-100, high=100, dtype=np.float32)),
drawdown_call=
None, # episode maximum drawdown threshold, default is 90% of initial value.
portfolio_actions=None,
# agent actions,
# should consist with BTgymStrategy order execution logic;
# defaults are: 0 - 'do nothing', 1 - 'buy', 2 - 'sell', 3 - 'close position'.
skip_frame=None,
# Number of environment steps to skip before returning next response,
# e.g. if set to 10 -- agent will interact with environment every 10th episode step;
# Every other step agent's action is assumed to be 'hold'.
# Note: INFO part of environment response is a list of all skipped frame's info's,
# i.e. [info[-9], info[-8], ..., info[0].
)
# Update self attributes, remove used kwargs:
for key in dir(self):
if key in kwargs.keys():
setattr(self, key, kwargs.pop(key))
self.metadata = {'render.modes': self.render_modes}
# Logging and verbosity control:
if self.log is None:
StreamHandler(sys.stdout).push_application()
if self.log_level is None:
log_levels = [(0, NOTICE), (1, INFO), (2, DEBUG)]
self.log_level = WARNING
for key, value in log_levels:
if key == self.verbose:
self.log_level = value
self.log = Logger('BTgymAPIshell_{}'.format(self.task),
level=self.log_level)
# Network parameters:
self.network_address += str(self.port)
self.data_network_address += str(self.data_port)
# Set server rendering:
if self.render_enabled:
self.renderer = BTgymRendering(self.metadata['render.modes'],
log_level=self.log_level,
**kwargs)
else:
self.renderer = BTgymNullRendering()
self.log.info(
'Rendering disabled. Call to render() will return null-plug image.'
)
# Append logging:
self.renderer.log = self.log
# Update params -1: pull from renderer, remove used kwargs:
self.params['render'].update(self.renderer.params)
for key in self.params['render'].keys():
if key in kwargs.keys():
_ = kwargs.pop(key)
if self.data_master:
# DATASET preparation, only data_master executes this:
#
if self.dataset is not None:
# If BTgymDataset instance has been passed:
# do nothing.
msg = 'Custom Dataset class used.'
else:
# If no BTgymDataset has been passed,
# Make default dataset with given CSV file:
try:
os.path.isfile(str(self.params['dataset']['filename']))
except:
raise FileNotFoundError(
'Dataset source data file not specified/not found')
# Use kwargs to instantiate dataset:
self.dataset = BTgymDataset(**kwargs)
msg = 'Base Dataset class used.'
# Append logging:
self.dataset.set_logger(self.log_level, self.task)
# Update params -2: pull from dataset, remove used kwargs:
self.params['dataset'].update(self.dataset.params)
for key in self.params['dataset'].keys():
if key in kwargs.keys():
_ = kwargs.pop(key)
self.log.info(msg)
# Connect/Start data server (and get dataset statistic):
self.log.info('Connecting data_server...')
self._start_data_server()
self.log.info('...done.')
# ENGINE preparation:
# Update params -3: pull engine-related kwargs, remove used:
for key in self.params['engine'].keys():
if key in kwargs.keys():
self.params['engine'][key] = kwargs.pop(key)
if self.engine is not None:
# If full-blown bt.Cerebro() subclass has been passed:
# Update info:
msg = 'Custom Cerebro class used.'
self.strategy = msg
for key in self.params['engine'].keys():
self.params['engine'][key] = msg
# Note: either way, bt.observers.DrawDown observer [and logger] will be added to any BTgymStrategy instance
# by BTgymServer process at runtime.
else:
# Default configuration for Backtrader computational engine (Cerebro),
# if no bt.Cerebro() custom subclass has been passed,
# get base class Cerebro(), using kwargs on top of defaults:
self.engine = bt.Cerebro()
msg = 'Base Cerebro class used.'
# First, set STRATEGY configuration:
if self.strategy is not None:
# If custom strategy has been passed:
msg2 = 'Custom Strategy class used.'
else:
# Base class strategy :
self.strategy = BTgymBaseStrategy
msg2 = 'Base Strategy class used.'
# Add, using kwargs on top of defaults:
#self.log.debug('kwargs for strategy: {}'.format(kwargs))
strat_idx = self.engine.addstrategy(self.strategy, **kwargs)
msg += ' ' + msg2
# Second, set Cerebro-level configuration:
self.engine.broker.setcash(self.params['engine']['start_cash'])
self.engine.broker.setcommission(
self.params['engine']['broker_commission'])
self.engine.addsizer(bt.sizers.SizerFix,
stake=self.params['engine']['fixed_stake'])
self.log.info(msg)
# Define observation space shape, minimum / maximum values and agent action space.
# Retrieve values from configured engine or...
# ...Update params -4:
# Pull strategy defaults to environment params dict :
for t_key, t_value in self.engine.strats[0][0][0].params._gettuple():
self.params['strategy'][t_key] = t_value
# Update it with values from strategy 'passed-to params':
for key, value in self.engine.strats[0][0][2].items():
self.params['strategy'][key] = value
# ... Push it all back (don't ask):
for key, value in self.params['strategy'].items():
self.engine.strats[0][0][2][key] = value
# For 'raw_state' min/max values,
# the only way is to infer from raw Dataset price values (we already got those from data_server):
if 'raw_state' in self.params['strategy']['state_shape'].keys():
# Exclude 'volume' from columns we count:
self.dataset_columns.remove('volume')
#print(self.params['strategy'])
#print('self.engine.strats[0][0][2]:', self.engine.strats[0][0][2])
#print('self.engine.strats[0][0][0].params:', self.engine.strats[0][0][0].params._gettuple())
# Override with absolute price min and max values:
self.params['strategy']['state_shape']['raw_state'].low =\
self.engine.strats[0][0][2]['state_shape']['raw_state'].low =\
np.zeros(self.params['strategy']['state_shape']['raw_state'].shape) +\
self.dataset_stat.loc['min', self.dataset_columns].min()
self.params['strategy']['state_shape']['raw_state'].high = \
self.engine.strats[0][0][2]['state_shape']['raw_state'].high = \
np.zeros(self.params['strategy']['state_shape']['raw_state'].shape) + \
self.dataset_stat.loc['max', self.dataset_columns].max()
self.log.info(
'Inferring `state_raw` high/low values form dataset: {:.6f} / {:.6f}.'
.format(
self.dataset_stat.loc['min', self.dataset_columns].min(),
self.dataset_stat.loc['max', self.dataset_columns].max()))
# Set observation space shape from engine/strategy parameters:
self.observation_space = DictSpace(
self.params['strategy']['state_shape'])
self.log.debug('Obs. shape: {}'.format(self.observation_space.spaces))
#self.log.debug('Obs. min:\n{}\nmax:\n{}'.format(self.observation_space.low, self.observation_space.high))
# Set action space and corresponding server messages:
self.action_space = spaces.Discrete(
len(self.params['strategy']['portfolio_actions']))
self.server_actions = self.params['strategy']['portfolio_actions']
# Finally:
self.server_response = None
self.env_response = None
#if not self.data_master:
self._start_server()
self.closed = False
self.log.info('Environment is ready.')
def __init__(
self,
filename=None,
parsing_params=None,
sampling_params=None,
name='base_data',
data_names=('default_asset',),
task=0,
frozen_time_split=None,
log_level=WARNING,
_config_stack=None,
**kwargs
):
"""
Args:
filename: Str or list of str, should be given either here or when calling read_csv(),
see `Notes`.
specific_params CSV to Pandas parsing
sep: ';'
header: 0
index_col: 0
parse_dates: True
names: ['open', 'high', 'low', 'close', 'volume']
specific_params Pandas to BT.feeds conversion
timeframe=1: 1 minute.
datetime: 0
open: 1
high: 2
low: 3
close: 4
volume: -1
openinterest: -1
specific_params Sampling
sample_class_ref: None - if not None, than sample() method will return instance of specified
class, which itself must be subclass of BaseBTgymDataset,
else returns instance of the base data class.
start_weekdays: [0, 1, 2, 3, ] - Only weekdays from the list will be used for sample start.
start_00: True - sample start time will be set to first record of the day
(usually 00:00).
sample_duration: {'days': 1, 'hours': 23, 'minutes': 55} - Maximum sample time duration
in days, hours, minutes
time_gap: {''days': 0, hours': 5, 'minutes': 0} - Data omittance threshold:
maximum no-data time gap allowed within sample in days, hours.
Thereby, if set to be < 1 day, samples containing weekends and holidays gaps
will be rejected.
test_period: {'days': 0, 'hours': 0, 'minutes': 0} - setting this param to non-zero
duration forces instance.data split to train / test subsets with test
subset duration equal to `test_period` with `time_gap` tolerance. Train data
always precedes test one:
[0_record<-train_data->split_point_record<-test_data->last_record].
sample_expanding: None, reserved for child classes.
Note:
- CSV file can contain duplicate records, checks will be performed and all duplicates will be removed;
- CSV file should be properly sorted by date_time in ascending order, no sorting checks performed.
- When supplying list of file_names, all files should be also listed ascending by their time period,
no correct sampling will be possible otherwise.
- Default parameters are source-specific and made to correctly parse 1 minute Forex generic ASCII
data files from www.HistData.com. Tune according to your data source.
"""
self.filename = filename
if parsing_params is None:
self.parsing_params = dict(
# Default parameters for source-specific CSV datafeed class,
# correctly parses 1 minute Forex generic ASCII
# data files from www.HistData.com:
# CSV to Pandas params.
sep=';',
header=0,
index_col=0,
parse_dates=True,
names=['open', 'high', 'low', 'close', 'volume'],
# Pandas to BT.feeds params:
timeframe=1, # 1 minute.
datetime=0,
open=1,
high=2,
low=3,
close=4,
volume=-1,
openinterest=-1,
)
else:
self.parsing_params = parsing_params
if sampling_params is None:
self.sampling_params = dict(
# Sampling params:
start_weekdays=[], # Only weekdays from the list will be used for episode start.
start_00=False, # Sample start time will be set to first record of the day (usually 00:00).
sample_duration=dict( # Maximum sample time duration in days, hours, minutes:
days=0,
hours=0,
minutes=0
),
time_gap=dict( # Maximum data time gap allowed within sample in days, hours. Thereby,
days=0, # if set to be < 1 day, samples containing weekends and holidays gaps will be rejected.
hours=0,
),
test_period=dict( # Time period to take test samples from, in days, hours, minutes:
days=0,
hours=0,
minutes=0
),
expanding=False,
)
else:
self.sampling_params = sampling_params
self.name = name
# String will be used as key name for bt_feed data-line:
self.task = task
self.log_level = log_level
self.data_names = data_names
self.data_name = self.data_names[0]
self.data = None # Will hold actual data as pandas dataframe
self.is_ready = False
self.global_timestamp = 0
self.start_timestamp = 0
self.final_timestamp = 0
self.data_stat = None # Dataset descriptive statistic as pandas dataframe
self.data_range_delta = None # Dataset total duration timedelta
self.max_time_gap = None
self.time_gap = None
self.max_sample_len_delta = None
self.sample_duration = None
self.sample_num_records = 0
self.start_weekdays = {0, 1, 2, 3, 4, 5, 6}
self.start_00 = False
self.expanding = False
self.sample_instance = None
self.test_range_delta = None
self.train_range_delta = None
self.test_num_records = 0
self.train_num_records = 0
self.total_num_records = 0
self.train_interval = [0, 0]
self.test_interval = [0, 0]
self.test_period = {'days': 0, 'hours': 0, 'minutes': 0}
self.train_period = {'days': 0, 'hours': 0, 'minutes': 0}
self._test_period_backshift_delta = datetime.timedelta(**{'days': 0, 'hours': 0, 'minutes': 0})
self.sample_num = 0
self.task = 0
self.metadata = {'sample_num': 0, 'type': None}
self.set_params(self.parsing_params)
self.set_params(self.sampling_params)
self._config_stack = copy.deepcopy(_config_stack)
try:
nested_config = self._config_stack.pop()
except (IndexError, AttributeError) as e:
# IF stack is empty, sample of this instance itself is not supposed to be sampled.
nested_config = dict(
class_ref=None,
kwargs=dict(
parsing_params=self.parsing_params,
sample_params=None,
name='data_stream',
task=self.task,
log_level=self.log_level,
_config_stack=None,
)
)
# Configure sample instance parameters:
self.nested_class_ref = nested_config['class_ref']
self.nested_params = nested_config['kwargs']
self.sample_name = '{}_w_{}_'.format(self.nested_params['name'], self.task)
self.nested_params['_config_stack'] = self._config_stack
# Logging:
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_{}'.format(self.name, self.task), level=self.log_level)
# Legacy parameter dictionary, left here for BTgym API_shell:
self.params = {}
self.params.update(self.parsing_params)
self.params.update(self.sampling_params)
if frozen_time_split is not None:
self.frozen_time_split = datetime.datetime(**frozen_time_split)
else:
self.frozen_time_split = None
self.frozen_split_timestamp = None
import asyncio
import configparser
import inspect
import sys
import time
import os
import discord
from discord.ext import commands
from logbook import Logger, StreamHandler, FileHandler
logger = Logger("Discord Music")
logger.handlers.append(StreamHandler(sys.stdout, bubble=True))
logger.handlers.append(FileHandler("last-run.log", bubble=True, mode="w"))
logger.debug("Loading config files")
default_config = "[Config]\ntoken = \nsnip = "
config = configparser.ConfigParser()
token = ""
snip = ""
if os.path.exists("config.ini"):
config.read("config.ini")
try:
token = config['Config']['token']
except KeyError:
logger.critical(
def main():
from logbook import StreamHandler
import sys
StreamHandler(sys.stdout).push_application()
"""
Module for building a complete dataset from local directory with csv files.
"""
import os
import sys
from logbook import Logger, StreamHandler
from numpy import empty
from pandas import DataFrame, read_csv, Index, Timedelta, NaT
from trading_calendars import register_calendar_alias
from zipline.utils.cli import maybe_show_progress
from . import core as bundles
handler = StreamHandler(sys.stdout, format_string=" | {record.message}")
logger = Logger(__name__)
logger.handlers.append(handler)
def csvdir_equities(tframes=None, csvdir=None):
"""
Generate an ingest function for custom data bundle
This function can be used in ~/.zipline/extension.py
to register bundle with custom parameters, e.g. with
a custom trading calendar.
Parameters
----------
tframes: tuple, optional
The data time frames, supported timeframes: 'daily' and 'minute'
def defLogging():
global debug
global logPath
global loggingLevel
global logging_setup
try:
if debug:
logging_setup = NestedSetup([
# make sure we never bubble up to the stderr handler
# if we run out of setup handling
NullHandler(),
StreamHandler(sys.stdout, bubble=False, level=loggingLevel),
TimedRotatingFileHandler(
logPath,
level=0,
backup_count=3,
bubble=True,
date_format='%Y-%m-%d',
),
])
else:
logging_setup = NestedSetup([
# make sure we never bubble up to the stderr handler
# if we run out of setup handling
NullHandler(),
FingersCrossedHandler(
TimedRotatingFileHandler(
logPath,
level=0,
backup_count=3,
bubble=False,
date_format='%Y-%m-%d',
),
action_level=ERROR,
buffer_size=1000,
# pull_information=True,
# reset=False,
)
])
except:
print(
"Critical error attempting to setup logging. Falling back to console only."
)
logging_setup = NestedSetup([
# make sure we never bubble up to the stderr handler
# if we run out of setup handling
NullHandler(),
StreamHandler(sys.stdout, bubble=False)
])
with logging_setup.threadbound():
# Output all stdout (print) messages as warnings
try:
sys.stdout = LoggerWriter(pyfalog.warning)
except:
pyfalog.critical(
"Cannot redirect. Continuing without writing stdout to log.")
# Output all stderr (stacktrace) messages as critical
try:
sys.stderr = LoggerWriter(pyfalog.critical)
except:
pyfalog.critical(
"Cannot redirect. Continuing without writing stderr to log.")
def run(self):
"""
Server process runtime body. This method is invoked by env._start_server().
"""
# Logging:
from logbook import Logger, StreamHandler, WARNING
import sys
StreamHandler(sys.stdout).push_application()
if self.log_level is None:
self.log_level = WARNING
self.log = Logger('BTgymServer_{}'.format(self.task),
level=self.log_level)
self.process = multiprocessing.current_process()
self.log.info('PID: {}'.format(self.process.pid))
# Runtime Housekeeping:
cerebro = None
episode_result = dict()
episode_sample = None
trial_sample = None
trial_stat = None
dataset_stat = None
# How long to wait for data_master to reset data:
self.wait_for_data_reset = 300 # seconds
connect_timeout = 60 # in seconds
# Set up a comm. channel for server as ZMQ socket
# to carry both service and data signal
# !! Reminder: Since we use REQ/REP - messages do go in pairs !!
self.context = zmq.Context()
self.socket = self.context.socket(zmq.REP)
self.socket.setsockopt(zmq.RCVTIMEO, -1)
self.socket.setsockopt(zmq.SNDTIMEO, connect_timeout * 1000)
self.socket.bind(self.network_address)
self.data_context = zmq.Context()
self.data_socket = self.data_context.socket(zmq.REQ)
self.data_socket.setsockopt(zmq.RCVTIMEO, connect_timeout * 1000)
self.data_socket.setsockopt(zmq.SNDTIMEO, connect_timeout * 1000)
self.data_socket.connect(self.data_network_address)
# Check connection:
self.log.debug('Pinging data_server at: {} ...'.format(
self.data_network_address))
data_server_response = self._comm_with_timeout(
socket=self.data_socket, message={'ctrl': 'ping!'})
if data_server_response['status'] in 'ok':
self.log.debug(
'Data_server seems ready with response: <{}>'.format(
data_server_response['message']))
else:
msg = 'Data_server unreachable with status: <{}>.'.\
format(data_server_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
# Init renderer:
self.render.initialize_pyplot()
# Mandatory DrawDown and auxillary plotting observers to add to data-master startegy instance:
# TODO: make plotters optional args
if self.render.enabled:
aux_obsrevers = [bt.observers.DrawDown, NormPnL, Position, Reward]
else:
aux_obsrevers = [bt.observers.DrawDown]
# Server 'Control Mode' loop:
for episode_number in itertools.count(0):
while True:
# Stuck here until '_reset' or '_stop':
service_input = self.socket.recv_pyobj()
msg = 'Control mode: received <{}>'.format(service_input)
self.log.debug(msg)
if 'ctrl' in service_input:
# It's time to exit:
if service_input['ctrl'] == '_stop':
# Server shutdown logic:
# send last run statistic, release comm channel and exit:
message = 'Exiting.'
self.log.info(message)
self.socket.send_pyobj(message)
self.socket.close()
self.context.destroy()
return None
# Start episode:
elif service_input['ctrl'] == '_reset':
message = 'Preparing new episode with kwargs: {}'.format(
service_input['kwargs'])
self.log.debug(message)
self.socket.send_pyobj(message) # pairs '_reset'
break
# Retrieve statistic:
elif service_input['ctrl'] == '_getstat':
self.socket.send_pyobj(episode_result)
self.log.debug('Episode statistic sent.')
# Send episode rendering:
elif service_input[
'ctrl'] == '_render' and 'mode' in service_input.keys(
):
# Just send what we got:
self.socket.send_pyobj(
self.render.render(service_input['mode'], ))
self.log.debug(
'Episode rendering for [{}] sent.'.format(
service_input['mode']))
else: # ignore any other input
# NOTE: response string must include 'ctrl' key
# for env.reset(), env.get_stat(), env.close() correct operation.
message = {
'ctrl':
'send control keys: <_reset>, <_getstat>, <_render>, <_stop>.'
}
self.log.debug('Control mode: sent: ' + str(message))
self.socket.send_pyobj(
message) # pairs any other input
else:
message = 'No <ctrl> key received:{}\nHint: forgot to call reset()?'.format(
msg)
self.log.debug(message)
self.socket.send_pyobj(message)
# Got '_reset' signal -> prepare Cerebro subclass and run episode:
start_time = time.time()
cerebro = copy.deepcopy(self.cerebro)
cerebro._socket = self.socket
cerebro._log = self.log
cerebro._render = self.render
# Add auxillary observers, if not already:
for aux in aux_obsrevers:
is_added = False
for observer in cerebro.observers:
if aux in observer:
is_added = True
if not is_added:
cerebro.addobserver(aux)
# Add communication utility:
cerebro.addanalyzer(
_BTgymAnalyzer,
_name='_env_analyzer',
)
# Data preparation:
# Parse args we got with _reset call:
sample_config = dict(
episode_config=copy.deepcopy(DataSampleConfig),
trial_config=copy.deepcopy(DataSampleConfig))
for key, config in sample_config.items():
try:
config.update(service_input['kwargs'][key])
except KeyError:
self.log.debug(
'_reset <{}> kwarg not found, using default values: {}'
.format(key, config))
# Get new Trial from data_server if requested,
# despite bult-in new/reuse data object sampling option, perform checks here to avoid
# redundant traffic:
if sample_config['trial_config']['get_new'] or trial_sample is None:
self.log.debug(
'Requesting new Trial sample with args: {}'.format(
sample_config['trial_config']))
trial_sample, trial_stat, dataset_stat = self.get_data(
**sample_config['trial_config'])
trial_sample.set_logger(self.log_level, self.task)
self.log.debug('Got new Trial: <{}>'.format(
trial_sample.filename))
else:
self.log.debug('Reusing Trial <{}>'.format(
trial_sample.filename))
# Get episode:
self.log.debug('Requesting episode from <{}>'.format(
trial_sample.filename))
episode_sample = trial_sample.sample(
**sample_config['episode_config'])
# Get episode data statistic and pass it to strategy params:
cerebro.strats[0][0][2]['trial_stat'] = trial_stat
cerebro.strats[0][0][2]['trial_metadata'] = trial_sample.metadata
cerebro.strats[0][0][2]['dataset_stat'] = dataset_stat
cerebro.strats[0][0][2]['episode_stat'] = episode_sample.describe()
cerebro.strats[0][0][2]['metadata'] = episode_sample.metadata
# Set nice broker cash plotting:
cerebro.broker.set_shortcash(False)
# Convert and add data to engine:
cerebro.adddata(episode_sample.to_btfeed())
# Finally:
episode = cerebro.run(stdstats=True,
preload=False,
oldbuysell=True)[0]
# Update episode rendering:
_ = self.render.render('just_render', cerebro=cerebro)
_ = None
# Recover that bloody analytics:
analyzers_list = episode.analyzers.getnames()
analyzers_list.remove('_env_analyzer')
elapsed_time = timedelta(seconds=time.time() - start_time)
self.log.debug('Episode elapsed time: {}.'.format(elapsed_time))
episode_result['episode'] = episode_number
episode_result['runtime'] = elapsed_time
episode_result['length'] = len(episode.data.close)
for name in analyzers_list:
episode_result[name] = episode.analyzers.getbyname(
name).get_analysis()
gc.collect()
# Just in case -- we actually shouldn't get there except by some error:
return None
def __init__(self, filament, **kwargs):
self._start = datetime.now()
try:
log_path = os.path.join(os.path.expanduser('~'), '.fibratus', 'fibratus.log')
FileHandler(log_path, mode='w+').push_application()
StreamHandler(sys.stdout).push_application()
except PermissionError:
panic("ERROR - Unable to open log file for writing due to permission error")
self.logger = Logger(Fibratus.__name__)
self._config = YamlConfig()
self.logger.info('Starting...')
enable_cswitch = kwargs.pop('cswitch', False)
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags(cswitch=enable_cswitch)
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
enum_handles = kwargs.pop('enum_handles', True)
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of the kernel trace
if enum_handles:
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
self.thread_registry = ThreadRegistry(self.handle_repository, self._handles)
self.kevt_streamc = KEventStreamCollector(etw.KERNEL_LOGGER_NAME.encode())
image_skips = self._config.image_skips
if len(image_skips) > 0:
self.logger.info("Adding skips for images %s" % image_skips)
for skip in image_skips:
self.kevt_streamc.add_skip(skip)
self.kevent = KEvent(self.thread_registry)
self.keventq = Queue()
self._output_classes = dict(console=ConsoleOutput,
amqp=AmqpOutput,
smtp=SmtpOutput,
elasticsearch=ElasticsearchOutput)
self._outputs = self._construct_outputs()
self.output_aggregator = OutputAggregator(self._outputs)
if filament:
filament.keventq = self.keventq
filament.logger = log_path
filament.setup_adapters(self._outputs)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.context_switch_registry = ContextSwitchRegistry(self.thread_registry, self.kevent)
self.output_kevents = {}
self.filters_count = 0
from discord.ratelimits.main import RatelimitManager
from discord.gateway import websocket_handler
from discord.errors import DiscordError
from discord.gateway.state_manager import StateManager
from discord.storage import Storage
from discord.user_storage import UserStorage
from discord.dispatcher import EventDispatcher
from discord.presence import PresenceManager
from discord.images import IconManager
from discord.jobs import JobManager
from discord.utils import DiscordJSONEncoder
# setup logbook
handler = StreamHandler(sys.stdout, level=logbook.INFO)
handler.push_application()
log = Logger('discord.boot')
redirect_logging()
def make_app():
app = Quart(__name__)
app.config.from_object(f'config.{config.MODE}')
is_debug = app.config.get('DEBUG', False)
app.debug = is_debug
if is_debug:
log.info('on debug')
handler.level = logbook.DEBUG
app.logger.level = logbook.DEBUG
def __init__(self, log_file):
super().__init__()
self.handlers.append(RotatingFileHandler(log_file, bubble=True))
self.handlers.append(StreamHandler(sys.stdout))
def run(self):
"""Worker runtime body.
"""
# Logging:
StreamHandler(sys.stdout).push_application()
self.log = Logger('Worker_{}'.format(self.task), level=self.log_level)
try:
tf.reset_default_graph()
if self.test_mode:
import gym
# Define cluster:
cluster = tf.train.ClusterSpec(self.cluster_spec).as_cluster_def()
# Start tf.server:
if self.job_name in 'ps':
server = tf.train.Server(
cluster,
job_name=self.job_name,
task_index=self.task,
config=tf.ConfigProto(device_filters=["/job:ps"]))
self.log.debug('parameters_server started.')
# Just block here:
server.join()
else:
server = tf.train.Server(
cluster,
job_name='worker',
task_index=self.task,
config=tf.ConfigProto(
intra_op_parallelism_threads=1, # original was: 1
inter_op_parallelism_threads=2 # original was: 2
))
self.log.debug('tf.server started.')
self.log.debug('making environments:')
# Making as many environments as many entries in env_config `port` list:
# TODO: Hacky-II: only one example over all parallel environments can be data-master [and renderer]
# TODO: measure data_server lags, maybe launch several instances
self.env_list = []
env_kwargs = self.env_kwargs.copy()
env_kwargs['log_level'] = self.log_level
port_list = env_kwargs.pop('port')
data_port_list = env_kwargs.pop('data_port')
data_master = env_kwargs.pop('data_master')
render_enabled = env_kwargs.pop('render_enabled')
render_list = [False for entry in port_list]
if render_enabled:
if self.render_last_env:
render_list[-1] = True
else:
render_list = [True for entry in port_list]
# render_list[0] = True
data_master_list = [False for entry in port_list]
if data_master:
data_master_list[0] = True
# Parallel envs. numbering:
if len(port_list) > 1:
task_id = 0.0
else:
task_id = 0
for port, data_port, is_render, is_master in zip(
port_list, data_port_list, render_list,
data_master_list):
# Get random seed for environments:
env_kwargs['random_seed'] = random.randint(0, 2**30)
if not self.test_mode:
# Assume BTgym env. class:
self.log.debug(
'setting env at port_{} is data_master: {}'.format(
port, data_master))
self.log.debug('env_kwargs:')
for k, v in env_kwargs.items():
self.log.debug('{}: {}'.format(k, v))
try:
self.env_list.append(
self.env_class(port=port,
data_port=data_port,
data_master=is_master,
render_enabled=is_render,
task=self.task + task_id,
**env_kwargs))
data_master = False
self.log.info(
'set BTGym environment {} @ port:{}, data_port:{}'
.format(self.task + task_id, port, data_port))
task_id += 0.01
except Exception as e:
self.log.exception(
'failed to make BTGym environment at port_{}.'.
format(port))
raise e
else:
# Assume atari testing:
try:
self.env_list.append(
self.env_class(env_kwargs['gym_id']))
self.log.debug('set Gyn/Atari environment.')
except Exception as e:
self.log.exception(
'failed to make Gym/Atari environment')
raise e
self.log.debug('Defining trainer...')
# Define trainer:
trainer = self.trainer_class(
env=self.env_list,
task=self.task,
policy_config=self.policy_config,
log_level=self.log_level,
cluster_spec=self.cluster_spec,
random_seed=self.random_seed,
**self.trainer_kwargs,
)
self.log.debug('trainer ok.')
# Saver-related:
variables_to_save = [
v for v in tf.global_variables() if not 'local' in v.name
]
local_variables = [
v for v in tf.global_variables() if 'local' in v.name
] + tf.local_variables()
init_op = tf.initializers.variables(variables_to_save)
local_init_op = tf.initializers.variables(local_variables)
init_all_op = tf.global_variables_initializer()
def init_fn(_sess):
self.log.notice("initializing all parameters...")
_sess.run(init_all_op)
# def init_fn_scaff(scaffold, _sess):
# self.log.notice("initializing all parameters...")
# _sess.run(init_all_op)
# self.log.warning('VARIABLES TO SAVE:')
# for v in variables_to_save:
# self.log.warning(v)
#
# self.log.warning('LOCAL VARS:')
# for v in local_variables:
# self.log.warning(v)
self.saver = FastSaver(var_list=variables_to_save,
max_to_keep=1,
save_relative_paths=True)
self.config = tf.ConfigProto(device_filters=[
"/job:ps", "/job:worker/task:{}/cpu:0".format(self.task)
])
sess_manager = tf.train.SessionManager(
local_init_op=local_init_op,
ready_op=None,
ready_for_local_init_op=tf.report_uninitialized_variables(
variables_to_save),
graph=None,
recovery_wait_secs=90,
)
with sess_manager.prepare_session(
master=server.target,
init_op=init_op,
config=self.config,
init_fn=init_fn,
) as sess:
# Try to restore pre-trained model
pre_trained_restored = self._restore_model_params(
sess, self.initial_ckpt_dir)
_ = sess.run(trainer.reset_global_step)
if not pre_trained_restored:
# If not - try to recover current checkpoint:
current_restored = self._restore_model_params(
sess, self.current_ckpt_dir)
else:
current_restored = False
if not pre_trained_restored and not current_restored:
self.log.notice('training from scratch...')
self.log.info("connecting to the parameter server... ")
self.summary_writer = tf.summary.FileWriter(
self.summary_dir, sess.graph)
trainer.start(sess, self.summary_writer)
# Note: `self.global_step` refers to number of environment steps
# summarized over all environment instances, not to number of policy optimizer train steps.
global_step = sess.run(trainer.global_step)
self.log.notice(
"started training at step: {}".format(global_step))
last_saved_time = datetime.datetime.now()
last_saved_step = global_step
while global_step < self.max_env_steps:
trainer.process(sess)
global_step = sess.run(trainer.global_step)
time_delta = datetime.datetime.now() - last_saved_time
if self.is_chief and time_delta.total_seconds(
) > self.save_secs:
self._save_model_params(sess, global_step)
train_speed = (global_step - last_saved_step) / (
time_delta.total_seconds() + 1)
self.log.notice(
'env. step: {}; cluster speed: {:.0f} step/sec; checkpoint saved.'
.format(global_step, train_speed))
last_saved_time = datetime.datetime.now()
last_saved_step = global_step
# Ask for all the services to stop:
for env in self.env_list:
env.close()
self.log.notice(
'reached {} steps, exiting.'.format(global_step))
except Exception as e:
self.log.exception(e)
raise e
def run(self):
"""Worker runtime body.
"""
# Logging:
StreamHandler(sys.stdout).push_application()
self.log = Logger('Worker_{}'.format(self.task), level=self.log_level)
tf.reset_default_graph()
if self.test_mode:
import gym
# Define cluster:
cluster = tf.train.ClusterSpec(self.cluster_spec).as_cluster_def()
# Start tf.server:
if self.job_name in 'ps':
server = tf.train.Server(
cluster,
job_name=self.job_name,
task_index=self.task,
config=tf.ConfigProto(device_filters=["/job:ps"]))
self.log.debug('parameters_server started.')
# Just block here:
server.join()
else:
server = tf.train.Server(
cluster,
job_name='worker',
task_index=self.task,
config=tf.ConfigProto(
intra_op_parallelism_threads=1, # original was: 1
inter_op_parallelism_threads=2 # original was: 2
))
self.log.debug('tf.server started.')
self.log.debug('making environments:')
# Making as many environments as many entries in env_config `port` list:
# TODO: Hacky-II: only one example of parallel [all] environments can be data-master and renderer
# TODO: measure data_server lags, maybe launch several instances
self.env_list = []
env_kwargs = self.env_kwargs.copy()
env_kwargs['log_level'] = self.log_level
port_list = env_kwargs.pop('port')
data_master = env_kwargs.pop('data_master')
render_enabled = env_kwargs.pop('render_enabled')
# Parallel envs. numbering:
if len(port_list) > 1:
task_id = 0.0
else:
task_id = 0
for port in port_list:
if not self.test_mode:
# Assume BTgym env. class:
self.log.debug('env at port_{} is data_master: {}'.format(
port, data_master))
try:
self.env_list.append(
self.env_class(port=port,
data_master=data_master,
render_enabled=render_enabled,
task=self.task + task_id,
**env_kwargs))
data_master = False
render_enabled = False
self.log.info(
'set BTGym environment {} at port_{}.'.format(
self.task + task_id, port))
task_id += 0.01
except:
self.log.exception(
'failed to make BTGym environment at port_{}.'.
format(port))
raise RuntimeError
else:
# Assume atari testing:
try:
self.env_list.append(
self.env_class(env_kwargs['gym_id']))
self.log.debug('set Gyn/Atari environment.')
except:
self.log.exception(
'failed to make Gym/Atari environment')
raise RuntimeError
# Define trainer:
trainer = self.trainer_class(
env=self.env_list,
task=self.task,
policy_config=self.policy_config,
log_level=self.log_level,
random_seed=self.random_seed,
**self.trainer_kwargs,
)
self.log.debug('trainer ok.')
# Saver-related:
variables_to_save = [
v for v in tf.global_variables()
if not v.name.startswith("local")
]
local_variables = [
v for v in tf.global_variables() if v.name.startswith("local")
]
init_op = tf.variables_initializer(variables_to_save)
local_init_op = tf.variables_initializer(local_variables)
init_all_op = tf.global_variables_initializer()
saver = _FastSaver(variables_to_save)
self.log.debug('vars_to_save:')
for v in variables_to_save:
self.log.debug('{}: {}'.format(v.name, v.get_shape()))
def init_fn(ses):
self.log.info("initializing all parameters.")
ses.run(init_all_op)
config = tf.ConfigProto(device_filters=[
"/job:ps", "/job:worker/task:{}/cpu:0".format(self.task)
])
logdir = os.path.join(self.log_dir, 'train')
summary_dir = logdir + "_{}".format(self.task)
summary_writer = tf.summary.FileWriter(summary_dir)
sv = tf.train.Supervisor(
is_chief=(self.task == 0),
logdir=logdir,
saver=saver,
summary_op=None,
init_op=init_op,
local_init_op=local_init_op,
init_fn=init_fn,
#ready_op=tf.report_uninitialized_variables(variables_to_save),
ready_op=tf.report_uninitialized_variables(),
global_step=trainer.global_step,
save_model_secs=300,
)
self.log.info("connecting to the parameter server... ")
with sv.managed_session(server.target,
config=config) as sess, sess.as_default():
#sess.run(trainer.sync)
trainer.start(sess, summary_writer)
# Note: `self.global_step` refers to number of environment steps
# summarized over all environment instances, not to number of policy optimizer train steps.
global_step = sess.run(trainer.global_step)
self.log.notice(
"started training at step: {}".format(global_step))
while not sv.should_stop(
) and global_step < self.max_env_steps:
trainer.process(sess)
global_step = sess.run(trainer.global_step)
# Ask for all the services to stop:
for env in self.env_list:
env.close()
sv.stop()
self.log.notice('reached {} steps, exiting.'.format(global_step))
def __init__(self, loggerName):
StreamHandler(sys.stdout).push_application()
self.logger = lg(loggerName)
def __init__(self,
env_config=None,
cluster_config=None,
policy_config=None,
trainer_config=None,
max_env_steps=None,
root_random_seed=None,
test_mode=False,
purge_previous=0,
log_level=None,
verbose=0):
"""
Args:
env_config (dict): environment class_config_dict, see 'Note' below.
cluster_config (dict): tf cluster configuration, see 'Note' below.
policy_config (dict): policy class_config_dict holding corr. policy class args.
trainer_config (dict): trainer class_config_dict holding corr. trainer class args.
max_env_steps (int): total number of environment steps to run training on.
root_random_seed (int): int or None
test_mode (bool): if True - use Atari gym env., BTGym otherwise.
purge_previous (int): keep or remove previous log files and saved checkpoints from log_dir:
{0 - keep, 1 - ask, 2 - remove}.
verbose (int): verbosity mode, {0 - WARNING, 1 - INFO, 2 - DEBUG}.
log_level (int): logbook level {DEBUG=10, INFO=11, NOTICE=12, WARNING=13},
overrides `verbose` arg.
Note:
class_config_dict: dictionary containing at least two keys:
- `class_ref`: reference to class constructor or function;
- `kwargs`: dictionary of keyword arguments, see corr. environment class args.
cluster_config: dictionary containing at least these keys:
- 'host': cluster host, def: '127.0.0.1'
- 'port': cluster port, def: 12222
- 'num_workers': number of workers to run, def: 1
- 'num_ps': number of parameter servers, def: 1
- 'num_envs': number of environments to run in parallel for each worker, def: 1
- 'log_dir': directory to save model and summaries, def: './tmp/btgym_aac_log'
"""
self.env_config = dict(class_ref=None,
kwargs=dict(
port=5000,
data_port=4999,
gym_id=None,
))
self.cluster_config = dict(
host='127.0.0.1',
port=12222,
num_workers=1,
num_ps=1,
log_dir='./tmp/btgym_aac_log',
num_envs=1,
)
self.policy_config = dict(class_ref=BaseAacPolicy,
kwargs=dict(lstm_layers=(256, )))
self.trainer_config = dict(class_ref=A3C, kwargs={})
self.max_env_steps = 100 * 10**6
self.ports_to_use = []
self.root_random_seed = root_random_seed
self.purge_previous = purge_previous
self.test_mode = test_mode
self.log_level = log_level
self.verbose = verbose
if max_env_steps is not None:
self.max_env_steps = max_env_steps
self.env_config = self._update_config_dict(self.env_config, env_config)
self.cluster_config = self._update_config_dict(self.cluster_config,
cluster_config)
self.policy_config = self._update_config_dict(self.policy_config,
policy_config)
self.trainer_config = self._update_config_dict(self.trainer_config,
trainer_config)
self.trainer_config['kwargs']['test_mode'] = self.test_mode
# Logging config:
StreamHandler(sys.stdout).push_application()
if self.log_level is None:
log_levels = [(0, NOTICE), (1, INFO), (2, DEBUG)]
self.log_level = WARNING
for key, value in log_levels:
if key == self.verbose:
self.log_level = value
self.log = Logger('LauncherShell', level=self.log_level)
# Seeding:
if self.root_random_seed is not None:
np.random.seed(self.root_random_seed)
self.log.info('Random seed: {}'.format(self.root_random_seed))
# Seeding for workers:
workers_rnd_seeds = list(
np.random.randint(
0, 2**30, self.cluster_config['num_workers'] +
self.cluster_config['num_ps']))
# Log_dir:
if os.path.exists(self.cluster_config['log_dir']):
# Remove previous log files and saved model if opted:
if self.purge_previous > 0:
confirm = 'y'
if self.purge_previous < 2:
confirm = input(
'<{}> already exists. Override[y/n]? '.format(
self.cluster_config['log_dir']))
if confirm in 'y':
files = glob.glob(self.cluster_config['log_dir'] + '/*')
p = psutil.Popen([
'rm',
'-R',
] + files,
stdout=PIPE,
stderr=PIPE)
self.log.notice('Files in <{}> purged.'.format(
self.cluster_config['log_dir']))
else:
self.log.notice('Appending to <{}>.'.format(
self.cluster_config['log_dir']))
else:
os.makedirs(self.cluster_config['log_dir'])
self.log.notice('<{}> created.'.format(
self.cluster_config['log_dir']))
for kwarg in ['port', 'data_port']:
assert kwarg in self.env_config['kwargs'].keys()
assert self.env_config['class_ref'] is not None
# Make cluster specification dict:
self.cluster_spec = self.make_cluster_spec(self.cluster_config)
# Configure workers:
self.workers_config_list = []
env_ports = np.arange(self.cluster_config['num_envs'])
worker_port = self.env_config['kwargs'][
'port'] # start value for BTGym comm. port
# TODO: Hacky, cause dataset is threadlocked; do: pass dataset as class_ref + kwargs_dict:
if self.test_mode:
dataset_instance = None
else:
dataset_instance = self.env_config['kwargs'].pop('dataset')
for key, spec_list in self.cluster_spec.items():
task_index = 0 # referenced farther as worker id
for _id in spec_list:
env_config = copy.deepcopy(self.env_config)
worker_config = {}
if key in 'worker':
# Configure worker BTgym environment:
if task_index == 0:
env_config['kwargs'][
'data_master'] = True # set worker_0 as chief and data_master
env_config['kwargs']['dataset'] = dataset_instance
env_config['kwargs']['render_enabled'] = True
else:
env_config['kwargs']['data_master'] = False
env_config['kwargs'][
'render_enabled'] = False # disable rendering for all but chief
# Add list of connection ports for every parallel env for each worker:
env_config['kwargs']['port'] = list(worker_port +
env_ports)
worker_port += self.cluster_config['num_envs']
worker_config.update({
'env_config': env_config,
'policy_config': self.policy_config,
'trainer_config': self.trainer_config,
'cluster_spec': self.cluster_spec,
'job_name': key,
'task': task_index,
'test_mode': self.test_mode,
'log_dir': self.cluster_config['log_dir'],
'max_env_steps': self.max_env_steps,
'log_level': self.log_level,
'random_seed': workers_rnd_seeds.pop()
})
self.clear_port(env_config['kwargs']['port'])
self.workers_config_list.append(worker_config)
task_index += 1
self.clear_port(self.env_config['kwargs']['data_port'])
self.log.debug('Launcher ready.')
import os
import re
import sys
import pymysql
import datetime
from pymongo import MongoClient
from logbook import Logger, StreamHandler, TimedRotatingFileHandler
stream_log = StreamHandler(sys.stdout)
stream_log.push_application()
log_file = TimedRotatingFileHandler(os.path.join(os.getcwd(), "index.log"),
backup_count=3)
logger = Logger('Logbook')
logger.handlers = []
# logger.handlers.append(log_file)
logger.handlers.append(stream_log)
env = os.environ.get
stock_format = [r'^[SI][ZHX]\d{6}$', r'^\d{6}\.[A-Z]{4}$']
def little8code(x):
"""
转换为前缀模式
:param x:
:return:
def create_logger():
""" Creates the logger fpr this module. """
StreamHandler(sys.stdout).push_application()
return Logger('PhotoBackup')
@ __mtime__: 2016/8/12 19:07
日志格式 测试
配置日志格式两种方式：
a) 传 格式 字符串
b）hook format function
"""
from logbook import StreamHandler, info
import sys
# 1. set format_string
sh = StreamHandler(
sys.stdout,
format_string=
"[{record.time}] {record.level_name}: {record.channel}：{record.message}")
# 2. set format_string
sh.format_string = "[{record.time}] {record.level_name}: {record.channel}：{record.message}"
# 3. invoke the format function
def my_format_fun(record, handler):
return " ".join([
"[" + str(record.time) + "]",
record.level_name + "：" + record.channel + "：", record.message
])
sh.formatter = my_format_fun
prefix = args.vocab[0]
args.vocab = [f'{prefix}.{args.langs[0]}', f'{prefix}.{args.langs[1]}']
return args, state_dict, resume
if __name__ == '__main__':
args, state_dict, resume = prepare()
# redirect stdout and stderr to log file
# redirection = open(log_name, 'a', buffering=1)
# sys.stdout = redirection
# sys.stderr = redirection
stdout_handler = StreamHandler(sys.stdout, bubble=True)
stderr_handler = StderrHandler(level=WARNING)
# write logs to log.MODEL file
# file_handler = FileHandler(log_name, bubble=True)
# file_handler.format_string = '{record.message},{record.extra[cwd]}'
# file_handler.format_string = '[{record.time:%Y-%m-%d %H:%M:%S.%f%z}] {record.level_name}: {record.message}'
# with file_handler.applicationbound():
stdout_handler.format_string = '[{record.time:%Y-%m-%d %H:%M:%S.%f%z}] ' \
'{record.level_name}: {record.message}'
with stdout_handler.applicationbound():
if resume:
logger.info(
f'Resume training from checkpoint: {Loader.get_latest(args.model)[1]}'
)
try:
def run():
out = StringIO()
with StreamHandler(out):
for x in xrange(500):
log.warning('this is not handled')
assert out.getvalue().count('\n') == 500
import sys
from flask import Blueprint, jsonify, request
from logbook import Logger, StreamHandler
import requests
from tldr.models import Article
from tldr.utils import summarize_text
apiv1_module = Blueprint('apiv1', __name__, template_folder='templates')
log = Logger(__name__)
log.handlers.append(StreamHandler(sys.stdout, level='INFO'))
# TODO: Make this more generic
# TODO: Move elsewhere
def json_requested():
"""Determines whether the requested content type is application/json."""
best = request.accept_mimetypes \
.best_match(['application/json', 'text/plain'])
return best == 'application/json' and \
request.accept_mimetypes[best] > \
request.accept_mimetypes['text/plain']
@apiv1_module.route('summarize', methods=['POST'])
def summarize():
"""Summarizes given text."""
text = request.form['text']
return summarize_text(text)
import re
import sys
from typing import AnyStr
from typing import Sequence
import progressbar
from dateutil import relativedelta
from logbook import Logger, StreamHandler
from rs2wat import db
DIR_LISTING_PAT = r"([0-9]{2}-[0-9]{2}-[0-9]{2}\s{2}[0-9]{2}:[0-9]{2}\w{2})\s*(\d*) (.*)"
DIR_LISTING_TIME_FMT = "%m-%d-%y %I:%M%p"
RS2_LOG_DATE_FMT = "%m/%d/%y %H:%M:%S"
StreamHandler(sys.stdout, level="INFO").push_application()
logger = Logger(__name__)
class ProgressFile(object):
"""
TODO: fix progress bar only updating once download is finished.
"""
def __init__(self, file_path, mode, max_value=progressbar.UnknownLength):
self.file_obj = open(file_path, mode)
fct = progressbar.FormatCustomText("%(f)s", dict(f=""))
self.progressbar = progressbar.ProgressBar(
max_value=max_value,
widgets=[
progressbar.Counter(format='[%(value)02d/%(max_value)d]'),
progressbar.Bar(marker=u'\u2588',
for index, txo in enumerate(tx_obj.txs_out):
if _script_hash(txo.puzzle_script()) in script_hashes:
utxos[(txid, index)] = txo
diff = sum(txo.coin_value for txo in utxos.values()) - balance
balance += diff
confirmations = tip['height'] - block_height + 1
rows.append([
txid, timestamps_map[block_height], block_height, confirmations,
f'{diff/1e5:,.5f}', f'{balance/1e5:,.5f}'
])
show_rows(rows, [
"txid", "block timestamp", "height", "confirmations", "delta (mBTC)",
"total (mBTC)"
])
tip_header = _parse_header(tip['hex'])
log.info('tip={}, height={} @ {}', tip_header.id(), tip['height'],
_parse_timestamp(tip_header))
unconfirmed = {
txs_map[txid]
for txid, height in txids_map.items() if height is None
}
# TODO: show unconfirmed balance
if __name__ == '__main__':
StreamHandler(sys.stderr).push_application()
main()
import subprocess
import sys
import time
from logbook import Logger
from logbook import StreamHandler
from rs2wapy import RS2WebAdmin
from simplediscordwh import DiscordWebhook
WEBHOOK_URL = os.environ["WEBHOOK_URL"]
WA_USERNAME = os.environ["WA_USERNAME"]
WA_PASSWORD = os.environ["WA_PASSWORD"]
WA_URL = os.environ["WA_URL"]
handler = StreamHandler(sys.stdout, level="INFO")
handler.format_string = (
"[{record.time}] {record.level_name}: {record.module}: "
"{record.func_name}: Process({record.process}): {record.message}")
logger = Logger(__name__)
logger.handlers.append(handler)
def write_exception(e):
try:
e = f"{datetime.datetime.now().isoformat()}: {type(e)}: {e}\n"
print(e)
with open("errors.log", "a") as f:
f.write(e)
f.write("----------------\n")
except Exception as e:
def run(
driver,
email,
password,
keep_creds,
output_dir,
scrape_only,
resume_only,
website_only,
profile_file,
timeout,
driver_path,
**kwargs,
):
# Setup logging
logbook.set_datetime_format("local")
format_string = (
"[{record.time:%Y-%m-%d %H:%M:%S}] {record.level_name}: {record.message}"
)
StreamHandler(sys.stdout, format_string=format_string).push_application()
log = Logger()
# Create output directory
make_dir(output_dir)
# Check if user has provided the profile json file
if profile_file is None:
if driver.lower() not in DRIVERS:
raise ValueError(
f'Browser driver has to be one of these: {", ".join(DRIVERS)}')
# Check if credentials file exists
credentials_file = os.path.expanduser(CREDENTIALS_FILE)
if os.path.exists(credentials_file):
with open(credentials_file) as f:
credentials = json.load(f)
email = credentials["email"]
password = credentials["password"]
else:
if email is None:
email = input("Enter your LinkedIn login email: ")
if password is None:
password = getpass("Enter your LinkedIn login password: "******"Scraping LinkedIn profile")
if driver not in [CHROME, FIREFOX]:
log.notice("Please keep the browser window on top")
profile = scrape(driver.lower(), driver_path, email, password,
output_dir, timeout)
if keep_creds:
store_creds(email, password, credentials_file)
else:
with open(profile_file) as f:
profile = json.load(f)
if not scrape_only:
if resume_only:
make_resume_files(profile, output_dir, timeout)
elif website_only:
make_website_files(profile, output_dir)
else:
make_resume_files(profile, output_dir, timeout)
make_website_files(profile, output_dir)
def main():
# Setup logging
logging.getLogger("pdfminer").setLevel(logging.WARNING)
logging.getLogger("ocrmypdf").setLevel(logging.WARNING)
redirect_logging()
format_string = "{record.level_name}: {record.message}"
StreamHandler(
sys.stdout, format_string=format_string, level="INFO"
).push_application()
log = Logger()
q = mq.MessageQueue(all_burst_limit=3, all_time_limit_ms=3000)
request = Request(con_pool_size=8)
pdf_bot = MQBot(TELE_TOKEN, request=request, mqueue=q)
# Create the EventHandler and pass it your bot's token.
updater = Updater(
bot=pdf_bot,
use_context=True,
request_kwargs={"connect_timeout": TIMEOUT, "read_timeout": TIMEOUT},
)
def stop_and_restart():
updater.stop()
os.execl(sys.executable, sys.executable, *sys.argv)
def restart(_):
Thread(target=stop_and_restart).start()
job_queue = updater.job_queue
job_queue.run_repeating(restart, interval=dt.timedelta(minutes=30))
# Get the dispatcher to register handlers
dispatcher = updater.dispatcher
# General commands handlers
dispatcher.add_handler(CommandHandler("start", start_msg, run_async=True))
dispatcher.add_handler(CommandHandler("help", help_msg, run_async=True))
dispatcher.add_handler(CommandHandler("setlang", send_lang, run_async=True))
dispatcher.add_handler(
CommandHandler("support", send_support_options, run_async=True)
)
dispatcher.add_handler(CommandHandler("send", send_msg, Filters.user(DEV_TELE_ID)))
dispatcher.add_handler(
CommandHandler("stats", get_stats, Filters.user(DEV_TELE_ID))
)
# Callback query handler
dispatcher.add_handler(CallbackQueryHandler(process_callback_query, run_async=True))
# Payment handlers
dispatcher.add_handler(PreCheckoutQueryHandler(precheckout_check, run_async=True))
dispatcher.add_handler(
MessageHandler(Filters.successful_payment, successful_payment, run_async=True)
)
# URL handler
dispatcher.add_handler(
MessageHandler(Filters.entity(MessageEntity.URL), url_to_pdf, run_async=True)
)
# PDF commands handlers
dispatcher.add_handler(compare_cov_handler())
dispatcher.add_handler(merge_cov_handler())
dispatcher.add_handler(photo_cov_handler())
dispatcher.add_handler(text_cov_handler())
dispatcher.add_handler(watermark_cov_handler())
# PDF file handler
dispatcher.add_handler(file_cov_handler())
# Feedback handler
dispatcher.add_handler(feedback_cov_handler())
# Log all errors
dispatcher.add_error_handler(error_callback)
# Start the Bot
if APP_URL is not None:
updater.start_webhook(
listen="0.0.0.0",
port=PORT,
url_path=TELE_TOKEN,
webhook_url=APP_URL + TELE_TOKEN,
)
log.notice("Bot started webhook")
else:
updater.start_polling()
log.notice("Bot started polling")
# Run the bot until the you presses Ctrl-C or the process receives SIGINT,
# SIGTERM or SIGABRT. This should be used most of the time, since
# start_polling() is non-blocking and will stop the bot gracefully.
updater.idle()
conn = client.Client(('localhost', 50001))
xpub, = sys.argv[1:]
total = 0
k = pycoin.ui.key_from_text.key_from_text(xpub)
for change in (0, 1):
empty = 0
for n in range(100):
address = k.subkey(change).subkey(n).address()
script = script_for_address(address)
script_hash = hashlib.sha256(script).digest()[::-1].hex()
log.debug('{}', conn.call('blockchain.scripthash.get_history',
script_hash))
reply = conn.call('blockchain.scripthash.get_balance', script_hash)
result = reply['result']
confirmed = result['confirmed'] / 1e8
total += confirmed
if confirmed:
log.info('{}/{} => {} has {:11.8f} BTC',
change, n, address, confirmed)
empty = 0
else:
empty += 1
if empty >= 10:
break
log.info('total balance: {} BTC', total)
if __name__ == '__main__':
with StreamHandler(sys.stderr, level='INFO').applicationbound():
main()
def logger():
StreamHandler(sys.stdout).push_application()
return Logger(LOG_BASE)
import csv
import os
import random
import sys
from logbook import Logger, StreamHandler
StreamHandler(sys.stdout).push_application()
log = Logger('TyphoonNames')
header = []
rows = []
with open(os.path.join(os.path.dirname(__file__), "retired-typhoon-names.csv"),
"r") as csvfile:
reader = csv.reader(csvfile)
header.extend(next(reader))
for row in reader:
info = {header[i]: v if v else None for i, v in enumerate(row)}
rows.append(info)
sample = {
'Name': 'Karen',
'Replacement name': None,
'Dates active': 'November 7\xa01 17, 1962',
'Peak classification': 'Category\xa05 super typhoon',
'Sustained wind speeds': '295\xa0km/h (185\xa0mph)',
'Pressure': '894\xa0hPa (26.40\xa0inHg)',
'Areas affected': 'Guam',
'Damage\xa0(USD)': '$250\xa0million',
class Mysql(object):
StreamHandler(sys.stdout).push_application()
logger = Logger('Mysql')
# 连接数据库
# 生产环境数据库
__config = {
'host': 'localhost',
'port': 3306,
'user': '******',
'password': '******',
'db': 'yilongspider',
'charset': 'utf8',
'cursorclass': pymysql.cursors.DictCursor,
}
# 数据库构造函数，从连接池中取出连接，并生成操作游标
def __init__(self):
"""
配置成员变量
"""
# 连接对象
self.__conn = self.__getConn()
self.__cursor = self.__getCursor()
def __getConn(self):
"""
获取con连接
:return: con
"""
self.__conn = pymysql.Connect(**Mysql.__config)
return self.__conn
def __getCursor(self):
"""
获取游标
:return: cursor
"""
self.__cursor = self.__conn.cursor()
return self.__cursor
@contextmanager
def __con_cursor(self):
"""
1、定义上下文管理器，连接后自动关闭连接
2、元组对象前面如果不带“*”、字典对象如果前面不带“**”，则作为普通的对象传递参数。
:return:
"""
# 打开连接
conn = self.__getConn()
# 打开游标
cursor = self.__getCursor()
try:
yield cursor
except Exception as ex:
conn.rollback()
Mysql.logger.error(repr(ex))
finally:
self.__conn.commit()
self.__cursor.close()
self.__conn.close()
# ---------- 搜索 ----------
def __query(self, cursor, sql, param=None):
if param:
count = cursor.execute(sql, param)
else:
count = cursor.execute(sql)
return count
def getOne(self, sql, param=None):
"""
@summary: 执行查询，并取出第一条
@param sql:查询ＳＱＬ，如果有查询条件，请只指定条件列表，并将条件值使用参数[param]传递进来
@param param: 可选参数，条件列表值（元组/列表）
@return: result list/boolean 查询到的结果集
"""
with self.__con_cursor() as cursor:
if self.__query(cursor, sql, param) > 0:
result = cursor.fetchone()
else:
result = False
return result
def getMany(self, sql, param=None, num=1):
"""
@summary: 执行查询，并取出num条结果
@param sql:查询ＳＱＬ，如果有查询条件，请只指定条件列表，并将条件值使用参数[param]传递进来
@param num:取得的结果条数
@param param: 可选参数，条件列表值（元组/列表）
@return: result list/boolean 查询到的结果集
"""
with self.__con_cursor() as cursor:
if self.__query(cursor, sql, param) > 0:
result = cursor.fetchmany(num)
else:
result = False
return result
def getAll(self, sql, param=None):
"""
@summary: 执行查询，并取出所有结果集
@param sql:查询ＳＱＬ，如果有查询条件，请只指定条件列表，并将条件值使用参数[param]传递进来
@param param: 可选参数，条件列表值（元组/列表）
@return: result list(字典对象)/boolean 查询到的结果集
"""
with self.__con_cursor() as cursor:
if self.__query(cursor, sql, param) > 0:
result = cursor.fetchall()
else:
result = False
return result
def __select(self, table, cond_dict=None, order=None):
"""
@summary: 执行条件查询，并取出所有结果集
@cond_dict:{'name':'xiaoming'...}
@order:'order by id desc'
@return: result ({"col":"val","":""},{})
"""
consql = ' '
if cond_dict != '':
for k, v in cond_dict.items():
consql = consql + k + '=' + v + ' and'
consql = consql + ' 1=1 '
sql = 'select * from %s where ' % table
sql = sql + consql + order
print('_select:' + sql)
return self.exeCute(sql)
# ---------- 更新 ----------
def update(self, sql, param=None):
"""
@summary: 更新数据表记录
@param sql: ＳＱＬ格式及条件，使用(%s,%s)
@param param: 要更新的 值 tuple/list
@return: count 受影响的行数
"""
with self.__con_cursor() as cursor:
return cursor.execute(sql, param)
# ---------- 删除 ----------
def delete(self, sql, param=None):
"""
@summary: 删除数据表记录
@param sql: ＳＱＬ格式及条件，使用(%s,%s)
@param param: 要删除的条件 值 tuple/list
@return: count 受影响的行数
"""
with self.__con_cursor() as cursor:
return cursor.execute(sql, param)
# ---------- 插入 ----------
# 插入一条/多条数据
def insert(self, sql, param):
"""
@summary: 向数据表插入一条记录
@param sql:要插入的ＳＱＬ格式
@param value:要插入的记录数据tuple/list
@return: insertId
"""
with self.__con_cursor() as cursor:
# 执行插入操作
cursor.executemany(sql, param)
# cursor.execute(sql)
# 获取最后更新的ID
return cursor.lastrowid
#if __name__ == '__main__':
#aaa = Mysql()
# print(aaa.getOne("select * from erp_users"))
# print(aaa.getOne("select * from erp_users WHERE id in(%s)", (19,)))
# for item in aaa.getMany("select * from erp_users WHERE id in(19,39)", None, 5):
# print(item)
# for item in aaa.getAll("select name from erp_users ORDER BY `name` asc"):
# print(item)
# for item in aaa.getAll("select * from erp_users WHERE id in(%s)", (19,)):
# print(item)
# print(aaa.update("UPDATE erp_users SET mail =%s WHERE id = %s", ('123@789', 19)))
# print(aaa.delete("DELETE erp_users WHERE id = %s", (19,)))
# print(aaa.getInsertId())
# 插入一条数据
# print(aaa.insert("INSERT `erp_areas` (`areaName`,`charge`,`areaCode`,`is_delete`,`commission`) VALUES (%s,%s,%s,%s,%s)", ('通州片区2222ssssssd', '片区经理3', '0', '0', '0.90')))
# 插入多条数据
# print(aaa.insert("INSERT `erp_areas` (`areaName`,`charge`,`areaCode`,`is_delete`,`commission`) VALUES (%s,%s,%s,%s,%s)", ('通州片区2222ssssssd', '片区经理3', '0', '0', '0.90'), ('通州片区2222ssssssd', '片区经理3', '0','0', '0.90')))
