def get_classes(code):
list(lex(code, PythonLexer()))
FMT = HtmlFormatter()
classes = [FMT.ttype2class.get(x) for x, y in lex(code, PythonLexer())]
classes = [c if c is not None else "" for c in classes]
return classes
def print_func(func):
print("--- Code start ---")
code = "".join(inspect.getsourcelines(func)[0][1:])
print(highlight(code, PythonLexer(), TerminalFormatter(bg='dark')))
print("--- Code end ---")
def _run(handle_data,
initialize,
before_trading_start,
analyze,
algofile,
algotext,
defines,
data_frequency,
capital_base,
bundle,
bundle_timestamp,
start,
end,
output,
trading_calendar,
print_algo,
metrics_set,
local_namespace,
environ,
blotter,
benchmark_spec):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
# 加载数据数据包到内存中
bundle_data = bundles.load(
bundle,
environ,
bundle_timestamp,
)
if trading_calendar is None:
trading_calendar = get_calendar('XNYS')
# date parameter validation
if trading_calendar.session_distance(start, end) < 1:
raise _RunAlgoError(
'There are no trading days between %s and %s' % (
start.date(),
end.date(),
),
)
benchmark_sid, benchmark_returns = benchmark_spec.resolve(
asset_finder=bundle_data.asset_finder,
start_date=start,
end_date=end,
)
# 通过 algotext & algofile 传递策略
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
first_trading_day = \
bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
bundle_data.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
pipeline_loader = USEquityPricingLoader.without_fx(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError(
"No PipelineLoader registered for column %s." % column
)
if isinstance(metrics_set, six.string_types):
try:
metrics_set = metrics.load(metrics_set)
except ValueError as e:
raise _RunAlgoError(str(e))
if isinstance(blotter, six.string_types):
try:
blotter = load(Blotter, blotter)
except ValueError as e:
raise _RunAlgoError(str(e))
try:
perf = TradingAlgorithm(
namespace=namespace,
data_portal=data,
get_pipeline_loader=choose_loader,
trading_calendar=trading_calendar,
sim_params=SimulationParameters(
start_session=start,
end_session=end,
trading_calendar=trading_calendar,
capital_base=capital_base,
data_frequency=data_frequency,
),
metrics_set=metrics_set,
blotter=blotter,
benchmark_returns=benchmark_returns,
benchmark_sid=benchmark_sid,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}
).run()
except NoBenchmark:
raise _RunAlgoError(
(
'No ``benchmark_spec`` was provided, and'
' ``zipline.api.set_benchmark`` was not called in'
' ``initialize``.'
),
(
"Neither '--benchmark-symbol' nor '--benchmark-sid' was"
" provided, and ``zipline.api.set_benchmark`` was not called"
" in ``initialize``. Did you mean to pass '--no-benchmark'?"
),
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
def generate_page(sourcefile, resultfile, genref):
# Generate html DOM from markdown.
markdown = open(sourcefile).read()
htmldoc = CommonMark.commonmark(markdown)
soup = BeautifulSoup(htmldoc, 'html.parser')
# Remove comments.
comments = soup.find_all(string=lambda text: isinstance(text, Comment))
for comment in comments:
comment.extract()
# All h4 sections are actually asides.
admonitions = soup.findAll("h4")
for admonition in admonitions:
p = admonition.find_next_sibling("p")
p['class'] = 'aside'
admonition.extract()
# Colorize the code blocks.
formatter = HtmlFormatter(style='tango')
snippets = soup.findAll("code", {"class": "language-python"})
for snippet in snippets:
code = snippet.contents[0]
highlighted = highlight(code, PythonLexer(), formatter)
newcode = BeautifulSoup(highlighted, 'html.parser')
snippet.parent.replace_with(newcode)
# Generate the HTML in its initial form, including <style>.
htmlfile = open(resultfile, 'w')
htmlfile.write(header)
htmlfile.write('<style>')
htmlfile.write(formatter.get_style_defs('.highlight'))
htmlfile.write('''
.highlight .mb, .highlight .mf, .highlight .mh, .highlight .mi,
.highlight .mo { color: #0063cf; }
''')
htmlfile.write('</style>')
htmlfile.write('<main>\n')
htmlfile.write(forkme)
htmlfile.write(str(soup))
# Generate quickref.
quickref = ''
if genref:
for member in inspect.getmembers(snowy):
name, value = member
if name.startswith('__'):
continue
if not inspect.isfunction(value):
continue
doc = inspect.getdoc(value)
src = inspect.getsource(value)
dsbegin = src.find(r'"""')
dsend = src.rfind(r'"""') + 4
dsbegin = src[:dsbegin].rfind('\n') + 1
src = src[:dsbegin] + src[dsend:]
nlines = len(src.split('\n'))
highlighted_src = highlight(src, PythonLexer(), formatter)
if doc:
doclines = doc.split('\n')
quickref += '<tr>\n'
quickref += f'<td><a href="#{name}">{name}</a></td>\n'
quickref += f'<td>{doclines[0]}</td>\n'
quickref += '<tr>\n'
htmlfile.write(f'<h3>{name}</h3>\n<p>\n')
htmlfile.write(' '.join(doclines))
htmlfile.write('\n</p>\n')
htmlfile.write(highlighted_src)
htmlfile.write('</main>\n')
htmlfile.close()
# Post process HTML by adding anchors, etc.
htmldoc = open(resultfile).read()
htmldoc = htmldoc.replace('$quickref$', quickref)
htmldoc = htmldoc.replace('<h1>', version + '\n<h1>')
soup = BeautifulSoup(htmldoc, 'html.parser')
for tag in 'h2 h3 h4'.split():
headings = soup.find_all(tag)
for heading in headings:
content = heading.contents[0].strip()
id = content.replace(' ', '_').lower()
heading["id"] = id
anchor = soup.new_tag('a', href='#' + id)
anchor.string = content
heading.contents[0].replace_with(anchor)
open(resultfile, 'w').write(str(soup))
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import TerminalFormatter
from pygments.filters import VisibleWhitespaceFilter
code = """
for i in range(1, 11):
\tprint("Hello world!")
if x and y:
print("yes")
if x or y:
\tprint("dunno")
"""
print(highlight(code, PythonLexer(), TerminalFormatter()))
print("-----------------------")
lexer = PythonLexer()
lexer.add_filter(VisibleWhitespaceFilter(tabs=True))
print(highlight(code, lexer, TerminalFormatter()))
from pygments import highlight as highlight
from pygments.lexers import PythonLexer as PythonLexer
from pygments.formatters import HtmlFormatter as HtmlFormatter
__all__ = ['doc']
_fn_base = dirname(__file__)
with open(join(_fn_base, 'odpydoc.js'), 'r') as _ifile:
_js = _ifile.read()
with open(join(_fn_base, 'odpydoc.css'), 'r') as _ifile:
_css = _ifile.read()
with open(join(_fn_base, 'one-dark-pygments.css'), 'r') as _ifile:
_css += _ifile.read()
del (_ifile)
_python_lexer = PythonLexer()
_html_formatter = HtmlFormatter(nowrap=False)
def _is_number(s):
"""Check if an element can be converted to a float, returning `True`
if it can and `False` if it can't"""
if ((s is False) or (s is True)):
return (False)
else:
try:
float(s)
except (ValueError, TypeError):
return (False)
else:
return (True)
def _run(handle_data,
initialize,
before_trading_start,
analyze,
algofile,
algotext,
defines,
data_frequency,
capital_base,
data,
bundle,
bundle_timestamp,
start,
end,
output,
print_algo,
local_namespace,
environ,
live,
exchange,
algo_namespace,
base_currency,
live_graph):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`catalyst.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
mode = 'live' if live else 'backtest'
log.info('running algo in {mode} mode'.format(mode=mode))
exchange_name = exchange
if exchange_name is None:
raise ValueError('Please specify at least one exchange.')
exchange_list = [x.strip().lower() for x in exchange.split(',')]
exchanges = dict()
for exchange_name in exchange_list:
# Looking for the portfolio from the cache first
portfolio = get_algo_object(
algo_name=algo_namespace,
key='portfolio_{}'.format(exchange_name),
environ=environ
)
if portfolio is None:
portfolio = ExchangePortfolio(
start_date=pd.Timestamp.utcnow()
)
# This corresponds to the json file containing api token info
exchange_auth = get_exchange_auth(exchange_name)
if live and (exchange_auth['key'] == '' \
or exchange_auth['secret'] == ''):
raise ExchangeAuthEmpty(
exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name, environ), 'auth.json'))
if exchange_name == 'bitfinex':
exchanges[exchange_name] = Bitfinex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
elif exchange_name == 'bittrex':
exchanges[exchange_name] = Bittrex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
elif exchange_name == 'poloniex':
exchanges[exchange_name] = Poloniex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
else:
raise ExchangeNotFoundError(exchange_name=exchange_name)
open_calendar = get_calendar('OPEN')
env = TradingEnvironment(
load=partial(
load_crypto_market_data,
environ=environ,
start_dt=start,
end_dt=end
),
environ=environ,
exchange_tz='UTC',
asset_db_path=None # We don't need an asset db, we have exchanges
)
env.asset_finder = AssetFinderExchange()
choose_loader = None # TODO: use the DataPortal for in the algorithm class for this
if live:
start = pd.Timestamp.utcnow()
# TODO: fix the end data.
end = start + timedelta(hours=8760)
data = DataPortalExchangeLive(
exchanges=exchanges,
asset_finder=env.asset_finder,
trading_calendar=open_calendar,
first_trading_day=pd.to_datetime('today', utc=True)
)
def fetch_capital_base(exchange, attempt_index=0):
"""
Fetch the base currency amount required to bootstrap
the algorithm against the exchange.
The algorithm cannot continue without this value.
:param exchange: the targeted exchange
:param attempt_index:
:return capital_base: the amount of base currency available for
trading
"""
try:
log.debug('retrieving capital base in {} to bootstrap '
'exchange {}'.format(base_currency, exchange_name))
balances = exchange.get_balances()
except ExchangeRequestError as e:
if attempt_index < 20:
log.warn(
'could not retrieve balances on {}: {}'.format(
exchange.name, e
)
)
sleep(5)
return fetch_capital_base(exchange, attempt_index + 1)
else:
raise ExchangeRequestErrorTooManyAttempts(
attempts=attempt_index,
error=e
)
if base_currency in balances:
base_currency_available = balances[base_currency]
log.info(
'base currency available in the account: {} {}'.format(
base_currency_available, base_currency
)
)
if capital_base is not None \
and capital_base < base_currency_available:
log.info(
'using capital base limit: {} {}'.format(
capital_base, base_currency
)
)
amount = capital_base
else:
amount = base_currency_available
return amount
else:
raise BaseCurrencyNotFoundError(
base_currency=base_currency,
exchange=exchange_name
)
combined_capital_base = 0
for exchange_name in exchanges:
exchange = exchanges[exchange_name]
combined_capital_base += fetch_capital_base(exchange)
sim_params = create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
emission_rate='minute',
data_frequency='minute'
)
# TODO: use the constructor instead
sim_params._arena = 'live'
algorithm_class = partial(
ExchangeTradingAlgorithmLive,
exchanges=exchanges,
algo_namespace=algo_namespace,
live_graph=live_graph
)
elif exchanges:
# Removed the existing Poloniex fork to keep things simple
# We can add back the complexity if required.
# I don't think that we should have arbitrary price data bundles
# Instead, we should center this data around exchanges.
# We still need to support bundles for other misc data, but we
# can handle this later.
data = DataPortalExchangeBacktest(
exchange_names=[exchange_name for exchange_name in exchanges],
asset_finder=None,
trading_calendar=open_calendar,
first_trading_day=start,
last_available_session=end
)
sim_params = create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
emission_rate=data_frequency,
)
algorithm_class = partial(
ExchangeTradingAlgorithmBacktest,
exchanges=exchanges
)
elif bundle is not None:
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url),
)
env = TradingEnvironment(asset_db_path=connstr, environ=environ)
first_trading_day = \
bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
env.asset_finder, open_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
perf = algorithm_class(
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
sim_params=sim_params,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}
).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the catalyst magic not write any data
perf.to_pickle(output)
return perf
def python_highlight(string):
return highlight(
string, PythonLexer(encoding="Utf-8"),
Terminal256Formatter(style='monokai', encoding="Utf-8"))
def python_html_highlight(string):
return highlight(string, PythonLexer(encode="Utf-8"),
HtmlFormatter(noclasses=True, encoding="UTf-8"))
def colorize_traceback(code, colors):
if colors:
code = highlight(code, PythonLexer(), TracebackFormatter())
return code
def _lex_python_result(tb):
"Return token list for Python string."
lexer = PythonLexer()
return lexer.get_tokens(tb)
def colorize(code, colors):
if colors:
code = highlight(code, PythonLexer(), DefaultFormatter())
return code
Keyword,
Name,
Comment,
String,
Error,
Number,
Operator,
Generic,
)
code = """
for i in range(1, 11):
print("Hello world!")
"""
print(highlight(code, PythonLexer(), Terminal256Formatter()))
print("-----------------------")
class NewStyle(Style):
default_style = ""
styles = {
Comment: "italic #ansidarkgray",
Keyword: "underline #ansired",
Name.Builtin: "bold #ansiyellow",
String: "#ansilightgray",
}
print(highlight(code, PythonLexer(), Terminal256Formatter(style=NewStyle)))
def parse_script(script, ns=None, infer=None, prev="", config=None):
"""
Parse a script into tokens and use Jedi to infer the fully qualified names
of each token.
Parameters
----------
script : str
the script to tokenize and infer types on
ns : dict
extra namespace to use with jedi's Interpreter.
infer : bool
whether to run jedi type inference that can be quite time consuming.
prev : str
previous lines that lead to this.
Yields
------
index
index in the tokenstream
type
pygments token type
text
text of the token
reference : str
fully qualified name of the type of current token
"""
jeds = []
import warnings
warnings.simplefilter("ignore", UserWarning)
l_delta = len(prev.split("\n"))
contextscript = prev + "\n" + script
if ns:
jeds.append(jedi.Interpreter(contextscript, namespaces=[ns]))
jeds.append(jedi.Script(prev + "\n" + script))
P = PythonLexer()
for index, type_, text in P.get_tokens_unprocessed(script):
line_n, col_n = pos_to_nl(script, index)
line_n += l_delta
try:
ref = None
for jed in jeds:
failed = ""
try:
if infer and (text
not in (" .=()[],")) and text.isidentifier():
inf = jed.infer(line_n + 1, col_n)
if inf:
ref = inf[0].full_name
# if ref.startswith('builtins'):
# ref = ''
else:
ref = ""
except (AttributeError, TypeError, Exception) as e:
raise type(
e
)(f"{contextscript}, {line_n=}, {col_n=}, {prev=}, {jed=}"
) from e
failed = "(jedi failed inference)"
print("failed inference on ", script, ns, jed, col_n,
line_n + 1)
break
except IndexError:
raise
ref = ""
yield text + failed, ref
warnings.simplefilter("default", UserWarning)
def source(obj):
"""Display the source code of an object.
Applies syntax highlighting if Pygments is available.
"""
import os
import sys
from inspect import findsource, getmodule, getsource, getsourcefile
from pydoc import pager
try:
# Check to see if the object is defined in a shared library, which
# findsource() doesn't do properly (see issue4050)
if not getsourcefile(obj):
raise TypeError
s = getsource(obj)
except TypeError:
sys.stderr.write("Source code unavailable (maybe it's part of "
"a C extension?)\n")
return
# Detect the module's file encoding. We could use
# tokenize.detect_encoding(), but it's only available in Python 3.
import re
enc = 'ascii'
for line in findsource(getmodule(obj))[0][:2]:
m = re.search(r'coding[:=]\s*([-\w.]+)', line)
if m:
enc = m.group(1)
if hasattr(s, 'decode'):
try:
s = s.decode(enc, 'replace')
except LookupError:
s = s.decode('ascii', 'replace')
try:
# For now, let's assume we'll never have a proper terminal on win32
if sys.platform == 'win32':
raise ImportError
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import TerminalFormatter
s = highlight(s, PythonLexer(), TerminalFormatter())
except (ImportError, UnicodeError):
pass
# Display the source code in the pager, and try to convince less not to
# escape color control codes.
has_lessopts = 'LESS' in os.environ
lessopts = os.environ.get('LESS', '')
try:
os.environ['LESS'] = lessopts + ' -R'
if hasattr(s, 'decode'):
pager(s.encode(sys.stdout.encoding, 'replace'))
else:
pager(s)
finally:
if has_lessopts:
os.environ['LESS'] = lessopts
else:
os.environ.pop('LESS', None)
def pylight(code):
return highlight(code, PythonLexer(), HtmlFormatter(noclasses=True))
def _vendor_wrap(self, colour_class, text):
"""Override in reporters that wrap snippet lines in vendor styles, e.g. pygments."""
if '-line' not in colour_class:
text = highlight(text, PythonLexer(),
HtmlFormatter(nowrap=True, lineseparator='', classprefix='pygments-'))
return text
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import Terminal256Formatter
code = """
for i in range(1, 11):
print("Hello world!")
"""
print(highlight(code, PythonLexer(), Terminal256Formatter()))
from sphinx.util.texescape import tex_hl_escape_map_new
from sphinx.ext import doctest
from pygments import highlight
from pygments.lexers import PythonLexer, Python3Lexer, PythonConsoleLexer, \
CLexer, TextLexer, RstLexer
from pygments.lexers import get_lexer_by_name, guess_lexer
from pygments.formatters import HtmlFormatter, LatexFormatter
from pygments.filters import ErrorToken
from pygments.styles import get_style_by_name
from pygments.util import ClassNotFound
from sphinx.pygments_styles import SphinxStyle, NoneStyle
lexers = dict(
none=TextLexer(stripnl=False),
python=PythonLexer(stripnl=False),
python3=Python3Lexer(stripnl=False),
pycon=PythonConsoleLexer(stripnl=False),
pycon3=PythonConsoleLexer(python3=True, stripnl=False),
rest=RstLexer(stripnl=False),
c=CLexer(stripnl=False),
)
for _lexer in lexers.values():
_lexer.add_filter('raiseonerror')
escape_hl_chars = {
ord(u'\\'): u'\\PYGZbs{}',
ord(u'{'): u'\\PYGZob{}',
ord(u'}'): u'\\PYGZcb{}'
}
def format_text_to_python_code(text: str) -> str:
"""Enhances text by markups specific for Python code. Output string is html."""
return highlight(text, PythonLexer(), HtmlFormatter(noclasses=True))
def run_pipeline(print_algo=True, **kwargs):
"""Runs a full zipline pipeline given configuration keyword
arguments.
1. Load data (start and end dates can be provided a strings as
well as the source and symobls).
2. Instantiate algorithm (supply either algo_text or algofile
kwargs containing initialize() and handle_data() functions). If
algofile is supplied, will try to look for algofile_analyze.py and
append it.
3. Run algorithm (supply capital_base as float).
4. Return performance dataframe.
:Arguments:
* print_algo : bool <default=True>
Whether to print the algorithm to command line. Will use
pygments syntax coloring if pygments is found.
"""
start = pd.Timestamp(kwargs['start'], tz='UTC')
end = pd.Timestamp(kwargs['end'], tz='UTC')
symbols = kwargs['symbols'].split(',')
if kwargs['source'] == 'yahoo':
source = zipline.data.load_bars_from_yahoo(stocks=symbols,
start=start,
end=end)
else:
raise NotImplementedError('Source %s not implemented.' %
kwargs['source'])
algo_text = kwargs.get('algo_text', None)
if algo_text is None:
# Expect algofile to be set
algo_fname = kwargs['algofile']
with open(algo_fname, 'r') as fd:
algo_text = fd.read()
analyze_fname = os.path.splitext(algo_fname)[0] + '_analyze.py'
if os.path.exists(analyze_fname):
with open(analyze_fname, 'r') as fd:
# Simply append
algo_text += fd.read()
if print_algo:
if PYGMENTS:
highlight(algo_text,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout)
else:
print_(algo_text)
algo = zipline.TradingAlgorithm(script=algo_text,
namespace=kwargs.get('namespace', {}),
capital_base=float(kwargs['capital_base']))
perf = algo.run(source)
output_fname = kwargs.get('output', None)
if output_fname is not None:
perf.to_pickle(output_fname)
return perf
def do_install(self, name, data):
if name in data:
utils.makedirs(self.output_dir)
LOGGER.notice('Downloading: ' + data[name])
zip_file = BytesIO()
zip_file.write(requests.get(data[name]).content)
LOGGER.notice('Extracting: {0} into plugins'.format(name))
utils.extract_all(zip_file, 'plugins')
dest_path = os.path.join('plugins', name)
else:
try:
plugin_path = utils.get_plugin_path(name)
except:
LOGGER.error("Can't find plugin " + name)
return False
utils.makedirs(self.output_dir)
dest_path = os.path.join(self.output_dir, name)
if os.path.exists(dest_path):
LOGGER.error("{0} is already installed".format(name))
return False
LOGGER.notice('Copying {0} into plugins'.format(plugin_path))
shutil.copytree(plugin_path, dest_path)
reqpath = os.path.join(dest_path, 'requirements.txt')
print(reqpath)
if os.path.exists(reqpath):
LOGGER.notice('This plugin has Python dependencies.')
LOGGER.notice('Installing dependencies with pip...')
try:
subprocess.check_call(('pip', 'install', '-r', reqpath))
except subprocess.CalledProcessError:
LOGGER.error('Could not install the dependencies.')
print('Contents of the requirements.txt file:\n')
with codecs.open(reqpath, 'rb', 'utf-8') as fh:
print(indent(fh.read(), 4 * ' '))
print('You have to install those yourself or through a '
'package manager.')
else:
LOGGER.notice('Dependency installation succeeded.')
reqnpypath = os.path.join(dest_path, 'requirements-nonpy.txt')
if os.path.exists(reqnpypath):
LOGGER.notice('This plugin has third-party '
'dependencies you need to install '
'manually.')
print('Contents of the requirements-nonpy.txt file:\n')
with codecs.open(reqnpypath, 'rb', 'utf-8') as fh:
for l in fh.readlines():
i, j = l.split('::')
print(indent(i.strip(), 4 * ' '))
print(indent(j.strip(), 8 * ' '))
print()
print('You have to install those yourself or through a package '
'manager.')
confpypath = os.path.join(dest_path, 'conf.py.sample')
if os.path.exists(confpypath):
LOGGER.notice('This plugin has a sample config file.')
print('Contents of the conf.py.sample file:\n')
with codecs.open(confpypath, 'rb', 'utf-8') as fh:
print(
indent(
pygments.highlight(fh.read(), PythonLexer(),
TerminalFormatter()), 4 * ' '))
return True
def show_python(text):
return highlight(source(text), PythonLexer(),
HtmlFormatter(style='colorful'))
def code(self):
code = inspect.getsource(self.value)
return highlight(code, PythonLexer(),
CodeHtmlFormatter(self.instance_class))
def format_source(debugger_ui, lines, breakpoints):
lineno_format = "%%%dd " % (len(str(len(lines))))
try:
import pygments
except ImportError:
return [
SourceLine(debugger_ui,
line.rstrip("\n\r").expandtabs(TABSTOP),
lineno_format % (i + 1),
None,
has_breakpoint=i + 1 in breakpoints)
for i, line in enumerate(lines)
]
else:
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatter import Formatter
import pygments.token as t
result = []
ATTR_MAP = {
t.Token:
"source",
t.Keyword:
"keyword",
t.Literal:
"literal",
t.Name.Function:
"name",
t.Name.Class:
"name",
t.Punctuation:
"punctuation",
t.String:
"string",
# XXX: Single and Double don't actually work yet.
# See https://bitbucket.org/birkenfeld/pygments-main/issue/685
t.String.Double:
"doublestring",
t.String.Single:
"singlestring",
t.String.Backtick:
"backtick",
t.String.Doc:
"docstring",
t.Comment:
"comment",
}
class UrwidFormatter(Formatter):
def __init__(subself, **options):
Formatter.__init__(subself, **options)
subself.current_line = ""
subself.current_attr = []
subself.lineno = 1
def format(subself, tokensource, outfile):
def add_snippet(ttype, s):
if not s:
return
while not ttype in ATTR_MAP:
if ttype.parent is not None:
ttype = ttype.parent
else:
raise RuntimeError("untreated token type: %s" %
str(ttype))
attr = ATTR_MAP[ttype]
subself.current_line += s
subself.current_attr.append((attr, len(s)))
def shipout_line():
result.append(
SourceLine(debugger_ui,
subself.current_line,
lineno_format % subself.lineno,
subself.current_attr,
has_breakpoint=subself.lineno
in breakpoints))
subself.current_line = ""
subself.current_attr = []
subself.lineno += 1
for ttype, value in tokensource:
while True:
newline_pos = value.find("\n")
if newline_pos == -1:
add_snippet(ttype, value)
break
else:
add_snippet(ttype, value[:newline_pos])
shipout_line()
value = value[newline_pos + 1:]
if subself.current_line:
shipout_line()
highlight("".join(l.expandtabs(TABSTOP) for l in lines),
PythonLexer(stripnl=False), UrwidFormatter())
return result
def syntax_highlight():
for token, content in lex(txt, PythonLexer()):
txt.insert("end", content, str(token))
def pprint_color(obj):
print(highlight(pformat(obj), PythonLexer(), Terminal256Formatter()))
def _run(handle_data,
initialize,
before_trading_start,
analyze,
algofile,
algotext,
defines,
data_frequency,
capital_base,
data,
bundle,
bundle_timestamp,
start,
end,
output,
print_algo,
local_namespace,
environ,
live,
exchange,
algo_namespace,
base_currency,
live_graph,
analyze_live,
simulate_orders,
auth_aliases,
stats_output):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`catalyst.run_algo`.
"""
# TODO: refactor for more granularity
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
log.warn(
'Catalyst is currently in ALPHA. It is going through rapid '
'development and it is subject to errors. Please use carefully. '
'We encourage you to report any issue on GitHub: '
'https://github.com/enigmampc/catalyst/issues'
)
log.info('Catalyst version {}'.format(catalyst.__version__))
sleep(3)
if live:
if simulate_orders:
mode = 'paper-trading'
else:
mode = 'live-trading'
else:
mode = 'backtest'
log.info('running algo in {mode} mode'.format(mode=mode))
exchange_name = exchange
if exchange_name is None:
raise ValueError('Please specify at least one exchange.')
if isinstance(auth_aliases, string_types):
aliases = auth_aliases.split(',')
if len(aliases) < 2 or len(aliases) % 2 != 0:
raise ValueError(
'the `auth_aliases` parameter must contain an even list '
'of comma-delimited values. For example, '
'"binance,auth2" or "binance,auth2,bittrex,auth2".'
)
auth_aliases = dict(zip(aliases[::2], aliases[1::2]))
exchange_list = [x.strip().lower() for x in exchange.split(',')]
exchanges = dict()
for name in exchange_list:
if auth_aliases is not None and name in auth_aliases:
auth_alias = auth_aliases[name]
else:
auth_alias = None
exchanges[name] = get_exchange(
exchange_name=name,
base_currency=base_currency,
must_authenticate=(live and not simulate_orders),
skip_init=True,
auth_alias=auth_alias,
)
open_calendar = get_calendar('OPEN')
env = TradingEnvironment(
load=partial(
load_crypto_market_data,
environ=environ,
start_dt=start,
end_dt=end
),
environ=environ,
exchange_tz='UTC',
asset_db_path=None # We don't need an asset db, we have exchanges
)
env.asset_finder = ExchangeAssetFinder(exchanges=exchanges)
def choose_loader(column):
bound_cols = TradingPairPricing.columns
if column in bound_cols:
return ExchangePricingLoader(data_frequency)
raise ValueError(
"No PipelineLoader registered for column %s." % column
)
if live:
start = pd.Timestamp.utcnow()
# TODO: fix the end data.
# is_end checks if an end date was specified by user
# needed for live clock
is_end = True
if end is None:
end = start + timedelta(hours=8760)
is_end = False
data = DataPortalExchangeLive(
exchanges=exchanges,
asset_finder=env.asset_finder,
trading_calendar=open_calendar,
first_trading_day=pd.to_datetime('today', utc=True)
)
sim_params = create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
emission_rate='minute',
data_frequency='minute'
)
# TODO: use the constructor instead
sim_params._arena = 'live'
algorithm_class = partial(
ExchangeTradingAlgorithmLive,
exchanges=exchanges,
algo_namespace=algo_namespace,
live_graph=live_graph,
simulate_orders=simulate_orders,
stats_output=stats_output,
analyze_live=analyze_live,
end=end,
is_end=is_end,
)
elif exchanges:
# Removed the existing Poloniex fork to keep things simple
# We can add back the complexity if required.
# I don't think that we should have arbitrary price data bundles
# Instead, we should center this data around exchanges.
# We still need to support bundles for other misc data, but we
# can handle this later.
if start != pd.tslib.normalize_date(start) or \
end != pd.tslib.normalize_date(end):
# todo: add to Sim_Params the option to start & end at specific times
log.warn(
"Catalyst currently starts and ends on the start and "
"end of the dates specified, respectively. We hope to "
"Modify this and support specific times in a future release."
)
data = DataPortalExchangeBacktest(
exchange_names=[exchange_name for exchange_name in exchanges],
asset_finder=None,
trading_calendar=open_calendar,
first_trading_day=start,
last_available_session=end
)
sim_params = create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
emission_rate=data_frequency,
)
algorithm_class = partial(
ExchangeTradingAlgorithmBacktest,
exchanges=exchanges
)
elif bundle is not None:
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url),
)
env = TradingEnvironment(asset_db_path=connstr, environ=environ)
first_trading_day = \
bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
env.asset_finder, open_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
perf = algorithm_class(
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
sim_params=sim_params,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}
).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the catalyst magic not write any data
perf.to_pickle(output)
return perf
class Agent(ACTR):
goal = Buffer()
imaginal = Buffer()
#dm_buffer = Buffer()
#dm = Memory(dm_buffer, latency=0.05, threshold=0)
#dm_noise = DMNoise(dm, noise=0.0, baseNoise=0.0)
#dm_base = DMBaseLevel(dm, decay=0.5, limit=None)
motor = Motor()
eye = MrChipsEye()
responses = []
lexer = PythonLexer()
goal.set("read-line")
def __init__(self, text_buffer):
ACTR.__init__(self)
self.eye.text_buffer = text_buffer
self.last_line = text_buffer.shape[0] - 1
self.fovea = [
i for i, s in enumerate(self.eye.retina.slots)
if s == Retina.SLOT_HIGH_RES
]
self.fovea_start = min(self.fovea)
self.fovea_stop = max(self.fovea)
self.fovea_idx = slice(self.fovea_start, self.fovea_stop + 1)
self.mem_locals = {}
self.mem_globals = {}
self.imaginal.set("line:")
def process_tokens(self, tokens):
return list(drop_spaces(combine_strings(simplify_tokens(tokens))))
def read_line(goal="read-line", eye="busy:False", motor="busy:False"):
code_str = eye.view()[self.fovea_idx]
tokens = self.process_tokens(lexer.get_tokens(code_str))
print tokens
goal.set("stop-task")
#imaginal.modify(line=imaginal["line"] + code_str)
#if len(code_str[(self.fovea_stop + 1):].strip()) == 0:
## Finished reading line, process it
#goal.set("understand-line")
#else:
## More text is left on the line. Move eye to the right.
#next_pos = (eye.pos[0] + len(self.fovea), eye.pos[1])
#eye.move_to(next_pos)
#goal.set("read-line")
#def understand_line(goal="understand-line"):
#full_line = imaginal["line"]
#imaginal.clear()
## Execute line in memory (if not blank)
#if len(full_line.strip()) > 0:
#response = ""
#with stdoutIO() as out:
#exec(full_line, self.mem_globals, self.mem_locals)
#response = out.getvalue().strip()
#if len(response) > 0:
#imaginal.set("keys:{0}".format(response))
#goal.set("type-response")
#responses.append(response)
#return
## Move to next line
#next_pos = (0, eye.pos[1] + 1)
#if next_pos[1] > self.last_line:
## Done reading
#goal.set("stop-task")
#else:
#eye.move_to(next_pos)
#goal.set("read-line")
def type_response(goal="type-response", motor="busy:False"):
keys = imaginal["keys"]
motor.press_key(keys[0])
if len(keys) > 1:
keys = keys[1:]
goal.set("type-response")
imaginal.modify(keys=keys)
else:
imaginal.clear()
imaginal.set("line:")
goal.set("read-line")
# TODO: Move eye to next line here?
def stop_task(goal="stop-task"):
self.stop()
def run_pipeline(print_algo=True, **kwargs):
"""Runs a full zipline pipeline given configuration keyword
arguments.
1. Load data (start and end dates can be provided a strings as
well as the source and symobls).
2. Instantiate algorithm (supply either algo_text or algofile
kwargs containing initialize() and handle_data() functions). If
algofile is supplied, will try to look for algofile_analyze.py and
append it.
3. Run algorithm (supply capital_base as float).
4. Return performance dataframe.
:Arguments:
* print_algo : bool <default=True>
Whether to print the algorithm to command line. Will use
pygments syntax coloring if pygments is found.
"""
start = kwargs['start']
end = kwargs['end']
# Compare against None because strings/timestamps may have been given
if start is not None:
start = pd.Timestamp(start, tz='UTC')
if end is not None:
end = pd.Timestamp(end, tz='UTC')
# Fail out if only one bound is provided
if ((start is None) or (end is None)) and (start != end):
raise PipelineDateError(start=start, end=end)
# Check if start and end are provided, and if the sim_params need to read
# a start and end from the DataSource
if start is None:
overwrite_sim_params = True
else:
overwrite_sim_params = False
symbols = kwargs['symbols'].split(',')
asset_identifier = kwargs['metadata_index']
# Pull asset metadata
asset_metadata = kwargs.get('asset_metadata', None)
asset_metadata_path = kwargs['metadata_path']
# Read in a CSV file, if applicable
if asset_metadata_path is not None:
if os.path.isfile(asset_metadata_path):
asset_metadata = pd.read_csv(asset_metadata_path,
index_col=asset_identifier)
source_arg = kwargs['source']
source_time_column = kwargs['source_time_column']
if source_arg is None:
raise NoSourceError()
elif source_arg == 'yahoo':
source = zipline.data.load_bars_from_yahoo(stocks=symbols,
start=start,
end=end)
elif os.path.isfile(source_arg):
source = zipline.data.load_prices_from_csv(
filepath=source_arg, identifier_col=source_time_column)
elif os.path.isdir(source_arg):
source = zipline.data.load_prices_from_csv_folder(
folderpath=source_arg, identifier_col=source_time_column)
else:
raise NotImplementedError('Source %s not implemented.' %
kwargs['source'])
algo_text = kwargs.get('algo_text', None)
if algo_text is None:
# Expect algofile to be set
algo_fname = kwargs['algofile']
with open(algo_fname, 'r') as fd:
algo_text = fd.read()
analyze_fname = os.path.splitext(algo_fname)[0] + '_analyze.py'
if os.path.exists(analyze_fname):
with open(analyze_fname, 'r') as fd:
# Simply append
algo_text += fd.read()
if print_algo:
if PYGMENTS:
highlight(algo_text,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout)
else:
print_(algo_text)
algo = zipline.TradingAlgorithm(script=algo_text,
namespace=kwargs.get('namespace', {}),
capital_base=float(kwargs['capital_base']),
algo_filename=kwargs.get('algofile'),
asset_metadata=asset_metadata,
identifiers=symbols,
start=start,
end=end)
perf = algo.run(source, overwrite_sim_params=overwrite_sim_params)
output_fname = kwargs.get('output', None)
if output_fname is not None:
perf.to_pickle(output_fname)
return perf
