def trade(strategies,
accounts=None,
review_date=None,
output="csv",
filepath_or_buffer=None):
"""
Run one or more strategies and generate orders.
Allocations are read from configuration (quantrocket.moonshot.allocations.yml).
Parameters
----------
strategies : list of str, required
one or more strategy codes
accounts : list of str, optional
limit to these accounts
review_date : str (YYYY-MM-DD), optional
generate orders as if it were this date, rather than using today's date
output : str, required
the output format (choices are csv or json)
filepath_or_buffer : str, optional
the location to write the orders file (omit to write to stdout)
Returns
-------
None
"""
params = {}
if strategies:
params["strategies"] = strategies
if accounts:
params["accounts"] = accounts
if review_date:
params["review_date"] = review_date
output = output or "csv"
if output not in ("csv", "json"):
raise ValueError(
"invalid output: {0} (choices are csv or json".format(output))
response = houston.get("/moonshot/orders.{0}".format(output),
params=params,
timeout=60 * 5)
houston.raise_for_status_with_json(response)
# Don't write a null response to file
if response.content[:4] == b"null":
return
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_positions(filepath_or_buffer=None, output="csv",
order_refs=None, accounts=None, conids=None):
"""
Query current positions and write results to file.
To return positions as a Python list, see list_positions.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, txt, default is csv)
order_refs : list of str, optional
limit to these order refs
accounts : list of str, optional
limit to these accounts
conids : list of int, optional
limit to these conids
Returns
-------
None
"""
params = {}
if order_refs:
params["order_refs"] = order_refs
if accounts:
params["accounts"] = accounts
if conids:
params["conids"] = conids
output = output or "csv"
if output not in ("csv", "json", "txt"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/blotter/positions.{0}".format(output), params=params)
houston.raise_for_status_with_json(response)
# Don't write a null response to file
if response.content[:4] == b"null":
return
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_executions(filepath_or_buffer=None,
order_refs=None,
accounts=None,
conids=None,
start_date=None,
end_date=None):
"""
Query executions from the executions database.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
order_refs : list of str, optional
limit to these order refs
accounts : list of str, optional
limit to these accounts
conids : list of int, optional
limit to these conids
start_date : str (YYYY-MM-DD), optional
limit to executions on or after this date
end_date : str (YYYY-MM-DD), optional
limit to executions on or before this date
Returns
-------
None
"""
params = {}
if order_refs:
params["order_refs"] = order_refs
if accounts:
params["accounts"] = accounts
if conids:
params["conids"] = conids
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
response = houston.get("/blotter/executions.csv", params=params)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def execute_command(cmd,
return_file=None,
filepath_or_buffer=None,
service="satellite"):
"""
Execute an abitrary command on a satellite service and optionally return a file.
Parameters
----------
cmd: str, required
the command to run
return_file : str, optional
the path of a file to be returned after the command completes
filepath_or_buffer : str, optional
the location to write the return_file (omit to write to stdout)
service : str, optional
the service name (default 'satellite')
Returns
-------
dict or None
None if return_file, otherwise status message
"""
params = {}
if not service:
raise ValueError("a service is required")
if not cmd:
raise ValueError("a command is required")
params["cmd"] = cmd
if return_file:
params["return_file"] = return_file
if not service.startswith("satellite"):
raise ValueError("service must start with 'satellite'")
response = houston.post("/{0}/commands".format(service),
params=params,
timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
if return_file:
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
else:
return response.json()
def download_history_availability_file(code,
filepath_or_buffer=None,
output="csv"):
"""
Query historical market data availability from a history database and download to file.
This function is normally called after running:
quantrocket history collect mydb --availability
Parameters
----------
code : str, required
the code of the database to query
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, default is csv)
Returns
-------
None
See Also
--------
get_history_availability : load historical availability into Series
"""
output = output or "csv"
if output not in ("csv", "json", "txt"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/history/availability/{0}.{1}".format(
code, output))
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def create_tearsheet(infilepath_or_buffer, outfilepath_or_buffer=None):
"""
Create a pyfolio PDF tear sheet from a Zipline backtest result.
Parameters
----------
infilepath_or_buffer : str, required
the CSV file from a Zipline backtest (specify '-' to read file from stdin)
outfilepath_or_buffer : str or file-like, optional
the location to write the pyfolio tear sheet (write to stdout if omitted)
Returns
-------
None
"""
url = "/zipline/tearsheets"
# Pyfolio can take a long time
timeout = 60 * 60 * 5
if infilepath_or_buffer == "-":
infilepath_or_buffer = sys.stdin.buffer if six.PY3 else sys.stdin
response = houston.post(url,
data=infilepath_or_buffer,
timeout=timeout)
elif infilepath_or_buffer and hasattr(infilepath_or_buffer, "read"):
if infilepath_or_buffer.seekable():
infilepath_or_buffer.seek(0)
response = houston.post(url,
data=infilepath_or_buffer,
timeout=timeout)
else:
with open(infilepath_or_buffer, "rb") as f:
response = houston.post(url, data=f, timeout=timeout)
houston.raise_for_status_with_json(response)
outfilepath_or_buffer = outfilepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(outfilepath_or_buffer, response)
def generate_orders(strategies, accounts=None, json=False, filepath_or_buffer=None):
"""
Run one or more strategies and generate orders.
Allocations are read from configuration (quantrocket.moonshot.allocations.yml).
Parameters
----------
strategies : list of str, required
one or more strategy codes
accounts : list of str, optional
limit to these accounts
json : bool
format orders as JSON (default is CSV)
filepath_or_buffer : str, optional
the location to write the orders file (omit to write to stdout)
Returns
-------
None
"""
params = {}
if strategies:
params["strategies"] = strategies
if accounts:
params["accounts"] = accounts
output = "json" if json else "csv"
response = houston.get("/moonshot/orders.{0}".format(output), params=params, timeout=60*5)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def run_algorithm(algofile,
data_frequency=None,
capital_base=None,
bundle=None,
bundle_timestamp=None,
start=None,
end=None,
filepath_or_buffer=None,
calendar=None):
"""
Run a Zipline backtest and write the test results to a CSV file.
The CSV result file contains several DataFrames stacked into one: the Zipline performance
results, plus the extracted returns, transactions, positions, and benchmark returns from those
results.
Parameters
----------
algofile : str, required
the file that contains the algorithm to run
data_frequency : str, optional
the data frequency of the simulation. Possible choices: daily, minute (default is daily)
capital_base : float, optional
the starting capital for the simulation (default is 10000000.0)
bundle : str, required
the data bundle to use for the simulation
bundle_timestamp : str, optional
the date to lookup data on or before (default is <current-time>)
start : str (YYYY-MM-DD), required
the start date of the simulation
end : str (YYYY-MM-DD), required
the end date of the simulation
filepath_or_buffer : str, optional
the location to write the output file (omit to write to stdout)
calendar : str, optional
the calendar you want to use e.g. LSE (default is to use the calendar
associated with the data bundle).
Returns
-------
None
Examples
--------
Run a backtest and save to CSV.
>>> from quantrocket.zipline import run_algorithm
>>> run_algorithm("momentum_pipeline.py", bundle="my-bundle",
start="2015-02-04", end="2015-12-31",
filepath_or_buffer="momentum_pipeline_results.csv")
Get a pyfolio tear sheet from the results:
>>> import pyfolio as pf
>>> pf.from_zipline_csv("momentum_pipeline_results.csv")
"""
params = {}
if data_frequency:
params["data_frequency"] = data_frequency
if capital_base:
params["capital_base"] = capital_base
if not bundle:
raise ValueError("must specify a bundle")
params["bundle"] = bundle
if bundle_timestamp:
params["bundle_timestamp"] = bundle_timestamp
if start:
params["start"] = start
if end:
params["end"] = end
if calendar:
params["calendar"] = calendar
response = houston.post("/zipline/backtests/{0}".format(algofile),
params=params,
timeout=60 * 60 * 3)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_reuters_estimates(codes, filepath_or_buffer=None, output="csv",
start_date=None, end_date=None,
universes=None, conids=None,
exclude_universes=None, exclude_conids=None,
period_types=None, fields=None):
"""
Query estimates and actuals from the Reuters estimates database and
download to file.
You can query one or more indicator codes. Use the `list_reuters_codes`
function to see available codes.
Parameters
----------
codes : list of str, required
the indicator code(s) to query
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, txt, default is csv)
start_date : str (YYYY-MM-DD), optional
limit to estimates and actuals on or after this fiscal period end date
end_date : str (YYYY-MM-DD), optional
limit to estimates and actuals on or before this fiscal period end date
universes : list of str, optional
limit to these universes
conids : list of int, optional
limit to these conids
exclude_universes : list of str, optional
exclude these universes
exclude_conids : list of int, optional
exclude these conids
period_types : list of str, optional
limit to these fiscal period types. Possible choices: A, Q, S, where
A=Annual, Q=Quarterly, S=Semi-Annual
fields : list of str, optional
only return these fields (pass ['?'] or any invalid fieldname to see
available fields)
Returns
-------
None
Examples
--------
Query EPS estimates and actuals for a universe of Australian stocks. You can use
StringIO to load the CSV into pandas.
>>> f = io.StringIO()
>>> download_reuters_estimates(["EPS"], f, universes=["asx-stk"],
start_date="2014-01-01"
end_date="2017-01-01")
>>> estimates = pd.read_csv(f, parse_dates=["FiscalPeriodEndDate", "AnnounceDate"])
"""
params = {}
if codes:
params["codes"] = codes
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
if exclude_universes:
params["exclude_universes"] = exclude_universes
if exclude_conids:
params["exclude_conids"] = exclude_conids
if period_types:
params["period_types"] = period_types
if fields:
params["fields"] = fields
output = output or "csv"
if output not in ("csv", "json", "txt"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/fundamental/reuters/estimates.{0}".format(output), params=params,
timeout=60*5)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_reuters_financials(codes, filepath_or_buffer=None, output="csv",
start_date=None, end_date=None,
universes=None, conids=None,
exclude_universes=None, exclude_conids=None,
interim=False, restatements=False, fields=None):
"""
Query financial statements from the Reuters financials database and
download to file.
You can query one or more COA codes. Use the `list_reuters_codes` function to see
available codes.
Annual or interim reports are available. Annual is the default and provides
deeper history.
By default restatements are excluded, but they can optionally be included.
Parameters
----------
codes : list of str, required
the Chart of Account (COA) code(s) to query
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, txt, default is csv)
start_date : str (YYYY-MM-DD), optional
limit to statements on or after this date (based on the
fiscal period end date if including restatements, otherwise the
filing date)
end_date : str (YYYY-MM-DD), optional
limit to statements on or before this date (based on the
fiscal period end date if including restatements, otherwise the
filing date)
universes : list of str, optional
limit to these universes
conids : list of int, optional
limit to these conids
exclude_universes : list of str, optional
exclude these universes
exclude_conids : list of int, optional
exclude these conids
interim : bool, optional
return interim reports (default is to return annual reports,
which provide deeper history)
restatements : bool, optional
include restatements (default is to exclude them)
fields : list of str, optional
only return these fields (pass ['?'] or any invalid fieldname to see
available fields)
Returns
-------
None
Examples
--------
Query total revenue (COA code RTLR) for a universe of Australian stocks. You can use
StringIO to load the CSV into pandas.
>>> f = io.StringIO()
>>> download_reuters_financials(["RTLR"], f, universes=["asx-stk"],
start_date="2014-01-01"
end_date="2017-01-01")
>>> financials = pd.read_csv(f, parse_dates=["StatementDate", "SourceDate", "FiscalPeriodEndDate"])
Query net income (COA code NINC) from interim reports for two securities
(identified by conid) and include restatements:
>>> download_reuters_financials(["NINC"], f, conids=[123456, 234567],
interim=True, restatements=True)
Query common and preferred shares outstanding (COA codes QTCO and QTPO) and return a
minimal set of fields (several required fields will always be returned):
>>> download_reuters_financials(["QTCO", "QTPO"], f, universes=["nyse-stk"],
fields=["Amount"])
"""
params = {}
if codes:
params["codes"] = codes
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
if exclude_universes:
params["exclude_universes"] = exclude_universes
if exclude_conids:
params["exclude_conids"] = exclude_conids
if interim:
params["interim"] = interim
if restatements:
params["restatements"] = restatements
if fields:
params["fields"] = fields
output = output or "csv"
if output not in ("csv", "json", "txt"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/fundamental/reuters/financials.{0}".format(output), params=params,
timeout=60*5)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def create_tearsheet(infilepath_or_buffer,
outfilepath_or_buffer=None,
simple=None,
live_start_date=None,
slippage=None,
hide_positions=None,
bayesian=None,
round_trips=None,
bootstrap=None):
"""
Create a pyfolio PDF tear sheet from a Zipline backtest result.
Parameters
----------
infilepath_or_buffer : str, required
the CSV file from a Zipline backtest (specify '-' to read file from stdin)
outfilepath_or_buffer : str or file-like, optional
the location to write the pyfolio tear sheet (write to stdout if omitted)
simple : bool
create a simple tear sheet (default is to create a full tear sheet)
live_start_date : str (YYYY-MM-DD), optional
date when the strategy began live trading
slippage : int or float, optional
basis points of slippage to apply to returns before generating tear sheet
stats and plots
hide_positions : bool
don't output any symbol names
bayesian : bool
include a Bayesian tear sheet
round_trips : bool
include a round-trips tear sheet
bootstrap : bool
perform bootstrap analysis for the performance metrics (takes a few minutes
longer)
Returns
-------
None
"""
params = {}
if simple:
params["simple"] = simple
if live_start_date:
params["live_start_date"] = live_start_date
if slippage:
params["slippage"] = slippage
if hide_positions:
params["hide_positions"] = hide_positions
if bayesian:
params["bayesian"] = bayesian
if round_trips:
params["round_trips"] = round_trips
if bootstrap:
params["bootstrap"] = bootstrap
url = "/zipline/tearsheets"
# Pyfolio can take a long time, particularly for Bayesian analysis
timeout = 60 * 60 * 5
if infilepath_or_buffer == "-":
infilepath_or_buffer = sys.stdin.buffer if six.PY3 else sys.stdin
response = houston.post(url,
data=infilepath_or_buffer,
params=params,
timeout=timeout)
elif infilepath_or_buffer and hasattr(infilepath_or_buffer, "read"):
if infilepath_or_buffer.seekable():
infilepath_or_buffer.seek(0)
response = houston.post(url,
data=infilepath_or_buffer,
params=params,
timeout=timeout)
else:
with open(infilepath_or_buffer, "rb") as f:
response = houston.post(url,
data=f,
params=params,
timeout=timeout)
houston.raise_for_status_with_json(response)
outfilepath_or_buffer = outfilepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(outfilepath_or_buffer, response)
def round_to_tick_sizes(infilepath_or_buffer, round_fields,
how=None, append_ticksize=False,
outfilepath_or_buffer=None):
"""
Round prices in a CSV file to valid tick sizes.
CSV should contain columns `ConId`, `Exchange`, and the columns to be rounded
(e.g. `LmtPrice`). Additional columns will be ignored and returned unchanged.
Parameters
----------
infilepath_or_buffer : str or file-like object, required
CSV file with prices to be rounded (specify '-' to read file from stdin)
round_fields : list of str, required
columns to be rounded
how : str, optional
which direction to round to. Possible choices: 'up', 'down', 'nearest'
(default is 'nearest')
append_ticksize : bool
append a column of tick sizes for each field to be rounded (default False)
outfilepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
Returns
-------
None
"""
params = {}
if round_fields:
params["round_fields"] = round_fields
if how:
params["how"] = how
if append_ticksize:
params["append_ticksize"] = append_ticksize
url = "/master/ticksizes.csv"
if infilepath_or_buffer == "-":
# No-op if an empty file is passed on stdin
f = six.StringIO(sys.stdin.read())
if not f.getvalue():
return
response = houston.get(url, params=params, data=to_bytes(f))
elif infilepath_or_buffer and hasattr(infilepath_or_buffer, "read"):
if infilepath_or_buffer.seekable():
infilepath_or_buffer.seek(0)
response = houston.get(url, params=params, data=to_bytes(infilepath_or_buffer))
elif infilepath_or_buffer:
with open(infilepath_or_buffer, "rb") as f:
response = houston.get(url, params=params, data=f)
else:
raise ValueError("infilepath_or_buffer is required")
houston.raise_for_status_with_json(response)
filepath_or_buffer = outfilepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_order_statuses(filepath_or_buffer=None,
output="csv",
order_ids=None,
conids=None,
order_refs=None,
accounts=None,
open_orders=None,
start_date=None,
end_date=None,
fields=None):
"""
Download order statuses.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json or csv, default is csv)
order_ids : list of str, optional
limit to these order IDs
conids : list of int, optional
limit to orders for these conids
order_refs : list of str, optional
limit to orders for these order refs
accounts : list of str, optional
limit to orders for these accounts
open_orders : bool
limit to open orders
start_date : str (YYYY-MM-DD), optional
limit to orders submitted on or after this date
end_date : str (YYYY-MM-DD), optional
limit to orders submitted on or before this date
fields : list of str, optional
return these fields in addition to the default fields (pass '?' or any invalid
fieldname to see available fields)
Returns
-------
None
Examples
--------
Download order status by order ID and load into Pandas:
>>> f = io.StringIO()
>>> download_order_statuses(f, order_ids=['6001:45','6001:46'])
>>> order_statuses = pd.read_csv(f)
Download order status for all open orders and include extra fields in output:
>>> download_order_statuses(open_orders=True, fields=["LmtPrice", "OcaGroup"])
Download order status of open orders by conid:
>>> download_order_statuses(conids=[123456], open_orders=True)
Download order status of open orders by order ref:
>>> download_order_statuses(order_refs=['my-strategy'], open_orders=True)
"""
params = {}
if order_ids:
params["order_ids"] = order_ids
if conids:
params["conids"] = conids
if order_refs:
params["order_refs"] = order_refs
if accounts:
params["accounts"] = accounts
if open_orders:
params["open_orders"] = open_orders
if fields:
params["fields"] = fields
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
output = output or "csv"
if output not in ("csv", "json"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/blotter/orders.{0}".format(output), params=params)
houston.raise_for_status_with_json(response)
# Don't write a null response to file
if response.content[:4] == b"null":
return
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def ml_walkforward(strategy,
start_date,
end_date,
train,
min_train=None,
rolling_train=None,
model_filepath=None,
force_nonincremental=None,
segment=None,
allocation=None,
nlv=None,
params=None,
details=None,
progress=False,
filepath_or_buffer=None):
"""
Run a walk-forward optimization of a machine learning strategy.
The date range will be split into segments of `train` size. For each
segment, the model will be trained with the data, then the trained model will
be backtested on the following segment.
By default, uses scikit-learn's StandardScaler+SGDRegressor. Also supports other
scikit-learn models/pipelines and Keras models. To customize model, instantiate
the model locally, serialize it to disk, and pass the path of the serialized
model as `model_filepath`.
Supports expanding walk-forward optimizations (the default), which use an anchored start date
for model training, or rolling walk-forward optimizations (by specifying `rolling_train`),
which use a rolling or non-anchored start date for model training.
Returns a backtest results CSV and a dump of the machine learning model
as of the end of the analysis.
Parameters
----------
strategy : str, required
the strategy code
start_date : str (YYYY-MM-DD), required
the analysis start date (note that model training will start on this date
but backtesting will not start until after the initial training period)
end_date : str (YYYY-MM-DD), required
the analysis end date
train : str, required
train model this frequently (use Pandas frequency string, e.g. 'A'
for annual training or 'Q' for quarterly training)
min_train : str, optional
don't backtest until at least this much model training has occurred;
defaults to the length of `train` if not specified (use Pandas frequency
string, e.g. '5Y' for 5 years of initial training)
rolling_train : str, optional
train model with a rolling window of this length; if omitted, train
model with an expanding window (use Pandas frequency string, e.g. '3Y' for
a 3-year rolling training window)
model_filepath : str, optional
filepath of serialized model to use, filename must end in ".joblib" or
".pkl" (if omitted, default model is scikit-learn's StandardScaler+SGDRegressor)
force_nonincremental : bool, optional
force the model to be trained non-incrementally (i.e. load entire training
data set into memory) even if it supports incremental learning. Must be True
in order to perform a rolling (as opposed to expanding) walk-forward optimization
with a model that supports incremental learning. Default False.
segment : str, optional
train and backtest in date segments of this size, to reduce memory usage;
must be smaller than `train`/`min_train` or will have no effect (use Pandas frequency string,
e.g. 'A' for annual segments or 'Q' for quarterly segments)
allocation : float, optional
the allocation for the strategy (default 1.0)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
details : bool
return detailed results for all securities instead of aggregating
progress : bool
log status and Sharpe ratios of each walk-forward segment during analysis
(default False)
filepath_or_buffer : str, optional
the location to write the ZIP file to; or, if path ends with "*", the
pattern to use for extracting the zipped files. For example, if the path is
my_ml*, files will extracted to my_ml_results.csv and my_ml_trained_model.joblib.
Returns
-------
None
Examples
--------
Run a walk-forward optimization using the default model and retrain the model
annually, writing the backtest results and trained model to demo_ml_results.csv
and demo_ml_trained_model.joblib, respectively:
>>> ml_walkforward(
"demo-ml",
"2007-01-01",
"2018-12-31",
train="A",
filepath_or_buffer="demo_ml*")
Create a scikit-learn model, serialize it with joblib, and use it to
run the walkforward backtest:
>>> from sklearn.linear_model import SGDClassifier
>>> import joblib
>>> clf = SGDClassifier()
>>> joblib.dump(clf, "my_model.joblib")
>>> ml_walkforward(
"demo-ml",
"2007-01-01",
"2018-12-31",
train="A",
model_filepath="my_model.joblib",
filepath_or_buffer="demo_ml*")
Run a walk-forward optimization using a custom model (serialized with joblib),
retrain the model annually, don't perform backtesting until after 5 years
of initial training, and further split the training and backtesting into
quarterly segments to reduce memory usage:
>>> ml_walkforward(
"demo-ml",
"2007-01-01",
"2018-12-31",
model_filepath="my_model.joblib",
train="A",
min_train="5Y",
segment="Q",
filepath_or_buffer="demo_ml*")
Create a Keras model, serialize it, and use it to run the walkforward backtest:
>>> from keras.models import Sequential
>>> from keras.layers import Dense
>>> model = Sequential()
>>> # input_dim should match number of features in training data
>>> model.add(Dense(units=4, activation='relu', input_dim=5))
>>> # last layer should have a single unit
>>> model.add(Dense(units=1, activation='softmax'))
>>> model.compile(loss='sparse_categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
>>> model.save('my_model.keras.h5')
>>> ml_walkforward(
"neuralnet-ml",
"2007-01-01",
"2018-12-31",
train="A",
model_filepath="my_model.keras.h5",
filepath_or_buffer="neuralnet_ml*")
"""
_params = {}
_params["start_date"] = start_date
_params["end_date"] = end_date
_params["train"] = train
if min_train:
_params["min_train"] = min_train
if rolling_train:
_params["rolling_train"] = rolling_train
if force_nonincremental:
_params["force_nonincremental"] = force_nonincremental
if segment:
_params["segment"] = segment
if allocation:
_params["allocation"] = allocation
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if details:
_params["details"] = details
if progress:
_params["progress"] = progress
if params:
_params["params"] = dict_to_dict_strs(params)
url = "/moonshot/ml/walkforward/{0}.zip".format(strategy)
if model_filepath:
# Send the filename as a hint how to open it
_params["model_filename"] = os.path.basename(model_filepath)
with open(model_filepath, "rb") as f:
response = houston.post(url,
data=f,
params=_params,
timeout=60 * 60 * 24)
else:
response = houston.post(url, params=_params, timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
auto_extract = isinstance(
filepath_or_buffer,
six.string_types) and filepath_or_buffer.endswith("*")
if auto_extract:
base_filepath = filepath_or_buffer[:-1]
zipfilepath = base_filepath + ".zip"
write_response_to_filepath_or_buffer(zipfilepath, response)
with ZipFile(zipfilepath, mode="r") as zfile:
model_filename = [
name for name in zfile.namelist() if "model" in name
][0]
model_filepath = base_filepath + "_" + model_filename
csv_filepath = base_filepath + "_results.csv"
with open(csv_filepath, "wb") as csvfile:
csvfile.write(zfile.read("results.csv"))
with open(model_filepath, "wb") as modelfile:
modelfile.write(zfile.read(model_filename))
os.remove(zipfilepath)
else:
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def scan_parameters(strategies,
start_date=None,
end_date=None,
segment=None,
param1=None,
vals1=None,
param2=None,
vals2=None,
allocations=None,
nlv=None,
params=None,
output="csv",
csv=None,
filepath_or_buffer=None):
"""
Run a parameter scan for one or more strategies.
By default returns a CSV of scan results but can also return a PDF tear sheet.
Parameters
----------
strategies : list of str, required
one or more strategy codes
start_date : str (YYYY-MM-DD), optional
the backtest start date (default is to use all available history)
end_date : str (YYYY-MM-DD), optional
the backtest end date (default is to use all available history)
segment : str, optional
backtest in date segments of this size, to reduce memory usage
(use Pandas frequency string, e.g. 'A' for annual segments or 'Q'
for quarterly segments)
param1 : str, required
the name of the parameter to test (a class attribute on the strategy)
vals1 : list of int/float/str/tuple, required
parameter values to test (values can be ints, floats, strings, False,
True, None, 'default' (to test current param value), or lists of
ints/floats/strings)
param2 : str, optional
name of a second parameter to test (for 2-D parameter scans)
vals2 : list of int/float/str/tuple, optional
values to test for parameter 2 (values can be ints, floats, strings,
False, True, None, 'default' (to test current param value), or lists
of ints/floats/strings)
allocations : dict of CODE:FLOAT, optional
the allocation for each strategy, passed as {code:allocation} (default
allocation is 1.0 / number of strategies)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
output : str, required
the output format (choices are csv or pdf)
csv : bool
DEPRECATED: this argument will be removed in a future version. This argument
may be omitted as CSV is the default.
filepath_or_buffer : str, optional
the location to write the results file (omit to write to stdout)
Returns
-------
None
Examples
--------
Run a parameter scan for several different moving averages on a strategy
called trend-friend and return a CSV (which can be rendered with Moonchart):
>>> scan_parameters("trend-friend",
param1="MAVG_WINDOW",
vals1=[20, 50, 100],
filepath_or_buffer="trend_friend_MAVG_WINDOW.csv")
Run a 2-D parameter scan for multiple strategies and return a CSV:
>>> scan_parameters(["strat1", "strat2", "strat3"],
param1="MIN_STD",
vals1=[1, 1.5, 2],
param2="STD_WINDOW",
vals2=[20, 50, 100, 200],
filepath_or_buffer="strategies_MIN_STD_and_STD_WINDOW.csv")
Run a parameter scan in 1-year segments to reduce memory usage:
>>> scan_parameters("big-strategy",
start_date="2000-01-01",
end_date="2018-01-01",
segment="A",
param1="MAVG_WINDOW",
vals1=[20, 50, 100],
filepath_or_buffer="big_strategy_MAVG_WINDOW.csv")
"""
output = output or "csv"
if output not in ("csv", "pdf"):
raise ValueError(
"invalid output: {0} (choices are csv or pdf".format(output))
if csv is not None:
import warnings
warnings.warn(
"the `csv` argument is deprecated and will removed in a future version; "
"this argument may be omitted as csv is the default",
DeprecationWarning)
_params = {}
if strategies:
_params["strategies"] = strategies
if start_date:
_params["start_date"] = start_date
if end_date:
_params["end_date"] = end_date
if segment:
_params["segment"] = segment
if param1:
_params["param1"] = param1
if vals1:
_params["vals1"] = [str(v) for v in vals1]
if param2:
_params["param2"] = param2
if vals2:
_params["vals2"] = [str(v) for v in vals2]
if allocations:
_params["allocations"] = dict_to_dict_strs(allocations)
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if params:
_params["params"] = dict_to_dict_strs(params)
response = houston.post("/moonshot/paramscans.{0}".format(output),
params=_params,
timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def scan_parameters(strategies, start_date=None, end_date=None,
param1=None, vals1=None, param2=None, vals2=None,
allocations=None, nlv=None, params=None, output="csv",
csv=None, filepath_or_buffer=None):
"""
Run a parameter scan for one or more strategies.
By default returns a CSV of scan results but can also return a PDF tear sheet.
Parameters
----------
strategies : list of str, required
one or more strategy codes
start_date : str (YYYY-MM-DD), optional
the backtest start date (default is to use all available history)
end_date : str (YYYY-MM-DD), optional
the backtest end date (default is to use all available history)
param1 : str, required
the name of the parameter to test (a class attribute on the strategy)
vals1 : list of int/float/str/tuple, required
parameter values to test (values can be ints, floats, strings, False,
True, None, 'default' (to test current param value), or lists of
ints/floats/strings)
param2 : str, optional
name of a second parameter to test (for 2-D parameter scans)
vals2 : list of int/float/str/tuple, optional
values to test for parameter 2 (values can be ints, floats, strings,
False, True, None, 'default' (to test current param value), or lists
of ints/floats/strings)
allocations : dict of CODE:FLOAT, optional
the allocation for each strategy, passed as {code:allocation} (default
allocation is 1.0 / number of strategies)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
output : str, required
the output format (choices are csv or pdf)
csv : bool
DEPRECATED: this argument will be removed in a future version. This argument
may be omitted as CSV is the default.
filepath_or_buffer : str, optional
the location to write the results file (omit to write to stdout)
Returns
-------
None
"""
output = output or "csv"
if output not in ("csv", "pdf"):
raise ValueError("invalid output: {0} (choices are csv or pdf".format(output))
if csv is not None:
import warnings
warnings.warn(
"the `csv` argument is deprecated and will removed in a future version; "
"this argument may be omitted as csv is the default", DeprecationWarning)
_params = {}
if strategies:
_params["strategies"] = strategies
if start_date:
_params["start_date"] = start_date
if end_date:
_params["end_date"] = end_date
if param1:
_params["param1"] = param1
if vals1:
_params["vals1"] = [str(v) for v in vals1]
if param2:
_params["param2"] = param2
if vals2:
_params["vals2"] = [str(v) for v in vals2]
if allocations:
_params["allocations"] = dict_to_dict_strs(allocations)
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if params:
_params["params"] = dict_to_dict_strs(params)
response = houston.post("/moonshot/paramscans.{0}".format(output), params=_params, timeout=60*60*24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_pnl(filepath_or_buffer=None,
order_refs=None,
accounts=None,
conids=None,
start_date=None,
end_date=None,
time=None,
details=False,
output="csv"):
"""
Query trading performance and return a CSV of results or PDF tearsheet.
Trading performance is broken down by account and order ref and optionally by
conid.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
order_refs : list of str, optional
limit to these order refs
accounts : list of str, optional
limit to these accounts
conids : list of int, optional
limit to these conids
start_date : str (YYYY-MM-DD), optional
limit to pnl on or after this date
end_date : str (YYYY-MM-DD), optional
limit to pnl on or before this date
time : str (HH:MM:SS [TZ]), optional
time of day (with optional timezone) for which to calculate daily PNL (default is
11:59:59 UTC)
details : bool
return detailed results for all securities instead of aggregating to
account/order ref level (only supported for a single account and order ref
at a time)
output : str, required
the output format (choices are csv or pdf, default is csv)
Returns
-------
None
"""
params = {}
if order_refs:
params["order_refs"] = order_refs
if accounts:
params["accounts"] = accounts
if conids:
params["conids"] = conids
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if time:
params["time"] = time
if details:
params["details"] = details
output = output or "csv"
if output not in ("csv", "pdf"):
raise ValueError(
"invalid output: {0} (choices are csv or pdf".format(output))
response = houston.get("/blotter/pnl.{0}".format(output),
params=params,
timeout=60 * 10)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def run_algorithm(algofile,
data_frequency=None,
capital_base=None,
bundle=None,
bundle_timestamp=None,
start=None,
end=None,
filepath_or_buffer=None,
calendar=None):
"""
Run a Zipline backtest and write the test results to a CSV file.
The CSV result file contains several DataFrames stacked into one: the Zipline performance
results, plus the extracted returns, transactions, positions, and benchmark returns from those
results.
Parameters
----------
algofile : str, required
the file that contains the algorithm to run
data_frequency : str, optional
the data frequency of the simulation. Possible choices: daily, minute (default is daily)
capital_base : float, optional
the starting capital for the simulation (default is 10000000.0)
bundle : str, optional
the data bundle to use for the simulation (default is quantopian-quandl)
bundle_timestamp : str, optional
the date to lookup data on or before (default is <current-time>)
start : str (YYYY-MM-DD), required
the start date of the simulation
end : str (YYYY-MM-DD), required
the end date of the simulation
filepath_or_buffer : str, optional
the location to write the output file (omit to write to stdout)
calendar : str, optional
the calendar you want to use e.g. LSE. NYSE is the default.
Returns
-------
None
Examples
--------
Run a backtest and load the results into pandas.
>>> from quantrocket.zipline import run_algorithm
>>> import pandas as pd
>>> import io
>>> f = io.StringIO()
>>> run_algorithm("momentum_pipeline.py", bundle="my-bundle", start="2015-02-04", end="2015-12-31", filepath_or_buffer=f)
>>> results = pd.read_csv(f, index_col=["dataframe", "index", "column"])["value"]
To use the results with pyfolio, extract and massage the returns, positions,
transactions, and benchmark returns:
>>> # Extract returns
>>> returns = results.loc["returns"].unstack()
>>> returns.index = returns.index.droplevel(0).tz_localize("UTC")
>>> returns = returns["returns"].astype(float)
>>> # Extract positions
>>> positions = results.loc["positions"].unstack()
>>> positions.index = positions.index.droplevel(0).tz_localize("UTC")
>>> positions = positions.astype(float)
>>> # Extract transactions
>>> transactions = results.loc["transactions"].unstack()
>>> transactions.index = transactions.index.droplevel(0).tz_localize("UTC")
>>> transactions = transactions.apply(pd.to_numeric, errors='ignore')
>>> # Extract benchmark
>>> benchmark_returns = results.loc["benchmark"].unstack()
>>> benchmark_returns.index = benchmark_returns.index.droplevel(0).tz_localize("UTC")
>>> benchmark_returns = benchmark_returns["benchmark"].astype(float)
Ready for pyfolio:
>>> pf.create_full_tear_sheet(returns, positions=positions, transactions=transactions, benchmark_rets=benchmark_returns)
"""
params = {}
if data_frequency:
params["data_frequency"] = data_frequency
if capital_base:
params["capital_base"] = capital_base
if bundle:
params["bundle"] = bundle
if bundle_timestamp:
params["bundle_timestamp"] = bundle_timestamp
if start:
params["start"] = start
if end:
params["end"] = end
if calendar:
params["calendar"] = calendar
response = houston.post("/zipline/backtests/{0}".format(algofile),
params=params,
timeout=60 * 60 * 3)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def close_positions(filepath_or_buffer=None,
output="csv",
order_refs=None,
accounts=None,
conids=None,
params=None):
"""
Generate orders to close positions.
Doesn't actually place any orders but returns an orders file that can be placed
separately. Additional order parameters can be appended with the `params` argument.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json or csv, default is csv)
order_refs : list of str, optional
limit to these order refs
accounts : list of str, optional
limit to these accounts
conids : list of int, optional
limit to these conids
params : dict of PARAM:VALUE, optional
additional parameters to append to each row in output (pass as {param:value},
for example {"OrderType":"MKT"})
Returns
-------
None
Examples
--------
Get orders to close positions, then place the orders:
>>> from quantrocket.blotter import place_orders, close_positions
>>> import io
>>> orders_file = io.StringIO()
>>> close_positions(orders_file, params={"OrderType":"MKT", "Tif":"DAY", "Exchange":"SMART"})
>>> place_orders(infilepath_or_buffer=orders_file)
"""
_params = {}
if order_refs:
_params["order_refs"] = order_refs
if accounts:
_params["accounts"] = accounts
if conids:
_params["conids"] = conids
if params:
_params["params"] = dict_to_dict_strs(params)
output = output or "csv"
if output not in ("csv", "json"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.delete("/blotter/positions.{0}".format(output),
params=_params)
houston.raise_for_status_with_json(response)
# Don't write a null response to file
if response.content[:4] == b"null":
return
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_positions(filepath_or_buffer=None,
output="csv",
order_refs=None,
accounts=None,
conids=None,
view="blotter",
diff=False):
"""
Query current positions and write results to file.
To return positions as a Python list, see list_positions.
There are two ways to view positions: blotter view (default) and broker view.
The default "blotter view" returns positions by account, conid, and order ref. Positions
are tracked based on execution records saved to the blotter database.
"Broker view" (view='broker') returns positions by account and conid (but
not order ref) as reported directly by IB. Broker view is more authoritative but less
informative than blotter view. Broker view is typically used to verify the accuracy
of blotter view.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json or csv, default is csv)
order_refs : list of str, optional
limit to these order refs (not supported with broker view)
accounts : list of str, optional
limit to these accounts
conids : list of int, optional
limit to these conids
view : str, optional
whether to return 'broker' view of positions (by account and conid) or
default 'blotter' view (by account, conid, and order ref). Choices are:
blotter, broker
diff : bool
limit to positions where the blotter quantity and broker quantity disagree
(requires `view='broker'`)
Returns
-------
None
See Also
--------
list_positions : load positions into Python list
"""
params = {}
if order_refs:
params["order_refs"] = order_refs
if accounts:
params["accounts"] = accounts
if conids:
params["conids"] = conids
if view:
params["view"] = view
if diff:
params["diff"] = diff
output = output or "csv"
if output not in ("csv", "json"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/blotter/positions.{0}".format(output),
params=params)
houston.raise_for_status_with_json(response)
# Don't write a null response to file
if response.content[:4] == b"null":
return
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_history_file(code, filepath_or_buffer=None, output="csv",
start_date=None, end_date=None,
universes=None, conids=None,
exclude_universes=None, exclude_conids=None,
times=None, cont_fut=None, fields=None, tz_naive=False):
"""
Query historical market data from a history database and download to file.
Parameters
----------
code : str, required
the code of the database to query
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, txt, default is csv)
start_date : str (YYYY-MM-DD), optional
limit to history on or after this date
end_date : str (YYYY-MM-DD), optional
limit to history on or before this date
universes : list of str, optional
limit to these universes (default is to return all securities in database)
conids : list of int, optional
limit to these conids
exclude_universes : list of str, optional
exclude these universes
exclude_conids : list of int, optional
exclude these conids
times: list of str (HH:MM:SS), optional
limit to these times
cont_fut : str
stitch futures into continuous contracts using this method (default is not
to stitch together). Possible choices: concat
fields : list of str, optional
only return these fields (pass ['?'] or any invalid fieldname to see
available fields)
tz_naive : bool
return timestamps without UTC offsets: 2018-02-01T10:00:00 (default is to
include UTC offsets: 2018-02-01T10:00:00-4000)
Returns
-------
None
Examples
--------
You can use StringIO to load the CSV into pandas.
>>> f = io.StringIO()
>>> download_history_file("my-db", f)
>>> history = pd.read_csv(f, parse_dates=["Date"])
See Also
--------
quantrocket.get_prices : load prices into a DataFrame
"""
params = {}
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
if exclude_universes:
params["exclude_universes"] = exclude_universes
if exclude_conids:
params["exclude_conids"] = exclude_conids
if times:
params["times"] = times
if cont_fut:
params["cont_fut"] = cont_fut
if fields:
params["fields"] = fields
if tz_naive:
params["tz_naive"] = tz_naive
output = output or "csv"
if output not in ("csv", "json", "txt"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/history/{0}.{1}".format(code, output), params=params,
timeout=60*30)
try:
houston.raise_for_status_with_json(response)
except requests.HTTPError as e:
# Raise a dedicated exception
if "no history matches the query parameters" in repr(e).lower():
raise NoHistoricalData(e)
raise
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_master_file(filepath_or_buffer=None, output="csv", exchanges=None, sec_types=None,
currencies=None, universes=None, symbols=None, conids=None,
exclude_universes=None, exclude_conids=None,
sectors=None, industries=None, categories=None,
delisted=False, frontmonth=False, fields=None):
"""
Query security details from the securities master database and download to file.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, txt, default is csv)
exchanges : list of str, optional
limit to these exchanges
sec_types : list of str, optional
limit to these security types. Possible choices: STK, ETF, FUT, CASH, IND, OPT, FOP
currencies : list of str, optional
limit to these currencies
universes : list of str, optional
limit to these universes
symbols : list of str, optional
limit to these symbols
conids : list of int, optional
limit to these conids
exclude_universes : list of str, optional
exclude these universes
exclude_conids : list of int, optional
exclude these conids
sectors : list of str, optional
limit to these sectors
industries : list of str, optional
limit to these industries
categories : list of str, optional
limit to these categories
delisted : bool
include delisted securities (default False)
frontmonth : bool
exclude backmonth and expired futures contracts (default False)
fields : list of str, optional
only return these fields (pass ['?'] or any invalid fieldname to see
available fields)
Returns
-------
None
Examples
--------
You can use StringIO to load the CSV into pandas.
>>> f = io.StringIO()
>>> download_master_file(f, universes=["my-universe"])
>>> securities = pd.read_csv(f)
"""
params = {}
if exchanges:
params["exchanges"] = exchanges
if sec_types:
params["sec_types"] = sec_types
if currencies:
params["currencies"] = currencies
if universes:
params["universes"] = universes
if symbols:
params["symbols"] = symbols
if conids:
params["conids"] = conids
if exclude_universes:
params["exclude_universes"] = exclude_universes
if exclude_conids:
params["exclude_conids"] = exclude_conids
if sectors:
params["sectors"] = sectors
if industries:
params["industries"] = industries
if categories:
params["categories"] = categories
if delisted:
params["delisted"] = delisted
if frontmonth:
params["frontmonth"] = frontmonth
if fields:
params["fields"] = fields
output = output or "csv"
if output not in ("csv", "json", "txt"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/master/securities.{0}".format(output), params=params)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def backtest(strategies,
start_date=None,
end_date=None,
segment=None,
allocations=None,
nlv=None,
params=None,
details=None,
output="csv",
csv=None,
filepath_or_buffer=None):
"""
Backtest one or more strategies.
By default returns a CSV of backtest results but can also return a PDF tear sheet
of performance charts.
If testing multiple strategies, each column in the CSV represents a strategy.
If testing a single strategy and `details=True`, each column in the CSV
represents a security in the strategy universe.
Parameters
----------
strategies : list of str, required
one or more strategy codes
start_date : str (YYYY-MM-DD), optional
the backtest start date (default is to use all available history)
end_date : str (YYYY-MM-DD), optional
the backtest end date (default is to use all available history)
segment : str, optional
backtest in date segments of this size, to reduce memory usage
(use Pandas frequency string, e.g. 'A' for annual segments or 'Q'
for quarterly segments)
allocations : dict of CODE:FLOAT, optional
the allocation for each strategy, passed as {code:allocation} (default
allocation is 1.0 / number of strategies)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
details : bool
return detailed results for all securities instead of aggregating to
strategy level (only supported for single-strategy backtests)
output : str, required
the output format (choices are csv or pdf)
csv : bool
DEPRECATED: this argument will be removed in a future version. This argument
may be omitted as CSV is the default.
filepath_or_buffer : str, optional
the location to write the results file (omit to write to stdout)
Returns
-------
None
Examples
--------
Backtest several HML (High Minus Low) strategies from 2005-2015 and return a
CSV of results:
>>> backtest(["hml-us", "hml-eur", "hml-asia"],
start_date="2005-01-01",
end_date="2015-12-31",
filepath_or_buffer="hml_results.csv")
Run a backtest in 1-year segments to reduce memory usage:
>>> backtest("big-strategy",
start_date="2000-01-01",
end_date="2018-01-01",
segment="A",
filepath_or_buffer="results.csv")
See Also
--------
read_moonshot_csv : load a Moonshot backtest CSV into a DataFrame
"""
output = output or "csv"
if output not in ("csv", "pdf"):
raise ValueError(
"invalid output: {0} (choices are csv or pdf".format(output))
if csv is not None:
import warnings
warnings.warn(
"the `csv` argument is deprecated and will removed in a future version; "
"this argument may be omitted as csv is the default",
DeprecationWarning)
_params = {}
if strategies:
_params["strategies"] = strategies
if start_date:
_params["start_date"] = start_date
if end_date:
_params["end_date"] = end_date
if segment:
_params["segment"] = segment
if allocations:
_params["allocations"] = dict_to_dict_strs(allocations)
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if details:
_params["details"] = details
if params:
_params["params"] = dict_to_dict_strs(params)
response = houston.post("/moonshot/backtests.{0}".format(output),
params=_params,
timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_market_data_file(code, filepath_or_buffer=None, output="csv",
start_date=None, end_date=None,
universes=None, conids=None,
exclude_universes=None, exclude_conids=None,
fields=None):
"""
Query market data from a tick database or aggregate database and download to file.
Parameters
----------
code : str, required
the code of the tick database or aggregate database to query
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, default is csv)
start_date : str (YYYY-MM-DD HH:MM:SS), optional
limit to market data on or after this datetime. Can pass a date (YYYY-MM-DD),
datetime with optional timezone (YYYY-MM-DD HH:MM:SS TZ), or time with
optional timezone. A time without date will be interpreted as referring to
today if the time is earlier than now, or yesterday if the time is later than
now.
end_date : str (YYYY-MM-DD HH:MM:SS), optional
limit to market data on or before this datetime. Can pass a date (YYYY-MM-DD),
datetime with optional timezone (YYYY-MM-DD HH:MM:SS TZ), or time with
optional timezone.
universes : list of str, optional
limit to these universes (default is to return all securities in database)
conids : list of int, optional
limit to these conids
exclude_universes : list of str, optional
exclude these universes
exclude_conids : list of int, optional
exclude these conids
fields : list of str, optional
only return these fields (pass '?' or any invalid fieldname to see
available fields)
Returns
-------
None
Examples
--------
Download a CSV of futures market data since 08:00 AM Chicago time:
>>> download_market_data_file("globex-fut-taq",
start_date="08:00:00 America/Chicago",
filepath_or_buffer="globex_taq.csv")
>>> market_data = pd.read_csv("globex_taq.csv", parse_dates=["Date"])
See Also
--------
quantrocket.get_prices : load prices into a DataFrame
"""
params = {}
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
if exclude_universes:
params["exclude_universes"] = exclude_universes
if exclude_conids:
params["exclude_conids"] = exclude_conids
if fields:
params["fields"] = fields
output = output or "csv"
if output not in ("csv", "json"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/realtime/{0}.{1}".format(code, output), params=params,
timeout=60*30)
try:
houston.raise_for_status_with_json(response)
except requests.HTTPError as e:
# Raise a dedicated exception
if "no market data matches the query parameters" in repr(e).lower():
raise NoRealtimeData(e)
raise
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_borrow_fees(filepath_or_buffer=None,
output="csv",
start_date=None,
end_date=None,
universes=None,
conids=None,
exclude_universes=None,
exclude_conids=None):
"""
Query borrow fees from the stockloan database and download to file.
Data timestamps are UTC.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, default is csv)
start_date : str (YYYY-MM-DD), optional
limit to data on or after this date
end_date : str (YYYY-MM-DD), optional
limit to data on or before this date
universes : list of str, optional
limit to these universes
conids : list of int, optional
limit to these conids
exclude_universes : list of str, optional
exclude these universes
exclude_conids : list of int, optional
exclude these conids
Returns
-------
None
Examples
--------
Query borrow fees for a universe of Australian stocks.
>>> f = io.StringIO()
>>> download_borrow_fees("asx_borrow_fees.csv", universes=["asx-stk"])
>>> borrow_fees = pd.read_csv("asx_borrow_fees.csv", parse_dates=["Date"])
"""
params = {}
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
if exclude_universes:
params["exclude_universes"] = exclude_universes
if exclude_conids:
params["exclude_conids"] = exclude_conids
output = output or "csv"
if output not in ("csv", "json"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/fundamental/stockloan/fees.{0}".format(output),
params=params,
timeout=60 * 5)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_exchange_rates(filepath_or_buffer=None,
output="csv",
start_date=None,
end_date=None,
latest=False,
base_currencies=None,
quote_currencies=None):
"""
Query exchange rates for the base currency.
The exchange rates in the exchange rate database are sourced from the
European Central Bank's reference rates, which are updated each day at 4 PM
CET.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, txt, default is csv)
start_date : str (YYYY-MM-DD), optional
limit to exchange rates on or after this date
end_date : str (YYYY-MM-DD), optional
limit to exchange rates on or before this date
latest : bool
rreturn the latest exchange rates
base_currencies : list of str, optional
limit to these base currencies
quote_currencies : list of str, optional
limit to these quote currencies
Returns
-------
None
Examples
--------
Query latest exchange rates. You can use StringIO to load the CSV into pandas.
>>> f = io.StringIO()
>>> download_exchange_rates(f, latest=True)
>>> rates = pd.read_csv(f, parse_dates=["Date"])
"""
params = {}
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if latest:
params["latest"] = latest
if base_currencies:
params["base_currencies"] = base_currencies
if quote_currencies:
params["quote_currencies"] = quote_currencies
output = output or "csv"
if output not in ("csv", "json", "txt"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/account/rates.{0}".format(output), params=params)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_master_file(filepath_or_buffer=None, output="csv", exchanges=None, sec_types=None,
currencies=None, universes=None, symbols=None, conids=None,
exclude_universes=None, exclude_conids=None,
sectors=None, industries=None, categories=None,
exclude_delisted=False, delisted=True, frontmonth=False, fields=None,
domain=None):
"""
Query security details from the securities master database and download to file.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, txt, default is csv)
exchanges : list of str, optional
limit to these exchanges
sec_types : list of str, optional
limit to these security types. Possible choices: STK, ETF, FUT, CASH, IND, OPT, FOP
currencies : list of str, optional
limit to these currencies
universes : list of str, optional
limit to these universes
symbols : list of str, optional
limit to these symbols
conids : list of int, optional
limit to these conids
exclude_universes : list of str, optional
exclude these universes
exclude_conids : list of int, optional
exclude these conids
sectors : list of str, optional
limit to these sectors
industries : list of str, optional
limit to these industries
categories : list of str, optional
limit to these categories
exclude_delisted : bool
exclude delisted securities (default is to include them)
delisted : bool
[DEPRECATED] include delisted securities; this parameter is deprecated
and will be removed in a future release; it has no effect as delisted
securities are included by default
frontmonth : bool
exclude backmonth and expired futures contracts (default False)
fields : list of str, optional
only return these fields (pass ['?'] or any invalid fieldname to see
available fields)
domain : str, optional
query against this domain (default is 'main', which runs against
quantrocket.master.main.sqlite. Possible choices: main, sharadar)
Returns
-------
None
Examples
--------
Download several exchanges to file:
>>> download_master_file("securities.csv", exchanges=["NYSE","NASDAQ"])
Download securities for a particular universe to in-memory file and
load the CSV into pandas.
>>> f = io.StringIO()
>>> download_master_file(f, universes=["my-universe"])
>>> securities = pd.read_csv(f)
Download Sharadar securities from quantrocket.master.sharadar.sqlite:
>>> download_master_file("sharadar_securities.csv", domain="sharadar")
"""
# Handle legacy param "delisted"
if delisted is False and exclude_delisted is None:
exclude_delisted = True
params = {}
if exchanges:
params["exchanges"] = exchanges
if sec_types:
params["sec_types"] = sec_types
if currencies:
params["currencies"] = currencies
if universes:
params["universes"] = universes
if symbols:
params["symbols"] = symbols
if conids:
params["conids"] = conids
if exclude_universes:
params["exclude_universes"] = exclude_universes
if exclude_conids:
params["exclude_conids"] = exclude_conids
if sectors:
params["sectors"] = sectors
if industries:
params["industries"] = industries
if categories:
params["categories"] = categories
if exclude_delisted:
params["exclude_delisted"] = exclude_delisted
if frontmonth:
params["frontmonth"] = frontmonth
if fields:
params["fields"] = fields
output = output or "csv"
url = "/master/{0}securities.{1}".format(
"{0}/".format(domain) if domain else "",
output)
if output not in ("csv", "json", "txt"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get(url, params=params)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def download_account_balances(filepath_or_buffer=None,
output="csv",
start_date=None,
end_date=None,
latest=False,
accounts=None,
below=None,
fields=None,
no_cache=False):
"""
Query IB account balances.
Parameters
----------
filepath_or_buffer : str or file-like object
filepath to write the data to, or file-like object (defaults to stdout)
output : str
output format (json, csv, txt, default is csv)
start_date : str (YYYY-MM-DD), optional
limit to account balance snapshots taken on or after this date
end_date : str (YYYY-MM-DD), optional
limit to account balance snapshots taken on or before this date
latest : bool
return the latest account balance snapshot
accounts : list of str, optional
limit to these accounts
below : dict of FIELD:AMOUNT, optional
limit to accounts where the specified field is below the specified
amount (pass as {field:amount}, for example {'Cushion':0.05})
fields : list of str, optional
only return these fields (pass ['?'] or any invalid fieldname to see
available fields)
no_cache : bool
fetch account balances directly from IB (default is to query the
database, which is updated every minute)
Returns
-------
None
Examples
--------
Query latest balances. You can use StringIO to load the CSV into pandas.
>>> f = io.StringIO()
>>> download_account_balances(f, latest=True)
>>> balances = pd.read_csv(f, parse_dates=["LastUpdated"])
"""
params = {}
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if latest:
params["latest"] = latest
if accounts:
params["accounts"] = accounts
if below:
params["below"] = dict_to_dict_strs(below)
if fields:
params["fields"] = fields
if no_cache:
params["no_cache"] = no_cache
output = output or "csv"
if output not in ("csv", "json", "txt"):
raise ValueError("Invalid ouput: {0}".format(output))
response = houston.get("/account/balances.{0}".format(output),
params=params)
houston.raise_for_status_with_json(response)
# Don't write a null response to file when using below filters
if below and response.content[:4] == b"null":
return
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def execute_command(cmd,
return_file=None,
filepath_or_buffer=None,
params=None,
service="satellite"):
"""
Execute a Python function or abitrary shell command on a satellite service.
Parameters
----------
cmd: str, required
the shell command to run, or the Python function in dot notation (must
start with "codeload." to be interpreted as a Python function).
return_file : str, optional
the path of a file to be returned after the command completes
filepath_or_buffer : str, optional
the location to write the return_file (omit to write to stdout)
params : dict of PARAM:VALUE, optional
one or more params to pass to the Python function (pass as {param:value})
service : str, optional
the service name (default 'satellite')
Returns
-------
dict or None
None if return_file, otherwise status message
Examples
--------
Run a Python function called 'create_calendar_spread' defined in '/codeload/scripts/combos.py'
and pass it arguments:
>>> execute_command("codeload.scripts.combos.create_calendar_spread",
params={"universe":"cl-fut", "contract_months":[1,2]})
Run a backtrader backtest and save the performance chart to file:
>>> execute_command("python /codeload/backtrader/dual_moving_average.py",
return_file="/tmp/backtrader-plot.pdf"
outfile="backtrader-plot.pdf")
"""
_params = {}
if not service:
raise ValueError("a service is required")
if not cmd:
raise ValueError("a command is required")
_params["cmd"] = cmd
if params:
_params["params"] = dict_to_dict_strs(params)
if return_file:
_params["return_file"] = return_file
if not service.startswith("satellite"):
raise ValueError("service must start with 'satellite'")
response = houston.post("/{0}/commands".format(service),
params=_params,
timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
if return_file:
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
else:
return response.json()
def backtest(strategies, start_date=None, end_date=None, allocations=None,
nlv=None, params=None, details=None, csv=None, filepath_or_buffer=None):
"""
Backtest one or more strategies.
By default returns a PDF tear sheet of performance charts but can also return a CSV of
backtest results.
Parameters
----------
strategies : list of str, required
one or more strategy codes
start_date : str (YYYY-MM-DD), optional
the backtest start date (default is to use all available history)
end_date : str (YYYY-MM-DD), optional
the backtest end date (default is to use all available history)
allocations : dict of CODE:FLOAT, optional
the allocation for each strategy, passed as {code:allocation} (default
allocation is 1.0 / number of strategies)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
details : bool
return detailed results for all securities instead of aggregating to
strategy level (only supported for single-strategy backtests)
csv : bool
return a CSV of performance data (default is to return a PDF
performance tear sheet)
filepath_or_buffer : str, optional
the location to write the results file (omit to write to stdout)
Returns
-------
None
"""
_params = {}
if strategies:
_params["strategies"] = strategies
if start_date:
_params["start_date"] = start_date
if end_date:
_params["end_date"] = end_date
if allocations:
_params["allocations"] = dict_to_dict_strs(allocations)
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if details:
_params["details"] = details
if csv:
_params["csv"] = csv
if params:
_params["params"] = dict_to_dict_strs(params)
response = houston.post("/moonshot/backtests", params=_params, timeout=60*60*24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
