def collect_option_chains(universes=None,
conids=None,
infilepath_or_buffer=None):
"""
Collect option chains for underlying securities.
Note: option chains often consist of hundreds, sometimes thousands of
options per underlying security. Be aware that requesting option chains
for large universes of underlying securities, such as all stocks on the
NYSE, can take numerous hours to complete, add hundreds of thousands of
rows to the securities master database, increase the database file size
by several hundred megabytes, and potentially add latency to database
queries.
Parameters
----------
universes : list of str, optional
collect options for these universes of underlying securities
conids : list of int, optional
collect options for these underlying conids
infilepath_or_buffer : str or file-like object, optional
collect options for the conids in this file (specify '-' to read file
from stdin)
Returns
-------
dict
status message
"""
params = {}
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
if infilepath_or_buffer == "-":
response = houston.post("/master/options",
params=params,
data=to_bytes(sys.stdin))
elif infilepath_or_buffer and hasattr(infilepath_or_buffer, "read"):
if infilepath_or_buffer.seekable():
infilepath_or_buffer.seek(0)
response = houston.post("/master/options",
params=params,
data=to_bytes(infilepath_or_buffer))
elif infilepath_or_buffer:
with open(infilepath_or_buffer, "rb") as f:
response = houston.post("/master/options", params=params, data=f)
else:
response = houston.post("/master/options", params=params)
houston.raise_for_status_with_json(response)
return response.json()
def fetch_reuters_financials(universes=None, conids=None):
"""
Fetch Reuters financial statements from IB and save to database.
This data provides cash flow, balance sheet, and income metrics.
Parameters
----------
universes : list of str, optional
limit to these universes (must provide universes, conids, or both)
conids : list of int, optional
limit to these conids (must provide universes, conids, or both)
Returns
-------
dict
status message
"""
params = {}
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
response = houston.post("/fundamental/reuters/financials", params=params)
houston.raise_for_status_with_json(response)
return response.json()
def fetch_reuters_estimates(universes=None, conids=None):
"""
Fetch Reuters estimates and actuals from IB and save to database.
This data provides analyst estimates and actuals for a variety of indicators.
Parameters
----------
universes : list of str, optional
limit to these universes (must provide universes, conids, or both)
conids : list of int, optional
limit to these conids (must provide universes, conids, or both)
Returns
-------
dict
status message
"""
params = {}
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
response = houston.post("/fundamental/reuters/estimates", params=params)
houston.raise_for_status_with_json(response)
return response.json()
def fetch_borrow_fees(countries=None):
"""
Fetch IB borrow fees data and save to database.
Data is organized by country and updated every 15 minutes. Historical
data is available from April 2018.
Parameters
----------
countries : list of str, optional
limit to these countries (pass '?' or any invalid country to see
available countries)
Returns
-------
dict
status message
"""
params = {}
if countries:
params["countries"] = countries
response = houston.post("/fundamental/stockloan/fees", params=params)
houston.raise_for_status_with_json(response)
return response.json()
def optimize_databases(service, codes=None):
"""
Optimize database file(s) to improve performance.
Parameters
----------
serivce : str, required
only optimize databases for this service (specify 'all' to
optimize all services)
codes: list of str, optional
only optimize databases identified by these codes (omit to
optimize all databases for service)
Returns
-------
json
status message
"""
data = {}
if codes:
data["codes"] = codes
response = houston.post("/db/optimizations/{0}".format(service), data=data)
houston.raise_for_status_with_json(response)
return response.json()
def fetch_listings(exchange=None,
sec_types=None,
currencies=None,
symbols=None,
universes=None,
conids=None):
"""
Fetch securities listings from IB into securities master database, either by exchange or by universes/conids.
Specify an exchange (optionally filtering by security type, currency, and/or symbol) to fetch
listings from the IB website and fetch associated contract details from the IB API. Or, specify universes
or conids to fetch details from the IB API, bypassing the website.
Parameters
----------
exchange : str
the exchange code to fetch listings for (required unless providing universes or conids)
sec_types : list of str, optional
limit to these security types. Possible choices: STK, ETF, FUT, CASH, IND
currencies : list of str, optional
limit to these currencies
symbols : list of str, optional
limit to these symbols
universes : list of str, optional
limit to these universes
conids : list of int, optional
limit to these conids
Returns
-------
dict
status message
"""
params = {}
if exchange:
params["exchange"] = exchange
if sec_types:
params["sec_types"] = sec_types
if currencies:
params["currencies"] = currencies
if symbols:
params["symbols"] = symbols
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
response = houston.post("/master/listings", params=params)
houston.raise_for_status_with_json(response)
return response.json()
def create_tearsheet(infilepath_or_buffer, outfilepath_or_buffer=None):
"""
Create a pyfolio PDF tear sheet from a Zipline backtest result.
Parameters
----------
infilepath_or_buffer : str, required
the CSV file from a Zipline backtest (specify '-' to read file from stdin)
outfilepath_or_buffer : str or file-like, optional
the location to write the pyfolio tear sheet (write to stdout if omitted)
Returns
-------
None
"""
url = "/zipline/tearsheets"
# Pyfolio can take a long time
timeout = 60 * 60 * 5
if infilepath_or_buffer == "-":
infilepath_or_buffer = sys.stdin.buffer if six.PY3 else sys.stdin
response = houston.post(url,
data=infilepath_or_buffer,
timeout=timeout)
elif infilepath_or_buffer and hasattr(infilepath_or_buffer, "read"):
if infilepath_or_buffer.seekable():
infilepath_or_buffer.seek(0)
response = houston.post(url,
data=infilepath_or_buffer,
timeout=timeout)
else:
with open(infilepath_or_buffer, "rb") as f:
response = houston.post(url, data=f, timeout=timeout)
houston.raise_for_status_with_json(response)
outfilepath_or_buffer = outfilepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(outfilepath_or_buffer, response)
def fetch_history(codes,
priority=False,
conids=None,
start_date=None,
end_date=None,
delist_missing=False):
"""
Fetch historical market data from IB and save it to a history database. The request is
queued and the data is fetched asynchronously.
Parameters
----------
codes : list of str, required
the database code(s) to fetch data for
priority : bool
use the priority queue (default is to use the standard queue)
conids : list of int, optional
fetch history for these conids (overrides config)
start_date : str (YYYY-MM-DD), optional
fetch history back to this start date (overrides config)
end_date : str (YYYY-MM-DD), optional
fetch history up to this end date (overrides config)
delist_missing : bool
auto-delist securities that are no longer available from IB
Returns
-------
dict
status message
"""
params = {}
if codes:
params["codes"] = codes
if priority:
params["priority"] = priority
if conids:
params["conids"] = conids
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if delist_missing:
params["delist_missing"] = delist_missing
response = houston.post("/history/queue", params=params)
houston.raise_for_status_with_json(response)
return response.json()
def execute_command(cmd,
return_file=None,
filepath_or_buffer=None,
service="satellite"):
"""
Execute an abitrary command on a satellite service and optionally return a file.
Parameters
----------
cmd: str, required
the command to run
return_file : str, optional
the path of a file to be returned after the command completes
filepath_or_buffer : str, optional
the location to write the return_file (omit to write to stdout)
service : str, optional
the service name (default 'satellite')
Returns
-------
dict or None
None if return_file, otherwise status message
"""
params = {}
if not service:
raise ValueError("a service is required")
if not cmd:
raise ValueError("a command is required")
params["cmd"] = cmd
if return_file:
params["return_file"] = return_file
if not service.startswith("satellite"):
raise ValueError("service must start with 'satellite'")
response = houston.post("/{0}/commands".format(service),
params=params,
timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
if return_file:
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
else:
return response.json()
def ingest_bundle(history_db=None,
calendar=None,
bundle=None,
assets_versions=None):
"""
Ingest a data bundle into Zipline for later backtesting.
You can ingest 1-minute or 1-day history databases from QuantRocket, or you
can ingest data using Zipline's built-in capabilities.
Parameters
----------
history_db : str, optional
the code of a history db to ingest
calendar : str, optional
the name of the calendar to use with this history db bundle (default is
NYSE; provide an invalid calendar name to see available choices)
bundle : str, optional
the data bundle to ingest (default is quantopian-quandl); don't provide
if specifying history_db
assets_versions : list of int, optional
versions of the assets db to which to downgrade
Returns
-------
dict
status message
"""
params = {}
if history_db:
params["history_db"] = history_db
if calendar:
params["calendar"] = calendar
if bundle:
params["bundle"] = bundle
if assets_versions:
params["assets_versions"] = assets_versions
response = houston.post("/zipline/bundles", params=params, timeout=60 * 30)
houston.raise_for_status_with_json(response)
return response.json()
def start_gateways(exchanges=None,
sec_type=None,
research_vendors=None,
gateways=None,
wait=False):
"""
Start one or more IB Gateway services.
Parameters
----------
exchanges : list of str, optional
limit to IB Gateway services with market data permission for these exchanges
sec_type : str, optional
limit to IB Gateway services with market data permission for this securitiy type (useful for disambiguating permissions for exchanges that trade multiple asset classes). Possible choices: STK, FUT, CASH, OPT
research_vendors : list of str, optional
limit to IB Gateway services with permission for these research vendors (choices: reuters, wsh)
gateways : list of str, optional
limit to these IB Gateway services
wait: bool
wait for the IB Gateway services to start before returning (default is to start the gateways asynchronously)
Returns
-------
dict
status message
"""
params = {"wait": wait}
if sec_type:
params["sec_type"] = sec_type
if exchanges:
params["exchanges"] = exchanges
if research_vendors:
params["research_vendors"] = research_vendors
if gateways:
params["gateways"] = gateways
response = houston.post("/launchpad/gateways", params=params, timeout=120)
houston.raise_for_status_with_json(response)
return response.json()
def fetch_calendar(exchanges=None):
"""
Fetch upcoming trading hours for exchanges and save to securites master database.
Parameters
----------
exchanges : list of str, optional
limit to these exchanges
Returns
-------
dict
status message
"""
params = {}
if exchanges:
params["exchanges"] = exchanges
response = houston.post("/master/calendar", params=params)
houston.raise_for_status_with_json(response)
return response.json()
def collect_sharadar_listings():
"""
Collect securities listings from Sharadar and save to
quantrocket.master.sharadar.sqlite.
Requires a Sharadar data plan. Collects NYSE, NASDAQ, or all US stock
listings, depending on your plan.
Sharadar listings have their own ConIds which are distinct from IB ConIds.
To facilitate using Sharadar and IB data together or separately, this command
also collects a list of IB<->Sharadar ConId translations and saves them
to quantrocket.master.translations.sqlite. They can be queried via
`translate_conids`.
Returns
-------
dict
status message
"""
response = houston.post("/master/sharadar/securities")
houston.raise_for_status_with_json(response)
return response.json()
def create_tearsheet(infilepath_or_buffer,
outfilepath_or_buffer=None,
simple=None,
live_start_date=None,
slippage=None,
hide_positions=None,
bayesian=None,
round_trips=None,
bootstrap=None):
"""
Create a pyfolio PDF tear sheet from a Zipline backtest result.
Parameters
----------
infilepath_or_buffer : str, required
the CSV file from a Zipline backtest (specify '-' to read file from stdin)
outfilepath_or_buffer : str or file-like, optional
the location to write the pyfolio tear sheet (write to stdout if omitted)
simple : bool
create a simple tear sheet (default is to create a full tear sheet)
live_start_date : str (YYYY-MM-DD), optional
date when the strategy began live trading
slippage : int or float, optional
basis points of slippage to apply to returns before generating tear sheet
stats and plots
hide_positions : bool
don't output any symbol names
bayesian : bool
include a Bayesian tear sheet
round_trips : bool
include a round-trips tear sheet
bootstrap : bool
perform bootstrap analysis for the performance metrics (takes a few minutes
longer)
Returns
-------
None
"""
params = {}
if simple:
params["simple"] = simple
if live_start_date:
params["live_start_date"] = live_start_date
if slippage:
params["slippage"] = slippage
if hide_positions:
params["hide_positions"] = hide_positions
if bayesian:
params["bayesian"] = bayesian
if round_trips:
params["round_trips"] = round_trips
if bootstrap:
params["bootstrap"] = bootstrap
url = "/zipline/tearsheets"
# Pyfolio can take a long time, particularly for Bayesian analysis
timeout = 60 * 60 * 5
if infilepath_or_buffer == "-":
infilepath_or_buffer = sys.stdin.buffer if six.PY3 else sys.stdin
response = houston.post(url,
data=infilepath_or_buffer,
params=params,
timeout=timeout)
elif infilepath_or_buffer and hasattr(infilepath_or_buffer, "read"):
if infilepath_or_buffer.seekable():
infilepath_or_buffer.seek(0)
response = houston.post(url,
data=infilepath_or_buffer,
params=params,
timeout=timeout)
else:
with open(infilepath_or_buffer, "rb") as f:
response = houston.post(url,
data=f,
params=params,
timeout=timeout)
houston.raise_for_status_with_json(response)
outfilepath_or_buffer = outfilepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(outfilepath_or_buffer, response)
def ml_walkforward(strategy,
start_date,
end_date,
train,
min_train=None,
rolling_train=None,
model_filepath=None,
force_nonincremental=None,
segment=None,
allocation=None,
nlv=None,
params=None,
details=None,
progress=False,
filepath_or_buffer=None):
"""
Run a walk-forward optimization of a machine learning strategy.
The date range will be split into segments of `train` size. For each
segment, the model will be trained with the data, then the trained model will
be backtested on the following segment.
By default, uses scikit-learn's StandardScaler+SGDRegressor. Also supports other
scikit-learn models/pipelines and Keras models. To customize model, instantiate
the model locally, serialize it to disk, and pass the path of the serialized
model as `model_filepath`.
Supports expanding walk-forward optimizations (the default), which use an anchored start date
for model training, or rolling walk-forward optimizations (by specifying `rolling_train`),
which use a rolling or non-anchored start date for model training.
Returns a backtest results CSV and a dump of the machine learning model
as of the end of the analysis.
Parameters
----------
strategy : str, required
the strategy code
start_date : str (YYYY-MM-DD), required
the analysis start date (note that model training will start on this date
but backtesting will not start until after the initial training period)
end_date : str (YYYY-MM-DD), required
the analysis end date
train : str, required
train model this frequently (use Pandas frequency string, e.g. 'A'
for annual training or 'Q' for quarterly training)
min_train : str, optional
don't backtest until at least this much model training has occurred;
defaults to the length of `train` if not specified (use Pandas frequency
string, e.g. '5Y' for 5 years of initial training)
rolling_train : str, optional
train model with a rolling window of this length; if omitted, train
model with an expanding window (use Pandas frequency string, e.g. '3Y' for
a 3-year rolling training window)
model_filepath : str, optional
filepath of serialized model to use, filename must end in ".joblib" or
".pkl" (if omitted, default model is scikit-learn's StandardScaler+SGDRegressor)
force_nonincremental : bool, optional
force the model to be trained non-incrementally (i.e. load entire training
data set into memory) even if it supports incremental learning. Must be True
in order to perform a rolling (as opposed to expanding) walk-forward optimization
with a model that supports incremental learning. Default False.
segment : str, optional
train and backtest in date segments of this size, to reduce memory usage;
must be smaller than `train`/`min_train` or will have no effect (use Pandas frequency string,
e.g. 'A' for annual segments or 'Q' for quarterly segments)
allocation : float, optional
the allocation for the strategy (default 1.0)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
details : bool
return detailed results for all securities instead of aggregating
progress : bool
log status and Sharpe ratios of each walk-forward segment during analysis
(default False)
filepath_or_buffer : str, optional
the location to write the ZIP file to; or, if path ends with "*", the
pattern to use for extracting the zipped files. For example, if the path is
my_ml*, files will extracted to my_ml_results.csv and my_ml_trained_model.joblib.
Returns
-------
None
Examples
--------
Run a walk-forward optimization using the default model and retrain the model
annually, writing the backtest results and trained model to demo_ml_results.csv
and demo_ml_trained_model.joblib, respectively:
>>> ml_walkforward(
"demo-ml",
"2007-01-01",
"2018-12-31",
train="A",
filepath_or_buffer="demo_ml*")
Create a scikit-learn model, serialize it with joblib, and use it to
run the walkforward backtest:
>>> from sklearn.linear_model import SGDClassifier
>>> import joblib
>>> clf = SGDClassifier()
>>> joblib.dump(clf, "my_model.joblib")
>>> ml_walkforward(
"demo-ml",
"2007-01-01",
"2018-12-31",
train="A",
model_filepath="my_model.joblib",
filepath_or_buffer="demo_ml*")
Run a walk-forward optimization using a custom model (serialized with joblib),
retrain the model annually, don't perform backtesting until after 5 years
of initial training, and further split the training and backtesting into
quarterly segments to reduce memory usage:
>>> ml_walkforward(
"demo-ml",
"2007-01-01",
"2018-12-31",
model_filepath="my_model.joblib",
train="A",
min_train="5Y",
segment="Q",
filepath_or_buffer="demo_ml*")
Create a Keras model, serialize it, and use it to run the walkforward backtest:
>>> from keras.models import Sequential
>>> from keras.layers import Dense
>>> model = Sequential()
>>> # input_dim should match number of features in training data
>>> model.add(Dense(units=4, activation='relu', input_dim=5))
>>> # last layer should have a single unit
>>> model.add(Dense(units=1, activation='softmax'))
>>> model.compile(loss='sparse_categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
>>> model.save('my_model.keras.h5')
>>> ml_walkforward(
"neuralnet-ml",
"2007-01-01",
"2018-12-31",
train="A",
model_filepath="my_model.keras.h5",
filepath_or_buffer="neuralnet_ml*")
"""
_params = {}
_params["start_date"] = start_date
_params["end_date"] = end_date
_params["train"] = train
if min_train:
_params["min_train"] = min_train
if rolling_train:
_params["rolling_train"] = rolling_train
if force_nonincremental:
_params["force_nonincremental"] = force_nonincremental
if segment:
_params["segment"] = segment
if allocation:
_params["allocation"] = allocation
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if details:
_params["details"] = details
if progress:
_params["progress"] = progress
if params:
_params["params"] = dict_to_dict_strs(params)
url = "/moonshot/ml/walkforward/{0}.zip".format(strategy)
if model_filepath:
# Send the filename as a hint how to open it
_params["model_filename"] = os.path.basename(model_filepath)
with open(model_filepath, "rb") as f:
response = houston.post(url,
data=f,
params=_params,
timeout=60 * 60 * 24)
else:
response = houston.post(url, params=_params, timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
auto_extract = isinstance(
filepath_or_buffer,
six.string_types) and filepath_or_buffer.endswith("*")
if auto_extract:
base_filepath = filepath_or_buffer[:-1]
zipfilepath = base_filepath + ".zip"
write_response_to_filepath_or_buffer(zipfilepath, response)
with ZipFile(zipfilepath, mode="r") as zfile:
model_filename = [
name for name in zfile.namelist() if "model" in name
][0]
model_filepath = base_filepath + "_" + model_filename
csv_filepath = base_filepath + "_results.csv"
with open(csv_filepath, "wb") as csvfile:
csvfile.write(zfile.read("results.csv"))
with open(model_filepath, "wb") as modelfile:
modelfile.write(zfile.read(model_filename))
os.remove(zipfilepath)
else:
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def collect_history(codes, priority=False, conids=None, universes=None,
start_date=None, end_date=None, availability_only=False,
delist_missing=False):
"""
Collect historical market data from IB and save it to a history database. The request is
queued and the data is collected asynchronously.
Parameters
----------
codes : list of str, required
the database code(s) to collect data for
priority : bool
use the priority queue (default is to use the standard queue)
conids : list of int, optional
collect history for these conids, overriding config (typically
used to collect a subset of securities)
universes : list of str, optional
collect history for these universes, overriding config (typically
used to collect a subset of securities)
start_date : str (YYYY-MM-DD), optional
collect history back to this start date, overriding config
end_date : str (YYYY-MM-DD), optional
collect history up to this end date, overriding config
availability_only : bool
determine and store how far back data is available but
don't yet collect the data
delist_missing : bool
auto-delist securities that are no longer available from IB
Returns
-------
dict
status message
"""
params = {}
if codes:
params["codes"] = codes
if priority:
params["priority"] = priority
if conids:
params["conids"] = conids
if universes:
params["universes"] = universes
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if availability_only:
params["availability_only"] = availability_only
if delist_missing:
params["delist_missing"] = delist_missing
response = houston.post("/history/queue", params=params)
houston.raise_for_status_with_json(response)
return response.json()
def scan_parameters(strategies,
start_date=None,
end_date=None,
segment=None,
param1=None,
vals1=None,
param2=None,
vals2=None,
allocations=None,
nlv=None,
params=None,
output="csv",
csv=None,
filepath_or_buffer=None):
"""
Run a parameter scan for one or more strategies.
By default returns a CSV of scan results but can also return a PDF tear sheet.
Parameters
----------
strategies : list of str, required
one or more strategy codes
start_date : str (YYYY-MM-DD), optional
the backtest start date (default is to use all available history)
end_date : str (YYYY-MM-DD), optional
the backtest end date (default is to use all available history)
segment : str, optional
backtest in date segments of this size, to reduce memory usage
(use Pandas frequency string, e.g. 'A' for annual segments or 'Q'
for quarterly segments)
param1 : str, required
the name of the parameter to test (a class attribute on the strategy)
vals1 : list of int/float/str/tuple, required
parameter values to test (values can be ints, floats, strings, False,
True, None, 'default' (to test current param value), or lists of
ints/floats/strings)
param2 : str, optional
name of a second parameter to test (for 2-D parameter scans)
vals2 : list of int/float/str/tuple, optional
values to test for parameter 2 (values can be ints, floats, strings,
False, True, None, 'default' (to test current param value), or lists
of ints/floats/strings)
allocations : dict of CODE:FLOAT, optional
the allocation for each strategy, passed as {code:allocation} (default
allocation is 1.0 / number of strategies)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
output : str, required
the output format (choices are csv or pdf)
csv : bool
DEPRECATED: this argument will be removed in a future version. This argument
may be omitted as CSV is the default.
filepath_or_buffer : str, optional
the location to write the results file (omit to write to stdout)
Returns
-------
None
Examples
--------
Run a parameter scan for several different moving averages on a strategy
called trend-friend and return a CSV (which can be rendered with Moonchart):
>>> scan_parameters("trend-friend",
param1="MAVG_WINDOW",
vals1=[20, 50, 100],
filepath_or_buffer="trend_friend_MAVG_WINDOW.csv")
Run a 2-D parameter scan for multiple strategies and return a CSV:
>>> scan_parameters(["strat1", "strat2", "strat3"],
param1="MIN_STD",
vals1=[1, 1.5, 2],
param2="STD_WINDOW",
vals2=[20, 50, 100, 200],
filepath_or_buffer="strategies_MIN_STD_and_STD_WINDOW.csv")
Run a parameter scan in 1-year segments to reduce memory usage:
>>> scan_parameters("big-strategy",
start_date="2000-01-01",
end_date="2018-01-01",
segment="A",
param1="MAVG_WINDOW",
vals1=[20, 50, 100],
filepath_or_buffer="big_strategy_MAVG_WINDOW.csv")
"""
output = output or "csv"
if output not in ("csv", "pdf"):
raise ValueError(
"invalid output: {0} (choices are csv or pdf".format(output))
if csv is not None:
import warnings
warnings.warn(
"the `csv` argument is deprecated and will removed in a future version; "
"this argument may be omitted as csv is the default",
DeprecationWarning)
_params = {}
if strategies:
_params["strategies"] = strategies
if start_date:
_params["start_date"] = start_date
if end_date:
_params["end_date"] = end_date
if segment:
_params["segment"] = segment
if param1:
_params["param1"] = param1
if vals1:
_params["vals1"] = [str(v) for v in vals1]
if param2:
_params["param2"] = param2
if vals2:
_params["vals2"] = [str(v) for v in vals2]
if allocations:
_params["allocations"] = dict_to_dict_strs(allocations)
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if params:
_params["params"] = dict_to_dict_strs(params)
response = houston.post("/moonshot/paramscans.{0}".format(output),
params=_params,
timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def backtest(strategies,
start_date=None,
end_date=None,
segment=None,
allocations=None,
nlv=None,
params=None,
details=None,
output="csv",
csv=None,
filepath_or_buffer=None):
"""
Backtest one or more strategies.
By default returns a CSV of backtest results but can also return a PDF tear sheet
of performance charts.
If testing multiple strategies, each column in the CSV represents a strategy.
If testing a single strategy and `details=True`, each column in the CSV
represents a security in the strategy universe.
Parameters
----------
strategies : list of str, required
one or more strategy codes
start_date : str (YYYY-MM-DD), optional
the backtest start date (default is to use all available history)
end_date : str (YYYY-MM-DD), optional
the backtest end date (default is to use all available history)
segment : str, optional
backtest in date segments of this size, to reduce memory usage
(use Pandas frequency string, e.g. 'A' for annual segments or 'Q'
for quarterly segments)
allocations : dict of CODE:FLOAT, optional
the allocation for each strategy, passed as {code:allocation} (default
allocation is 1.0 / number of strategies)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
details : bool
return detailed results for all securities instead of aggregating to
strategy level (only supported for single-strategy backtests)
output : str, required
the output format (choices are csv or pdf)
csv : bool
DEPRECATED: this argument will be removed in a future version. This argument
may be omitted as CSV is the default.
filepath_or_buffer : str, optional
the location to write the results file (omit to write to stdout)
Returns
-------
None
Examples
--------
Backtest several HML (High Minus Low) strategies from 2005-2015 and return a
CSV of results:
>>> backtest(["hml-us", "hml-eur", "hml-asia"],
start_date="2005-01-01",
end_date="2015-12-31",
filepath_or_buffer="hml_results.csv")
Run a backtest in 1-year segments to reduce memory usage:
>>> backtest("big-strategy",
start_date="2000-01-01",
end_date="2018-01-01",
segment="A",
filepath_or_buffer="results.csv")
See Also
--------
read_moonshot_csv : load a Moonshot backtest CSV into a DataFrame
"""
output = output or "csv"
if output not in ("csv", "pdf"):
raise ValueError(
"invalid output: {0} (choices are csv or pdf".format(output))
if csv is not None:
import warnings
warnings.warn(
"the `csv` argument is deprecated and will removed in a future version; "
"this argument may be omitted as csv is the default",
DeprecationWarning)
_params = {}
if strategies:
_params["strategies"] = strategies
if start_date:
_params["start_date"] = start_date
if end_date:
_params["end_date"] = end_date
if segment:
_params["segment"] = segment
if allocations:
_params["allocations"] = dict_to_dict_strs(allocations)
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if details:
_params["details"] = details
if params:
_params["params"] = dict_to_dict_strs(params)
response = houston.post("/moonshot/backtests.{0}".format(output),
params=_params,
timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def backtest(strategies, start_date=None, end_date=None, allocations=None,
nlv=None, params=None, details=None, csv=None, filepath_or_buffer=None):
"""
Backtest one or more strategies.
By default returns a PDF tear sheet of performance charts but can also return a CSV of
backtest results.
Parameters
----------
strategies : list of str, required
one or more strategy codes
start_date : str (YYYY-MM-DD), optional
the backtest start date (default is to use all available history)
end_date : str (YYYY-MM-DD), optional
the backtest end date (default is to use all available history)
allocations : dict of CODE:FLOAT, optional
the allocation for each strategy, passed as {code:allocation} (default
allocation is 1.0 / number of strategies)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
details : bool
return detailed results for all securities instead of aggregating to
strategy level (only supported for single-strategy backtests)
csv : bool
return a CSV of performance data (default is to return a PDF
performance tear sheet)
filepath_or_buffer : str, optional
the location to write the results file (omit to write to stdout)
Returns
-------
None
"""
_params = {}
if strategies:
_params["strategies"] = strategies
if start_date:
_params["start_date"] = start_date
if end_date:
_params["end_date"] = end_date
if allocations:
_params["allocations"] = dict_to_dict_strs(allocations)
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if details:
_params["details"] = details
if csv:
_params["csv"] = csv
if params:
_params["params"] = dict_to_dict_strs(params)
response = houston.post("/moonshot/backtests", params=_params, timeout=60*60*24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def scan_parameters(strategies, start_date=None, end_date=None,
param1=None, vals1=None, param2=None, vals2=None,
allocations=None, nlv=None, params=None, output="csv",
csv=None, filepath_or_buffer=None):
"""
Run a parameter scan for one or more strategies.
By default returns a CSV of scan results but can also return a PDF tear sheet.
Parameters
----------
strategies : list of str, required
one or more strategy codes
start_date : str (YYYY-MM-DD), optional
the backtest start date (default is to use all available history)
end_date : str (YYYY-MM-DD), optional
the backtest end date (default is to use all available history)
param1 : str, required
the name of the parameter to test (a class attribute on the strategy)
vals1 : list of int/float/str/tuple, required
parameter values to test (values can be ints, floats, strings, False,
True, None, 'default' (to test current param value), or lists of
ints/floats/strings)
param2 : str, optional
name of a second parameter to test (for 2-D parameter scans)
vals2 : list of int/float/str/tuple, optional
values to test for parameter 2 (values can be ints, floats, strings,
False, True, None, 'default' (to test current param value), or lists
of ints/floats/strings)
allocations : dict of CODE:FLOAT, optional
the allocation for each strategy, passed as {code:allocation} (default
allocation is 1.0 / number of strategies)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
output : str, required
the output format (choices are csv or pdf)
csv : bool
DEPRECATED: this argument will be removed in a future version. This argument
may be omitted as CSV is the default.
filepath_or_buffer : str, optional
the location to write the results file (omit to write to stdout)
Returns
-------
None
"""
output = output or "csv"
if output not in ("csv", "pdf"):
raise ValueError("invalid output: {0} (choices are csv or pdf".format(output))
if csv is not None:
import warnings
warnings.warn(
"the `csv` argument is deprecated and will removed in a future version; "
"this argument may be omitted as csv is the default", DeprecationWarning)
_params = {}
if strategies:
_params["strategies"] = strategies
if start_date:
_params["start_date"] = start_date
if end_date:
_params["end_date"] = end_date
if param1:
_params["param1"] = param1
if vals1:
_params["vals1"] = [str(v) for v in vals1]
if param2:
_params["param2"] = param2
if vals2:
_params["vals2"] = [str(v) for v in vals2]
if allocations:
_params["allocations"] = dict_to_dict_strs(allocations)
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if params:
_params["params"] = dict_to_dict_strs(params)
response = houston.post("/moonshot/paramscans.{0}".format(output), params=_params, timeout=60*60*24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def place_orders(orders=None, infilepath_or_buffer=None):
"""
Place one or more orders.
Returns a list of order IDs, which can be used to cancel the orders or check
their status.
Parameters
----------
orders : list of dict of PARAM:VALUE, optional
a list of one or more orders, where each order is a dict specifying the
order parameters (see examples)
infilepath_or_buffer : str or file-like object, optional
place orders from this CSV or JSON file (specify '-' to read file
from stdin). Mutually exclusive with `orders` argument.
Returns
-------
list
order IDs
Examples
--------
>>> orders = []
>>> order1 = {
'ConId':123456,
'Action':'BUY',
'Exchange':'SMART',
'TotalQuantity':100,
'OrderType':'MKT',
'Tif':'Day',
'Account':'DU12345',
'OrderRef':'my-strategy'
}
>>> orders.append(order1)
>>> order_ids = place_orders(orders)
"""
if orders and infilepath_or_buffer:
raise ValueError(
"orders and infilepath_or_buffer are mutually exclusive")
url = "/blotter/orders"
if orders:
response = houston.post(url, json=orders)
elif infilepath_or_buffer == "-":
response = houston.post(url, data=to_bytes(sys.stdin))
elif infilepath_or_buffer and hasattr(infilepath_or_buffer, "read"):
if infilepath_or_buffer.seekable():
infilepath_or_buffer.seek(0)
response = houston.post(url, data=to_bytes(infilepath_or_buffer))
elif infilepath_or_buffer:
with open(infilepath_or_buffer, "rb") as f:
response = houston.post(url, data=f)
else:
response = houston.post(url)
houston.raise_for_status_with_json(response)
return response.json()
def run_algorithm(algofile,
data_frequency=None,
capital_base=None,
bundle=None,
bundle_timestamp=None,
start=None,
end=None,
filepath_or_buffer=None,
calendar=None):
"""
Run a Zipline backtest and write the test results to a CSV file.
The CSV result file contains several DataFrames stacked into one: the Zipline performance
results, plus the extracted returns, transactions, positions, and benchmark returns from those
results.
Parameters
----------
algofile : str, required
the file that contains the algorithm to run
data_frequency : str, optional
the data frequency of the simulation. Possible choices: daily, minute (default is daily)
capital_base : float, optional
the starting capital for the simulation (default is 10000000.0)
bundle : str, required
the data bundle to use for the simulation
bundle_timestamp : str, optional
the date to lookup data on or before (default is <current-time>)
start : str (YYYY-MM-DD), required
the start date of the simulation
end : str (YYYY-MM-DD), required
the end date of the simulation
filepath_or_buffer : str, optional
the location to write the output file (omit to write to stdout)
calendar : str, optional
the calendar you want to use e.g. LSE (default is to use the calendar
associated with the data bundle).
Returns
-------
None
Examples
--------
Run a backtest and save to CSV.
>>> from quantrocket.zipline import run_algorithm
>>> run_algorithm("momentum_pipeline.py", bundle="my-bundle",
start="2015-02-04", end="2015-12-31",
filepath_or_buffer="momentum_pipeline_results.csv")
Get a pyfolio tear sheet from the results:
>>> import pyfolio as pf
>>> pf.from_zipline_csv("momentum_pipeline_results.csv")
"""
params = {}
if data_frequency:
params["data_frequency"] = data_frequency
if capital_base:
params["capital_base"] = capital_base
if not bundle:
raise ValueError("must specify a bundle")
params["bundle"] = bundle
if bundle_timestamp:
params["bundle_timestamp"] = bundle_timestamp
if start:
params["start"] = start
if end:
params["end"] = end
if calendar:
params["calendar"] = calendar
response = houston.post("/zipline/backtests/{0}".format(algofile),
params=params,
timeout=60 * 60 * 3)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def collect_listings(exchanges=None, sec_types=None, currencies=None, symbols=None,
universes=None, conids=None, exchange=None):
"""
Collect securities listings from IB and store in securities master
database (quantrocket.master.main.sqlite).
Specify an exchange (optionally filtering by security type, currency,
and/or symbol) to collect listings from the IB website and collect
associated contract details from the IB API. Or, specify universes or
conids to collect details from the IB API, bypassing the website.
Parameters
----------
exchanges : list or str
one or more exchange codes to collect listings for (required unless providing
universes or conids)
sec_types : list of str, optional
limit to these security types. Possible choices: STK, ETF, FUT, CASH, IND
currencies : list of str, optional
limit to these currencies
symbols : list of str, optional
limit to these symbols
universes : list of str, optional
limit to these universes
conids : list of int, optional
limit to these conids
exchange : str
DEPRECATED, this option will be removed in a future release, please use
`exchanges` instead (previously only a single exchange was supported but
now multiple exchanges are supported)
Returns
-------
dict
status message
"""
params = {}
if exchanges:
params["exchanges"] = exchanges
if sec_types:
params["sec_types"] = sec_types
if currencies:
params["currencies"] = currencies
if symbols:
params["symbols"] = symbols
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
if exchange:
import warnings
# DeprecationWarning is ignored by default but we want the user
# to see it
warnings.simplefilter("always", DeprecationWarning)
warnings.warn(
"the `exchange` option is deprecated and will be removed in a "
"future release, please use `exchanges` instead (previously only "
"a single exchange was supported but now multiple exchanges are "
"supported)", DeprecationWarning)
params["exchange"] = exchange
response = houston.post("/master/securities", params=params)
houston.raise_for_status_with_json(response)
return response.json()
def run_algorithm(algofile,
data_frequency=None,
capital_base=None,
bundle=None,
bundle_timestamp=None,
start=None,
end=None,
filepath_or_buffer=None,
calendar=None):
"""
Run a Zipline backtest and write the test results to a CSV file.
The CSV result file contains several DataFrames stacked into one: the Zipline performance
results, plus the extracted returns, transactions, positions, and benchmark returns from those
results.
Parameters
----------
algofile : str, required
the file that contains the algorithm to run
data_frequency : str, optional
the data frequency of the simulation. Possible choices: daily, minute (default is daily)
capital_base : float, optional
the starting capital for the simulation (default is 10000000.0)
bundle : str, optional
the data bundle to use for the simulation (default is quantopian-quandl)
bundle_timestamp : str, optional
the date to lookup data on or before (default is <current-time>)
start : str (YYYY-MM-DD), required
the start date of the simulation
end : str (YYYY-MM-DD), required
the end date of the simulation
filepath_or_buffer : str, optional
the location to write the output file (omit to write to stdout)
calendar : str, optional
the calendar you want to use e.g. LSE. NYSE is the default.
Returns
-------
None
Examples
--------
Run a backtest and load the results into pandas.
>>> from quantrocket.zipline import run_algorithm
>>> import pandas as pd
>>> import io
>>> f = io.StringIO()
>>> run_algorithm("momentum_pipeline.py", bundle="my-bundle", start="2015-02-04", end="2015-12-31", filepath_or_buffer=f)
>>> results = pd.read_csv(f, index_col=["dataframe", "index", "column"])["value"]
To use the results with pyfolio, extract and massage the returns, positions,
transactions, and benchmark returns:
>>> # Extract returns
>>> returns = results.loc["returns"].unstack()
>>> returns.index = returns.index.droplevel(0).tz_localize("UTC")
>>> returns = returns["returns"].astype(float)
>>> # Extract positions
>>> positions = results.loc["positions"].unstack()
>>> positions.index = positions.index.droplevel(0).tz_localize("UTC")
>>> positions = positions.astype(float)
>>> # Extract transactions
>>> transactions = results.loc["transactions"].unstack()
>>> transactions.index = transactions.index.droplevel(0).tz_localize("UTC")
>>> transactions = transactions.apply(pd.to_numeric, errors='ignore')
>>> # Extract benchmark
>>> benchmark_returns = results.loc["benchmark"].unstack()
>>> benchmark_returns.index = benchmark_returns.index.droplevel(0).tz_localize("UTC")
>>> benchmark_returns = benchmark_returns["benchmark"].astype(float)
Ready for pyfolio:
>>> pf.create_full_tear_sheet(returns, positions=positions, transactions=transactions, benchmark_rets=benchmark_returns)
"""
params = {}
if data_frequency:
params["data_frequency"] = data_frequency
if capital_base:
params["capital_base"] = capital_base
if bundle:
params["bundle"] = bundle
if bundle_timestamp:
params["bundle_timestamp"] = bundle_timestamp
if start:
params["start"] = start
if end:
params["end"] = end
if calendar:
params["calendar"] = calendar
response = houston.post("/zipline/backtests/{0}".format(algofile),
params=params,
timeout=60 * 60 * 3)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def collect_market_data(codes, conids=None, universes=None, fields=None, until=None,
snapshot=False, wait=False):
"""
Collect real-time market data and save it to a tick database.
A single snapshot of market data or a continuous stream of market data can
be collected, depending on the `snapshot` parameter.
Streaming real-time data is collected until cancelled, or can be scheduled
for cancellation using the `until` parameter.
Parameters
----------
codes : list of str, required
the tick database code(s) to collect data for
conids : list of int, optional
collect market data for these conids, overriding db config (typically
used to collect a subset of securities)
universes : list of str, optional
collect market data for these universes, overriding db config (typically
used to collect a subset of securities)
fields : list of str, optional
limit to these fields, overriding db config
until : str, optional
schedule data collection to end at this time. Can be a datetime
(YYYY-MM-DD HH:MM:SS), a time (HH:MM:SS), or a Pandas timedelta
string (e.g. 2h or 30min). If not provided, market data is collected
until cancelled.
snapshot : bool
collect a snapshot of market data (default is to collect a continuous
stream of market data)
wait : bool
wait for market data snapshot to complete before returning (default is
to return immediately). Requires 'snapshot=True'
Returns
-------
dict
status message
Examples
--------
Collect market data for all securities in a tick database called 'japan-banks-trades':
>>> collect_market_data("japan-banks-trades")
Collect market data for a subset of securities in a tick database called 'usa-stk-trades'
and automatically cancel the data collection in 30 minutes:
>>> collect_market_data("usa-stk-trades", conids=[12345,23456,34567], until="30m")
Collect a market data snapshot and wait until it completes:
>>> collect_market_data("usa-stk-trades", snapshot=True, wait=True)
"""
params = {}
if codes:
params["codes"] = codes
if conids:
params["conids"] = conids
if universes:
params["universes"] = universes
if fields:
params["fields"] = fields
if until:
params["until"] = until
if snapshot:
params["snapshot"] = snapshot
if wait:
params["wait"] = wait
response = houston.post("/realtime/collections", params=params, timeout=3600 if wait else 30)
houston.raise_for_status_with_json(response)
return response.json()
def scan_parameters(strategies, start_date=None, end_date=None,
param1=None, vals1=None, param2=None, vals2=None,
allocations=None, nlv=None, params=None, csv=None,
filepath_or_buffer=None):
"""
Run a parameter scan for one or more strategies.
By default returns a PDF tear sheet of results but can also return a CSV.
Parameters
----------
strategies : list of str, required
one or more strategy codes
start_date : str (YYYY-MM-DD), optional
the backtest start date (default is to use all available history)
end_date : str (YYYY-MM-DD), optional
the backtest end date (default is to use all available history)
param1 : str, required
the name of the parameter to test (a class attribute on the strategy)
vals1 : list of int/float/str/tuple, required
parameter values to test (values can be ints, floats, strings, False,
True, None, 'default' (to test current param value), or lists of
ints/floats/strings)
param2 : str, optional
name of a second parameter to test (for 2-D parameter scans)
vals2 : list of int/float/str/tuple, optional
values to test for parameter 2 (values can be ints, floats, strings,
False, True, None, 'default' (to test current param value), or lists
of ints/floats/strings)
allocations : dict of CODE:FLOAT, optional
the allocation for each strategy, passed as {code:allocation} (default
allocation is 1.0 / number of strategies)
nlv : dict of CURRENCY:NLV, optional
the NLV (net liquidation value, i.e. account balance) to assume for
the backtest, expressed in each currency represented in the backtest (pass
as {currency:nlv})
params : dict of PARAM:VALUE, optional
one or more strategy params to set on the fly before backtesting
(pass as {param:value})
csv : bool
return a CSV of performance data (default is to return a PDF
tear sheet)
filepath_or_buffer : str, optional
the location to write the results file (omit to write to stdout)
Returns
-------
None
"""
_params = {}
if strategies:
_params["strategies"] = strategies
if start_date:
_params["start_date"] = start_date
if end_date:
_params["end_date"] = end_date
if param1:
_params["param1"] = param1
if vals1:
_params["vals1"] = [str(v) for v in vals1]
if param2:
_params["param2"] = param2
if vals2:
_params["vals2"] = [str(v) for v in vals2]
if allocations:
_params["allocations"] = dict_to_dict_strs(allocations)
if nlv:
_params["nlv"] = dict_to_dict_strs(nlv)
if csv:
_params["csv"] = csv
if params:
_params["params"] = dict_to_dict_strs(params)
response = houston.post("/moonshot/paramscans", params=_params, timeout=60*60*24)
houston.raise_for_status_with_json(response)
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
def clone(repo, branch=None, replace=None, skip_existing=None):
"""
Clone files from a Git repository.
Only the files are copied, not the Git metadata. Can be run multiple
times to clone files from multiple repositories. Won't overwrite
any existing files unless `replace=True`.
Parameters
----------
repo : str, required
the name or URL of the repo. Can be the name of a QuantRocket demo
repo (e.g. 'umd'), a GitHub username/repo (e.g. 'myuser/myrepo'),
or the URL of any Git repository
branch : str, optional
the branch to clone (default 'master')
replace : bool, optional
if a file already exists locally, replace it with the remote file
(mutually exclusive with skip_existing)
skip_existing : bool, optional
if a file already exists locally, skip it (mutually exclusive with
replace)
Returns
-------
dict
status message
Examples
--------
Clone QuantRocket's "umd" demo repository:
>>> clone("umd")
Clone a GitHub repo and skip files that already exist locally:
>>> clone("myuser/myrepo", skip_existing=True)
Clone a Bitbucket repo:
>>> clone("https://bitbucket.org/myuser/myrepo.git")
Clone a private GitHub repo by including authentication credentials
in the URL (also works for Bitbucket):
>>> clone("https://*****:*****@github.com/myuser/myrepo.git")
"""
data = {
"repo": repo
}
if branch:
data["branch"] = branch
if replace:
data["replace"] = replace
if skip_existing:
data["skip_existing"] = skip_existing
response = houston.post("/codeload/repo", data=data)
houston.raise_for_status_with_json(response)
return response.json()
def execute_command(cmd,
return_file=None,
filepath_or_buffer=None,
params=None,
service="satellite"):
"""
Execute a Python function or abitrary shell command on a satellite service.
Parameters
----------
cmd: str, required
the shell command to run, or the Python function in dot notation (must
start with "codeload." to be interpreted as a Python function).
return_file : str, optional
the path of a file to be returned after the command completes
filepath_or_buffer : str, optional
the location to write the return_file (omit to write to stdout)
params : dict of PARAM:VALUE, optional
one or more params to pass to the Python function (pass as {param:value})
service : str, optional
the service name (default 'satellite')
Returns
-------
dict or None
None if return_file, otherwise status message
Examples
--------
Run a Python function called 'create_calendar_spread' defined in '/codeload/scripts/combos.py'
and pass it arguments:
>>> execute_command("codeload.scripts.combos.create_calendar_spread",
params={"universe":"cl-fut", "contract_months":[1,2]})
Run a backtrader backtest and save the performance chart to file:
>>> execute_command("python /codeload/backtrader/dual_moving_average.py",
return_file="/tmp/backtrader-plot.pdf"
outfile="backtrader-plot.pdf")
"""
_params = {}
if not service:
raise ValueError("a service is required")
if not cmd:
raise ValueError("a command is required")
_params["cmd"] = cmd
if params:
_params["params"] = dict_to_dict_strs(params)
if return_file:
_params["return_file"] = return_file
if not service.startswith("satellite"):
raise ValueError("service must start with 'satellite'")
response = houston.post("/{0}/commands".format(service),
params=_params,
timeout=60 * 60 * 24)
houston.raise_for_status_with_json(response)
if return_file:
filepath_or_buffer = filepath_or_buffer or sys.stdout
write_response_to_filepath_or_buffer(filepath_or_buffer, response)
else:
return response.json()
def ingest_bundle(history_db=None,
calendar=None,
bundle=None,
start_date=None,
end_date=None,
universes=None,
conids=None,
exclude_universes=None,
exclude_conids=None):
"""
Ingest a history database into Zipline for later backtesting.
You can ingest 1-minute or 1-day history databases.
Re-ingesting a previously ingested database will create a new version of the
ingested data, while preserving the earlier version. See
`quantrocket.zipline.clean_bundles` to remove earlier versions.
Ingestion parameters (start_date, end_date, universes, conids, exclude_universes,
exclude_conids) can only be specified the first time a bundle is ingested, and
will be reused for subsequent ingestions. You must remove the bundle and start
over to change the parameters.
Parameters
----------
history_db : str, optional
the code of a history db to ingest
calendar : str, optional
the name of the calendar to use with this history db bundle (provide '?' or
any invalid calendar name to see available choices)
bundle : str, optional
the name to assign to the bundle (defaults to the history database code)
start_date : str (YYYY-MM-DD), optional
limit to history on or after this date
end_date : str (YYYY-MM-DD), optional
limit to history on or before this date
universes : list of str, optional
limit to these universes
conids : list of int, optional
limit to these conids
exclude_universes : list of str, optional
exclude these universes
exclude_conids : list of int, optional
exclude these conids
Returns
-------
dict
status message
Examples
--------
Ingest a history database called "arca-etf-eod" into Zipline:
>>> from quantrocket.zipline import ingest_bundle
>>> ingest_bundle(history_db="arca-etf-eod", calendar="NYSE")
Re-ingest "arca-etf-eod" (calendar and other ingestion parameters aren't
needed as they will be re-used from the first ingestion):
>>> ingest_bundle(history_db="arca-etf-eod")
Ingest a history database called "lse-stk" into Zipline and associate it with
the LSE calendar:
>>> ingest_bundle(history_db="lse-stk", calendar="LSE")
Ingest a single year of US 1-minute stock data and name the bundle usa-stk-2017:
>>> ingest_bundle(history_db="usa-stk-1min", calendar="NYSE",
>>> start_date="2017-01-01", end_date="2017-12-31",
>>> bundle="usa-stk-2017")
Re-ingest the bundle usa-stk-2017:
>>> ingest_bundle(bundle="usa-stk-2017")
"""
params = {}
if history_db:
params["history_db"] = history_db
if calendar:
params["calendar"] = calendar
if bundle:
params["bundle"] = bundle
if start_date:
params["start_date"] = start_date
if end_date:
params["end_date"] = end_date
if universes:
params["universes"] = universes
if conids:
params["conids"] = conids
if exclude_universes:
params["exclude_universes"] = exclude_universes
if exclude_conids:
params["exclude_conids"] = exclude_conids
response = houston.post("/zipline/bundles",
params=params,
timeout=60 * 60 * 48)
houston.raise_for_status_with_json(response)
return response.json()
