def get_equities(self):
assets = []
t = normalize_date(pd.Timestamp('now', tz=NY))
raw_assets = self._api.list_assets(asset_class='us_equity')
for raw_asset in raw_assets:
asset = Equity(
raw_asset.id, raw_asset.exchange,
symbol=raw_asset.symbol,
asset_name=raw_asset.symbol,
)
asset.start_date = t - one_day_offset
if raw_asset.status == 'active' and raw_asset.tradable:
asset.end_date = t + end_offset
else:
# if asset is not tradable, set end_date = day before
asset.end_date = t - one_day_offset
asset.auto_close_date = asset.end_date
assets.append(asset)
# register all unseen exchange name as
# alias of NYSE (e.g. AMEX, ARCA, NYSEARCA.)
if not default_calendar.has_calendar(raw_asset.exchange):
register_calendar_alias(raw_asset.exchange,
'NYSE', force=True)
return assets
def get_equities(self):
assets = []
t = normalize_date(pd.Timestamp('now', tz=NY))
raw_assets = self._api.list_assets(asset_class='us_equity')
for raw_asset in raw_assets:
asset = Equity(
raw_asset.id,
raw_asset.exchange,
symbol=raw_asset.symbol,
asset_name=raw_asset.symbol,
)
asset.start_date = t - one_day_offset
if raw_asset.status == 'active' and raw_asset.tradable:
asset.end_date = t + end_offset
else:
# this is an experimental change, if an asset is not active or
# tradable, don't include it in the asset list. why?
# first the logical thing - if it's not tradable - we don't
# need it. now why bother?
# some symbols are included more than once in the Alpaca list.
# e.g VXX. one is tradable, one is not. as it happens, the
# first one we iterate on is the not tradable asset. which
# means when someone tries to trade it, even though there's a
# tradable asset, it rejects it because we test it against the
# untradable one. by doing this - we avoid this issue.
# but, it fear it may cause issues (e.g if an asset was
# tradable yesterday, but not tradable today. just a thought)
# so I do this with caution.
continue
# if asset is not tradable, set end_date = day before
asset.end_date = t - one_day_offset
asset.auto_close_date = asset.end_date
assets.append(asset)
# register all unseen exchange name as
# alias of NYSE (e.g. AMEX, ARCA, NYSEARCA.)
if not default_calendar.has_calendar(raw_asset.exchange):
register_calendar_alias(raw_asset.exchange, 'NYSE', force=True)
return assets
data = BytesIO()
with progressbar(length=total_size, **progress_kwargs) as pbar:
for chunk in resp.iter_content(chunk_size=chunk_size):
data.write(chunk)
pbar.update(len(chunk))
data.seek(0)
return data
def download_without_progress(url):
"""
Download data from a URL, returning a BytesIO containing the loaded data.
Parameters
----------
url : str
A URL that can be understood by ``requests.get``.
Returns
-------
data : BytesIO
A BytesIO containing the downloaded data.
"""
resp = requests.get(url)
resp.raise_for_status()
return BytesIO(resp.content)
register_calendar_alias("QUANDL-EOD", "NYSE")
split = DataFrame(data=tmp.index.tolist(),
columns=['effective_date'])
split['ratio'] = tmp.tolist()
split['sid'] = sid
splits = divs_splits['splits']
index = Index(
range(splits.shape[0], splits.shape[0] + split.shape[0]))
split.set_index(index, inplace=True)
divs_splits['splits'] = splits.append(split)
if 'dividend' in dfr.columns:
# ex_date amount sid record_date declared_date pay_date
tmp = dfr[dfr['dividend'] != 0.0]['dividend']
div = DataFrame(data=tmp.index.tolist(), columns=['ex_date'])
div['record_date'] = NaT
div['declared_date'] = NaT
div['pay_date'] = NaT
div['amount'] = tmp.tolist()
div['sid'] = sid
divs = divs_splits['divs']
ind = Index(range(divs.shape[0], divs.shape[0] + div.shape[0]))
div.set_index(ind, inplace=True)
divs_splits['divs'] = divs.append(div)
yield sid, dfr
register_calendar_alias("CSVDIR", "NYSE")
split = DataFrame(data=tmp.index.tolist(),
columns=['effective_date'])
split['ratio'] = tmp.tolist()
split['sid'] = sid
splits = divs_splits['splits']
index = Index(range(splits.shape[0],
splits.shape[0] + split.shape[0]))
split.set_index(index, inplace=True)
divs_splits['splits'] = splits.append(split)
if 'dividend' in dfr.columns:
# ex_date amount sid record_date declared_date pay_date
tmp = dfr[dfr['dividend'] != 0.0]['dividend']
div = DataFrame(data=tmp.index.tolist(), columns=['ex_date'])
div['record_date'] = NaT
div['declared_date'] = NaT
div['pay_date'] = NaT
div['amount'] = tmp.tolist()
div['sid'] = sid
divs = divs_splits['divs']
ind = Index(range(divs.shape[0], divs.shape[0] + div.shape[0]))
div.set_index(ind, inplace=True)
divs_splits['divs'] = divs.append(div)
yield sid, dfr
register_calendar_alias("CSVDIR", "NYSE")
_quantrocket_mic_aliases = {
# Other US Mics
"XNAS": "XNYS",
"ARCX": "XNYS",
"XASE": "XNYS",
"BATS": "XNYS",
"PINX": "XNYS",
"PSGM": "XNYS",
"OTCB": "XNYS",
"OTCQ": "XNYS",
}
quantrocket_calendar_factories = {}
for name, calendar in _quantrocket_ibkr_calendar_factories.items():
register_calendar_type(name, calendar, force=True)
quantrocket_calendar_factories[name] = calendar
for alias, real_name in _quantrocket_ibkr_calendar_aliases.items():
register_calendar_alias(alias, real_name, force=True)
quantrocket_calendar_factories[alias] = quantrocket_calendar_factories[
real_name]
for alias, real_name in _quantrocket_mic_aliases.items():
register_calendar_alias(alias, real_name, force=True)
quantrocket_calendar_factories[alias] = _default_calendar_factories[
real_name]
quantrocket_calendar_names = sorted(quantrocket_calendar_factories.keys())
def run_algorithm(initialize,
start=None,
end=None,
capital_base=1e6,
handle_data=None,
before_trading_start=None,
analyze=None,
data_frequency='daily',
bundle='sharadar',
bundle_timestamp=None,
trading_calendar=None,
metrics_set='default_daily',
benchmark_symbol='SPY',
default_extension=True,
extensions=(),
strict_extensions=True,
environ=os.environ,
blotter='default',
broker=None,
state_filename=None):
"""
Run a trading algorithm.
Parameters
----------
start : datetime
The start date of the backtest.
end : datetime
The end date of the backtest..
initialize : callable[context -> None]
The initialize function to use for the algorithm. This is called once
at the very begining of the backtest and should be used to set up
any state needed by the algorithm.
capital_base : float
The starting capital for the backtest.
handle_data : callable[(context, BarData) -> None], optional
The handle_data function to use for the algorithm. This is called
every minute when ``data_frequency == 'minute'`` or every day
when ``data_frequency == 'daily'``.
before_trading_start : callable[(context, BarData) -> None], optional
The before_trading_start function for the algorithm. This is called
once before each trading day (after initialize on the first day).
analyze : callable[(context, pd.DataFrame) -> None], optional
The analyze function to use for the algorithm. This function is called
once at the end of the backtest and is passed the context and the
performance data.
data_frequency : {'daily', 'minute'}, optional
The data frequency to run the algorithm at. For live trading the default is 'minute', otherwise 'daily'
bundle : str, optional
The name of the data bundle to use to load the data to run the backtest
with. This defaults to 'quantopian-quandl'.
bundle_timestamp : datetime, optional
The datetime to lookup the bundle data for. This defaults to the
current time.
trading_calendar : TradingCalendar, optional
The trading calendar to use for your backtest.
metrics_set : iterable[Metric] or str, optional
The set of metrics to compute in the simulation. If a string is passed,
resolve the set with :func:`zipline.finance.metrics.load`.
benchmark_symbol: The symbol of the benchmark. For live trading the default None, otherwise 'SPY'.
default_extension : bool, optional
Should the default zipline extension be loaded. This is found at
``$ZIPLINE_ROOT/extension.py``
extensions : iterable[str], optional
The names of any other extensions to load. Each element may either be
a dotted module path like ``a.b.c`` or a path to a python file ending
in ``.py`` like ``a/b/c.py``.
strict_extensions : bool, optional
Should the run fail if any extensions fail to load. If this is false,
a warning will be raised instead.
environ : mapping[str -> str], optional
The os environment to use. Many extensions use this to get parameters.
This defaults to ``os.environ``.
blotter : str or zipline.finance.blotter.Blotter, optional
Blotter to use with this algorithm. If passed as a string, we look for
a blotter construction function registered with
``zipline.extensions.register`` and call it with no parameters.
Default is a :class:`zipline.finance.blotter.SimulationBlotter` that
never cancels orders.
broker : instance of zipline.gens.brokers.broker.Broker
state_filename : path to pickle file storing the algorithm "context" (similar to self)
Returns
-------
perf : pd.DataFrame
The daily performance of the algorithm.
See Also
--------
zipline.data.bundles.bundles : The available data bundles.
"""
load_extensions(default_extension, extensions, strict_extensions, environ)
register_calendar_alias('NYSEMKT', 'XNYS')
register_calendar_alias('OTC', 'XNYS')
return _run(handle_data=handle_data,
initialize=initialize,
before_trading_start=before_trading_start,
analyze=analyze,
algofile=None,
algotext=None,
defines=(),
data_frequency=data_frequency,
capital_base=capital_base,
bundle=bundle,
bundle_timestamp=bundle_timestamp,
start=start,
end=end,
output=os.devnull,
trading_calendar=trading_calendar,
print_algo=False,
metrics_set=metrics_set,
local_namespace=False,
environ=environ,
blotter=blotter,
benchmark_symbol=benchmark_symbol,
broker=broker,
state_filename=state_filename)
'will be removed in a future release.'
)
def quantopian_quandl_bundle(environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
if show_progress:
data = download_with_progress(
QUANTOPIAN_QUANDL_URL,
chunk_size=ONE_MEGABYTE,
label="Downloading Bundle: quantopian-quandl",
)
else:
data = download_without_progress(QUANTOPIAN_QUANDL_URL)
with tarfile.open('r', fileobj=data) as tar:
if show_progress:
log.info("Writing data to %s." % output_dir)
tar.extractall(output_dir)
register_calendar_alias("QUANDL", "NYSE")
split = DataFrame(data=tmp.index.tolist(),
columns=['effective_date'])
split['ratio'] = tmp.tolist()
split['sid'] = sid
splits = divs_splits['splits']
index = Index(
range(splits.shape[0], splits.shape[0] + split.shape[0]))
split.set_index(index, inplace=True)
divs_splits['splits'] = splits.append(split)
if 'dividend' in dfr.columns:
# ex_date amount sid record_date declared_date pay_date
tmp = dfr[dfr['dividend'] != 0.0]['dividend']
div = DataFrame(data=tmp.index.tolist(), columns=['ex_date'])
div['record_date'] = NaT
div['declared_date'] = NaT
div['pay_date'] = NaT
div['amount'] = tmp.tolist()
div['sid'] = sid
divs = divs_splits['divs']
ind = Index(range(divs.shape[0], divs.shape[0] + div.shape[0]))
div.set_index(ind, inplace=True)
divs_splits['divs'] = divs.append(div)
yield sid, dfr
register_calendar_alias("NSECSVDIR", "XBOM")
df = cache[path]
except KeyError:
query = "select stat_date date,open,high,close,vol volume from stock_history where symbol='%s' order by date desc" % symbol
df = cache[path] = pd.read_sql(sql=query, con=conn, index_col='date', parse_dates=['date']).sort_index()
if boDebug:
print ("read_sqllite df", type(df), "length", len(df))
start_date = df.index[0]
end_date = df.index[-1]
ac_date = end_date + pd.Timedelta(days=1)
if boDebug:
print ("start_date", type(start_date), start_date)
print ("end_date", type(end_date), end_date)
print ("ac_date", type(ac_date), ac_date)
metadata.iloc[sid] = start_date, end_date, ac_date, symbol
new_index = ['open', 'high', 'low', 'close', 'volume']
df = df.reindex(columns = new_index, copy=False)
sessions = calendar.sessions_in_range(start_date, end_date)
df = df.reindex(
sessions.tz_localize(None),
copy=False,
).fillna(0.0)
yield sid, df
sid += 1
register_calendar_alias("SQLITEDB", "SHSZ")
data = BytesIO()
with progressbar(length=total_size, **progress_kwargs) as pbar:
for chunk in resp.iter_content(chunk_size=chunk_size):
data.write(chunk)
pbar.update(len(chunk))
data.seek(0)
return data
def download_without_progress(url):
"""
Download data from a URL, returning a BytesIO containing the loaded data.
Parameters
----------
url : str
A URL that can be understood by ``requests.get``.
Returns
-------
data : BytesIO
A BytesIO containing the downloaded data.
"""
resp = requests.get(url)
resp.raise_for_status()
return BytesIO(resp.content)
register_calendar_alias("sep", "NYSE")
split = DataFrame(data=tmp.index.tolist(),
columns=['effective_date'])
split['ratio'] = tmp.tolist()
split['sid'] = sid
splits = divs_splits['splits']
index = Index(
range(splits.shape[0], splits.shape[0] + split.shape[0]))
split.set_index(index, inplace=True)
divs_splits['splits'] = splits.append(split)
if 'dividend' in dfr.columns:
# ex_date amount sid record_date declared_date pay_date
tmp = dfr[dfr['dividend'] != 0.0]['dividend']
div = DataFrame(data=tmp.index.tolist(), columns=['ex_date'])
div['record_date'] = NaT
div['declared_date'] = NaT
div['pay_date'] = NaT
div['amount'] = tmp.tolist()
div['sid'] = sid
divs = divs_splits['divs']
ind = Index(range(divs.shape[0], divs.shape[0] + div.shape[0]))
div.set_index(ind, inplace=True)
divs_splits['divs'] = divs.append(div)
yield sid, dfr
register_calendar_alias("CSVDIR", "XSHG")
data = BytesIO()
with progressbar(length=total_size, **progress_kwargs) as pbar:
for chunk in resp.iter_content(chunk_size=chunk_size):
data.write(chunk)
pbar.update(len(chunk))
data.seek(0)
return data
def download_without_progress(url):
"""
Download data from a URL, returning a BytesIO containing the loaded data.
Parameters
----------
url : str
A URL that can be understood by ``requests.get``.
Returns
-------
data : BytesIO
A BytesIO containing the downloaded data.
"""
resp = requests.get(url)
resp.raise_for_status()
return BytesIO(resp.content)
register_calendar_alias("sharadar-prices", "NYSE")
data.append(it)
df = DataFrame(data)
df["date"] = to_datetime(df["date"], utc=True)
df.set_index("date", inplace=True)
# print(df.index)
sessions = get_calendar('XSHG').sessions_in_range("19900101", "20181231")
# print(sessions)
df = df.reindex(sessions).fillna(method="ffill")
df.dropna(inplace=True)
df["volume"] = list(map(lambda x: int(x), df["volume"]))
# print(df.dtypes)
return df
# sessions = get_calendar('XSHG').sessions_in_range("19900101", "20181231")
register_calendar_alias("MONGO", "XSHG")
def main():
mongo_db = client["daily"]["600578.SH"]
# mongo_db.get_collection("")
# print(read_mongo(mongo_db))
mongo_equities([{"db": "daily", "collections": ["000001.SH"]}], client)
#
#
if __name__ == '__main__':
main()
