def populate_candles():
import tqdm
class_names = ['OHLC', 'HeikinAshi']
tickers = ['BTCUSD.SPOT.BITS', 'BTCUSD.SPOT.BITF', 'BTCUSD.PERP.BMEX']
frequencies = ['5T', '15T', '30T', '1H', '4H']
from sgmtradingcore.analytics.features.crypto.runner import CryptoFeatureRunner
runner = CryptoFeatureRunner()
combos = itertools.product(class_names, tickers, frequencies)
valid_dt_dict = {}
for cls, ticker, freq in tqdm.tqdm(list(combos)):
if ticker not in valid_dt_dict:
valid_dt_dict[ticker] = get_first_valid_datetime(ticker)
start_date = get_first_valid_datetime(ticker)
end_date = datetime.datetime.now(tz=pytz.UTC)
params = {'input': FeatureRequest('CryptoTrades', {'source': CryptoDataSource.ARCTIC,
'ticker': ticker}),
'columns': [TSInput.TRADE_PRICE],
'start': start_date,
'clock': FeatureRequest('FixedPeriodClock', {'frequency': freq})}
fr = FeatureRequest(cls, params)
runner.precompute([fr], start_date, end_date, n_jobs=1)
def main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(name)s - '
'%(levelname)s - %(message)s')
tickers = ['BTCUSD.SPOT.BITS', 'BTCUSD.SPOT.BITF', 'BTCUSD.PERP.BMEX']
frequency = '1d'
runner = CryptoFeatureRunner()
request = [
FeatureRequest(
'OHLCTrades',
{
'frequency': frequency,
},
{},
),
# FeatureRequest('OHLC',
# {
# 'frequency': frequency,
# 'base_ts': {
# 'is_feature_base_ts': False,
# 'base_ts_name': TSInput.L1_MID,
# }
# },
# {},
# prefix='t_'
# )
]
_FIRST_VALID_DATETIMES = get_first_valid_datetime()
_LAST_VALID_DATETIMES = get_last_valid_datetime()
for ticker in tickers:
runner = CryptoFeatureRunner()
start_dt = _FIRST_VALID_DATETIMES[ticker]
end_dt = _LAST_VALID_DATETIMES[ticker]
df = runner.get_merged_dataframes(request, start_dt, end_dt)
column_names = ['open', 'high', 'low', 'close']
column_names_t = ['t_open', 't_high', 't_low', 't_close']
column_names_a = ['a_open', 'a_high', 'a_low', 'a_close']
mongo_client = MongoPersister.init_from_config('arctic_crypto',
auto_connect=True)
arctic = ArcticStorage(mongo_client.client)
arctic_trades_df = arctic.load_trades(ticker, start_dt, end_dt)
timestamps = arctic_trades_df.index.to_pydatetime().tolist()
prices = arctic_trades_df['price'].tolist()
arctic_candles = make_ohlc_candles(timestamps=timestamps,
values=prices,
period=timedelta(days=1))
arctic_candles.columns = [
'a_' + str(col) for col in arctic_candles.columns
]
df1 = pd.merge(df, arctic_candles)
compare_candles(ticker, df1, column_names, column_names_a)
'columns': ['size']}
def _compute_ticks(self, groups):
return groups.agg({'size': 'sum'}).rename(columns={'size': 'traded_volume'})
@classmethod
def get_empty_feature(cls):
return pd.DataFrame(columns=['traded_volume'])
if __name__ == '__main__':
from sgmtradingcore.analytics.features.crypto.runner import CryptoFeatureRunner
import datetime
import pytz
runner_ = CryptoFeatureRunner()
ticker = 'BTCUSD.PERP.BMEX'
clock = FeatureRequest('FixedVolumeClock', {'start': datetime.datetime(2018, 5, 30, tzinfo=pytz.UTC),
'volume': 100000,
'offset': 0,
'units': CryptoCrossSide.ASSET,
'ticker': ticker,
'source': CryptoDataSource.ARCTIC})
input = FeatureRequest('CryptoTrades', {'source': CryptoDataSource.ARCTIC,
'ticker': 'BTCUSD.PERP.BMEX',
'aggregate': True})
fr = FeatureRequest('VolumeTraded', {'clock': clock,
'input': input,
}
result_df = pd.DataFrame(
[self._transform_function(tick) for tick in data_for_range])
if not result_df.empty:
result_df.set_index('timestamp', inplace=True)
return result_df
if __name__ == '__main__':
from sgmtradingcore.analytics.features.request import FeatureRequest
from sgmtradingcore.analytics.features.crypto.runner import CryptoFeatureRunner
import datetime
from stratagemdataprocessing.crypto.enums import get_first_valid_datetime
runner_ = CryptoFeatureRunner()
tickers_ = ['BTCUSD.SPOT.BITF']
start_dt_ = get_first_valid_datetime()[tickers_[0]]
end_dt_ = start_dt_ + datetime.timedelta(
days=2) # datetime.datetime(2014, 6, 30, 0, 0, tzinfo=pytz.UTC)
requests = [
FeatureRequest(
'CryptoTrades', {
'source': CryptoDataSource.RAW_FILES,
'ticker': tickers_[0],
'aggregate': True
})
]
cache_trades = runner_.get_merged_dataframes(requests, start_dt_, end_dt_)
ohlc_data['lower_band'] = ohlc_data['middle_band'] - 2 * ohlc_close_std
# now drop whatever you used to facilitate computation:
ohlc_data = ohlc_data[ohlc_data.index >= start_dt]
return ohlc_data[['middle_band', 'lower_band', 'upper_band']]
if __name__ == '__main__':
from sgmtradingcore.analytics.features.crypto.runner import CryptoFeatureRunner
import pytz
import datetime as dt
# first valid timestamp BITS 17/3/2014
tickers = ['BTCUSD.SPOT.BITS']
runner = CryptoFeatureRunner(env='dev')
start_dt = dt.datetime(2018, 1, 1, 0, 0, tzinfo=pytz.UTC)
end_dt = dt.datetime(2018, 2, 28, 0, 0, tzinfo=pytz.UTC)
input0 = FeatureRequest('CryptoLOB', {
'ts_name': TSInput.L1_MID,
'ticker': 'BTCUSD.SPOT.BITS'
})
inputRequest = FeatureRequest('OHLC', {
'frequency': '15T',
'input': input0
})
requests = [
class PriceMoveRegression(OHLC):
"""
Similar to PriceMoveBinary except it measures the value of the price movement
over the bar
"""
def _compute_ticks(self, groups):
ret = super(PriceMoveRegression, self)._compute_ticks(groups)
ret.index = groups['previous'].first()
return pd.DataFrame(ret['close'] - ret['open'], columns=['target'])
if __name__ == '__main__':
from sgmtradingcore.analytics.features.crypto.runner import CryptoFeatureRunner
runner = CryptoFeatureRunner()
start_date = datetime.datetime(2018, 6, 1, tzinfo=pytz.utc)
end_date = start_date + datetime.timedelta(days=5)
ticker = 'BTCUSD.PERP.BMEX'
params = {
'input':
FeatureRequest('CryptoTrades', {
'source': CryptoDataSource.ARCTIC,
'ticker': ticker
}),
'columns': [TSInput.TRADE_PRICE],
'start':
start_date,
}
fr = FeatureRequest('PriceMoveRegression', params)
def get_empty_candles_report(ticker,
start_dt,
end_dt,
freq,
lob_or_trades="trades"):
"""
:param ticker: crypto exchange ticker
:param start_dt (datetime): start date (must be rounded to the nearest day)
:param end_dt (datetime): end_date (must be rounded to the nearest day)
:param freq (str): candles' frequency, currently supported: '15m', '1h', '4h', '1d'.
:param lob_or_trades (str): 'trades' if we want the candles based on trades, 'lob' if we want candles
based on lob
:return: The function finds the missing candles for the specified interval and frequency. It returns a
dataframe containing the following columns:
- expected_candles (number)
- missing_number ( how many candles we are missing)
- missing_candles (the actual intervals/single timestamps we are missing)
"""
if lob_or_trades == "lob":
request = [
FeatureRequest(
'OHLC', {
'frequency': freq,
'base_ts': {
'is_feature_base_ts': False,
'base_ts_name': 'l1_mid'
}
}, {})
]
elif lob_or_trades == "trades":
request = [FeatureRequest(
'OHLCTrades',
{'frequency': freq},
{},
)]
else:
raise ValueError(
"lob_or_trades {} not supported yet".format(lob_or_trades))
runner = CryptoFeatureRunner()
df = runner.get_merged_dataframes(request, start_dt, end_dt)
# drop None/Nan values, ie. no candles
df = df.dropna()
if freq == '15m':
freq = '15min'
# round dates ro nearest hour
start_dt = datetime(*start_dt.timetuple()[:4])
end_dt = datetime(*end_dt.timetuple()[:4])
expected_timestamps = pd.date_range(start=start_dt, end=end_dt,
freq=freq)[:-1]
actual_timestamps = df.index
expected_candles_nr = len(expected_timestamps)
actual_candles_nr = len(actual_timestamps)
missing_number = expected_candles_nr - actual_candles_nr
missing_candles = list(set(expected_timestamps) - set(actual_timestamps))
missing_candles = sorted(missing_candles)
if freq != '1d':
missing_candles = get_date_range(missing_candles, freq)
report_dict = {
'expected_candles': expected_candles_nr,
'missing_number': missing_number,
'missing_candles': [missing_candles]
}
df = pd.DataFrame.from_dict(report_dict, orient='index')
df = df.reindex(
index=['expected_candles', 'missing_candles', 'missing_number'])
return df