def __init__(self,
engine='plotly',
data_source='bloomberg',
market_data_generator=MarketDataGenerator()):
self._chart = Chart(engine=engine)
self._market = Market(market_data_generator=market_data_generator)
self._data_source = data_source
def plot_animated_vol_market():
market = Market(market_data_generator=MarketDataGenerator())
cross = ['EURUSD']; start_date = '01 Mar 2017'; finish_date = '21 Apr 2017'; sampling = 'no'
md_request = MarketDataRequest(start_date=start_date, finish_date=finish_date,
data_source='bloomberg', cut='NYC', category='fx-implied-vol',
tickers=cross, cache_algo='cache_algo_return')
df = market.fetch_market(md_request)
if sampling != 'no': df = df.resample(sampling).mean()
fxvf = FXVolFactory()
df_vs = []
# Grab the vol surface for each date and create a dataframe for each date (could have used a panel)
for i in range(0, len(df.index)): df_vs.append(fxvf.extract_vol_surface_for_date(df, cross[0], i))
# Do static plot for first day using Plotly
style = Style(title="FX vol surface of " + cross[0], source="chartpy", color='Blues')
Chart(df=df_vs[0], chart_type='surface', style=style).plot(engine='plotly')
# Now do animation (TODO: need to fix animation in chartpy for matplotlib)
style = Style(title="FX vol surface of " + cross[0], source="chartpy", color='Blues',
animate_figure=True, animate_titles=df.index,
animate_frame_ms=500, normalize_colormap=False)
Chart(df=df_vs, chart_type='surface', style=style).plot(engine='plotly')
# Chart object is initialised with the dataframe and our chart style
Chart(df=df_vs, chart_type='surface', style=style).plot(engine='plotly')
def process_batch(start, end, symbol):
market = Market(market_data_generator=MarketDataGenerator())
md_request = MarketDataRequest(start_date=start, finish_date=end,
category='fx', fields=['bid', 'ask'], freq='tick',
data_source='dukascopy', tickers=[symbol.upper()])
df = market.fetch_market(md_request)
print(df.tail(n=5))
df.reset_index(level=0, inplace=True)
df.columns = ['_id','bid','ask']
print(df.tail(n=5))
dc = df.to_dict('records')
print("converted to dict")
if config.mongo['user'] == "" or config.mongo['pasw'] == "":
mongouri = "mongodb://"+config.mongo['uri']+":27017"
else:
mongouri = "mongodb://"+config.mongo['user']+":"+config.mongo['pasw']+"@"+config.mongo['uri']+":27017/admin"
print(mongouri)
client = MongoClient(mongouri)
db = client['prices-data']
# db = client['tests']
collection = db[symbol.lower()+'_ticks']
insert = collection.insert_many(dc)
# doc= collection.find_one({})
# print(insert)
return insert
def test_huobi():
market = Market(market_data_generator=MarketDataGenerator())
# test daily historical data
md_request = MarketDataRequest(start_date='11 Apr 2018',
finish_date='20 Apr 2018',
cut='LOC',
freq='daily',
data_source='huobi',
category='crypto',
fields=['high', 'low'],
tickers=['XBTUSD'])
df = market.fetch_market(md_request)
assert not df.empty
# test historical tick (second) data, last 5 mins from 1 min ago
finish_dt = dt.datetime.utcnow() - dt.timedelta(minutes=1)
start_dt = finish_dt - dt.timedelta(minutes=5)
md_request = MarketDataRequest(start_date=start_dt,
finish_date=finish_dt,
cut='LOC',
freq='tick',
data_source='huobi',
category='crypto',
fields=['high', 'low'],
tickers=['XBTUSD'])
df = market.fetch_market(md_request)
assert not df.empty
def update_graph(
graph_id,
graph_title,
stock,
ticker,
yaxis_title,
source,
):
"""
:param graph_id: ID for Dash callbacks
:param graph_title: Displayed on layout
:param y_train_index: name of column index for y train we want to retrieve
:param y_val_index: name of column index for y val we want to retrieve
:param run_log_json: the json file containing the data
:param display_mode: 'separate' or 'overlap'
:param checklist_smoothing_options: 'train' or 'val'
:param slider_smoothing: value between 0 and 1, at interval of 0.05
:return: dcc Graph object containing the updated figures
"""
market = Market(market_data_generator=MarketDataGenerator())
# download equities data from Yahoo
md_request = MarketDataRequest(
start_date="decade", # start date
data_source='yahoo', # use Bloomberg as data source
tickers=[stock, 'Citigroup'], # ticker (findatapy)
fields=[ticker], # which fields to download
vendor_tickers=[stock, 'c'], # ticker (Yahoo)
vendor_fields=[ticker]) # which Bloomberg fields to download)
quandl.ApiConfig.api_key = "545Z2yCpQxyZ4WH9Q55E"
data=None
stockString=None
if(source==1):
data = market.fetch_market(md_request)
stockString=data[stock+'.'+ticker]
else:
data = quandl.get("EOD/"+stock, start_date='2017-12-28', end_date='2017-12-28')
stockString=data
scatterGraph = go.Scatter(
x=data.index,
y=stockString,
name="Training",
line=dict(color="rgb(54, 218, 170)"),
showlegend=False,
)
layout = go.Layout(
title=graph_title,
margin=go.layout.Margin(l=50, r=50, b=50, t=50),
yaxis={"title": yaxis_title},
)
figure = go.Figure(data=[scatterGraph], layout=layout)
return dcc.Graph(
figure=figure,
id=graph_id
)
def test_bitcoincharts():
market = Market(market_data_generator=MarketDataGenerator())
md_request = MarketDataRequest(start_date='11 Nov 2015', finish_date='02 Feb 2018', cut='LOC',
freq='tick', data_source='bitcoincharts', category='crypto',
fields=['close', 'volume'], tickers=['XBTUSD_itbit'])
df = market.fetch_market(md_request)
assert not df.empty
def test_binance():
market = Market(market_data_generator=MarketDataGenerator())
md_request = MarketDataRequest(start_date='18 Feb 2017', finish_date='20 Feb 2018', cut='LOC',
freq='daily', data_source='binance', category='crypto',
fields=['close', 'volume', 'quote-asset-volume'],
tickers=['WTCXBT'])
df = market.fetch_market(md_request)
assert not df.empty
def __init__(self):
super(TradingModel, self).__init__()
##### FILL IN WITH YOUR OWN PARAMETERS FOR display, dumping, TSF etc.
self.market = Market(market_data_generator=MarketDataGenerator())
self.DUMP_PATH = ''
self.FINAL_STRATEGY = 'FX trend'
self.SCALE_FACTOR = 1
self.DEFAULT_PLOT_ENGINE = 'matplotlib'
self.br = self.load_parameters()
return
def construct_backtest(ticker, vendor_ticker, sma_period, data_source, start_date, quandl_api_key):
backtest = Backtest()
br = BacktestRequest()
# Set all the parameters for the backtest
br.start_date = start_date
br.finish_date = datetime.datetime.utcnow()
br.spot_tc_bp = 2.5 # 2.5 bps bid/ask spread
br.ann_factor = 252
tech_params = TechParams()
tech_params.sma_period = sma_period
indicator = 'SMA'
md_request = MarketDataRequest(
start_date=start_date,
finish_date=datetime.date.today(),
freq='daily',
data_source=data_source,
tickers=ticker,
fields=['close'],
vendor_tickers=vendor_ticker,
quandl_api_key=quandl_api_key)
market = Market(market_data_generator=MarketDataGenerator())
# Download the market data (the asset we are trading is also
# being used to generate the signal)
asset_df = market.fetch_market(md_request)
spot_df = asset_df
# Use technical indicator to create signals
# (we could obviously create whatever function we wanted for generating the signal dataframe)
# However, finmarketpy has some technical indicators built in (and some signals too)
tech_ind = TechIndicator()
tech_ind.create_tech_ind(spot_df, indicator, tech_params);
signal_df = tech_ind.get_signal()
# use the same data for generating signals
backtest.calculate_trading_PnL(br, asset_df, signal_df, None, False)
# Get the returns and signals for the portfolio
port = backtest.portfolio_cum()
port.columns = [indicator + ' = ' + str(tech_params.sma_period) + ' ' + str(backtest.portfolio_pnl_desc()[0])]
signals = backtest.portfolio_signal()
# returns = backtest.pnl()
return port, signals
def getFxData(self,
startDate='14 Jun 2016',
endDate='15 Jun 2016',
tickers=['EURUSD'],
fields=['close'],
frequency='tick'):
md_request = MarketDataRequest(start_date=startDate,
finish_date=endDate,
category=self.category,
fields=fields,
freq=frequency,
data_source=self.datasource,
tickers=tickers)
market = Market(market_data_generator=MarketDataGenerator())
return market.fetch_market(md_request)
def generate_market_data_for_tests(start_date, finish_date):
md_request = MarketDataRequest(start_date=start_date,
finish_date=finish_date,
cut='NYC',
category='fx',
fields=['bid'],
freq='tick',
data_source='dukascopy',
tickers=['EURUSD'])
market = Market(market_data_generator=MarketDataGenerator())
try:
df = market.fetch_market(md_request)
return df
except:
return None
def load_minute_data(ticker,
start_date='01 Jan 2019',
finish_date='30 Jun 2019'):
# Load tick data from DukasCopy (if doesn't exist on disk) and then save to disk as 1 minute data
# This is in UTC timezone
# By default the path is the working director but we can change that
raw_data_path = ''
# Imports of various findatapy libraries for market data downloads
from findatapy.market import Market, MarketDataRequest, MarketDataGenerator
import os
# First we can do it by defining all the vendor fields, tickers etc. so we bypass the configuration file
md_request = MarketDataRequest(start_date=start_date,
finish_date=finish_date,
fields=['bid', 'ask'],
vendor_fields=['bid', 'ask'],
freq='tick',
data_source='dukascopy',
tickers=ticker,
vendor_tickers=ticker,
category='fx')
market = Market(market_data_generator=MarketDataGenerator())
compression_type = 'gzip' # you can change this to 'snappy' if you want!
# Only download file if not on disk (slow to download)
if not (os.path.exists(raw_data_path + ticker + '_1min.gzip')):
df_tick = market.fetch_market(md_request)
df_tick['mid'] = (df_tick[ticker + '.bid'] +
df_tick[ticker + '.ask']) / 2.0
df_minute = pd.DataFrame(
df_tick['mid'].resample("1min").first()).dropna()
df_minute.to_parquet(raw_data_path + ticker + '_1min.gzip',
compression=compression_type,
engine='fastparquet')
else:
# Edit the below line if you want to pick only one of the yearly Parquet files
# If you load the whole amount might run out of memory!
df_minute = pd.read_parquet(raw_data_path + ticker + '_1min.gzip',
engine='fastparquet')
return df_minute
def _download(self, md_request, folder_prefix):
from findatapy.market import MarketDataRequest, MarketDataGenerator, Market
logger = LoggerManager.getLogger(__name__)
market = Market(market_data_generator=MarketDataGenerator())
ticker = md_request.ticker[0]
df = market.fetch_market(md_request=md_request)
df.columns = ['bid', 'ask', 'bidv', 'askv']
df['venue'] = 'dukascopy'
df['ticker'] = ticker
df['mid'] = (df['bid'].values + df['ask'].values) / 2.0
self.dump_hdf5_file(df, folder_prefix + "_" + ticker + ".h5")
logger.info('Dumped to ' + folder_prefix + "_" + ticker + ".h5")
def load_une_data():
md_request = MarketDataRequest(
start_date='01 Jan 2001', # Start date
finish_date='12 Aug 2019', # Finish date
tickers=us_states, # What we want the ticker to look like once download
vendor_tickers=us_states_fred, # The ticker used by the vendor
fields=[
'close'
], # What fields we want (usually close, we can also define vendor fields)
data_source='alfred', # What is the data source?
# vendor_fields=['actual-release', 'first-revision', 'close'],
fred_api_key=FRED_API_KEY
) # Most data sources will require us to specify an API key/password
market = Market(market_data_generator=MarketDataGenerator())
df_une = market.fetch_market(md_request)
df_une.columns = [x.replace('.close', '') for x in df_une.columns]
return df_une
def __init__(self):
super(TradingModel, self).__init__()
##### FILL IN WITH YOUR OWN PARAMETERS FOR display, dumping, TSF etc.
self.market = Market(market_data_generator=MarketDataGenerator())
self.DUMP_PATH = ''
self.FINAL_STRATEGY = 'FX trend'
self.SCALE_FACTOR = 1
self.DEFAULT_PLOT_ENGINE = 'matplotlib'
self.br = self.load_parameters()
return
def get_figure(ticker, data_source, start_date, api_key):
import matplotlib.pyplot as plt
# Fetch market data from Quandl
md_request = MarketDataRequest(start_date=start_date,
tickers=ticker,
vendor_tickers=ticker,
data_source=data_source)
if data_source == 'quandl':
md_request.quandl_api_key = api_key
df = Market(market_data_generator=MarketDataGenerator()).fetch_market(md_request)
scale = 1.0
# Plot using Matplotlib
fig = plt.figure(dpi=90, figsize=(scale * 3.0, scale * 2.0))
df.plot(ax=plt.gca())
return fig, df
def generate_market_data_for_tests():
# generate daily S&P500 data from Quandl
md_request = MarketDataRequest(start_date='01 Jan 2001',
finish_date='01 Dec 2008',
tickers=['S&P500'],
vendor_tickers=['YAHOO/INDEX_GSPC'],
fields=['close'],
data_source='quandl')
market = Market(market_data_generator=MarketDataGenerator())
df = market.fetch_market(md_request)
df.to_csv("S&P500.csv")
# generate tick data from DukasCopy for EURUSD
md_request = MarketDataRequest(start_date='14 Jun 2016',
finish_date='15 Jun 2016',
cut='NYC',
category='fx',
fields=['bid'],
freq='tick',
data_source='dukascopy',
tickers=['EURUSD'])
market = Market(market_data_generator=MarketDataGenerator())
df = market.fetch_market(md_request)
df.to_csv("EURUSD_tick.csv")
def generate_market_data_for_tests():
# generate daily S&P500 data from Quandl
md_request = MarketDataRequest(start_date="01 Jan 2001",
finish_date="01 Dec 2008",
tickers=["S&P500"],
vendor_tickers=["YAHOO/INDEX_GSPC"],
fields=["close"],
data_source="quandl")
market = Market(market_data_generator=MarketDataGenerator())
df = market.fetch_market(md_request)
df.to_csv("S&P500.csv")
# generate tick data from DukasCopy for EURUSD
md_request = MarketDataRequest(start_date="14 Jun 2016",
finish_date="15 Jun 2016",
cut="NYC",
category="fx",
fields=["bid"],
freq="tick",
data_source="dukascopy",
tickers=["EURUSD"])
market = Market(market_data_generator=MarketDataGenerator())
df = market.fetch_market(md_request)
df.to_csv("EURUSD_tick.csv")
def get_mid_price(raw_data_path, ticker='EURUSD'):
# First we can do it by defining all the vendor fields, tickers etc. so we bypass the configuration file
# We use findatapy
md_request = MarketDataRequest(
start_date='01 Jan 2007', finish_date='30 Jun 2019',
fields=['bid', 'ask'], vendor_fields=['bid', 'ask'],
freq='tick', data_source='dukascopy',
tickers=[ticker], vendor_tickers=[ticker], category='fx')
market = Market(market_data_generator=MarketDataGenerator())
compression_type = 'gzip' # you can change this to 'snappy' if you want!
# Only download file if not on disk (slow to download),
# then write to disk as parquet and CSV
# Note: writing to CSV takes a long time, so we have commented it here!
if not (os.path.exists(os.path.join(raw_data_path, ticker + '.gzip'))):
df_tick = market.fetch_market(md_request)
df_tick.to_parquet(os.path.join(raw_data_path, ticker + '.gzip'), compression=compression_type,
engine='fastparquet')
start_year = df_tick.index[0].year
finish_year = df_tick.index[-1].year
for i in range(start_year, finish_year + 1):
df_year = df_tick[df_tick.index.year == i]
df_year.to_parquet(raw_data_path + ticker + '_' + str(i) + '.gzip',
compression=compression_type, engine='fastparquet')
else:
# Edit the below line if you want to pick only one of the yearly Parquet files
# If you load the whole amount might run out of memory!
df_tick = pd.read_parquet(os.path.join(raw_data_path, ticker + '_2019.gzip'),
engine='fastparquet')
# calculate mid-price
df_tick['mid'] = (df_tick[ticker + '.ask'] + df_tick[ticker + '.bid']) / 2.0
# get 1 minute data
return pd.DataFrame(df_tick['mid'].resample("1min").first()).dropna()
def plot_animated_vol_market():
market = Market(market_data_generator=MarketDataGenerator())
cross = ['EURUSD']; start_date = '01 Mar 2017'; finish_date = '21 Apr 2017'; sampling = 'no'
md_request = MarketDataRequest(start_date=start_date, finish_date=finish_date,
data_source='bloomberg', cut='LDN', category='fx-implied-vol',
tickers=cross, cache_algo='cache_algo_return')
df = market.fetch_market(md_request)
if sampling != 'no': df = df.resample(sampling).mean()
fxvf = FXVolFactory()
df_vs = []
# grab the vol surface for each date and create a dataframe for each date (could have used a panel)
for i in range(0, len(df.index)): df_vs.append(fxvf.extract_vol_surface_for_date(df, cross[0], i))
style = Style(title="FX vol surface of " + cross[0], source="chartpy", color='Blues',
animate_figure=True, animate_titles=df.index,
animate_frame_ms=500, normalize_colormap=False)
# Chart object is initialised with the dataframe and our chart style
Chart(df=df_vs, chart_type='surface', style=style).plot(engine='matplotlib')
def load_tickers():
logger = LoggerManager.getLogger(__name__)
market = Market(market_data_generator=MarketDataGenerator())
DataConstants.market_thread_technique = 'thread'
# load S&P 500 ticker via wikipedia
snp = pd.read_html(
'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
tickers = snp[0]['Symbol'].to_list()
# download equities data from Yahoo
md_request = MarketDataRequest(
start_date=START_DATE,
data_source='yahoo', # use Bloomberg as data source
tickers=tickers, # ticker (findatapy)
fields=['close', 'open', 'high', 'low',
'volume'], # which fields to download
vendor_tickers=tickers, # ticker (Yahoo)
vendor_fields=['Close', 'Open', 'High', 'Low',
'Volume']) # which Bloomberg fields to download)
logger.info("Loading data with threading")
df = market.fetch_market(md_request)
logger.info("Loading data with multiprocessing")
DataConstants.market_thread_technique = 'multiprocessing'
df = market.fetch_market(md_request)
logger.info("Loaded data with multiprocessing")
df.to_csv("temp_downloads/snp.csv")
def plot_animated_vol_market():
market = Market(market_data_generator=MarketDataGenerator())
cross = ['GBPUSD']
start_date = '01 Jun 2016'
finish_date = '01 Aug 2016'
sampling = 'no'
md_request = MarketDataRequest(start_date=start_date,
finish_date=finish_date,
data_source='bloomberg',
cut='LDN',
category='fx-implied-vol',
tickers=cross,
cache_algo='internet_load_return')
df = market.fetch_market(md_request)
if sampling != 'no': df = df.resample(sampling).mean()
fxvf = FXVolFactory()
df_vs = []
# grab the vol surface for each date and create a dataframe for each date (could have used a panel)
for i in range(0, len(df.index)):
df_vs.append(fxvf.extract_vol_surface_for_date(df, cross[0], i))
style = Style(title="FX vol surface of " + cross[0],
source="chartpy",
color='Blues',
animate_figure=True,
animate_titles=df.index,
animate_frame_ms=500,
normalize_colormap=False)
# Chart object is initialised with the dataframe and our chart style
Chart(df=df_vs, chart_type='surface',
style=style).plot(engine='matplotlib')
def load_data():
# Download the historical spot data once and store in memory, we'll process later
market = Market(market_data_generator=MarketDataGenerator())
market_data_request = MarketDataRequest(
start_date='01 Jan 2000', # Start date
freq='daily', # Daily data
data_source='quandl', # Use Quandl as data source
tickers=tickers, # Ticker (Cuemacro)
fields=['close'], # Which fields to download
vendor_tickers=vendor_tickers, # Ticker (Quandl)
vendor_fields=['close'], # Which Bloomberg fields to download
cache_algo='cache_algo_return') # How to return data
# You need to type your Quandl API below (or modify the DataCred file)
# market_data_request.quandl_api_key = None
df = market.fetch_market(market_data_request)
df = df.fillna(method='ffill')
df_ret = df / df.shift(1)
return df, df_ret
__author__ = 'saeedamen'
# loading data
import datetime
from chartpy import Chart, Style
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from findatapy.util.loggermanager import LoggerManager
logger = LoggerManager().getLogger(__name__)
chart = Chart(engine='matplotlib')
market = Market(market_data_generator=MarketDataGenerator())
# choose run_example = 0 for everything
# run_example = 1 - download BoE data from quandl
run_example = 0
###### fetch data from Quandl for BoE rate (using Bloomberg data)
if run_example == 1 or run_example == 0:
# Monthly average of UK resident monetary financial institutions' (excl. Central Bank) sterling
# Weighted average interest rate, other loans, new advances, on a fixed rate to private non-financial corporations (in percent)
# not seasonally adjusted
md_request = MarketDataRequest(
start_date="01 Jan 2000", # start date
data_source='quandl', # use Quandl as data source
tickers=['Weighted interest rate'],
fields=['close'], # which fields to download
class TradingModelFXTrend_Example(TradingModel):
def __init__(self):
super(TradingModel, self).__init__()
##### FILL IN WITH YOUR OWN PARAMETERS FOR display, dumping, TSF etc.
self.market = Market(market_data_generator=MarketDataGenerator())
self.DUMP_PATH = ''
self.FINAL_STRATEGY = 'FX trend'
self.SCALE_FACTOR = 1
self.DEFAULT_PLOT_ENGINE = 'matplotlib'
self.br = self.load_parameters()
return
###### Parameters and signal generations (need to be customised for every model)
def load_parameters(self):
##### FILL IN WITH YOUR OWN BACKTESTING PARAMETERS
br = BacktestRequest()
# get all asset data
br.start_date = "04 Jan 1989"
br.finish_date = datetime.datetime.utcnow().date()
br.spot_tc_bp = 0.5
br.ann_factor = 252
br.plot_start = "01 Apr 2015"
br.calc_stats = True
br.write_csv = False
br.plot_interim = True
br.include_benchmark = True
# have vol target for each signal
br.signal_vol_adjust = True
br.signal_vol_target = 0.1
br.signal_vol_max_leverage = 5
br.signal_vol_periods = 20
br.signal_vol_obs_in_year = 252
br.signal_vol_rebalance_freq = 'BM'
br.signal_vol_resample_freq = None
# have vol target for portfolio
br.portfolio_vol_adjust = True
br.portfolio_vol_target = 0.1
br.portfolio_vol_max_leverage = 5
br.portfolio_vol_periods = 20
br.portfolio_vol_obs_in_year = 252
br.portfolio_vol_rebalance_freq = 'BM'
br.portfolio_vol_resample_freq = None
# tech params
br.tech_params.sma_period = 200
return br
def load_assets(self):
##### FILL IN WITH YOUR ASSET DATA
# for FX basket
full_bkt = [
'EURUSD', 'USDJPY', 'GBPUSD', 'AUDUSD', 'USDCAD', 'NZDUSD',
'USDCHF', 'USDNOK', 'USDSEK'
]
basket_dict = {}
for i in range(0, len(full_bkt)):
basket_dict[full_bkt[i]] = [full_bkt[i]]
basket_dict['FX trend'] = full_bkt
br = self.load_parameters()
self.logger.info("Loading asset data...")
vendor_tickers = [
'FRED/DEXUSEU', 'FRED/DEXJPUS', 'FRED/DEXUSUK', 'FRED/DEXUSAL',
'FRED/DEXCAUS', 'FRED/DEXUSNZ', 'FRED/DEXSZUS', 'FRED/DEXNOUS',
'FRED/DEXSDUS'
]
market_data_request = MarketDataRequest(
start_date=br.start_date, # start date
finish_date=br.finish_date, # finish date
freq='daily', # daily data
data_source='quandl', # use Quandl as data source
tickers=full_bkt, # ticker (Thalesians)
fields=['close'], # which fields to download
vendor_tickers=vendor_tickers, # ticker (Quandl)
vendor_fields=['close'], # which Bloomberg fields to download
cache_algo='internet_load_return') # how to return data
asset_df = self.market.fetch_market(market_data_request)
# if web connection fails read from CSV
if asset_df is None:
import pandas
asset_df = pandas.read_csv(
"d:/fxcta.csv",
index_col=0,
parse_dates=['Date'],
date_parser=lambda x: pandas.datetime.strptime(x, '%Y-%m-%d'))
# signalling variables
spot_df = asset_df
spot_df2 = None
# asset_df
return asset_df, spot_df, spot_df2, basket_dict
def construct_signal(self, spot_df, spot_df2, tech_params, br):
##### FILL IN WITH YOUR OWN SIGNALS
# use technical indicator to create signals
# (we could obviously create whatever function we wanted for generating the signal dataframe)
tech_ind = TechIndicator()
tech_ind.create_tech_ind(spot_df, 'SMA', tech_params)
signal_df = tech_ind.get_signal()
return signal_df
def construct_strategy_benchmark(self):
###### FILL IN WITH YOUR OWN BENCHMARK
tsr_indices = MarketDataRequest(
start_date=self.br.start_date, # start date
finish_date=self.br.finish_date, # finish date
freq='daily', # intraday data
data_source='quandl', # use Bloomberg as data source
tickers=["EURUSD"], # tickers to download
vendor_tickers=['FRED/DEXUSEU'],
fields=['close'], # which fields to download
vendor_fields=['close'],
cache_algo='cache_algo_return') # how to return data)
df = self.market.fetch_market(tsr_indices)
df.columns = [x.split(".")[0] for x in df.columns]
return df
import pandas as pd
# For plotting
from chartpy import Chart, Style
# For loading market data
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from findatapy.timeseries import Calculations, Calendar
from findatapy.util.loggermanager import LoggerManager
logger = LoggerManager().getLogger(__name__)
chart = Chart(engine='plotly')
market = Market(market_data_generator=MarketDataGenerator())
calculations = Calculations()
# Choose run_example = 0 for everything
# run_example = 1 - get forwards prices for AUDUSD interpolated for an odd date/broken date
# run_example = 2 - get implied deposit rate
run_example = 2
from finmarketpy.curve.rates.fxforwardspricer import FXForwardsPricer
###### Value forwards for AUDUSD for odd days
if run_example == 1 or run_example == 0:
cross = 'AUDUSD'
fx_forwards_tenors = ['1W', '2W', '3W', '1M']
vendor_tickers = ['FRED/DEXUSEU', 'FRED/DEXJPUS', 'FRED/DEXUSUK', 'FRED/DEXUSAL', 'FRED/DEXCAUS',
'FRED/DEXUSNZ', 'FRED/DEXSZUS', 'FRED/DEXNOUS', 'FRED/DEXSDUS']
md_request = MarketDataRequest(
start_date = "01 Jan 1989", # start date
finish_date = datetime.date.today(), # finish date
freq = 'daily', # daily data
data_source = 'quandl', # use Quandl as data source
tickers = tickers, # ticker (findatapy)
fields = ['close'], # which fields to download
vendor_tickers = vendor_tickers, # ticker (Quandl)
vendor_fields = ['close'], # which Bloomberg fields to download
cache_algo = 'internet_load_return') # how to return data
market = Market(market_data_generator=MarketDataGenerator())
asset_df = market.fetch_market(md_request)
spot_df = asset_df
logger.info("Running backtest...")
# use technical indicator to create signals
# (we could obviously create whatever function we wanted for generating the signal dataframe)
tech_ind = TechIndicator()
tech_ind.create_tech_ind(spot_df, indicator, tech_params); signal_df = tech_ind.get_signal()
# use the same data for generating signals
backtest.calculate_trading_PnL(br, asset_df, signal_df)
port = backtest.get_cumportfolio()
port.columns = [indicator + ' = ' + str(tech_params.sma_period) + ' ' + str(backtest.get_portfolio_pnl_desc()[0])]
class TradingModelFXTrend_Example(TradingModel):
def __init__(self):
super(TradingModel, self).__init__()
##### FILL IN WITH YOUR OWN PARAMETERS FOR display, dumping, TSF etc.
self.market = Market(market_data_generator=MarketDataGenerator())
self.DUMP_PATH = ''
self.FINAL_STRATEGY = 'FX trend'
self.SCALE_FACTOR = 1
self.DEFAULT_PLOT_ENGINE = 'matplotlib'
self.br = self.load_parameters()
return
###### Parameters and signal generations (need to be customised for every model)
def load_parameters(self):
##### FILL IN WITH YOUR OWN BACKTESTING PARAMETERS
br = BacktestRequest()
# get all asset data
br.start_date = "04 Jan 1989"
br.finish_date = datetime.datetime.utcnow().date()
br.spot_tc_bp = 0.5
br.ann_factor = 252
br.plot_start = "01 Apr 2015"
br.calc_stats = True
br.write_csv = False
br.plot_interim = True
br.include_benchmark = True
# have vol target for each signal
br.signal_vol_adjust = True
br.signal_vol_target = 0.1
br.signal_vol_max_leverage = 5
br.signal_vol_periods = 20
br.signal_vol_obs_in_year = 252
br.signal_vol_rebalance_freq = 'BM'
br.signal_vol_resample_freq = None
# have vol target for portfolio
br.portfolio_vol_adjust = True
br.portfolio_vol_target = 0.1
br.portfolio_vol_max_leverage = 5
br.portfolio_vol_periods = 20
br.portfolio_vol_obs_in_year = 252
br.portfolio_vol_rebalance_freq = 'BM'
br.portfolio_vol_resample_freq = None
# tech params
br.tech_params.sma_period = 200
return br
def load_assets(self):
##### FILL IN WITH YOUR ASSET DATA
# for FX basket
full_bkt = ['EURUSD', 'USDJPY', 'GBPUSD', 'AUDUSD', 'USDCAD',
'NZDUSD', 'USDCHF', 'USDNOK', 'USDSEK']
basket_dict = {}
for i in range(0, len(full_bkt)):
basket_dict[full_bkt[i]] = [full_bkt[i]]
basket_dict['FX trend'] = full_bkt
br = self.load_parameters()
self.logger.info("Loading asset data...")
vendor_tickers = ['FRED/DEXUSEU', 'FRED/DEXJPUS', 'FRED/DEXUSUK', 'FRED/DEXUSAL', 'FRED/DEXCAUS',
'FRED/DEXUSNZ', 'FRED/DEXSZUS', 'FRED/DEXNOUS', 'FRED/DEXSDUS']
market_data_request = MarketDataRequest(
start_date = br.start_date, # start date
finish_date = br.finish_date, # finish date
freq = 'daily', # daily data
data_source = 'quandl', # use Quandl as data source
tickers = full_bkt, # ticker (Thalesians)
fields = ['close'], # which fields to download
vendor_tickers = vendor_tickers, # ticker (Quandl)
vendor_fields = ['close'], # which Bloomberg fields to download
cache_algo = 'internet_load_return') # how to return data
asset_df = self.market.fetch_market(market_data_request)
# if web connection fails read from CSV
if asset_df is None:
import pandas
asset_df = pandas.read_csv("d:/fxcta.csv", index_col=0, parse_dates=['Date'],
date_parser = lambda x: pandas.datetime.strptime(x, '%Y-%m-%d'))
# signalling variables
spot_df = asset_df
spot_df2 = None
# asset_df
return asset_df, spot_df, spot_df2, basket_dict
def construct_signal(self, spot_df, spot_df2, tech_params, br):
##### FILL IN WITH YOUR OWN SIGNALS
# use technical indicator to create signals
# (we could obviously create whatever function we wanted for generating the signal dataframe)
tech_ind = TechIndicator()
tech_ind.create_tech_ind(spot_df, 'SMA', tech_params); signal_df = tech_ind.get_signal()
return signal_df
def construct_strategy_benchmark(self):
###### FILL IN WITH YOUR OWN BENCHMARK
tsr_indices = MarketDataRequest(
start_date = self.br.start_date, # start date
finish_date = self.br.finish_date, # finish date
freq = 'daily', # intraday data
data_source = 'quandl', # use Bloomberg as data source
tickers = ["EURUSD"], # tickers to download
vendor_tickers=['FRED/DEXUSEU'],
fields = ['close'], # which fields to download
vendor_fields = ['close'],
cache_algo = 'cache_algo_return') # how to return data)
df = self.market.fetch_market(tsr_indices)
df.columns = [x.split(".")[0] for x in df.columns]
return df
import pytest
import pandas as pd
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from findatapy.util.dataconstants import DataConstants
market = Market(market_data_generator=MarketDataGenerator())
data_constants = DataConstants()
quandl_api_key = data_constants.quandl_api_key
def test_quandl_download():
md_request = MarketDataRequest(start_date="year",
category="fx",
data_source="quandl",
tickers=["AUDJPY"],
quandl_api_key=quandl_api_key)
df = market.fetch_market(md_request)
assert df is not None
def test_yahoo_download():
md_request = MarketDataRequest(
start_date="year", # start date
data_source="yahoo", # use Bloomberg as data source
tickers=["Apple", "Citigroup"], # ticker (findatapy)
fields=["close"], # which fields to download
vendor_tickers=["aapl", "c"], # ticker (Yahoo)
import pandas as pd
# For plotting
from chartpy import Chart, Style
# For loading market data
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from findatapy.util.loggermanager import LoggerManager
# For doing the various volatility _calculations
from finmarketpy.volatility.volstats import VolStats
logger = LoggerManager().getLogger(__name__)
chart = Chart(engine='plotly')
market = Market(market_data_generator=MarketDataGenerator())
# Choose run_example = 0 for everything
# run_example = 1 - calculating difference between realized and implied volatility over Brexit for GBPUSD
# run_example = 2 - calculating realized volatility using different minute frequencies over Brexit for GBPUSD
# run_example = 3 - calculating implied volatility addon associated with days
# run_example = 4 - compare recent implied vs realized volatility for EURUSD
run_example = 0
###### Looking at realized and implied volatility over GBPUSD in the overnight (ON) tenor
if run_example == 1 or run_example == 0:
# Download the whole all market data for GBPUSD for pricing options (vol surface)
md_request = MarketDataRequest(start_date='01 Jun 2016', finish_date='02 Jul 2016',
data_source='bloomberg', cut='10AM', category='fx-vol-market',
tickers=['GBPUSD'],
# License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and limitations under the License.
#
if __name__ == '__main__':
###### below line CRUCIAL when running Windows, otherwise multiprocessing doesn't work! (not necessary on Linux)
from findatapy.util import SwimPool
SwimPool()
from findatapy.market import Market, MarketDataRequest, MarketDataGenerator
market = Market(market_data_generator=MarketDataGenerator())
# choose run_example = 0 for everything
# run_example = 1 - download free tick data from DukasCopy example
# run_example = 2 - download free FX daily data from Quandl
# run_example = 3 - download FX data from FRED
# run_example = 4 - download FX data from Bloomberg
# run_example = 5 - download second FX data from Bloomberg
# run_example = 6 - download free tick data from FXCM example (compare with DukasCopy)
run_example = 6
if run_example == 1 or run_example == 0:
####### DukasCopy examples
# let's download data for 14 Jun 2016 for EUR/USD - the raw data has bid/ask, if we specify close, we calculate
class QuickChart(object):
"""Displays charts from downloaded data, in a single line code call. Ideal for quickly generating charts from sources
including Bloomberg, Quandl, ALFRED/FRED etc.
"""
def __init__(self,
engine='plotly',
data_source='bloomberg',
market_data_generator=MarketDataGenerator()):
self._chart = Chart(engine=engine)
self._market = Market(market_data_generator=market_data_generator)
self._data_source = data_source
def plot_chart(self,
tickers=None,
tickers_rhs=None,
start_date=None,
finish_date=None,
chart_file=None,
chart_type='line',
title='',
fields={'close': 'PX_LAST'},
freq='daily',
source='Web',
brand_label='Cuemacro',
display_brand_label=True,
reindex=False,
additive_index=False,
yoy=False,
plotly_plot_mode='offline_png',
quandl_api_key=dataconstants.quandl_api_key,
fred_api_key=dataconstants.fred_api_key,
alpha_vantage_api_key=dataconstants.alpha_vantage_api_key,
df=None):
if start_date is None:
start_date = datetime.datetime.utcnow().date() - timedelta(days=60)
if finish_date is None:
finish_date = datetime.datetime.utcnow()
if isinstance(tickers, str):
tickers = {tickers: tickers}
elif isinstance(tickers, list):
tickers_dict = {}
for t in tickers:
tickers_dict[t] = t
tickers = tickers_dict
if tickers_rhs is not None:
if isinstance(tickers_rhs, str):
tickers_rhs = {tickers_rhs: tickers_rhs}
elif isinstance(tickers, list):
tickers_rhs_dict = {}
for t in tickers_rhs:
tickers_rhs_dict[t] = t
tickers_rhs = tickers_rhs_dict
tickers.update(tickers_rhs)
else:
tickers_rhs = {}
if df is None:
md_request = MarketDataRequest(
start_date=start_date,
finish_date=finish_date,
freq=freq,
data_source=self._data_source,
tickers=list(tickers.keys()),
vendor_tickers=list(tickers.values()),
fields=list(fields.keys()),
vendor_fields=list(fields.values()),
quandl_api_key=quandl_api_key,
fred_api_key=fred_api_key,
alpha_vantage_api_key=alpha_vantage_api_key)
df = self._market.fetch_market(md_request=md_request)
df = df.fillna(method='ffill')
df.columns = [x.split('.')[0] for x in df.columns]
style = Style(title=title,
chart_type=chart_type,
html_file_output=chart_file,
scale_factor=-1,
height=400,
width=600,
file_output=datetime.date.today().strftime("%Y%m%d") +
" " + title + ".png",
plotly_plot_mode=plotly_plot_mode,
source=source,
brand_label=brand_label,
display_brand_label=display_brand_label)
if reindex:
df = Calculations().create_mult_index_from_prices(df)
style.y_title = 'Reindexed from 100'
if additive_index:
df = (df - df.shift(1)).cumsum()
style.y_title = 'Additive changes from 0'
if yoy:
if freq == 'daily':
obs_in_year = 252
elif freq == 'intraday':
obs_in_year = 1440
df_rets = Calculations().calculate_returns(df)
df = Calculations().average_by_annualised_year(
df_rets, obs_in_year=obs_in_year) * 100
style.y_title = 'Annualized % YoY'
if list(tickers_rhs.keys()) != []:
style.y_axis_2_series = list(tickers_rhs.keys())
style.y_axis_2_showgrid = False
style.y_axis_showgrid = False
return self._chart.plot(df, style=style), df
###### then plot intraday price action around NFP for EUR/USD
start_date = datetime.date.today() - timedelta(days=180)
finish_date = datetime.datetime.utcnow()
md_request = MarketDataRequest(
start_date=start_date, # start date
finish_date=finish_date, # finish date
category='fx',
freq='intraday', # intraday
data_source='bloomberg', # use Bloomberg as data source
tickers=['USDJPY'], # ticker (finmarketpy)
fields=['close'], # which fields to download
cache_algo='internet_load_return') # how to return data
market = Market(market_data_generator=MarketDataGenerator())
df = None
df = market.fetch_market(md_request)
calc = Calculations()
df = calc.calculate_returns(df)
# fetch NFP times from Bloomberg
md_request = MarketDataRequest(
start_date=start_date, # start date
finish_date=finish_date, # finish date
category="events",
freq='daily', # daily data
data_source='bloomberg', # use Bloomberg as data source
tickers=['NFP'],
def fetch_continuous_time_series(self,
md_request,
market_data_generator,
fx_vol_surface=None,
enter_trading_dates=None,
fx_options_trading_tenor=None,
roll_days_before=None,
roll_event=None,
construct_via_currency=None,
fx_options_tenor_for_interpolation=None,
base_depos_tenor=None,
roll_months=None,
cum_index=None,
strike=None,
contract_type=None,
premium_output=None,
position_multiplier=None,
depo_tenor_for_option=None,
freeze_implied_vol=None,
tot_label=None,
cal=None,
output_calculation_fields=None):
if fx_vol_surface is None: fx_vol_surface = self._fx_vol_surface
if enter_trading_dates is None:
enter_trading_dates = self._enter_trading_dates
if market_data_generator is None:
market_data_generator = self._market_data_generator
if fx_options_trading_tenor is None:
fx_options_trading_tenor = self._fx_options_trading_tenor
if roll_days_before is None: roll_days_before = self._roll_days_before
if roll_event is None: roll_event = self._roll_event
if construct_via_currency is None:
construct_via_currency = self._construct_via_currency
if fx_options_tenor_for_interpolation is None:
fx_options_tenor_for_interpolation = self._fx_options_tenor_for_interpolation
if base_depos_tenor is None: base_depos_tenor = self._base_depos_tenor
if roll_months is None: roll_months = self._roll_months
if strike is None: strike = self._strike
if contract_type is None: contract_type = self._contact_type
if premium_output is None: premium_output = self._premium_output
if position_multiplier is None:
position_multiplier = self._position_multiplier
if depo_tenor_for_option is None:
depo_tenor_for_option = self._depo_tenor_for_option
if freeze_implied_vol is None:
freeze_implied_vol = self._freeze_implied_vol
if tot_label is None: tot_label = self._tot_label
if cal is None: cal = self._cal
if output_calculation_fields is None:
output_calculation_fields = self._output_calculation_fields
# Eg. we construct EURJPY via EURJPY directly (note: would need to have sufficient options/forward data for this)
if construct_via_currency == 'no':
if fx_vol_surface is None:
# Download FX spot, FX forwards points and base depos etc.
market = Market(market_data_generator=market_data_generator)
md_request_download = MarketDataRequest(md_request=md_request)
fx_conv = FXConv()
# CAREFUL: convert the tickers to correct notation, eg. USDEUR => EURUSD, because our data
# should be fetched in correct convention
md_request_download.tickers = [
fx_conv.correct_notation(x) for x in md_request.tickers
]
md_request_download.category = 'fx-vol-market'
md_request_download.fields = 'close'
md_request_download.abstract_curve = None
md_request_download.fx_options_tenor = fx_options_tenor_for_interpolation
md_request_download.base_depos_tenor = base_depos_tenor
# md_request_download.base_depos_currencies = []
forwards_market_df = market.fetch_market(md_request_download)
else:
forwards_market_df = None
# Now use the original tickers
return self.construct_total_return_index(
md_request.tickers,
forwards_market_df,
fx_vol_surface=fx_vol_surface,
enter_trading_dates=enter_trading_dates,
fx_options_trading_tenor=fx_options_trading_tenor,
roll_days_before=roll_days_before,
roll_event=roll_event,
fx_options_tenor_for_interpolation=
fx_options_tenor_for_interpolation,
roll_months=roll_months,
cum_index=cum_index,
strike=strike,
contract_type=contract_type,
premium_output=premium_output,
position_multiplier=position_multiplier,
freeze_implied_vol=freeze_implied_vol,
depo_tenor_for_option=depo_tenor_for_option,
tot_label=tot_label,
cal=cal,
output_calculation_fields=output_calculation_fields)
else:
# eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency
# Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns)
total_return_indices = []
for tick in md_request.tickers:
base = tick[0:3]
terms = tick[3:6]
md_request_base = MarketDataRequest(md_request=md_request)
md_request_base.tickers = base + construct_via_currency
md_request_terms = MarketDataRequest(md_request=md_request)
md_request_terms.tickers = terms + construct_via_currency
# Construct the base and terms separately (ie. AUDJPY => AUDUSD & JPYUSD)
base_vals = self.fetch_continuous_time_series(
md_request_base,
market_data_generator,
fx_vol_surface=fx_vol_surface,
enter_trading_dates=enter_trading_dates,
fx_options_trading_tenor=fx_options_trading_tenor,
roll_days_before=roll_days_before,
roll_event=roll_event,
fx_options_tenor_for_interpolation=
fx_options_tenor_for_interpolation,
base_depos_tenor=base_depos_tenor,
roll_months=roll_months,
cum_index=cum_index,
strike=strike,
contract_type=contract_type,
premium_output=premium_output,
position_multiplier=position_multiplier,
depo_tenor_for_option=depo_tenor_for_option,
freeze_implied_vol=freeze_implied_vol,
tot_label=tot_label,
cal=cal,
output_calculation_fields=output_calculation_fields,
construct_via_currency='no')
terms_vals = self.fetch_continuous_time_series(
md_request_terms,
market_data_generator,
fx_vol_surface=fx_vol_surface,
enter_trading_dates=enter_trading_dates,
fx_options_trading_tenor=fx_options_trading_tenor,
roll_days_before=roll_days_before,
roll_event=roll_event,
fx_options_tenor_for_interpolation=
fx_options_tenor_for_interpolation,
base_depos_tenor=base_depos_tenor,
roll_months=roll_months,
cum_index=cum_index,
strike=strike,
contract_type=contract_type,
position_multiplier=position_multiplier,
depo_tenor_for_option=depo_tenor_for_option,
freeze_implied_vol=freeze_implied_vol,
tot_label=tot_label,
cal=cal,
output_calculation_fields=output_calculation_fields,
construct_via_currency='no')
# Special case for USDUSD case (and if base or terms USD are USDUSD
if base + terms == construct_via_currency + construct_via_currency:
base_rets = self._calculations.calculate_returns(base_vals)
cross_rets = pd.DataFrame(0,
index=base_rets.index,
columns=base_rets.columns)
elif base + construct_via_currency == construct_via_currency + construct_via_currency:
cross_rets = -self._calculations.calculate_returns(
terms_vals)
elif terms + construct_via_currency == construct_via_currency + construct_via_currency:
cross_rets = self._calculations.calculate_returns(
base_vals)
else:
base_rets = self._calculations.calculate_returns(base_vals)
terms_rets = self._calculations.calculate_returns(
terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# First returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self._calculations.create_mult_index(cross_rets)
cross_vals.columns = [tick + '-option-tot.close']
total_return_indices.append(cross_vals)
return self._calculations.join(total_return_indices, how='outer')
def plot_vol_curves(self):
if self._fin_fx_vol_surface is not None:
self._fin_fx_vol_surface.plotVolCurves()
if __name__ == '__main__':
from findatapy.market import Market, MarketDataRequest, MarketDataGenerator
ticker = 'EURUSD'
start_date = '06 Oct 2020'
md_request = MarketDataRequest(start_date=start_date,
finish_date=start_date,
data_source='bloomberg',
cut='LDN',
category='fx-vol-market',
tickers=ticker)
import os
import pandas as pd
if os.path.exists(ticker + '.parquet'):
market_df = pd.read_parquet(ticker + '.parquet')
else:
market_df = Market(market_data_generator=MarketDataGenerator()
).fetch_market(md_request)
market_df.to_parquet(ticker + '.parquet')
fx_vol_surface = FXVolSurface(market_df=market_df)
fx_vol_surface.build_vol_surface(start_date, asset='EURUSD')
