def create_test_raw_data(self,
ticker_list=None,
start_date=None,
finish_date=None,
folder_prefix=None):
"""Downloads FX tick data from DukasCopy and then dumps each ticker in a separate HDF5 file if a folder is specified.
If no folder is specified returns a list of DataFrames (note: can be a very large list in memory)
Parameters
----------
ticker_list : str (list)
List of FX tickers to download
start_date : datetime/str
Start date of FX tick data download
finish_date : datetime/str
Finish date of FX tick data download
folder_prefix : str
Folder to dump everything
Returns
-------
DataFrame (list)
"""
from findatapy.market import MarketDataRequest, MarketDataGenerator, Market
if start_date is None and finish_date is None:
finish_date = datetime.datetime.utcnow().date() - timedelta(
days=30)
start_date = finish_date - timedelta(days=30 * 15)
start_date = self._compute_random_date(start_date, finish_date)
finish_date = start_date + timedelta(days=90)
df_list = []
result = []
# From multiprocessing.dummy import Pool # threading
from multiprocess.pool import Pool # actuall new processes
import time
# If we don't specify a folder
if folder_prefix is None:
mini_ticker_list = self._split_list(ticker_list, 2)
# Use multiprocess to speed up the download
for mini in mini_ticker_list:
pool = Pool(processes=2)
for ticker in mini:
time.sleep(1)
self.logger.info("Loading " + ticker)
md_request = MarketDataRequest(
start_date=start_date,
finish_date=finish_date,
category='fx',
tickers=ticker,
fields=['bid', 'ask', 'bidv', 'askv'],
data_source='dukascopy',
freq='tick')
# self._download(md_request)
result.append(
pool.apply_async(self._download,
args=(
md_request,
folder_prefix,
)))
pool.close()
pool.join()
else:
market = Market(market_data_generator=MarketDataGenerator())
for ticker in ticker_list:
md_request = MarketDataRequest(
start_date=start_date,
finish_date=finish_date,
category='fx',
tickers=ticker,
fields=['bid', 'ask', 'bidv', 'askv'],
data_source='dukascopy',
freq='tick')
df = market.fetch_market(md_request=md_request)
df.columns = ['bid', 'ask', 'bidv', 'askv']
df['venue'] = 'dukascopy'
df['ticker'] = ticker
# print(df)
if folder_prefix is not None:
self.dump_hdf5_file(df,
folder_prefix + "_" + ticker + ".h5")
# df.to_csv(folder_prefix + "_" + ticker + ".csv") # CSV files can be very large, so try to avoid
else:
df_list.append(df)
return df_list
"""
# For plotting
from chartpy import Chart, Style
# For loading market data
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from findatapy.util.loggermanager import LoggerManager
from finmarketpy.curve.volatility.fxvolsurface import FXVolSurface
logger = LoggerManager().getLogger(__name__)
chart = Chart(engine='plotly')
market = Market(market_data_generator=MarketDataGenerator())
# Choose run_example = 0 for everything
# run_example = 1 - plot GBPUSD 1W implied vol
# run_example = 2 - get GBPUSD vol surface for a date and plot interpolated vol surface and implied PDF
# run_example = 3 - do an animation of GBPUSD implied vol surface over this period
# run_example = 4 - get implied vol for a particular strike, interpolating the surface
# run_example = 5 - get USDJPY vol surface around US presidential election and plot
run_example = 3
###### Fetch market data for pricing GBPUSD FX options over Brexit vote (ie. FX spot, FX forwards, FX deposits and FX vol quotes)
###### Show how to plot ATM 1M implied_vol vol time series
if run_example == 1 or run_example == 0:
# Download the whole all market data for GBPUSD for pricing options (vol surface)
def load_tickers():
logger = LoggerManager.getLogger(__name__)
market = Market(market_data_generator=MarketDataGenerator())
DataConstants.market_thread_technique = 'thread'
# get recent list of S&P500 (some of these will fail eg. BRK.B because incorrect ticker - findatapy should handle
# this error gracefully)
tickers = ["MMM",
"ABT",
"ABBV",
"ACN",
"ATVI",
"AYI",
"ADBE",
"AAP",
"AES",
"AET",
"AMG",
"AFL",
"A",
"APD",
"AKAM",
"ALK",
"ALB",
"AGN",
"LNT",
"ALXN",
"ALLE",
"ADS",
"ALL",
"GOOGL",
"GOOG",
"MO",
"AMZN",
"AEE",
"AAL",
"AEP",
"AXP",
"AIG",
"AMT",
"AWK",
"AMP",
"ABC",
"AME",
"AMGN",
"APH",
"APC",
"ADI",
"ANTM",
"AON",
"APA",
"AIV",
"AAPL",
"AMAT",
"ADM",
"ARNC",
"AJG",
"AIZ",
"T",
"ADSK",
"ADP",
"AN",
"AZO",
"AVB",
"AVY",
"BHI",
"BLL",
"BAC",
"BK",
"BCR",
"BAX",
"BBT",
"BDX",
"BBBY",
"BRK.B",
"BBY",
"BIIB",
"BLK",
"HRB",
"BA",
"BWA",
"BXP",
"BSX",
"BMY",
"AVGO",
"BF.B",
"CHRW",
"CA",
"COG",
"CPB",
"COF",
"CAH",
"HSIC",
"KMX",
"CCL",
"CAT",
"CBG",
"CBS",
"CELG",
"CNC",
"CNP",
"CTL",
"CERN",
"CF",
"SCHW",
"CHTR",
"CHK",
"CVX",
"CMG",
"CB",
"CHD",
"CI",
"XEC",
"CINF",
"CTAS",
"CSCO",
"C",
"CFG",
"CTXS",
"CLX",
"CME",
"CMS",
"COH",
"KO",
"CTSH",
"CL",
"CMCSA",
"CMA",
"CAG",
"CXO",
"COP",
"ED",
"STZ",
"GLW",
"COST",
"COTY",
"CCI",
"CSRA",
"CSX",
"CMI",
"CVS",
"DHI",
"DHR",
"DRI",
"DVA",
"DE",
"DLPH",
"DAL",
"XRAY",
"DVN",
"DLR",
"DFS",
"DISCA",
"DISCK",
"DG",
"DLTR",
"D",
"DOV",
"DOW",
"DPS",
"DTE",
"DD",
"DUK",
"DNB",
"ETFC",
"EMN",
"ETN",
"EBAY",
"ECL",
"EIX",
"EW",
"EA",
"EMR",
"ENDP",
"ETR",
"EVHC",
"EOG",
"EQT",
"EFX",
"EQIX",
"EQR",
"ESS",
"EL",
"ES",
"EXC",
"EXPE",
"EXPD",
"ESRX",
"EXR",
"XOM",
"FFIV",
"FB",
"FAST",
"FRT",
"FDX",
"FIS",
"FITB",
"FSLR",
"FE",
"FISV",
"FLIR",
"FLS",
"FLR",
"FMC",
"FTI",
"FL",
"F",
"FTV",
"FBHS",
"BEN",
"FCX",
"FTR",
"GPS",
"GRMN",
"GD",
"GE",
"GGP",
"GIS",
"GM",
"GPC",
"GILD",
"GPN",
"GS",
"GT",
"GWW",
"HAL",
"HBI",
"HOG",
"HAR",
"HRS",
"HIG",
"HAS",
"HCA",
"HCP",
"HP",
"HES",
"HPE",
"HOLX",
"HD",
"HON",
"HRL",
"HST",
"HPQ",
"HUM",
"HBAN",
"IDXX",
"ITW",
"ILMN",
"IR",
"INTC",
"ICE",
"IBM",
"IP",
"IPG",
"IFF",
"INTU",
"ISRG",
"IVZ",
"IRM",
"JEC",
"JBHT",
"SJM",
"JNJ",
"JCI",
"JPM",
"JNPR",
"KSU",
"K",
"KEY",
"KMB",
"KIM",
"KMI",
"KLAC",
"KSS",
"KHC",
"KR",
"LB",
"LLL",
"LH",
"LRCX",
"LEG",
"LEN",
"LVLT",
"LUK",
"LLY",
"LNC",
"LLTC",
"LKQ",
"LMT",
"L",
"LOW",
"LYB",
"MTB",
"MAC",
"M",
"MNK",
"MRO",
"MPC",
"MAR",
"MMC",
"MLM",
"MAS",
"MA",
"MAT",
"MKC",
"MCD",
"MCK",
"MJN",
"MDT",
"MRK",
"MET",
"MTD",
"KORS",
"MCHP",
"MU",
"MSFT",
"MAA",
"MHK",
"TAP",
"MDLZ",
"MON",
"MNST",
"MCO",
"MS",
"MOS",
"MSI",
"MUR",
"MYL",
"NDAQ",
"NOV",
"NAVI",
"NTAP",
"NFLX",
"NWL",
"NFX",
"NEM",
"NWSA",
"NWS",
"NEE",
"NLSN",
"NKE",
"NI",
"NBL",
"JWN",
"NSC",
"NTRS",
"NOC",
"NRG",
"NUE",
"NVDA",
"ORLY",
"OXY",
"OMC",
"OKE",
"ORCL",
"PCAR",
"PH",
"PDCO",
"PAYX",
"PYPL",
"PNR",
"PBCT",
"PEP",
"PKI",
"PRGO",
"PFE",
"PCG",
"PM",
"PSX",
"PNW",
"PXD",
"PBI",
"PNC",
"RL",
"PPG",
"PPL",
"PX",
"PCLN",
"PFG",
"PG",
"PGR",
"PLD",
"PRU",
"PEG",
"PSA",
"PHM",
"PVH",
"QRVO",
"PWR",
"QCOM",
"DGX",
"RRC",
"RTN",
"O",
"RHT",
"REGN",
"RF",
"RSG",
"RAI",
"RHI",
"ROK",
"COL",
"ROP",
"ROST",
"RCL",
"R",
"CRM",
"SCG",
"SLB",
"SNI",
"STX",
"SEE",
"SRE",
"SHW",
"SIG",
"SPG",
"SWKS",
"SLG",
"SNA",
"SO",
"LUV",
"SWN",
"SE",
"SPGI",
"SWK",
"SPLS",
"SBUX",
"STT",
"SRCL",
"SYK",
"STI",
"SYMC",
"SYF",
"SYY",
"TROW",
"TGT",
"TEL",
"TGNA",
"TDC",
"TSO",
"TXN",
"TXT",
"COO",
"HSY",
"TRV",
"TMO",
"TIF",
"TWX",
"TJX",
"TMK",
"TSS",
"TSCO",
"TDG",
"RIG",
"TRIP",
"FOXA",
"FOX",
"TSN",
"UDR",
"ULTA",
"USB",
"UA",
"UAA",
"UNP",
"UAL",
"UNH",
"UPS",
"URI",
"UTX",
"UHS",
"UNM",
"URBN",
"VFC",
"VLO",
"VAR",
"VTR",
"VRSN",
"VRSK",
"VZ",
"VRTX",
"VIAB",
"V",
"VNO",
"VMC",
"WMT",
"WBA",
"DIS",
"WM",
"WAT",
"WEC",
"WFC",
"HCN",
"WDC",
"WU",
"WRK",
"WY",
"WHR",
"WFM",
"WMB",
"WLTW",
"WYN",
"WYNN",
"XEL",
"XRX",
"XLNX",
"XL",
"XYL",
"YHOO",
"YUM",
"ZBH",
"ZION",
"ZTS",
]
# download equities data from Yahoo
md_request = MarketDataRequest(
start_date="decade", # start date
data_source='yahoo', # use Bloomberg as data source
tickers=tickers, # ticker (findatapy)
fields=['close'], # which fields to download
vendor_tickers=tickers, # ticker (Yahoo)
vendor_fields=['Close']) # which Bloomberg fields to download)
logger.info("Loading data with threading")
df = market.fetch_market(md_request)
logger.info("Loading data with multiprocessing")
DataConstants.market_thread_technique = 'multiprocessing'
df = market.fetch_market(md_request)
logger.info("Loaded data with multiprocessing")