def __init__(self):
DataBase.__init__(self)
self.months = { 'jan': '01', 'feb': '02', 'mar': '03', 'apr': '04', 'may': '05', 'jun': '06', \
'jul': '07', 'aug': '08', 'sep': '09', 'oct': '10', 'nov': '11', 'dec': '12' }
self.weekdays = [ 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun' ]
self.dayz = [ 'Today', 'Yesterday' ]
self.filterz = [ ' ' ]
def __init__(self, ticker, freq='A', bulk_option=False):
DataBase.__init__(self)
self.ticker = ticker
self.freq = freq
self.verbose = False
self.bulk_option = bulk_option
self.bulk_bool = bulk_option
if type(self.bulk_bool) != type(True):
self.bulk_bool = True
def cboe_option_chain(ticker, head=False):
db = DataBase()
db.head = head
url = 'http://www.cboe.com/delayedquote/quote-table-download'
try:
driver = db._load_driver(caps='none')
driver.get(url)
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@id="txtTicker"]')))
driver.find_element_by_xpath('//input[@id="txtTicker"]').send_keys(
ticker)
driver.find_element_by_xpath('//input[@id="txtTicker"]').send_keys(
Keys.ENTER)
db._downloads_done('quotedata.dat')
driver.close()
driver.quit()
except:
print('Failed to load data...')
driver.close()
driver.quit()
return None
df = pd.read_csv(db.download_path + '/quotedata.dat',
error_bad_lines=False,
warn_bad_lines=False)
underlying_price = float(df.columns[-2])
df = pd.read_csv(db.download_path + '/quotedata.dat',
skiprows=[0, 1, 3],
error_bad_lines=False,
warn_bad_lines=False)
df['underlying'] = underlying_price
os.remove(db.download_path + '/quotedata.dat')
df.columns = [
col.replace(' ', '_').lower().replace('_date', '')
for col in df.columns
]
puts = df.loc[:, ['expiration', 'puts'] +
[col for col in df.columns if '.1' in col] +
['strike', 'underlying']]
puts.columns = [col.replace('.1', '') for col in puts.columns]
calls = df.loc[:, [col for col in df.columns if '.1' not in col]]
calls.drop('puts', inplace=True, axis=1)
return calls, puts
def __init__(self, ticker, freq = 'A'):
DataBase.__init__(self)
self.ticker = ticker
self.freq = freq
def _download_prices(date):
'''
input: datetime object
output: pandas dataframe with prices for all available futures for the
specified date
'''
db = DataBase()
errors = []
if type(date) == type('str'):
date = pd.to_datetime(date, format='%Y-%m-%d')
y = str(date.year)
if len(str(date.month)) == 2:
m = str(date.month)
else:
m = '0' + str(date.month)
if len(str(date.day)) == 2:
d = str(date.day)
else:
d = '0' + str(date.day)
try:
url = f'https://www.mrci.com/ohlc/{y}/{y[-2:]+m+d}.php'
soup = db._get_session(url)
df = pd.read_html(str(soup.find('map').find_next('table')))[0]
try:
futures_lookup = pd.read_csv(
os.path.dirname(__file__) +
'/futures_lookup.csv').name.tolist()
except:
futures_lookup = pd.read_csv(
os.path.dirname(__file__) +
'\\futures_lookup.csv').name.tolist()
indices = [
i for i, j in enumerate(df.iloc[:, 0]) if j in futures_lookup
]
columns = [
'month', 'date', 'open', 'high', 'low', 'close', 'change',
'volume', 'open_interest', 'change_in_oi'
]
if len(df.columns) == 11:
df = df.iloc[indices[0]:-2, :len(df.columns) - 1]
else:
df = df.iloc[indices[0]:-2, :]
#session.close()
except:
errors.append(date)
#session.close()
return errors
df.columns = columns
#[ i for i in np.unique(df.month).tolist() if i not in futures_lookup ]
first = True
for i in range(1, len(indices)):
temp = df.loc[indices[i - 1] + 1:indices[i] - 2].copy()
temp['future'] = df.loc[indices[i - 1], 'month']
if first:
out = temp.copy()
first = False
else:
out = out.append(temp)
out = out[out.iloc[:, 1] != 'Total Volume and Open Interest']
# out.to_csv('futures.csv')
out.index = [date] * len(
out
) #pd.to_datetime( [ f'{i[-2:]}/{i[2:4]}/{i[:2]}' for i in out.date ] )
out.replace('\+', '', regex=True, inplace=True)
out.replace('unch', np.nan, inplace=True)
out = db._col_to_float(out)
return dd.from_pandas(out, npartitions=1)
def __init__(self, ticker):
DataBase.__init__(self)
self.ticker = ticker
def cftc(report_type='futures_traders', year=2000):
'''
# report type options:
'disaggregated_futures'
'disaggregated_combined'
'futures_traders'
'futures_traders_combined'
'futures'
'futures_combined'
'commodity_index'
'''
db = DataBase()
if report_type == 'disaggregated_futures':
if year < 2016:
last_url = [
'https://cftc.gov/files/dea/history/fut_disagg_txt_hist_2006_2016.zip'
]
url_list = last_url + [
f'https://cftc.gov/files/dea/history/fut_disagg_txt_{i}.zip'
for i in range(2016,
dt.datetime.today().year + 1)
]
else:
url_list = [
f'https://cftc.gov/files/dea/history/fut_disagg_txt_{i}.zip'
for i in range(year,
dt.datetime.today().year + 1)
]
elif report_type == 'disaggregated_combined':
# Disaggregated Futures-and-Options Combined Reports:
if year < 2016:
last_url = [
'https://cftc.gov/files/dea/history/com_disagg_txt_hist_2006_2016.zip'
]
url_list = last_url + [
f'https://cftc.gov/files/dea/history/com_disagg_txt_{i}.zip'
for i in range(2016,
dt.datetime.today().year + 1)
]
else:
url_list = [
f'https://cftc.gov/files/dea/history/com_disagg_txt_{i}.zip'
for i in range(year,
dt.datetime.today().year + 1)
]
elif report_type == 'futures_traders':
# Disaggregated Futures-and-Options Combined Reports:
if year < 2016:
last_url = [
'https://cftc.gov/files/dea/history/fin_fut_txt_2006_2016.zip'
]
url_list = last_url + [
f'https://cftc.gov/files/dea/history/fut_fin_txt_{i}.zip'
for i in range(2016,
dt.datetime.today().year + 1)
]
else:
url_list = [
f'https://cftc.gov/files/dea/history/fut_fin_txt_{i}.zip'
for i in range(year,
dt.datetime.today().year + 1)
]
elif report_type == 'futures_traders_combined':
# Disaggregated Futures-and-Options Combined Reports:
if year < 2016:
last_url = [
'https://cftc.gov/files/dea/history/fin_com_txt_2006_2016.zip'
]
url_list = last_url + [
f'https://cftc.gov/files/dea/history/com_fin_txt_{i}.zip'
for i in range(2016,
dt.datetime.today().year + 1)
]
else:
url_list = [
f'https://cftc.gov/files/dea/history/com_fin_txt_{i}.zip'
for i in range(year,
dt.datetime.today().year + 1)
]
elif report_type == 'futures':
# Disaggregated Futures-and-Options Combined Reports:
if year < 2016:
last_url = [
'https://cftc.gov/files/dea/history/deacot1986_2016.zip'
]
url_list = last_url + [
f'https://cftc.gov/files/dea/history/deacot{i}.zip'
for i in range(2016,
dt.datetime.today().year + 1)
]
else:
url_list = [
f'https://cftc.gov/files/dea/history/deacot{i}.zip'
for i in range(year,
dt.datetime.today().year + 1)
]
elif report_type == 'futures_combined':
# Disaggregated Futures-and-Options Combined Reports:
if year < 2016:
last_url = [
'https://cftc.gov/files/dea/history/deahistfo_1995_2016.zip'
]
url_list = last_url + [
f'https://cftc.gov/files/dea/history/deahistfo{i}.zip'
for i in range(2016,
dt.datetime.today().year + 1)
]
else:
url_list = [
f'https://cftc.gov/files/dea/history/deahistfo{i}.zip'
for i in range(year,
dt.datetime.today().year + 1)
]
elif report_type == 'commodity_index':
# Disaggregated Futures-and-Options Combined Reports:
if year < 2016:
last_url = [
'https://cftc.gov/files/dea/history/dea_cit_txt_2006_2016.zip'
]
url_list = last_url + [
f'https://cftc.gov/files/dea/history/dea_cit_txt_{i}.zip'
for i in range(2016,
dt.datetime.today().year + 1)
]
else:
url_list = [
f'https://cftc.gov/files/dea/history/dea_cit_txt_{i}.zip'
for i in range(year,
dt.datetime.today().year + 1)
]
else:
print('Report type not valied.')
return None
df = pd.concat([db._load_zip_file(url) for url in url_list])
df.iloc[:, 2] = [i.split(' ')[0] for i in df.iloc[:, 2]]
try:
df.index = pd.to_datetime(df.iloc[:, 2], format='%Y-%m-%d')
except:
df.index = [
pd.to_datetime(i, format='%Y-%m-%d')
if '/' not in i else pd.to_datetime(i, format='%m/%d/%Y')
for i in df.iloc[:, 2]
]
df.index.name = 'date'
df.drop(df.columns[2], axis=1, inplace=True)
df.drop(df.columns[1], axis=1, inplace=True)
df.columns = [col.replace(' ', '_').lower() for col in df.columns]
df.iloc[1, :] = df.iloc[1, :].astype('str')
tmp = [i for i, col in enumerate(df.columns) if col == 'contract_units'][0]
df.iloc[:, 3:tmp] = df.iloc[:, 3:tmp].replace('\.', '0',
regex=True).astype('float')
return df
def __init__(self, ticker, freq = 'A', countryCode = ''):
DataBase.__init__(self)
self.ticker = ticker
self.countryCode = countryCode
self.freq = freq