def jpy_func(d_pref, pyc_pref, exec_path):
driver = webdriver.Chrome(executable_path=exec_path)
# Gets home page of jpn data
driver.get('http://www3.boj.or.jp/market/en/menu_m.htm')
first = driver.find_element_by_xpath('/html/body/ul/li[2]/a[1]').text # str
second = driver.find_element_by_xpath('/html/body/ul/li[2]/a[2]').text # str
year_list = [first, second]
for year in year_list:
tup = data_from_page_of_link_lists(driver, year)
dates_text_init = tup[0]
data_entries_init = tup[1]
data_entries = limit_dp(removes_percentage_signs(data_entries_init))
dates_text = format_jpy_dates(dates_text_init, year)
with open(pyc_pref + 'jpy_' + year + '.csv', 'w') as csv_file:
write = writer(csv_file)
for x, y in zip(dates_text, data_entries):
write.writerow([x, y])
join_csvs(pyc_pref + 'jpy_' + first + '.csv', pyc_pref + 'jpy_' + second + '.csv', pyc_pref + 'jpy_on_int.csv')
if path.exists(pyc_pref + 'jpy_on_data.csv'):
join_csvs(pyc_pref + 'jpy_on_int.csv', pyc_pref + 'jpy_on_data.csv', pyc_pref + 'jpy_on.csv')
file_path = pyc_pref + 'jpy_on.csv'
output_func(file_path, 1, pyc_pref + 'jpy_on')
else:
output_func(pyc_pref + 'jpy_on_int.csv', 1, pyc_pref + 'jpy_on')
driver.close()
def usd_func(d_pref, pyc_pref, exec_path):
column_index = 2
csv_name = pyc_pref + 'usd_on'
driver = webdriver.Chrome(executable_path=exec_path)
driver.get('https://apps.newyorkfed.org/markets/autorates/fed%20funds')
driver.find_element_by_xpath('/html/body/div/div[2]/div[2]/div/table[1]/tbody/tr/td/table[1]/'
'tbody/tr[1]/td/table[2]/tbody/tr/td[1]/p[1]/a[2]').click()
first_date_field = driver.find_element_by_xpath('/html/body/div/div[2]/div[2]/div/table/tbody/'
'tr/td/div/table/tbody/tr/td/form/table[1]/tbody/tr[3]/td/input[1]')
first_date_field.send_keys('07/03/2000')
last_date_field = driver.find_element_by_xpath('/html/body/div/div[2]/div[2]/div/table/tbody/'
'tr/td/div/table/tbody/tr/td/form/table[1]/tbody/tr[3]/td/input[2]')
# gets today's date and writes it in the necessary fashion
today = date.today()
date_today_string = today.strftime('%m/%d/%Y')
# writes today's date in the right field
last_date_field.send_keys(date_today_string)
# clicks find
driver.find_element_by_xpath('/html/body/div/div[2]/div[2]/div/table/tbody/tr/td/div/table/tbody/'
'tr/td/form/table[2]/tbody/tr[3]/td/input').click()
# clicks the Excel button
driver.find_element_by_xpath('/html/body/div/div[2]/div[2]/div/table/tbody/tr/td/div[1]/table/tbody/'
'tr/td/table/tbody/tr/td/table[1]/tbody/tr[3]/td/table/tbody/tr/td[2]/a[1]').click()
sleep(5)
# * means all if need specific format then *.csv
list_of_files = glob(d_pref + '*.xls')
latest_file = max(list_of_files, key=path.getctime)
output_func(latest_file, column_index, csv_name)
driver.close()
def chf_formatting(file_path, int_csv_name, final_csv_name, p_pref):
columns = defaultdict(list)
with open(file_path) as f:
read = reader(f, delimiter=';')
max_row_count = 0
row_list = []
for row in read:
if len(row) > max_row_count:
max_row_count = len(row)
# Tracks back to the start of the csv
f.seek(0)
for row1 in read:
if not row1 == []:
while len(row1) < max_row_count:
row1.append(' ')
row_list.append(row1)
for row in row_list:
for (i, v) in enumerate(row):
columns[i].append(v)
dates_init = columns[0]
data_init = columns[2]
with open(int_csv_name + '.csv', 'w') as csv_file:
write = writer(csv_file)
for x, y in zip(dates_init, data_init):
write.writerow([x, y])
output_func(int_csv_name + '.csv', 1, final_csv_name)
def eur_func(d_pref, pyc_pref, exec_path):
file_name = 'data.csv'
column_index = 1
csv_name = pyc_pref + 'eur_on'
driver = webdriver.Chrome(executable_path=exec_path)
driver.get('https://sdw.ecb.europa.eu/quickviewexport.do?SERIES_KEY=198.EON.D.EONIA_TO.RATE&type=csv')
sleep(5)
output_func(d_pref + file_name, column_index, csv_name)
driver.close()
def cad_func(d_pref, pyc_pref, exec_path):
downloaded_file = 'lookup.csv'
column_index_input = 1
new_csv_name = pyc_pref + 'cad_on'
old_data_file_name_input = pyc_pref + 'cad_on'
driver = webdriver.Chrome(executable_path=exec_path)
driver.get('https://www.bankofcanada.ca/rates/interest-rates/canadian-interest-rates/')
# selects option for range
driver.find_element_by_xpath('//*[@id="rangeType_dates"]').click()
# finds first date field
first_date_field = driver.find_element_by_xpath('//*[@id="dFrom"]')
# gets the date 10 years ago
now = datetime.now()
current_year = now.year
current_month = now.month
current_day = now.day
past_year = current_year - 10
past_date = date(past_year, current_month, current_day).isoformat()
# fills in the field
first_date_field.send_keys(past_date)
# finds last date field
last_date_field = driver.find_element_by_xpath('//*[@id="dTo"]')
# gets today's date
today = date.today()
date_today_string = today.strftime('%Y-%m-%d')
# fills field in
last_date_field.send_keys(date_today_string)
# selects overnight money market
driver.find_element_by_xpath('//*[@id="V39050"]').click()
# submits
driver.find_element_by_xpath('//*[@id="lookupForm"]/div[4]/button[2]').click()
sleep(5)
# downloads csv file
driver.find_element_by_xpath('//*[@id="cfct-build-39876"]/div/div/div[4]/div/span/a[3]').click()
sleep(5)
if path.exists(old_data_file_name_input + '_data.csv'):
join_hist_with_pres(downloaded_file, column_index_input, new_csv_name, old_data_file_name_input, d_pref)
else:
output_func(downloaded_file, column_index_input, new_csv_name)
if path.exists(d_pref + downloaded_file):
remove(d_pref + downloaded_file)
driver.close()
def gbp_func(d_pref, pyc_pref, exec_path):
file_name = 'Bank of England Database.csv'
column_index = 1
csv_name = pyc_pref + 'gbp_on'
driver = webdriver.Chrome(executable_path=exec_path)
driver.get('https://www.bankofengland.co.uk/boeapps/database/fromshowcolumns.asp?'
'Travel=NIxSUx&FromSeries=1&ToSeries=50&DAT=ALL&FNY=&CSVF=TT&html.x=191&html.y=39&C=5JK&Filter=N')
csv_button = driver.find_element_by_xpath('/html/body/div[2]/div/section[2]/div/div[1]/div[1]/a[2]')
sleep(2)
csv_button.click()
sleep(5)
output_func(d_pref + file_name, column_index, csv_name)
driver.close()
def sek_func(d_pref, pyc_pref, exec_path):
column_index_to_format_init = 3
column_index_for_pathway = 1
csv_name = pyc_pref + 'sek_on'
driver = webdriver.Chrome(executable_path=exec_path)
driver.get('https://www.riksbank.se/en-gb/statistics/search-interest--exchange-rates/')
driver.maximize_window()
# Scrolls down half a window (1080 is a fullHD monitor)
driver.execute_script("window.scrollTo(0, 540)")
sleep(5)
# Dropdown Riksbank interest rates
driver.find_element_by_xpath('//*[@id="swea-searchform"]/fieldset[1]/div[1]/div[1]/a').click()
sleep(5)
# Dropdown Riksbank key interest rates
driver.find_element_by_xpath('//*[@id="swea-searchform"]/fieldset[1]/div[1]/div[1]/div[2]/a').click()
sleep(5)
# Selects repo rate
driver.find_element_by_xpath('//*[@id="swea-searchform"]/fieldset[1]/div[1]/div[1]/div[2]/div/label[4]').click()
sleep(5)
# Past date
past_date_field = driver.find_element_by_xpath('//*[@id="datetime-from"]')
past_date_field.clear()
past_date_field.send_keys('01/01/2009')
# Toggles dropdown menu so that it disappears
driver.find_element_by_xpath('//*[@id="swea-searchform"]/fieldset[1]/div[1]/div[1]/a').click()
driver.execute_script("window.scrollTo(0, 1080)")
sleep(5)
# Shows result
driver.find_element_by_xpath('//*[@id="swea-searchform"]/fieldset[2]/div/button').click()
sleep(5)
# Downloads csv
driver.find_element_by_xpath('//*[@id="main-content"]/div/div[1]/div[1]/article/div/div[3]/div[1]/a[2]').click()
sleep(8)
list_of_files = glob(d_pref + '*.csv') # * means all if need specific format then *.csv
latest_file = max(list_of_files, key=path.getctime)
swedish_csv_formatter(latest_file, column_index_to_format_init)
output_func(latest_file, column_index_for_pathway, csv_name)
driver.close()
def sgd_func(d_pref, pyc_pref, exec_path):
file_name = 'Domestic Interest Rates.csv'
csv_name = pyc_pref + 'sgd'
final_csv_name = pyc_pref + 'sgd_on'
driver = webdriver.Chrome(executable_path=exec_path)
driver.get('https://secure.mas.gov.sg/dir/domesticinterestrates.aspx')
sleep(5)
first_fields_xpath = ['//*[@id="ctl00_ContentPlaceHolder1_StartYearDropDownList"]',
'//*[@id="ctl00_ContentPlaceHolder1_StartMonthDropDownList"]',
'//*[@id="ctl00_ContentPlaceHolder1_FrequencyDropDownList"]']
first_fields_entries = ['1987', 'Jan', 'Daily']
for a in range(len(first_fields_entries)):
driver.find_element_by_xpath(first_fields_xpath[a]).send_keys(first_fields_entries[a])
now = datetime.now()
current_month = month_name[now.month]
sleep(5)
second_fields_xpath = ['//*[@id="ctl00_ContentPlaceHolder1_EndYearDropDownList"]',
'//*[@id="ctl00_ContentPlaceHolder1_EndMonthDropDownList"]']
second_fields_entries = [now.year, current_month]
for b in range(len(second_fields_entries)):
driver.find_element_by_xpath(second_fields_xpath[b]).send_keys(second_fields_entries[b])
driver.find_element_by_xpath('//*[@id="ctl00_ContentPlaceHolder1_ColumnsCheckBoxList_9"]').click()
# Submits
driver.find_element_by_xpath('//*[@id="ctl00_ContentPlaceHolder1_Button2"]').click()
sleep(5)
columns = defaultdict(list)
file_path = d_pref + file_name
with open(file_path) as f:
read = reader(f)
for row in read:
if len(row) == 4 and row[2] != '':
for (i, v) in enumerate(row):
columns[i].append(v)
years = columns[0]
months = columns[1]
days = columns[2]
data = columns[3]
years_step1 = []
year_store = ''
for year in years:
if year != '':
year_store = year
years_step1.append(year_store)
months_step1 = []
month_store = []
for month in months:
if month != '':
month_store = month
months_step1.append(month_store)
dates_step1 = []
for i, e in enumerate(years_step1):
dates_step1.append(e + '-' + months_step1[i] + '-' + days[i])
data_tuples_list = list(zip(dates_step1, data))
with open(csv_name + '.csv', 'w') as f1:
write = writer(f1)
for tup in data_tuples_list:
write.writerow(tup)
output_func(csv_name + '.csv', 1, final_csv_name)
driver.close()