def visit_ibmbraceorjp(
user: UserInformation,
driver: webdriver.remote.webdriver.WebDriver,
timeout: int = 15,
) -> bool:
"""Visit and login to im.mbrace.or.jp.
Args:
user (UserInformation): user information
driver (webdriver.remote.webdriver.WebDriver): webdriver
timeout (int, optional): timeout parameter. Defaults to 15.
Returns:
bool: whether login is succeeded.
"""
# check
if not check_login_status(driver):
login(driver, user, timeout)
# visit
driver.get(IBMBRACEORJP)
# check
return driver.title == 'トップ - BOAT RACE インターネット投票'
def go_to_reports_page(driver: webdriver.remote.webdriver.WebDriver) -> None:
driver.get(
"https://www.interactivebrokers.co.uk/AccountManagement/AmAuthentication?action=Statements"
)
# Wait for the page to load
driver.find_element_by_xpath(
"//*[normalize-space(text()) = 'MTM Summary']/../../..")
def mail_transactions(driver: webdriver.remote.webdriver.WebDriver) -> None:
driver.get('https://app.finpension.ch/documents/transactions')
wait = WebDriverWait(driver, 30)
mail_the_csv_button = wait.until(
expected_conditions.element_to_be_clickable(
(By.CSS_SELECTOR, 'ion-icon.ux-documents-item__mail')))
mail_the_csv_button.click()
def login(driver: webdriver.remote.webdriver.WebDriver,
user: UserInformation,
timeout: int = 15,
logger: Logger = getLogger(__name__)) -> bool:
"""Login to boatrace.jp
Args:
driver (webdriver.remote.webdriver.WebDriver): webdriver.
user (UserInformation): User information.
timeout (int, optional): Timeout parameter. Defaults to 15.
logger (Logger, optional): logger. Defaults to getLogger(__name__).
Raises:
LoginFailException: Occurred when login failure.
Returns:
bool: Whether login is succeeded.
NOTE:
Behavior:
Return True when login is succeeded;
Raise LoginFailException when login is failed.
"""
# get
driver.get(BOATRACEJP_LOGIN_URL)
# send keys
WebDriverWait(driver, timeout).until(
EC.presence_of_element_located((By.NAME, 'in_KanyusyaNo')))
driver.find_element_by_css_selector('input[name="in_KanyusyaNo"]')\
.send_keys(user.userid)
logger.debug('put userid')
WebDriverWait(driver, timeout).until(
EC.presence_of_element_located((By.NAME, 'in_AnsyoNo')))
driver.find_element_by_css_selector('input[name="in_AnsyoNo"]')\
.send_keys(user.pin)
logger.debug('put user pin')
WebDriverWait(driver, timeout).until(
EC.presence_of_element_located((By.NAME, 'in_PassWord')))
driver.find_element_by_css_selector('input[name="in_PassWord"]')\
.send_keys(user.auth_pass)
logger.debug('put authentification password')
# press button
WebDriverWait(driver, timeout).until(
EC.presence_of_element_located((By.CLASS_NAME, 'is-type3_2')))
driver.find_element_by_css_selector('button[class="btn is-type3_2"]')\
.click()
logger.debug('Pressed login button')
is_successed = check_login_status(driver)
if not is_successed:
raise LoginFailException()
return is_successed
def fetch_portfolio_statement(driver: webdriver.remote.webdriver.WebDriver,
creds: Credentials) -> bytes:
"""Fetches Degiro's portfolio statement using Selenium
Returns:
A CSV UTF-8 encoded statement.
"""
driver.implicitly_wait(30)
login(creds, driver)
wait_for_login(driver)
return fetch_portfolio(driver)
def fetch_data(driver: webdriver.remote.webdriver.WebDriver,
creds: Credentials) -> bytes:
"""Fetches Interactive Brokers's transaction data using Selenium
Returns:
A CSV with the fetched transactions.
"""
driver.implicitly_wait(60)
login(creds, driver)
wait_for_logged_in_state(driver)
return fetch_account_statement(driver)
def fetch_account_history(driver: webdriver.remote.webdriver.WebDriver,
creds: Credentials) -> bytes:
"""Fetches Charles Schwab's account history using Selenium
Returns:
A CSV UTF-8 encoded statement.
"""
driver.implicitly_wait(30)
login(creds, driver)
cookies = driver_cookie_jar_to_requests_cookies(driver.get_cookies())
return fetch_account_history_csv(cookies)
def fetch_data(creds: Credentials, gmail_creds: gmail.Credentials,
driver: webdriver.remote.webdriver.WebDriver) -> bytes:
"""Fetches Finpensions' transaction data.
Returns:
A CSV UTF-8 encoded string with the fetched transactions.
"""
driver.implicitly_wait(30)
login(creds, driver)
wait_for_login(driver)
time.sleep(2)
mail_transactions(driver)
return retry_n_times(lambda: fetch_csv_from_gmail(gmail_creds))
def login(creds: Credentials,
driver: webdriver.remote.webdriver.WebDriver) -> None:
driver.implicitly_wait(10)
LOGIN_PAGE = 'https://app.revolut.com/start'
driver.get(LOGIN_PAGE)
form = driver.find_element(By.CSS_SELECTOR, 'form')
country_code_input, phone_number_input = form.find_elements(
By.CSS_SELECTOR, 'input')
set_value(driver, country_code_input, creds.country_code)
phone_number_input.send_keys(creds.phone_number + Keys.RETURN)
fields = find_pin_input_fields(driver)
fields[0].send_keys(creds.pin)
def login(creds: Credentials,
driver: webdriver.remote.webdriver.WebDriver) -> None:
"""Logs into the Charles Schwab website"""
LOGIN_PAGE = 'https://client.schwab.com/Login/SignOn/CustomerCenterLogin.aspx'
driver.get(LOGIN_PAGE)
driver.switch_to.frame("lmsSecondaryLogin")
login_id = driver.find_element(By.ID, "loginIdInput")
wait = WebDriverWait(driver, 30)
wait.until(expected_conditions.visibility_of(login_id))
login_id.send_keys(creds.id + Keys.TAB)
driver.find_element(By.ID,
"passwordInput").send_keys(creds.pwd + Keys.RETURN)
wait_for_pin_number_and_select_it(wait, driver)
wait_for_user_to_provide_pin_and_login(driver)
def login(creds: Credentials,
driver: webdriver.remote.webdriver.WebDriver) -> None:
username_field_name = 'Benutzername'
driver.get(LOGIN_PAGE)
wait = WebDriverWait(driver, 30)
wait.until(
expected_conditions.presence_of_element_located(
(By.ID, username_field_name)))
driver.find_element(By.ID,
username_field_name).send_keys(creds.id + Keys.TAB)
time.sleep(1)
pwd_field = driver.find_element(By.ID, "Passwort")
pwd_field.send_keys(creds.pwd)
time.sleep(1)
pwd_field.send_keys(Keys.RETURN)
def get_account_id(driver: webdriver.remote.webdriver.WebDriver) -> str:
wait = WebDriverWait(driver, 30)
wait.until(
expected_conditions.presence_of_element_located(
(By.TAG_NAME, 'iframe')))
driver.switch_to.frame(0)
try:
# We want the page to load fully. Otherwise it seems that statement
# fetching doesn't work. That's why we wait for the export icon to
# appear and some currency figures, because they appear relatively late
# in the loading process.
wait.until(
expected_conditions.presence_of_element_located(
(By.CSS_SELECTOR,
'fin-icon-link[label="Exportieren als PDF"]')))
wait.until(
expected_conditions.presence_of_element_located(
(By.CSS_SELECTOR, 'a.ribbon')))
wait.until(
expected_conditions.presence_of_element_located(
(By.CSS_SELECTOR, '.currency')))
ribbons = driver.find_elements(By.CSS_SELECTOR, 'a.ribbon')
elem_with_account_id = ribbons[0]
params = json.loads(
elem_with_account_id.get_attribute('fin-ui-sref-params'))
finally:
driver.switch_to.parent_frame()
return params['accountId']
def fetch_portfolio(driver: webdriver.remote.webdriver.WebDriver) -> bytes:
logging.info("Fetching portfolio.")
portfolioSideBarLink = driver.find_element(By.CSS_SELECTOR,
'[href="#/portfolio"]')
portfolioSideBarLink.click()
exportButton = driver.find_element(By.CSS_SELECTOR,
'[data-name="exportButton"]')
exportButton.click()
time.sleep(1)
reportExportForm = driver.find_element(By.CSS_SELECTOR,
'[data-name="reportExportForm"]')
csvLink = reportExportForm.find_element(
By.XPATH, "//a[normalize-space(text()) = 'CSV']")
csvFetchUrl = csvLink.get_attribute('href')
cookies = driver_cookie_jar_to_requests_cookies(driver.get_cookies())
return fetch_csv(csvFetchUrl, cookies)
def logout(
driver: webdriver.remote.webdriver.WebDriver,
logger: Logger = getLogger(__name__)
) -> bool:
"""Logout from boatrace.jp
Args:
driver (webdriver.remote.webdriver.WebDriver): webdriver
logger (Logger, optional): logger. Defaults to getLogger(__name__).
Returns:
bool: Whether logout is succeeded.
"""
# logout
driver.get(BOATRACEJP_LOGOUT_URL)
return not check_login_status(driver)
def login(creds: Credentials,
driver: webdriver.remote.webdriver.WebDriver) -> None:
LOGIN_PAGE = 'https://app.finpension.ch/login'
driver.get(LOGIN_PAGE)
phone_country_selection = driver.find_element(By.CSS_SELECTOR,
".PhoneInputCountrySelect")
set_value(driver, phone_country_selection, 'CH')
wait = WebDriverWait(driver, 30)
mobile_number_field = wait.until(
expected_conditions.element_to_be_clickable((By.ID, 'mobile_number')))
password_field = wait.until(
expected_conditions.element_to_be_clickable((By.NAME, 'password')))
mobile_number_field.send_keys(creds.id + Keys.TAB)
password_field.send_keys(creds.pwd + Keys.RETURN)
code_field = wait.until(
expected_conditions.element_to_be_clickable(
(By.CSS_SELECTOR, 'ion-input[name=code]')))
code_field.click()
def fetch_account(driver: webdriver.remote.webdriver.WebDriver) -> bytes:
logging.info("Fetching account.")
driver.get(
get_account_overview_url(
from_date=get_three_months_ago(date.today()),
to_date=date.today(),
))
exportButton = driver.find_element(By.CSS_SELECTOR,
'[data-name="exportButton"]')
exportButton.click()
time.sleep(1)
reportExportForm = driver.find_element(By.CSS_SELECTOR,
'[data-name="reportExportForm"]')
csvLink = reportExportForm.find_element(
By.XPATH, "//a[normalize-space(text()) = 'CSV']")
csvFetchUrl = csvLink.get_attribute('href')
cookies = driver_cookie_jar_to_requests_cookies(driver.get_cookies())
return fetch_csv(csvFetchUrl, cookies)
def check_login_status(
driver: webdriver.remote.webdriver.WebDriver,
logger: Logger = getLogger(__name__),
) -> bool:
"""[summary]
Args:
driver (webdriver.remote.webdriver.WebDriver): webdriver
logger (Logger, optional): logger. Defaults to getLogger(__name__).
Returns:
bool: Whether you are loggined in boatrace.jp
"""
# get
driver.get(BOATRACEJP_MAIN_URL)
if driver.find_elements_by_class_name('is-logout1'):
return True
else:
return False
def wait_for_pin_number_and_select_it(
wait: WebDriverWait, driver: webdriver.remote.webdriver.WebDriver):
wait.until(
expected_conditions.url_matches('https://sws-gateway.schwab.com/.*'))
driver.switch_to.window(driver.window_handles[0])
pin_element = driver.find_element(By.ID, "placeholderCode")
# Initially the element is findable but not clickable. When that happens,
# the click operation fails, so wait for the element to be visible.
wait.until(expected_conditions.visibility_of(pin_element))
pin_element.click()
return pin_element
def fetch_account_statement_csv(driver: webdriver.remote.webdriver.WebDriver,
download_url: str) -> bytes:
fetch_page = 'https://www.bcge.ch/next/' + download_url
cookies = driver_cookie_jar_to_requests_cookies(driver.get_cookies())
headers = prepare_headers(cookies, get_user_agent(driver))
response = requests.get(fetch_page, headers=headers, cookies=cookies)
if not response.ok:
raise Exception("The statement fetch request has failed. " +
('Response reason: {0}, parameters: {1}'
).format(response.reason, (fetch_page, cookies)))
return response.content
def login(creds: Credentials,
driver: webdriver.remote.webdriver.WebDriver) -> None:
logging.info("Logging in.")
LOGIN_PAGE = 'https://trader.degiro.nl/login/chde/#/login'
driver.get(LOGIN_PAGE)
driver.find_element(By.ID, "username").send_keys(creds.id + Keys.TAB)
driver.find_element(By.ID, "password").send_keys(creds.pwd + Keys.RETURN)
def fetch_latest_bill(driver: webdriver.remote.webdriver.WebDriver) -> bytes:
driver.get('https://one.viseca.ch/de/rechnungen')
driver.implicitly_wait(30)
latest_bill_a_elem = driver.find_element(
By.CSS_SELECTOR, '#statement-list-statement-date0 a')
latest_bill_id = extract_bid(latest_bill_a_elem.get_attribute('id'))
return fetch_bill(latest_bill_id, driver.get_cookies())
def fetch_all_transactions_since_2018(
driver: webdriver.remote.webdriver.WebDriver) -> bytes:
from_date = datetime.date(2018, 1, 1)
to_date = datetime.date.today()
resp = requests.post(
FETCH_PAGE,
json=download_request_json_payload(from_date, to_date),
cookies=driver_cookie_jar_to_requests_cookies(driver.get_cookies()))
if not resp.ok:
raise Exception(
"The CSV fetch request has failed. Response reason: {0}".format(
resp.reason))
return resp.content
def fetch_account_statement(
driver: webdriver.remote.webdriver.WebDriver) -> bytes:
logging.info("Going to the reports page.")
go_to_reports_page(driver)
logging.info("Reports page loaded, fetching cookies.")
last_get = get_last_get_request(driver)
if last_get is None:
raise Exception(
'Could not fetch the account statement, because we did not ' +
'find any previous GET requests in the session to initialize ' +
'the fetch requests ')
cookies = driver_cookie_jar_to_requests_cookies(driver.get_cookies())
logging.info("Fetching the CSV file.")
return fetch_account_statement_csv(dict(last_get.headers), cookies)
def login(creds: Credentials,
driver: webdriver.remote.webdriver.WebDriver) -> None:
LOGIN_PAGE = 'https://www.bcge.ch/authen/login?lang=de'
driver.get(LOGIN_PAGE)
wait = WebDriverWait(driver, 30)
wait.until(
expected_conditions.presence_of_element_located((By.ID, 'username')))
driver.find_element(By.ID, "username").send_keys(creds.id + Keys.TAB)
driver.find_element(By.ID, "password").send_keys(creds.pwd + Keys.RETURN)
def login_to_mbank(creds: Credentials,
driver: webdriver.remote.webdriver.WebDriver) -> None:
driver.get(MBANK_LOGIN_PAGE)
driver.find_element(By.ID, "userID").send_keys(creds.id + Keys.TAB)
driver.find_element(By.ID, "pass").send_keys(creds.pwd + Keys.RETURN)
wait = WebDriverWait(driver, 30)
wait.until(
expected_conditions.element_to_be_clickable(
(By.CSS_SELECTOR,
'[data-test-id="SCA:UnknownDevice:OneTimeAccess"]'))).click()
wait = WebDriverWait(driver, 30)
wait.until(expected_conditions.url_matches(HISTORY_PAGE))
def fetch_download_url(driver: webdriver.remote.webdriver.WebDriver,
account_id: str) -> str:
fetch_page = ('https://www.bcge.ch/' +
'next/api/accounts/{0}/bookings/export'.format(account_id))
json_payload = download_url_request_json_payload(
account_id,
from_date=datetime.date(2018, 1, 1),
to_date=datetime.date.today())
cookies = driver_cookie_jar_to_requests_cookies(driver.get_cookies())
headers = prepare_headers(cookies, get_user_agent(driver))
response = requests.post(fetch_page,
headers=headers,
cookies=cookies,
json=json_payload)
if not response.ok:
raise Exception("The URL fetch request has failed. " +
('Response reason: {0}, parameters: {1}'
).format(response.reason, (fetch_page, headers,
cookies, json_payload)))
return json.loads(response.content)['data']['downloadUrl']
def get_bet_limit(user: UserInformation,
driver: webdriver.remote.webdriver.WebDriver,
timeout: int = 15) -> int:
"""Get the amount of deposit.
Args:
user (UserInformation): user information.
driver (webdriver.remote.webdriver.WebDriver): webdriver
timeout (int, optional): timeout parameter. Defaults to 15.
Returns:
int: the amount of deposit.
"""
# visit
visit_ibmbraceorjp(user, driver, timeout)
# get vote limit
WebDriverWait(driver, timeout).until(
EC.presence_of_element_located((By.ID, 'currentBetLimitAmount')))
limit = driver.find_element_by_id('currentBetLimitAmount').text
return int(limit.replace(',', ''))
def set_value(driver: webdriver.remote.webdriver.WebDriver, element,
value: str) -> None:
driver.execute_script(set_value_js(value), element)
def get_next_element_sibling(driver: webdriver.remote.webdriver.WebDriver,
elem):
return driver.execute_script("return arguments[0].nextElementSibling",
elem)
def get_user_agent(driver: webdriver.remote.webdriver.WebDriver) -> str:
return driver.execute_script("return navigator.userAgent;")