def get(self, bank_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
bank_config = banking_configuration.banks[bank_id]
access_code = database.get_bank_access_code(db, bank_config)
return jsonify(io.encode_object(access_code))
def get(self, account_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
account = account_from_config(
banking_configuration.accounts[account_id])
access_code = database.get_account_access_code(db, account)
return jsonify(io.encode_object(access_code))
def get(self, account_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
account = account_from_config(
banking_configuration.accounts[account_id])
encoded_account = io.encode_account(account)
encoded_account['balance'] = database.get_account_balance(db, account)
return jsonify(encoded_account)
def get(self, account_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
account = account_from_config(
banking_configuration.accounts[account_id])
transactions = database.find_transactions(
db,
account,
sort_direction=database.SortDirection.NEWEST_TRANSACTION_FIRST)
return jsonify(list(map(io.encode_object, transactions)))
def put(self, bank_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
code = request.get_json()['code']
bank_config = banking_configuration.banks[bank_id]
access_code = datatypes.BankAccessCode(code=code,
date=datetime.utcnow(),
bank_id=bank_config.id)
database.update_bank_access_code(db, bank_config, access_code)
return jsonify(io.encode_object(access_code))
def run(config):
execute_update_all = partial(bank.update_all, config, bank.env())
for scrapping_hour in config.scheduler.scrapping_hours:
schedule.every().day.at(scrapping_hour).do(execute_update_all).tag(
'Update all transactions from banks', 'run_at_start')
# Start scheduling thread
_thread.start_new_thread(schedule_loop, ())
# Run the scrapping job once, non blocking
if bank.env()['update_accounts_on_start']:
run_once(execute_update_all, 'Update all transactions from banks')
def put(self, account_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
code = request.get_json()['code']
account = account_from_config(
banking_configuration.accounts[account_id])
access_code = datatypes.AccountAccessCode(code=code,
date=datetime.utcnow(),
account_id=account.id)
database.update_account_access_code(db, account, access_code)
return jsonify(io.encode_object(access_code))
def get(self):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
encoded_accounts = map(
lambda account: io.encode_account(account, include_children=False),
list(banking_configuration.accounts.values()))
return jsonify(list(encoded_accounts))
from .domain import Rule
from .io import Match, MatchAll, MatchAny, MatchNumeric
from .io import Set, SetFromCapture, Add
from datatypes import TransactionType
from bank import load_categories, env
categories = load_categories(env()['categories_file'])
_rules = [
# ---------------------------------------------------------
# Rules to fix or improve source or destination description
# And on some cases, set tags or related comment
# ---------------------------------------------------------
Rule(conditions=[
Match('type', TransactionType.RECEIVED_TRANSFER),
MatchAll('keywords', 'GIL', 'AEAT', 'DEDUCCION')
],
actions=[
Set('source', 'Agencia Tributaria'),
Set('comment', 'Deducció Maternitat Gil')
]),
Rule(conditions=[
Match('type', TransactionType.RECEIVED_TRANSFER),
MatchAll('keywords', 'GIL', 'AEAT', 'DEDUCCION')
],
actions=[
Set('source', 'Agencia Tributaria'),
Set('comment', 'Deducció Maternitat Gil')
]),
def login(browser, username, password):
log('Loading BBVA main page')
browser.get('https://www.bbva.es')
gdpr_button = browser.find_elements_by_css_selector('.cookiesgdpr button').filter(lambda button: 'aceptar' in button.text.lower())
if gdpr_button:
gdpr_button[0].click()
log('Opening login form')
browser.find_element_by_css_selector('.header__actions .header__access').forced_click()
log('Filling login form')
browser.driver.switch_to_frame(browser.find_element_by_id('tab-personas-iframe').result)
# Even with waiting for visible, still get some misses sometimes
user_input = browser.find_element_by_name('user', timeout=15, visible=True)
user_input.focus().clear().send_keys(username)
password_input = browser.find_element_by_name('password', visible=True)
password_input.focus().clear().send_keys(password)
log('Submitting login')
browser.find_element_by_css_selector("button[type=submit]").focus().forced_click()
# Fill SMS OTP if any
log('Waiting for code request, if any')
code_request_input = browser.find_element_by_css_selector('input[name=otp][type=password]', visible=True, timeout=15, do_raise=False)
if code_request_input:
received_code = None
t0 = int(time.time())
def sms_timeout():
return (int(time.time()) - t0) > SMS_TIMEOUT
while not received_code and not sms_timeout():
log('Waiting for SMS code')
access_code = database.get_bank_access_code(
database.load(env()['database_folder']),
load_config(env()['main_config_file']).banks['bbva']
)
if access_code is None:
time.sleep(0.5)
continue
age = (datetime.utcnow() - access_code.date).total_seconds()
if age < 30:
log('Valid SMS code arrived! {} seconds old'.format(age))
received_code = access_code.code
time.sleep(0.5)
if received_code is None:
log('No recent SMS code received')
raise SMSOTPException('No SMS received in time')
if received_code:
log('Submitting SMS code')
code_request_input.focus().clear().send_keys(received_code)
browser.find_element_by_css_selector('#cuentas_buscador_firma input[type=submit]').click()
else:
log('No code requested or already entered recently')
# Close popup if any
log('Waiting for popups to close them')
modal_close_button = browser.find_element_by_css_selector('#ui-dialog-campania-bloqueante #noInteresa', timeout=15, do_raise=False)
if modal_close_button:
modal_close_button.forced_click()
log('Popup closed')
else:
log('No popups found')
log('Waiting login to finish')
browser.find_element_by_css_selector('#t-main-content', visible=True, timeout=20)
def get_account_transactions(browser, account_number, from_date, to_date):
browser.get('https://web.bbva.es/index.html')
log('Loading BBVA account page')
browser.find_elements_by_css_selector('p[role=link').filter(lambda element: account_number in element.text)[0].forced_click()
log('Loading account advanced search')
browser.find_element_by_css_selector('ul.menuPestanas span.consultas').forced_click()
log('Filling date query parameters')
browser.find_element_by_name('filtros.fechas.inicio', visible=True).focus().clear().send_keys(encode_date(from_date))
browser.find_element_by_name('filtros.fechas.fin', visible=True).focus().clear().send_keys(encode_date(to_date))
time.sleep(2) # To try to avoid the null values in the date filter request
log('Setting up XHR request interceptor')
script = xhr_intercept_response(
match_url="accountTransactionsAdvancedSearch",
output="interceptedResponse",
# This will modify the date in the request so the full hour is set and we get all transactions
request_intercept_script=FIX_NULL_DATE_ACCOUNT.format(
from_date=from_date.strftime('%Y-%m-%dT00:00:00Z'),
to_date=to_date.strftime('%Y-%m-%dT%H:%M:%SZ')
)
)
browser.driver.execute_script(script)
log('Launching the initial search')
browser.find_elements_by_css_selector('*[role=button').filter(lambda element: 'Buscar' in element.text)[0].focus().click()
# Close popup if any
log('Waiting for code request, if any')
code_request_input = browser.find_element_by_css_selector('#cuentas_buscador_firma input[type=password]', visible=True, timeout=15, do_raise=False)
if code_request_input:
received_code = None
t0 = int(time.time())
def sms_timeout():
return (int(time.time()) - t0) > SMS_TIMEOUT
while not received_code and not sms_timeout():
log('Waiting for SMS code')
access_code = database.get_account_access_code(
database.load(env()['database_folder']),
load_config(env()['main_config_file']).accounts[account_number]
)
if access_code is None:
time.sleep(0.5)
continue
age = (datetime.utcnow() - access_code.date).total_seconds()
if age < 30:
log('Valid SMS code arrived! {} seconds old'.format(age))
received_code = access_code.code
time.sleep(0.5)
if received_code is None:
log('No recent SMS code received')
raise SMSOTPException('No SMS received in time')
if received_code:
log('Submitting SMS code')
code_request_input.focus().clear().send_keys(received_code)
browser.find_element_by_css_selector('#cuentas_buscador_firma input[type=submit]').click()
else:
log('No code requested or already entered recently')
intercepted_responses = []
intercepted_responses_count = 0
still_have_results = True
# Iterate trough all pagination
while still_have_results:
while intercepted_responses_count == len(intercepted_responses):
# Inner Loop to wait for the page to load and push the new transactions
browser.find_element_by_id("interceptedResponse")
intercepted_json = browser.execute_script("return JSON.stringify(document.getElementById('interceptedResponse').responses)")
intercepted_responses = list(filter(lambda response: 'pagination' in response, json.loads(intercepted_json)))
time.sleep(0.1)
intercepted_responses_count = len(intercepted_responses)
still_have_results = False if intercepted_responses[-1] is None else intercepted_responses[-1]['pagination'].get('nextPage', False)
if still_have_results:
browser.find_element_by_css_selector('#moreButton', visible=True).click()
log('Loading more results')
time.sleep(0.1)
fixed_date = browser.execute_script("return document.getElementById('fixed_date')")
log('Fixed date: {}'.format(fixed_date))
# Results come from newer to older, we want it the other way around, that why we reverse them
results = list(reversed(list((chain.from_iterable([response['accountTransactions'] for response in intercepted_responses if response is not None])))))
return results
def parse_date(date_string):
"""
Dates from cli are expected as yyyy-mm-dd
"""
if not date_string:
return None
year, month, day = map(int, date_string.split('-'))
return datetime(year, month, day)
if __name__ == '__main__':
arguments = docopt(__doc__, version='Banking 1.0')
banking_configuration = bank.load_config(bank.env()['main_config_file'])
action = list(
filter(lambda action: arguments[action] is True,
['get', 'load', 'apply', 'update', 'run', 'remove']))[0]
target = list(
filter(lambda target: arguments[target] is True,
['account', 'card', 'server', 'all']))[0]
load_raw = arguments['raw']
load_all = arguments['all']
headless = not arguments['--debug-browser']
close_browser = not arguments['--keep-browser-open']
update_accounts_on_start = not arguments['--no-initial-update']
def parse_credit_card_transaction(bank_config, account_config, card_config,
transaction):
amount = transaction['amount']['amount']
transaction_code = get_nested_item(transaction, 'concept.id')
if transaction_code == '0000':
# code 0000 seems like an error, as it's really a regular purcharse,
# so we fake the code
transaction_code = '0005'
transation_direction = TransactionDirection.CHARGE if amount < 0 else TransactionDirection.INCOME
transaction_type = get_type(transaction_code, transation_direction)
# Transactions have a _PT or _TT termination, that changes once over time and makes id unusable
# this is an attempt to fix this and still being able to use the id as an unique id
transaction_id = re.sub(r'_[PT]T$', '', transaction['id'])
details = get_card_transaction_details(transaction, transaction_type)
details['account'] = Account.from_config(account_config)
details['bank'] = Bank.from_config(bank_config)
# As we are processing a concrete card, and transaction doesn't have this
# information, we set it to be able to process all transactions equally
card_used = Card.from_config(card_config)
keywords = extract_keywords(
chain(extract_literals(transaction, KEYWORD_FIELDS),
filter(lambda value: isinstance(value, str), details.values())))
comment = get_comment(details, transaction_type)
source = get_source(details, transaction_type)
destination = get_destination(details, transaction_type)
del details['bank']
del details['account']
is_debit_operation = transaction.get('operationTypeIndicator') == 'D'
is_consolidated = transaction.get('status', {}).get('id') == '7'
notify_not_added = False
status_flags = datatypes.StatusFlags()
if is_debit_operation:
if notify_not_added:
from common.notifications import get_notifier
import bank
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
notifier = get_notifier(banking_configuration.notifications)
notifier(
'Debit transaction found, not adding {bank.name} card transaction: {date} {amount}, {source}->{destination}'
.format(bank=bank_config,
amount=amount,
date=transaction['valueDate'],
source=str(source),
destination=str(destination)))
status_flags.invalid = True
if not is_consolidated and notify_not_added:
if notify_not_added:
from common.notifications import get_notifier
import bank
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
notifier = get_notifier(banking_configuration.notifications)
notifier(
'Non consolidated transaction found, not adding {bank.name} card transaction: {date} {amount}, {source}->{destination}'
.format(bank=bank_config,
amount=amount,
date=transaction['valueDate'],
source=str(source),
destination=str(destination)))
return None
return ParsedCreditCardTransaction(
transaction_id=transaction_id,
type=transaction_type,
currency=transaction['amount']['currency']['code'],
amount=amount,
value_date=decode_date(transaction['valueDate']),
transaction_date=decode_date(transaction['transactionDate']),
source=source,
destination=destination,
card=card_used,
details=details,
keywords=keywords,
comment=comment if comment is not None else '',
status_flags=status_flags)