def set_bank_account_login():
"""
This will ask for a username and Password.
The password will be retrieved via getpass lib.
Requires the password length to be at least 5 characters.
"""
# Getting the login name from a raw_input.
obp_logger.info("get username")
username = raw_input("Username: "******"username is set")
# Now getting the password via getpass lib.
obp_logger.info("get password")
password = getpass.getpass()
obp_logger.debug("password has set")
# We know from the Web Page that we need at least 5 characters.
# This will check for the right length of the password.
obp_logger.debug("start while loop")
while len(password) < 5:
obp_logger.error("Password was not 5 character long")
print "Password has to contain at least 5 letters"
password = getpass.getpass()
obp_logger.debug("Password has set")
obp_logger.debug("Will return username: %s ,password is set" % username)
# Return username and password.
return username, password
def create_hash(value_to_hash):
"""This will create a hash and return it"""
obp_logger.info("Creating hash with sha512")
data_hash = hashlib.sha512(value_to_hash).hexdigest()
obp_logger.debug("created hash is in hexdigest: %s" % data_hash)
return check_hash(data_hash)
def main(csv_file_path):
"""Will check for a valid CSV and import it to the Scala API"""
obp_logger.info("Start Main")
obp_logger.debug("csv_file_path is: %s" % csv_file_path)
obp_logger.debug("Check that csv_file_path is valid path")
check_for_existing_csv(csv_file_path)
obp_logger.debug("Start parse_row_of_csv")
parse_row_of_csv(csv_file_path)
def main(csv_file_path):
"""Will check for a valid CSV and import it to the Scala API"""
obp_logger.info("Start Main")
obp_logger.debug("csv_file_path is: %s" % csv_file_path)
obp_logger.debug("Check that csv_file_path is valid path")
check_for_existing_csv(csv_file_path)
obp_logger.debug("Start parse_row_of_csv")
parse_row_of_csv(csv_file_path)
def insert_into_scala(scala_api_host, scala_api_port, JSON_to_insert):
"""
Inserting JSON via HTTP POST into the API.
"""
obp_logger.info("Insert JSON to Scala API")
obp_logger.debug("test connection to Scala API Host")
check_scala_host_reachable(scala_api_host, scala_api_port)
# Set content-type to JSON in the HTTP Header
headers = {'content-type': 'application/json'}
obp_logger.debug("Set HTTP headers to: %s" % headers)
post_request = requests.post("http://" + scala_api_host + ":" + scala_api_port + "/api/transactions",
data=JSON_to_insert, headers=headers)
# Return the http status code.
obp_logger.debug("Inserted to SCALA API")
return post_request
def insert_into_scala(scala_api_host, scala_api_port, JSON_to_insert):
"""
Inserting JSON via HTTP POST into the API.
"""
obp_logger.info("Insert JSON to Scala API")
obp_logger.debug("test connection to Scala API Host")
check_scala_host_reachable(scala_api_host, scala_api_port)
# Set content-type to JSON in the HTTP Header
headers = {'content-type': 'application/json'}
obp_logger.debug("Set HTTP headers to: %s" % headers)
post_request = requests.post("http://" + scala_api_host + ":" +
scala_api_port + "/api/transactions",
data=JSON_to_insert,
headers=headers)
# Return the http status code.
obp_logger.debug("Inserted to SCALA API")
return post_request
def insert_hash_to_cache(hash_to_insert, file):
"""Will insert the Hash_input into the file, if it isn't already there"""
obp_logger.debug("start insert_hash")
obp_logger.debug("check for valid_hash")
valid_hash = check_hash(hash_to_insert)
obp_logger.debug("read cache file for existing hash")
if check_existing_hashs(valid_hash, file) != True:
obp_logger.debug("Opening cache file, appending")
file_to_write = open(file, 'a')
obp_logger.debug("writing hash to file, hash is: %s" % valid_hash)
file_to_write.write(valid_hash + '\n')
obp_logger.debug("closing cache file")
file_to_write.close()
obp_logger.debug("returning True")
return True
else:
obp_logger.info("Hash already inserted")
return False
def check_for_clean_folder(check_path):
"""Check for an empty folder. Else create it."""
obp_logger.info("Check for empty and exists folder")
obp_logger.debug("Check that %s exists" % check_path)
if os.path.exists(check_path) != True:
obp_logger.debug("%s no there, create it")
os.makedirs(check_path)
# Check for an empty folder
obp_logger.debug("Check for empty folder")
if len(os.listdir(check_path)) != 0:
obp_logger.debug("Getting length of items in %s" % check_path)
obp_logger.debug("Try to remove items in check_path")
for item in os.listdir(check_path):
item_to_remove = os.path.join(check_path, item)
obp_logger.debug("Item is %s" % item_to_remove)
try:
os.remove(item_to_remove)
except OSError, e:
obp_logger.warn("Can't remove %s" % e)
def parse_row_of_csv(csv_file_to_parse):
"""Gets rows from CSV file"""
obp_logger.info("start reading the csv file line by line")
# This is to set the separators in the CSV file
# TODO: This should be in the obp_config
delimiter = ';'
quote_char = '"'
obp_logger.debug("Set CSV delimiter to: %s" % delimiter)
obp_logger.debug("Set CSV quote_char to: %s" % quote_char)
# re : \d\d\.\d\d\.\d\d\d\d
# This will check if date is formatted like this: 23.01.2001
obp_logger.debug("Set regular expression to: \d\d\.\d\d\.\d\d")
data_expression = re.compile('\d\d\.\d\d\.\d\d\d\d')
obp_logger.debug("starting csv reader")
transaction_reader = csv.reader(open(csv_file_to_parse, 'rb'),
delimiter=delimiter,
quotechar=quote_char)
obp_logger.debug("Start of for loop of transaction_reader")
for row in transaction_reader:
# The first valid entry always has a date: check for it.
# If it doesn't exist, add this row to the csv_header_info and then continue.
obp_logger.debug("checking for date in first row from csv")
if data_expression.match(row[0]) == None:
csv_header_info.append(row)
obp_logger.debug("append row to csv_header_info, row is: %s" % row)
continue
else:
# When we have a valid date, call get_info_from_row.
obp_transaction_dict = get_info_from_row(row)
obp_logger.debug("call get_info_from_row")
# This will create a hash and return it.
json_hash = create_hash(json_formatter(obp_transaction_dict))
obp_logger.debug("create json_hash from obp_transaction_dict")
# Some debug output. So that we may can see the content of the JSON
# and the hash.
obp_logger.info("The hash of the JSON is: %s" % json_hash)
print "%s:The hash of the JSON is: %s" % (date_now_formatted(),
json_hash)
# Insert the hash into the cache. If it worked (the hash did not yet exist)
# send it to the API.
result = insert_hash_to_cache(json_hash, HASH_FILE)
if result == True:
result = insert_into_scala(SCALA_HOST, SCALA_PORT,
json_formatter(obp_transaction_dict))
obp_logger.debug("HTTP POST result is: %s" % result)
#obp_logger.debug("HTTP POST text from result is: %s" % result.text)
else:
obp_logger.info(
"Transaction is already in hash file, not inserting")
print "%s:Transaction is already in hash file, not inserting" % date_now_formatted(
)
def parse_row_of_csv(csv_file_to_parse):
"""Gets rows from CSV file"""
obp_logger.info("start reading the csv file line by line")
# This is to set the separators in the CSV file
# TODO: This should be in the obp_config
delimiter = ';'
quote_char = '"'
obp_logger.debug("Set CSV delimiter to: %s" % delimiter)
obp_logger.debug("Set CSV quote_char to: %s" % quote_char)
# re : \d\d\.\d\d\.\d\d\d\d
# This will check if date is formatted like this: 23.01.2001
obp_logger.debug("Set regular expression to: \d\d\.\d\d\.\d\d")
data_expression = re.compile('\d\d\.\d\d\.\d\d\d\d')
obp_logger.debug("starting csv reader")
transaction_reader = csv.reader(
open(csv_file_to_parse, 'rb'),
delimiter=delimiter,
quotechar=quote_char)
obp_logger.debug("Start of for loop of transaction_reader")
for row in transaction_reader:
# The first valid entry always has a date: check for it.
# If it doesn't exist, add this row to the csv_header_info and then continue.
obp_logger.debug("checking for date in first row from csv")
if data_expression.match(row[0]) == None:
csv_header_info.append(row)
obp_logger.debug("append row to csv_header_info, row is: %s" % row)
continue
else:
# When we have a valid date, call get_info_from_row.
obp_transaction_dict = get_info_from_row(row)
obp_logger.debug("call get_info_from_row")
# This will create a hash and return it.
json_hash = create_hash(json_formatter(obp_transaction_dict))
obp_logger.debug("create json_hash from obp_transaction_dict")
# Some debug output. So that we may can see the content of the JSON
# and the hash.
obp_logger.info("The hash of the JSON is: %s" % json_hash)
print "%s:The hash of the JSON is: %s" % (date_now_formatted(), json_hash)
# Insert the hash into the cache. If it worked (the hash did not yet exist)
# send it to the API.
result = insert_hash_to_cache(json_hash, HASH_FILE)
if result == True:
result = insert_into_scala(
SCALA_HOST,
SCALA_PORT,
json_formatter(obp_transaction_dict))
obp_logger.debug("HTTP POST result is: %s" % result)
#obp_logger.debug("HTTP POST text from result is: %s" % result.text)
else:
obp_logger.info("Transaction is already in hash file, not inserting")
print "%s:Transaction is already in hash file, not inserting" % date_now_formatted()
def get_info_from_row(input_row):
"""Read rows and get the transaction data, print as JSON"""
obp_logger.info("Start get_info_from_row")
# The Germans format money like 1.200,12 Eur. We
# need a English format, i.e. 1200.12 Eur
# So have to remove the dot and replace the , with a dot.
# This will turn . to ""
obp_logger.debug("replace . with empty string")
dotless_amount = re.sub('\.', '', input_row[6])
dotless_new_balance = re.sub('\.', '', input_row[7])
obp_logger.debug("replace , with .")
comma_to_dot_amount = re.sub(',', '.', dotless_amount)
comma_to_dot_new_balance = re.sub(',', '.', dotless_new_balance)
# This regular expression searches for all kind of numbers in a string.
# Also covering + and -
#obp_logger.debug("")
amount = re.match("[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?",
comma_to_dot_amount)
new_balance = re.match("[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?",
comma_to_dot_new_balance)
obp_logger.debug("set this_account_holder")
this_account_holder = csv_header_info[1]
obp_logger.debug("set this_account_IBAN")
this_account_IBAN = csv_header_info[4]
obp_logger.debug("set this_account_number")
this_account_number = csv_header_info[3]
# There is still some value inside that we need to remove
this_account_unclean_currency = csv_header_info[5]
this_account_currency = re.search('\xe2\x82\xac',
this_account_unclean_currency[1])
obp_logger.debug("set this_account_currency")
obp_logger.debug("set this_account_kind")
this_account_kind = 'current'
obp_logger.debug("set this_account_ni")
this_account_ni = "" # ni = national_identifier
obp_logger.debug("set this_account_bank_name")
this_account_bank_name = 'Postbank'
# Need to use row 4 if we're sending money,
# and row 5 when we're getting money.
obp_logger.debug(
"check that this_account_holder is not other_account_holder")
if input_row[5].rstrip() != this_account_holder[1]:
other_account_holder = input_row[5].rstrip()
obp_logger.debug("set other_account_holder")
else:
other_account_holder = input_row[4].rstrip()
obp_logger.debug("set other_account_holder")
# Don't print out the JSON, to ensure no sensitive data gets displayed.
obp_logger.debug("create json dump")
obp_transaction_data = json.dumps(
[{
"obp_transaction": {
"this_account": {
"holder": this_account_holder[1],
"number": this_account_number[1],
"kind": this_account_kind,
"bank": {
"IBAN": this_account_IBAN[1],
"national_identifier": this_account_ni,
"name": this_account_bank_name
}
},
"other_account": {
"holder": other_account_holder,
"number": input_row[3].rstrip(),
"kind": "",
"bank": {
"IBAN": "",
"national_identifier": "",
"name": ""
}
},
"details": {
"type_en": "",
"type_de": input_row[2],
"posted": {
"$dt": convert_date(
input_row[0]
) # Have to set to $dt so Scala can work with it.
},
"completed": {
"$dt": convert_date(
input_row[1]
) # Have to set to $dt so Scala can work with it.
},
"new_balance": {
"currency":
currency_sign_to_text(this_account_currency.group()),
"amount":
new_balance.group()
},
"value": {
"currency":
currency_sign_to_text(this_account_currency.group()),
"amount":
amount.group()
},
"other_data": input_row[5]
}
}
}],
sort_keys=False)
obp_logger.debug("Done filling json, returning obp_transaction_data")
return obp_transaction_data
def get_csv_with_selenium(path_to_save_csv, username, password):
"""Getting CSV file via Firefox, controlled by Selenium webdriver"""
# TODO: When no username and password is set, use the demo login.
# Clean up the OBP temp folder (delete all csv files there).
# LINK: http://seleniumhq.org/docs/03_webdriver.html#getting-started-with-selenium-webdriver
obp_logger.info("Setting csv_folder...")
# Check for existing and empty tmp
check_for_clean_folder(path_to_save_csv)
csv_save_path = os.path.join(os.getcwd(), path_to_save_csv, TMP_CSV_SUFFIX)
obp_logger.debug("csv_folder is: %s" % csv_save_path)
check_for_clean_folder(csv_save_path)
obp_logger.info("Start Selenium")
obp_logger.debug("csv_save_path: %s" % csv_save_path)
obp_logger.debug("username is set")
obp_logger.debug("password is set")
# Setting up a Profile for Firefox.
# Proxy is disabled and download files without asking.
obp_logger.info("Setup Firefox Profile")
fp = webdriver.FirefoxProfile()
obp_logger.debug("webdriver firefox")
fp.set_preference("network.proxy.type", 0)
obp_logger.debug("network.proxy.type 0")
fp.set_preference("browser.download.folderList", 2)
obp_logger.debug("rowser.download.fold 2")
fp.set_preference("browser.download.manager.showWhenStarting", False)
obp_logger.debug("rowser.download.manager.showWhenStarting False ")
fp.set_preference("browser.download.dir", csv_save_path)
obp_logger.debug("browser.download.dir %s" % csv_save_path)
# Need to set CSV to saveToDisk, else it's unknown to FF and it will ask for it
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv")
obp_logger.debug("browser.helperApps.neverAsk.saveToDisk text/csv")
obp_logger.info("Start Firefox")
browser = webdriver.Firefox(firefox_profile=fp) # Get local session of firefox
obp_logger.debug("Open URL: %s" % postbank_main_url_login_page)
browser.get(postbank_main_url_login_page) # Load page
assert "Postbank Online-Banking" in browser.title
#selenium.set_browser_log_level("error")
# Here we will insert the username and password:
# find the element that's name attribute is nutzername and kennwort
obp_logger.info("Inserting Username and Password to Login")
obp_logger.debug("searching for login box")
inputElement_username = browser.find_element_by_name("nutzername")
obp_logger.debug("searching for password box")
inputElement_password = browser.find_element_by_name("kennwort")
# send Username and Password
obp_logger.debug("Inserting username into login box: %s " % username)
inputElement_username.send_keys(username)
obp_logger.debug("Inserting password into login box")
inputElement_password.send_keys(password)
# submit the Username and Password to Postbank.
obp_logger.info("submitting login_data to login")
inputElement_password.submit()
# This opens the main page for accounts, and checks the name.
# Call the Transaction Page
obp_logger.debug("Open URL: %s" % postbank_main_url_value_page)
browser.get(postbank_main_url_value_page)
assert "Postbank Online-Banking" in browser.title
# Call the CSV Link.
# Warning!
# The Postbank uses a :page counter, and when the URL doesn't have the right page counter it will return
# an error message.
obp_logger.debug("Open URL: %s" % postbank_main_url_value_download)
result = browser.get(postbank_main_url_value_download)
obp_logger.info("closing Firefox")
browser.close()
return csv_save_path
def main():
obp_logger.info("Start main")
check_for_clean_folder()
get_csv_with_selenium(TMP)
obp_logger.info("Done main")
def get_info_from_row(input_row):
"""Read rows and get the transaction data, print as JSON"""
obp_logger.info("Start get_info_from_row")
# The Germans format money like 1.200,12 Eur. We
# need a English format, i.e. 1200.12 Eur
# So have to remove the dot and replace the , with a dot.
# This will turn . to ""
obp_logger.debug("replace . with empty string")
dotless_amount = re.sub('\.', '', input_row[6])
dotless_new_balance = re.sub('\.', '', input_row[7])
obp_logger.debug("replace , with .")
comma_to_dot_amount = re.sub(',', '.', dotless_amount)
comma_to_dot_new_balance = re.sub(',', '.', dotless_new_balance)
# This regular expression searches for all kind of numbers in a string.
# Also covering + and -
#obp_logger.debug("")
amount = re.match(
"[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?",
comma_to_dot_amount)
new_balance = re.match(
"[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?",
comma_to_dot_new_balance)
obp_logger.debug("set this_account_holder")
this_account_holder = csv_header_info[1]
obp_logger.debug("set this_account_IBAN")
this_account_IBAN = csv_header_info[4]
obp_logger.debug("set this_account_number")
this_account_number = csv_header_info[3]
# There is still some value inside that we need to remove
this_account_unclean_currency = csv_header_info[5]
this_account_currency = re.search(
'\xe2\x82\xac',
this_account_unclean_currency[1])
obp_logger.debug("set this_account_currency")
obp_logger.debug("set this_account_kind")
this_account_kind = 'current'
obp_logger.debug("set this_account_ni")
this_account_ni = "" # ni = national_identifier
obp_logger.debug("set this_account_bank_name")
this_account_bank_name = 'Postbank'
# Need to use row 4 if we're sending money,
# and row 5 when we're getting money.
obp_logger.debug("check that this_account_holder is not other_account_holder")
if input_row[5].rstrip() != this_account_holder[1]:
other_account_holder = input_row[5].rstrip()
obp_logger.debug("set other_account_holder")
else:
other_account_holder = input_row[4].rstrip()
obp_logger.debug("set other_account_holder")
# Don't print out the JSON, to ensure no sensitive data gets displayed.
obp_logger.debug("create json dump")
obp_transaction_data = json.dumps([
{
"obp_transaction": {
"this_account": {
"holder": this_account_holder[1],
"number": this_account_number[1],
"kind": this_account_kind,
"bank": {
"IBAN": this_account_IBAN[1],
"national_identifier": this_account_ni,
"name": this_account_bank_name
}
},
"other_account": {
"holder": other_account_holder,
"number": input_row[3].rstrip(),
"kind": "",
"bank": {
"IBAN": "",
"national_identifier": "",
"name": ""
}
},
"details": {
"type_en": "",
"type_de": input_row[2],
"posted": {
"$dt": convert_date(input_row[0]) # Have to set to $dt so Scala can work with it.
},
"completed": {
"$dt": convert_date(input_row[1]) # Have to set to $dt so Scala can work with it.
},
"new_balance":{
"currency": currency_sign_to_text(this_account_currency.group()),
"amount": new_balance.group()
},
"value": {
"currency": currency_sign_to_text(this_account_currency.group()),
"amount": amount.group()
},
"other_data": input_row[5]
}
}
}], sort_keys=False)
obp_logger.debug("Done filling json, returning obp_transaction_data")
return obp_transaction_data
def get_csv_with_selenium(path_to_save_csv, username, password):
"""Getting CSV file via Firefox, controlled by Selenium webdriver"""
# TODO: When no username and password is set, use the demo login.
# Clean up the OBP temp folder (delete all csv files there).
# LINK: http://seleniumhq.org/docs/03_webdriver.html#getting-started-with-selenium-webdriver
obp_logger.info("Setting csv_folder...")
# Check for existing and empty tmp
check_for_clean_folder(path_to_save_csv)
csv_save_path = os.path.join(os.getcwd(), path_to_save_csv, TMP_CSV_SUFFIX)
obp_logger.debug("csv_folder is: %s" % csv_save_path)
check_for_clean_folder(csv_save_path)
obp_logger.info("Start Selenium")
obp_logger.debug("csv_save_path: %s" % csv_save_path)
obp_logger.debug("username is set")
obp_logger.debug("password is set")
# Setting up a Profile for Firefox.
# Proxy is disabled and download files without asking.
obp_logger.info("Setup Firefox Profile")
fp = webdriver.FirefoxProfile()
obp_logger.debug("webdriver firefox")
fp.set_preference("network.proxy.type", 0)
obp_logger.debug("network.proxy.type 0")
fp.set_preference("browser.download.folderList", 2)
obp_logger.debug("rowser.download.fold 2")
fp.set_preference("browser.download.manager.showWhenStarting", False)
obp_logger.debug("rowser.download.manager.showWhenStarting False ")
fp.set_preference("browser.download.dir", csv_save_path)
obp_logger.debug("browser.download.dir %s" % csv_save_path)
# Need to set CSV to saveToDisk, else it's unknown to FF and it will ask for it
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv")
obp_logger.debug("browser.helperApps.neverAsk.saveToDisk text/csv")
obp_logger.info("Start Firefox")
browser = webdriver.Firefox(
firefox_profile=fp) # Get local session of firefox
obp_logger.debug("Open URL: %s" % postbank_main_url_login_page)
browser.get(postbank_main_url_login_page) # Load page
assert "Postbank Online-Banking" in browser.title
#selenium.set_browser_log_level("error")
# Here we will insert the username and password:
# find the element that's name attribute is nutzername and kennwort
obp_logger.info("Inserting Username and Password to Login")
obp_logger.debug("searching for login box")
inputElement_username = browser.find_element_by_name("nutzername")
obp_logger.debug("searching for password box")
inputElement_password = browser.find_element_by_name("kennwort")
# send Username and Password
obp_logger.debug("Inserting username into login box: %s " % username)
inputElement_username.send_keys(username)
obp_logger.debug("Inserting password into login box")
inputElement_password.send_keys(password)
# submit the Username and Password to Postbank.
obp_logger.info("submitting login_data to login")
inputElement_password.submit()
# This opens the main page for accounts, and checks the name.
# Call the Transaction Page
obp_logger.debug("Open URL: %s" % postbank_main_url_value_page)
browser.get(postbank_main_url_value_page)
assert "Postbank Online-Banking" in browser.title
# Call the CSV Link.
# Warning!
# The Postbank uses a :page counter, and when the URL doesn't have the right page counter it will return
# an error message.
obp_logger.debug("Open URL: %s" % postbank_main_url_value_download)
result = browser.get(postbank_main_url_value_download)
obp_logger.info("closing Firefox")
browser.close()
return csv_save_path
def main():
obp_logger.info("Start main")
check_for_clean_folder()
get_csv_with_selenium(TMP)
obp_logger.info("Done main")
