def create_hash(value_to_hash):
"""This will create a hash and return it"""
obp_logger.info("Creating hash with sha512")
data_hash = hashlib.sha512(value_to_hash).hexdigest()
obp_logger.debug("created hash is in hexdigest: %s" % data_hash)
return check_hash(data_hash)
def set_bank_account_login():
"""
This will ask for a username and Password.
The password will be retrieved via getpass lib.
Requires the password length to be at least 5 characters.
"""
# Getting the login name from a raw_input.
obp_logger.info("get username")
username = raw_input("Username: "******"username is set")
# Now getting the password via getpass lib.
obp_logger.info("get password")
password = getpass.getpass()
obp_logger.debug("password has set")
# We know from the Web Page that we need at least 5 characters.
# This will check for the right length of the password.
obp_logger.debug("start while loop")
while len(password) < 5:
obp_logger.error("Password was not 5 character long")
print "Password has to contain at least 5 letters"
password = getpass.getpass()
obp_logger.debug("Password has set")
obp_logger.debug("Will return username: %s ,password is set" % username)
# Return username and password.
return username, password
def json_formatter(json):
"""
This is needed for the API! The API can't handle[ and ].
This will remove the first [ and the last ] of the JSON.
"""
obp_logger.debug("removing [ and ] from json")
return re.sub(r'^\[|\]$', ' ', json)
def date_now_formatted():
"""
Print out nice formatted dates.
In following order: Day, Month Year, Hour:Minute:Second
"""
obp_logger.debug("setting nicely formatted datetime")
return datetime.datetime.now().strftime('[%d, %h %Y, %H:%M:%S]')
def check_for_existing_cache(cache_file):
"""This will check for a existing cache_file"""
obp_logger.debug("checking for existing cache_file: %s" % cache_file)
if os.path.exists(cache_file) == False:
print "ERROR! - NO CACHE FILE!"
obp_logger.error("ERROR! - NO CACHE FILE!")
raise
else:
obp_logger.debug("cache file exist")
def check_hash(hash_to_check):
"""Checks the hash for the correct size"""
obp_logger.debug("Checking length of hash")
if len(hash_to_check) != 128:
obp_logger.critiacl("Value does not have enough characters for a hash")
print "Value does not have enough characters for a hash!"
raise
else:
obp_logger.debug("Hash length is fine, returning hash_to_check: %s" % hash_to_check)
return hash_to_check
def currency_sign_to_text(currency_sign):
"""
This will convert the currency sign to ISO 4217 currency codes.
TODO: Add more currencies.
#E.S. This will get tricky as it depends on locale. For example $
#E.S. can be USD, CAD, AUD, etc.
"""
obp_logger.debug("looking up key: %s" % currency_sign)
currency_sign_text_dic = {'\xe2\x82\xac': 'EUR'}
obp_logger.debug("returning value: %s text currency" % currency_sign_text_dic[currency_sign])
return currency_sign_text_dic[currency_sign]
def remove_empty_lines(file):
"""
There is a problem reading the CSV File with a empty
newline, so this will remove it.
This edits the file in place and does not save it.
"""
#Don't remove the print statement, else everything will be removed!
obp_logger.debug("starting for loop to remove newlines")
for lines in fileinput.FileInput(file, inplace=1):
lines = lines.strip()
if lines == '':
continue
print lines
def check_for_existing_csv(csv_file_path):
"""
This function will check that the csv file exists.
TODO:
Increase the detail of this check, to ensure there
really is a CSV file.
"""
obp_logger.debug("checking if csv_file_path %s exists" % csv_file_path)
if os.path.exists(csv_file_path) == False:
print "ERROR! - NO CSV FILE!"
obp_logger.error("ERROR! - NO CSV FILE!")
raise
else:
obp_logger.debug("csv_file_path %s exist" % csv_file_path)
def main(csv_file_path):
"""Will check for a valid CSV and import it to the Scala API"""
obp_logger.info("Start Main")
obp_logger.debug("csv_file_path is: %s" % csv_file_path)
obp_logger.debug("Check that csv_file_path is valid path")
check_for_existing_csv(csv_file_path)
obp_logger.debug("Start parse_row_of_csv")
parse_row_of_csv(csv_file_path)
def main(csv_file_path):
"""Will check for a valid CSV and import it to the Scala API"""
obp_logger.info("Start Main")
obp_logger.debug("csv_file_path is: %s" % csv_file_path)
obp_logger.debug("Check that csv_file_path is valid path")
check_for_existing_csv(csv_file_path)
obp_logger.debug("Start parse_row_of_csv")
parse_row_of_csv(csv_file_path)
def insert_into_scala(scala_api_host, scala_api_port, JSON_to_insert):
"""
Inserting JSON via HTTP POST into the API.
"""
obp_logger.info("Insert JSON to Scala API")
obp_logger.debug("test connection to Scala API Host")
check_scala_host_reachable(scala_api_host, scala_api_port)
# Set content-type to JSON in the HTTP Header
headers = {'content-type': 'application/json'}
obp_logger.debug("Set HTTP headers to: %s" % headers)
post_request = requests.post("http://" + scala_api_host + ":" + scala_api_port + "/api/transactions",
data=JSON_to_insert, headers=headers)
# Return the http status code.
obp_logger.debug("Inserted to SCALA API")
return post_request
def check_scala_host_reachable(scala_api_host, scala_api_port):
"""
Check for a reachable System of the Scala API Host.
"""
obp_logger.debug("try to connect to api host")
# Try to call the web root of the scala_api_host
try:
obp_logger.debug("requested http://%s:%s/" % (scala_api_host, scala_api_port))
result = requests.get('http://' + scala_api_host + ':' + scala_api_port + '/', timeout=5)
obp_logger.debug("request result is: %s" % result)
return result
except:
obp_logger.critical("ERROR! - Can't connect to Scala API")
obp_logger.critical("Check the API Host!")
print "ERROR! - Can't connect to Scala API!"
print "Check the API Host!"
raise
def insert_into_scala(scala_api_host, scala_api_port, JSON_to_insert):
"""
Inserting JSON via HTTP POST into the API.
"""
obp_logger.info("Insert JSON to Scala API")
obp_logger.debug("test connection to Scala API Host")
check_scala_host_reachable(scala_api_host, scala_api_port)
# Set content-type to JSON in the HTTP Header
headers = {'content-type': 'application/json'}
obp_logger.debug("Set HTTP headers to: %s" % headers)
post_request = requests.post("http://" + scala_api_host + ":" +
scala_api_port + "/api/transactions",
data=JSON_to_insert,
headers=headers)
# Return the http status code.
obp_logger.debug("Inserted to SCALA API")
return post_request
def check_scala_host_reachable(scala_api_host, scala_api_port):
"""
Check for a reachable System of the Scala API Host.
"""
obp_logger.debug("try to connect to api host")
# Try to call the web root of the scala_api_host
try:
obp_logger.debug("requested http://%s:%s/" %
(scala_api_host, scala_api_port))
result = requests.get('http://' + scala_api_host + ':' +
scala_api_port + '/',
timeout=5)
obp_logger.debug("request result is: %s" % result)
return result
except:
obp_logger.critical("ERROR! - Can't connect to Scala API")
obp_logger.critical("Check the API Host!")
print "ERROR! - Can't connect to Scala API!"
print "Check the API Host!"
raise
def check_and_return_csv_file_name(path_to_saved_csv):
"""
Check that there is only one csv file, and then return it.
"""
obp_logger.debug("check and return filename")
#obp_logger.debug("path_to_saved_csv is: %s" % path_to_saved_csv)
#We expect that in the folder is only one file:
csv_folder = os.listdir(path_to_saved_csv)
# We expect only one file in this folder
file_count = len(csv_folder)
obp_logger.debug("File count in csv_folder: %s" % file_count)
if file_count == 0:
obp_logger.error("ERROR - We didn't get the CSV file.")
print "ERROR - We didn't get the CSV file."
elif file_count != 1:
obp_logger.error("ERROR - We found too many files.")
print "ERROR - We found too many files."
obp_logger.debug("return csv_folder")
return csv_folder[0]
def get_info_from_row(input_row):
"""Read rows and get the transaction data, print as JSON"""
obp_logger.info("Start get_info_from_row")
# The Germans format money like 1.200,12 Eur. We
# need a English format, i.e. 1200.12 Eur
# So have to remove the dot and replace the , with a dot.
# This will turn . to ""
obp_logger.debug("replace . with empty string")
dotless_amount = re.sub('\.', '', input_row[6])
dotless_new_balance = re.sub('\.', '', input_row[7])
obp_logger.debug("replace , with .")
comma_to_dot_amount = re.sub(',', '.', dotless_amount)
comma_to_dot_new_balance = re.sub(',', '.', dotless_new_balance)
# This regular expression searches for all kind of numbers in a string.
# Also covering + and -
#obp_logger.debug("")
amount = re.match("[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?",
comma_to_dot_amount)
new_balance = re.match("[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?",
comma_to_dot_new_balance)
obp_logger.debug("set this_account_holder")
this_account_holder = csv_header_info[1]
obp_logger.debug("set this_account_IBAN")
this_account_IBAN = csv_header_info[4]
obp_logger.debug("set this_account_number")
this_account_number = csv_header_info[3]
# There is still some value inside that we need to remove
this_account_unclean_currency = csv_header_info[5]
this_account_currency = re.search('\xe2\x82\xac',
this_account_unclean_currency[1])
obp_logger.debug("set this_account_currency")
obp_logger.debug("set this_account_kind")
this_account_kind = 'current'
obp_logger.debug("set this_account_ni")
this_account_ni = "" # ni = national_identifier
obp_logger.debug("set this_account_bank_name")
this_account_bank_name = 'Postbank'
# Need to use row 4 if we're sending money,
# and row 5 when we're getting money.
obp_logger.debug(
"check that this_account_holder is not other_account_holder")
if input_row[5].rstrip() != this_account_holder[1]:
other_account_holder = input_row[5].rstrip()
obp_logger.debug("set other_account_holder")
else:
other_account_holder = input_row[4].rstrip()
obp_logger.debug("set other_account_holder")
# Don't print out the JSON, to ensure no sensitive data gets displayed.
obp_logger.debug("create json dump")
obp_transaction_data = json.dumps(
[{
"obp_transaction": {
"this_account": {
"holder": this_account_holder[1],
"number": this_account_number[1],
"kind": this_account_kind,
"bank": {
"IBAN": this_account_IBAN[1],
"national_identifier": this_account_ni,
"name": this_account_bank_name
}
},
"other_account": {
"holder": other_account_holder,
"number": input_row[3].rstrip(),
"kind": "",
"bank": {
"IBAN": "",
"national_identifier": "",
"name": ""
}
},
"details": {
"type_en": "",
"type_de": input_row[2],
"posted": {
"$dt": convert_date(
input_row[0]
) # Have to set to $dt so Scala can work with it.
},
"completed": {
"$dt": convert_date(
input_row[1]
) # Have to set to $dt so Scala can work with it.
},
"new_balance": {
"currency":
currency_sign_to_text(this_account_currency.group()),
"amount":
new_balance.group()
},
"value": {
"currency":
currency_sign_to_text(this_account_currency.group()),
"amount":
amount.group()
},
"other_data": input_row[5]
}
}
}],
sort_keys=False)
obp_logger.debug("Done filling json, returning obp_transaction_data")
return obp_transaction_data
def parse_row_of_csv(csv_file_to_parse):
"""Gets rows from CSV file"""
obp_logger.info("start reading the csv file line by line")
# This is to set the separators in the CSV file
# TODO: This should be in the obp_config
delimiter = ';'
quote_char = '"'
obp_logger.debug("Set CSV delimiter to: %s" % delimiter)
obp_logger.debug("Set CSV quote_char to: %s" % quote_char)
# re : \d\d\.\d\d\.\d\d\d\d
# This will check if date is formatted like this: 23.01.2001
obp_logger.debug("Set regular expression to: \d\d\.\d\d\.\d\d")
data_expression = re.compile('\d\d\.\d\d\.\d\d\d\d')
obp_logger.debug("starting csv reader")
transaction_reader = csv.reader(open(csv_file_to_parse, 'rb'),
delimiter=delimiter,
quotechar=quote_char)
obp_logger.debug("Start of for loop of transaction_reader")
for row in transaction_reader:
# The first valid entry always has a date: check for it.
# If it doesn't exist, add this row to the csv_header_info and then continue.
obp_logger.debug("checking for date in first row from csv")
if data_expression.match(row[0]) == None:
csv_header_info.append(row)
obp_logger.debug("append row to csv_header_info, row is: %s" % row)
continue
else:
# When we have a valid date, call get_info_from_row.
obp_transaction_dict = get_info_from_row(row)
obp_logger.debug("call get_info_from_row")
# This will create a hash and return it.
json_hash = create_hash(json_formatter(obp_transaction_dict))
obp_logger.debug("create json_hash from obp_transaction_dict")
# Some debug output. So that we may can see the content of the JSON
# and the hash.
obp_logger.info("The hash of the JSON is: %s" % json_hash)
print "%s:The hash of the JSON is: %s" % (date_now_formatted(),
json_hash)
# Insert the hash into the cache. If it worked (the hash did not yet exist)
# send it to the API.
result = insert_hash_to_cache(json_hash, HASH_FILE)
if result == True:
result = insert_into_scala(SCALA_HOST, SCALA_PORT,
json_formatter(obp_transaction_dict))
obp_logger.debug("HTTP POST result is: %s" % result)
#obp_logger.debug("HTTP POST text from result is: %s" % result.text)
else:
obp_logger.info(
"Transaction is already in hash file, not inserting")
print "%s:Transaction is already in hash file, not inserting" % date_now_formatted(
)
def get_csv_with_selenium(path_to_save_csv, username, password):
"""Getting CSV file via Firefox, controlled by Selenium webdriver"""
# TODO: When no username and password is set, use the demo login.
# Clean up the OBP temp folder (delete all csv files there).
# LINK: http://seleniumhq.org/docs/03_webdriver.html#getting-started-with-selenium-webdriver
obp_logger.info("Setting csv_folder...")
# Check for existing and empty tmp
check_for_clean_folder(path_to_save_csv)
csv_save_path = os.path.join(os.getcwd(), path_to_save_csv, TMP_CSV_SUFFIX)
obp_logger.debug("csv_folder is: %s" % csv_save_path)
check_for_clean_folder(csv_save_path)
obp_logger.info("Start Selenium")
obp_logger.debug("csv_save_path: %s" % csv_save_path)
obp_logger.debug("username is set")
obp_logger.debug("password is set")
# Setting up a Profile for Firefox.
# Proxy is disabled and download files without asking.
obp_logger.info("Setup Firefox Profile")
fp = webdriver.FirefoxProfile()
obp_logger.debug("webdriver firefox")
fp.set_preference("network.proxy.type", 0)
obp_logger.debug("network.proxy.type 0")
fp.set_preference("browser.download.folderList", 2)
obp_logger.debug("rowser.download.fold 2")
fp.set_preference("browser.download.manager.showWhenStarting", False)
obp_logger.debug("rowser.download.manager.showWhenStarting False ")
fp.set_preference("browser.download.dir", csv_save_path)
obp_logger.debug("browser.download.dir %s" % csv_save_path)
# Need to set CSV to saveToDisk, else it's unknown to FF and it will ask for it
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv")
obp_logger.debug("browser.helperApps.neverAsk.saveToDisk text/csv")
obp_logger.info("Start Firefox")
browser = webdriver.Firefox(firefox_profile=fp) # Get local session of firefox
obp_logger.debug("Open URL: %s" % postbank_main_url_login_page)
browser.get(postbank_main_url_login_page) # Load page
assert "Postbank Online-Banking" in browser.title
#selenium.set_browser_log_level("error")
# Here we will insert the username and password:
# find the element that's name attribute is nutzername and kennwort
obp_logger.info("Inserting Username and Password to Login")
obp_logger.debug("searching for login box")
inputElement_username = browser.find_element_by_name("nutzername")
obp_logger.debug("searching for password box")
inputElement_password = browser.find_element_by_name("kennwort")
# send Username and Password
obp_logger.debug("Inserting username into login box: %s " % username)
inputElement_username.send_keys(username)
obp_logger.debug("Inserting password into login box")
inputElement_password.send_keys(password)
# submit the Username and Password to Postbank.
obp_logger.info("submitting login_data to login")
inputElement_password.submit()
# This opens the main page for accounts, and checks the name.
# Call the Transaction Page
obp_logger.debug("Open URL: %s" % postbank_main_url_value_page)
browser.get(postbank_main_url_value_page)
assert "Postbank Online-Banking" in browser.title
# Call the CSV Link.
# Warning!
# The Postbank uses a :page counter, and when the URL doesn't have the right page counter it will return
# an error message.
obp_logger.debug("Open URL: %s" % postbank_main_url_value_download)
result = browser.get(postbank_main_url_value_download)
obp_logger.info("closing Firefox")
browser.close()
return csv_save_path
def get_info_from_row(input_row):
"""Read rows and get the transaction data, print as JSON"""
obp_logger.info("Start get_info_from_row")
# The Germans format money like 1.200,12 Eur. We
# need a English format, i.e. 1200.12 Eur
# So have to remove the dot and replace the , with a dot.
# This will turn . to ""
obp_logger.debug("replace . with empty string")
dotless_amount = re.sub('\.', '', input_row[6])
dotless_new_balance = re.sub('\.', '', input_row[7])
obp_logger.debug("replace , with .")
comma_to_dot_amount = re.sub(',', '.', dotless_amount)
comma_to_dot_new_balance = re.sub(',', '.', dotless_new_balance)
# This regular expression searches for all kind of numbers in a string.
# Also covering + and -
#obp_logger.debug("")
amount = re.match(
"[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?",
comma_to_dot_amount)
new_balance = re.match(
"[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?",
comma_to_dot_new_balance)
obp_logger.debug("set this_account_holder")
this_account_holder = csv_header_info[1]
obp_logger.debug("set this_account_IBAN")
this_account_IBAN = csv_header_info[4]
obp_logger.debug("set this_account_number")
this_account_number = csv_header_info[3]
# There is still some value inside that we need to remove
this_account_unclean_currency = csv_header_info[5]
this_account_currency = re.search(
'\xe2\x82\xac',
this_account_unclean_currency[1])
obp_logger.debug("set this_account_currency")
obp_logger.debug("set this_account_kind")
this_account_kind = 'current'
obp_logger.debug("set this_account_ni")
this_account_ni = "" # ni = national_identifier
obp_logger.debug("set this_account_bank_name")
this_account_bank_name = 'Postbank'
# Need to use row 4 if we're sending money,
# and row 5 when we're getting money.
obp_logger.debug("check that this_account_holder is not other_account_holder")
if input_row[5].rstrip() != this_account_holder[1]:
other_account_holder = input_row[5].rstrip()
obp_logger.debug("set other_account_holder")
else:
other_account_holder = input_row[4].rstrip()
obp_logger.debug("set other_account_holder")
# Don't print out the JSON, to ensure no sensitive data gets displayed.
obp_logger.debug("create json dump")
obp_transaction_data = json.dumps([
{
"obp_transaction": {
"this_account": {
"holder": this_account_holder[1],
"number": this_account_number[1],
"kind": this_account_kind,
"bank": {
"IBAN": this_account_IBAN[1],
"national_identifier": this_account_ni,
"name": this_account_bank_name
}
},
"other_account": {
"holder": other_account_holder,
"number": input_row[3].rstrip(),
"kind": "",
"bank": {
"IBAN": "",
"national_identifier": "",
"name": ""
}
},
"details": {
"type_en": "",
"type_de": input_row[2],
"posted": {
"$dt": convert_date(input_row[0]) # Have to set to $dt so Scala can work with it.
},
"completed": {
"$dt": convert_date(input_row[1]) # Have to set to $dt so Scala can work with it.
},
"new_balance":{
"currency": currency_sign_to_text(this_account_currency.group()),
"amount": new_balance.group()
},
"value": {
"currency": currency_sign_to_text(this_account_currency.group()),
"amount": amount.group()
},
"other_data": input_row[5]
}
}
}], sort_keys=False)
obp_logger.debug("Done filling json, returning obp_transaction_data")
return obp_transaction_data
def convert_date(date_to_convert):
"""This will convert a German formatted date(17.0.1.2012) to UTC Time."""
#E.S. It's probably better to find a library to do this than to use regexes.
# Will replace the dots of the date with a space.
# Then split it into 3 parts.
new_form = re.sub('\.', ' ', date_to_convert)
obp_logger.debug("replace . with ' ' in date_to_convert")
date_parts = new_form.split()
obp_logger.debug("split date_to_convert: %s" % date_to_convert)
# Getting now the single date, day, month and year
day = date_parts[0]
obp_logger.debug("Setting day")
month = date_parts[1]
obp_logger.debug("setting month")
year = date_parts[2]
obp_logger.debug("setting year")
# We can't set the exact date as we only have the day, month, and year,
# so we'll just set it to midnight.
zero_time = datetime.time(0, 0, 0)
obp_logger.debug("setting time to midnight")
to_convert = datetime.date(
int(year),
int(month),
int(day))
obp_logger.debug("Merge single date items together in to_convert")
datetime.datetime.combine(to_convert, zero_time)
obp_logger.debug("convert to_convert to datetime object")
# Will return UTC Date with GMT+1 option.
obp_logger.debug("return UTC formatted datetime")
return datetime.datetime.strftime(to_convert, "%Y-%m-%dT%H:%M:%S.001Z")
def parse_row_of_csv(csv_file_to_parse):
"""Gets rows from CSV file"""
obp_logger.info("start reading the csv file line by line")
# This is to set the separators in the CSV file
# TODO: This should be in the obp_config
delimiter = ';'
quote_char = '"'
obp_logger.debug("Set CSV delimiter to: %s" % delimiter)
obp_logger.debug("Set CSV quote_char to: %s" % quote_char)
# re : \d\d\.\d\d\.\d\d\d\d
# This will check if date is formatted like this: 23.01.2001
obp_logger.debug("Set regular expression to: \d\d\.\d\d\.\d\d")
data_expression = re.compile('\d\d\.\d\d\.\d\d\d\d')
obp_logger.debug("starting csv reader")
transaction_reader = csv.reader(
open(csv_file_to_parse, 'rb'),
delimiter=delimiter,
quotechar=quote_char)
obp_logger.debug("Start of for loop of transaction_reader")
for row in transaction_reader:
# The first valid entry always has a date: check for it.
# If it doesn't exist, add this row to the csv_header_info and then continue.
obp_logger.debug("checking for date in first row from csv")
if data_expression.match(row[0]) == None:
csv_header_info.append(row)
obp_logger.debug("append row to csv_header_info, row is: %s" % row)
continue
else:
# When we have a valid date, call get_info_from_row.
obp_transaction_dict = get_info_from_row(row)
obp_logger.debug("call get_info_from_row")
# This will create a hash and return it.
json_hash = create_hash(json_formatter(obp_transaction_dict))
obp_logger.debug("create json_hash from obp_transaction_dict")
# Some debug output. So that we may can see the content of the JSON
# and the hash.
obp_logger.info("The hash of the JSON is: %s" % json_hash)
print "%s:The hash of the JSON is: %s" % (date_now_formatted(), json_hash)
# Insert the hash into the cache. If it worked (the hash did not yet exist)
# send it to the API.
result = insert_hash_to_cache(json_hash, HASH_FILE)
if result == True:
result = insert_into_scala(
SCALA_HOST,
SCALA_PORT,
json_formatter(obp_transaction_dict))
obp_logger.debug("HTTP POST result is: %s" % result)
#obp_logger.debug("HTTP POST text from result is: %s" % result.text)
else:
obp_logger.info("Transaction is already in hash file, not inserting")
print "%s:Transaction is already in hash file, not inserting" % date_now_formatted()
def get_csv_with_selenium(path_to_save_csv, username, password):
"""Getting CSV file via Firefox, controlled by Selenium webdriver"""
# TODO: When no username and password is set, use the demo login.
# Clean up the OBP temp folder (delete all csv files there).
# LINK: http://seleniumhq.org/docs/03_webdriver.html#getting-started-with-selenium-webdriver
obp_logger.info("Setting csv_folder...")
# Check for existing and empty tmp
check_for_clean_folder(path_to_save_csv)
csv_save_path = os.path.join(os.getcwd(), path_to_save_csv, TMP_CSV_SUFFIX)
obp_logger.debug("csv_folder is: %s" % csv_save_path)
check_for_clean_folder(csv_save_path)
obp_logger.info("Start Selenium")
obp_logger.debug("csv_save_path: %s" % csv_save_path)
obp_logger.debug("username is set")
obp_logger.debug("password is set")
# Setting up a Profile for Firefox.
# Proxy is disabled and download files without asking.
obp_logger.info("Setup Firefox Profile")
fp = webdriver.FirefoxProfile()
obp_logger.debug("webdriver firefox")
fp.set_preference("network.proxy.type", 0)
obp_logger.debug("network.proxy.type 0")
fp.set_preference("browser.download.folderList", 2)
obp_logger.debug("rowser.download.fold 2")
fp.set_preference("browser.download.manager.showWhenStarting", False)
obp_logger.debug("rowser.download.manager.showWhenStarting False ")
fp.set_preference("browser.download.dir", csv_save_path)
obp_logger.debug("browser.download.dir %s" % csv_save_path)
# Need to set CSV to saveToDisk, else it's unknown to FF and it will ask for it
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv")
obp_logger.debug("browser.helperApps.neverAsk.saveToDisk text/csv")
obp_logger.info("Start Firefox")
browser = webdriver.Firefox(
firefox_profile=fp) # Get local session of firefox
obp_logger.debug("Open URL: %s" % postbank_main_url_login_page)
browser.get(postbank_main_url_login_page) # Load page
assert "Postbank Online-Banking" in browser.title
#selenium.set_browser_log_level("error")
# Here we will insert the username and password:
# find the element that's name attribute is nutzername and kennwort
obp_logger.info("Inserting Username and Password to Login")
obp_logger.debug("searching for login box")
inputElement_username = browser.find_element_by_name("nutzername")
obp_logger.debug("searching for password box")
inputElement_password = browser.find_element_by_name("kennwort")
# send Username and Password
obp_logger.debug("Inserting username into login box: %s " % username)
inputElement_username.send_keys(username)
obp_logger.debug("Inserting password into login box")
inputElement_password.send_keys(password)
# submit the Username and Password to Postbank.
obp_logger.info("submitting login_data to login")
inputElement_password.submit()
# This opens the main page for accounts, and checks the name.
# Call the Transaction Page
obp_logger.debug("Open URL: %s" % postbank_main_url_value_page)
browser.get(postbank_main_url_value_page)
assert "Postbank Online-Banking" in browser.title
# Call the CSV Link.
# Warning!
# The Postbank uses a :page counter, and when the URL doesn't have the right page counter it will return
# an error message.
obp_logger.debug("Open URL: %s" % postbank_main_url_value_download)
result = browser.get(postbank_main_url_value_download)
obp_logger.info("closing Firefox")
browser.close()
return csv_save_path
def check_existing_hashs(hash_to_check, file):
"""
Will open a file and read it.
line by line it will compare it with the input hash
Return true when it hit something
"""
#E.S. Confusingly named function? Should it be hash_exists?
obp_logger.debug("checking for existing hash")
valid_hash = check_hash(hash_to_check)
obp_logger.debug("check for valid hash, hash is: %s" % valid_hash)
obp_logger.debug("Checking for existing cache_file")
check_for_existing_cache(file)
obp_logger.debug("opening cache file readonly: %s" % file)
with open(file, 'r') as file_where_hash_is:
obp_logger.debug("looping through the cache file")
for saved_hashes in file_where_hash_is.readlines():
obp_logger.debug("line is: %s" % saved_hashes.strip())
obp_logger.debug("Comparing valid_hash with line from cache file")
if valid_hash == saved_hashes.strip():
obp_logger.debug("Found valid_hash, returning True")
return True
def insert_hash_to_cache(hash_to_insert, file):
"""Will insert the Hash_input into the file, if it isn't already there"""
obp_logger.debug("start insert_hash")
obp_logger.debug("check for valid_hash")
valid_hash = check_hash(hash_to_insert)
obp_logger.debug("read cache file for existing hash")
if check_existing_hashs(valid_hash, file) != True:
obp_logger.debug("Opening cache file, appending")
file_to_write = open(file, 'a')
obp_logger.debug("writing hash to file, hash is: %s" % valid_hash)
file_to_write.write(valid_hash + '\n')
obp_logger.debug("closing cache file")
file_to_write.close()
obp_logger.debug("returning True")
return True
else:
obp_logger.info("Hash already inserted")
return False
def check_for_clean_folder(check_path):
"""Check for an empty folder. Else create it."""
obp_logger.info("Check for empty and exists folder")
obp_logger.debug("Check that %s exists" % check_path)
if os.path.exists(check_path) != True:
obp_logger.debug("%s no there, create it")
os.makedirs(check_path)
# Check for an empty folder
obp_logger.debug("Check for empty folder")
if len(os.listdir(check_path)) != 0:
obp_logger.debug("Getting length of items in %s" % check_path)
obp_logger.debug("Try to remove items in check_path")
for item in os.listdir(check_path):
item_to_remove = os.path.join(check_path, item)
obp_logger.debug("Item is %s" % item_to_remove)
try:
os.remove(item_to_remove)
except OSError, e:
obp_logger.warn("Can't remove %s" % e)
def show_here():
"""Showing current working directory."""
obp_logger.debug("return current working directory")
return os.getcwd()
def clean_up(path_to_clean):
"""This function will clean up all files from tmp/"""
here = show_here()
obp_logger.debug("here is: %s" % here)
os.chdir(path_to_clean)
obp_logger.debug("changing to path_to_clean: %s" % path_to_clean)
obp_logger.debug("for loop on every element in dir")
for item in os.listdir(path_to_clean):
obp_logger.debug("checking that item is not a folder")
if os.path.isdir(item) == False:
obp_logger.debug("item is not a dir")
obp_logger.debug("deleting item: %s" % item)
os.remove(item)
obp_logger.debug("chdir to here: %s" % here)
os.chdir(here)
