def set_bank_account_login(): """ This will ask for a username and Password. The password will be retrieved via getpass lib. Requires the password length to be at least 5 characters. """ # Getting the login name from a raw_input. obp_logger.info("get username") username = raw_input("Username: "******"username is set") # Now getting the password via getpass lib. obp_logger.info("get password") password = getpass.getpass() obp_logger.debug("password has set") # We know from the Web Page that we need at least 5 characters. # This will check for the right length of the password. obp_logger.debug("start while loop") while len(password) < 5: obp_logger.error("Password was not 5 character long") print "Password has to contain at least 5 letters" password = getpass.getpass() obp_logger.debug("Password has set") obp_logger.debug("Will return username: %s ,password is set" % username) # Return username and password. return username, password
def create_hash(value_to_hash): """This will create a hash and return it""" obp_logger.info("Creating hash with sha512") data_hash = hashlib.sha512(value_to_hash).hexdigest() obp_logger.debug("created hash is in hexdigest: %s" % data_hash) return check_hash(data_hash)
def main(csv_file_path): """Will check for a valid CSV and import it to the Scala API""" obp_logger.info("Start Main") obp_logger.debug("csv_file_path is: %s" % csv_file_path) obp_logger.debug("Check that csv_file_path is valid path") check_for_existing_csv(csv_file_path) obp_logger.debug("Start parse_row_of_csv") parse_row_of_csv(csv_file_path)
def insert_into_scala(scala_api_host, scala_api_port, JSON_to_insert): """ Inserting JSON via HTTP POST into the API. """ obp_logger.info("Insert JSON to Scala API") obp_logger.debug("test connection to Scala API Host") check_scala_host_reachable(scala_api_host, scala_api_port) # Set content-type to JSON in the HTTP Header headers = {'content-type': 'application/json'} obp_logger.debug("Set HTTP headers to: %s" % headers) post_request = requests.post("http://" + scala_api_host + ":" + scala_api_port + "/api/transactions", data=JSON_to_insert, headers=headers) # Return the http status code. obp_logger.debug("Inserted to SCALA API") return post_request
def insert_hash_to_cache(hash_to_insert, file): """Will insert the Hash_input into the file, if it isn't already there""" obp_logger.debug("start insert_hash") obp_logger.debug("check for valid_hash") valid_hash = check_hash(hash_to_insert) obp_logger.debug("read cache file for existing hash") if check_existing_hashs(valid_hash, file) != True: obp_logger.debug("Opening cache file, appending") file_to_write = open(file, 'a') obp_logger.debug("writing hash to file, hash is: %s" % valid_hash) file_to_write.write(valid_hash + '\n') obp_logger.debug("closing cache file") file_to_write.close() obp_logger.debug("returning True") return True else: obp_logger.info("Hash already inserted") return False
def check_for_clean_folder(check_path): """Check for an empty folder. Else create it.""" obp_logger.info("Check for empty and exists folder") obp_logger.debug("Check that %s exists" % check_path) if os.path.exists(check_path) != True: obp_logger.debug("%s no there, create it") os.makedirs(check_path) # Check for an empty folder obp_logger.debug("Check for empty folder") if len(os.listdir(check_path)) != 0: obp_logger.debug("Getting length of items in %s" % check_path) obp_logger.debug("Try to remove items in check_path") for item in os.listdir(check_path): item_to_remove = os.path.join(check_path, item) obp_logger.debug("Item is %s" % item_to_remove) try: os.remove(item_to_remove) except OSError, e: obp_logger.warn("Can't remove %s" % e)
def parse_row_of_csv(csv_file_to_parse): """Gets rows from CSV file""" obp_logger.info("start reading the csv file line by line") # This is to set the separators in the CSV file # TODO: This should be in the obp_config delimiter = ';' quote_char = '"' obp_logger.debug("Set CSV delimiter to: %s" % delimiter) obp_logger.debug("Set CSV quote_char to: %s" % quote_char) # re : \d\d\.\d\d\.\d\d\d\d # This will check if date is formatted like this: 23.01.2001 obp_logger.debug("Set regular expression to: \d\d\.\d\d\.\d\d") data_expression = re.compile('\d\d\.\d\d\.\d\d\d\d') obp_logger.debug("starting csv reader") transaction_reader = csv.reader(open(csv_file_to_parse, 'rb'), delimiter=delimiter, quotechar=quote_char) obp_logger.debug("Start of for loop of transaction_reader") for row in transaction_reader: # The first valid entry always has a date: check for it. # If it doesn't exist, add this row to the csv_header_info and then continue. obp_logger.debug("checking for date in first row from csv") if data_expression.match(row[0]) == None: csv_header_info.append(row) obp_logger.debug("append row to csv_header_info, row is: %s" % row) continue else: # When we have a valid date, call get_info_from_row. obp_transaction_dict = get_info_from_row(row) obp_logger.debug("call get_info_from_row") # This will create a hash and return it. json_hash = create_hash(json_formatter(obp_transaction_dict)) obp_logger.debug("create json_hash from obp_transaction_dict") # Some debug output. So that we may can see the content of the JSON # and the hash. obp_logger.info("The hash of the JSON is: %s" % json_hash) print "%s:The hash of the JSON is: %s" % (date_now_formatted(), json_hash) # Insert the hash into the cache. If it worked (the hash did not yet exist) # send it to the API. result = insert_hash_to_cache(json_hash, HASH_FILE) if result == True: result = insert_into_scala(SCALA_HOST, SCALA_PORT, json_formatter(obp_transaction_dict)) obp_logger.debug("HTTP POST result is: %s" % result) #obp_logger.debug("HTTP POST text from result is: %s" % result.text) else: obp_logger.info( "Transaction is already in hash file, not inserting") print "%s:Transaction is already in hash file, not inserting" % date_now_formatted( )
def parse_row_of_csv(csv_file_to_parse): """Gets rows from CSV file""" obp_logger.info("start reading the csv file line by line") # This is to set the separators in the CSV file # TODO: This should be in the obp_config delimiter = ';' quote_char = '"' obp_logger.debug("Set CSV delimiter to: %s" % delimiter) obp_logger.debug("Set CSV quote_char to: %s" % quote_char) # re : \d\d\.\d\d\.\d\d\d\d # This will check if date is formatted like this: 23.01.2001 obp_logger.debug("Set regular expression to: \d\d\.\d\d\.\d\d") data_expression = re.compile('\d\d\.\d\d\.\d\d\d\d') obp_logger.debug("starting csv reader") transaction_reader = csv.reader( open(csv_file_to_parse, 'rb'), delimiter=delimiter, quotechar=quote_char) obp_logger.debug("Start of for loop of transaction_reader") for row in transaction_reader: # The first valid entry always has a date: check for it. # If it doesn't exist, add this row to the csv_header_info and then continue. obp_logger.debug("checking for date in first row from csv") if data_expression.match(row[0]) == None: csv_header_info.append(row) obp_logger.debug("append row to csv_header_info, row is: %s" % row) continue else: # When we have a valid date, call get_info_from_row. obp_transaction_dict = get_info_from_row(row) obp_logger.debug("call get_info_from_row") # This will create a hash and return it. json_hash = create_hash(json_formatter(obp_transaction_dict)) obp_logger.debug("create json_hash from obp_transaction_dict") # Some debug output. So that we may can see the content of the JSON # and the hash. obp_logger.info("The hash of the JSON is: %s" % json_hash) print "%s:The hash of the JSON is: %s" % (date_now_formatted(), json_hash) # Insert the hash into the cache. If it worked (the hash did not yet exist) # send it to the API. result = insert_hash_to_cache(json_hash, HASH_FILE) if result == True: result = insert_into_scala( SCALA_HOST, SCALA_PORT, json_formatter(obp_transaction_dict)) obp_logger.debug("HTTP POST result is: %s" % result) #obp_logger.debug("HTTP POST text from result is: %s" % result.text) else: obp_logger.info("Transaction is already in hash file, not inserting") print "%s:Transaction is already in hash file, not inserting" % date_now_formatted()
def get_info_from_row(input_row): """Read rows and get the transaction data, print as JSON""" obp_logger.info("Start get_info_from_row") # The Germans format money like 1.200,12 Eur. We # need a English format, i.e. 1200.12 Eur # So have to remove the dot and replace the , with a dot. # This will turn . to "" obp_logger.debug("replace . with empty string") dotless_amount = re.sub('\.', '', input_row[6]) dotless_new_balance = re.sub('\.', '', input_row[7]) obp_logger.debug("replace , with .") comma_to_dot_amount = re.sub(',', '.', dotless_amount) comma_to_dot_new_balance = re.sub(',', '.', dotless_new_balance) # This regular expression searches for all kind of numbers in a string. # Also covering + and - #obp_logger.debug("") amount = re.match("[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?", comma_to_dot_amount) new_balance = re.match("[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?", comma_to_dot_new_balance) obp_logger.debug("set this_account_holder") this_account_holder = csv_header_info[1] obp_logger.debug("set this_account_IBAN") this_account_IBAN = csv_header_info[4] obp_logger.debug("set this_account_number") this_account_number = csv_header_info[3] # There is still some value inside that we need to remove this_account_unclean_currency = csv_header_info[5] this_account_currency = re.search('\xe2\x82\xac', this_account_unclean_currency[1]) obp_logger.debug("set this_account_currency") obp_logger.debug("set this_account_kind") this_account_kind = 'current' obp_logger.debug("set this_account_ni") this_account_ni = "" # ni = national_identifier obp_logger.debug("set this_account_bank_name") this_account_bank_name = 'Postbank' # Need to use row 4 if we're sending money, # and row 5 when we're getting money. obp_logger.debug( "check that this_account_holder is not other_account_holder") if input_row[5].rstrip() != this_account_holder[1]: other_account_holder = input_row[5].rstrip() obp_logger.debug("set other_account_holder") else: other_account_holder = input_row[4].rstrip() obp_logger.debug("set other_account_holder") # Don't print out the JSON, to ensure no sensitive data gets displayed. obp_logger.debug("create json dump") obp_transaction_data = json.dumps( [{ "obp_transaction": { "this_account": { "holder": this_account_holder[1], "number": this_account_number[1], "kind": this_account_kind, "bank": { "IBAN": this_account_IBAN[1], "national_identifier": this_account_ni, "name": this_account_bank_name } }, "other_account": { "holder": other_account_holder, "number": input_row[3].rstrip(), "kind": "", "bank": { "IBAN": "", "national_identifier": "", "name": "" } }, "details": { "type_en": "", "type_de": input_row[2], "posted": { "$dt": convert_date( input_row[0] ) # Have to set to $dt so Scala can work with it. }, "completed": { "$dt": convert_date( input_row[1] ) # Have to set to $dt so Scala can work with it. }, "new_balance": { "currency": currency_sign_to_text(this_account_currency.group()), "amount": new_balance.group() }, "value": { "currency": currency_sign_to_text(this_account_currency.group()), "amount": amount.group() }, "other_data": input_row[5] } } }], sort_keys=False) obp_logger.debug("Done filling json, returning obp_transaction_data") return obp_transaction_data
def get_csv_with_selenium(path_to_save_csv, username, password): """Getting CSV file via Firefox, controlled by Selenium webdriver""" # TODO: When no username and password is set, use the demo login. # Clean up the OBP temp folder (delete all csv files there). # LINK: http://seleniumhq.org/docs/03_webdriver.html#getting-started-with-selenium-webdriver obp_logger.info("Setting csv_folder...") # Check for existing and empty tmp check_for_clean_folder(path_to_save_csv) csv_save_path = os.path.join(os.getcwd(), path_to_save_csv, TMP_CSV_SUFFIX) obp_logger.debug("csv_folder is: %s" % csv_save_path) check_for_clean_folder(csv_save_path) obp_logger.info("Start Selenium") obp_logger.debug("csv_save_path: %s" % csv_save_path) obp_logger.debug("username is set") obp_logger.debug("password is set") # Setting up a Profile for Firefox. # Proxy is disabled and download files without asking. obp_logger.info("Setup Firefox Profile") fp = webdriver.FirefoxProfile() obp_logger.debug("webdriver firefox") fp.set_preference("network.proxy.type", 0) obp_logger.debug("network.proxy.type 0") fp.set_preference("browser.download.folderList", 2) obp_logger.debug("rowser.download.fold 2") fp.set_preference("browser.download.manager.showWhenStarting", False) obp_logger.debug("rowser.download.manager.showWhenStarting False ") fp.set_preference("browser.download.dir", csv_save_path) obp_logger.debug("browser.download.dir %s" % csv_save_path) # Need to set CSV to saveToDisk, else it's unknown to FF and it will ask for it fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv") obp_logger.debug("browser.helperApps.neverAsk.saveToDisk text/csv") obp_logger.info("Start Firefox") browser = webdriver.Firefox(firefox_profile=fp) # Get local session of firefox obp_logger.debug("Open URL: %s" % postbank_main_url_login_page) browser.get(postbank_main_url_login_page) # Load page assert "Postbank Online-Banking" in browser.title #selenium.set_browser_log_level("error") # Here we will insert the username and password: # find the element that's name attribute is nutzername and kennwort obp_logger.info("Inserting Username and Password to Login") obp_logger.debug("searching for login box") inputElement_username = browser.find_element_by_name("nutzername") obp_logger.debug("searching for password box") inputElement_password = browser.find_element_by_name("kennwort") # send Username and Password obp_logger.debug("Inserting username into login box: %s " % username) inputElement_username.send_keys(username) obp_logger.debug("Inserting password into login box") inputElement_password.send_keys(password) # submit the Username and Password to Postbank. obp_logger.info("submitting login_data to login") inputElement_password.submit() # This opens the main page for accounts, and checks the name. # Call the Transaction Page obp_logger.debug("Open URL: %s" % postbank_main_url_value_page) browser.get(postbank_main_url_value_page) assert "Postbank Online-Banking" in browser.title # Call the CSV Link. # Warning! # The Postbank uses a :page counter, and when the URL doesn't have the right page counter it will return # an error message. obp_logger.debug("Open URL: %s" % postbank_main_url_value_download) result = browser.get(postbank_main_url_value_download) obp_logger.info("closing Firefox") browser.close() return csv_save_path
def main(): obp_logger.info("Start main") check_for_clean_folder() get_csv_with_selenium(TMP) obp_logger.info("Done main")
def get_info_from_row(input_row): """Read rows and get the transaction data, print as JSON""" obp_logger.info("Start get_info_from_row") # The Germans format money like 1.200,12 Eur. We # need a English format, i.e. 1200.12 Eur # So have to remove the dot and replace the , with a dot. # This will turn . to "" obp_logger.debug("replace . with empty string") dotless_amount = re.sub('\.', '', input_row[6]) dotless_new_balance = re.sub('\.', '', input_row[7]) obp_logger.debug("replace , with .") comma_to_dot_amount = re.sub(',', '.', dotless_amount) comma_to_dot_new_balance = re.sub(',', '.', dotless_new_balance) # This regular expression searches for all kind of numbers in a string. # Also covering + and - #obp_logger.debug("") amount = re.match( "[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?", comma_to_dot_amount) new_balance = re.match( "[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?", comma_to_dot_new_balance) obp_logger.debug("set this_account_holder") this_account_holder = csv_header_info[1] obp_logger.debug("set this_account_IBAN") this_account_IBAN = csv_header_info[4] obp_logger.debug("set this_account_number") this_account_number = csv_header_info[3] # There is still some value inside that we need to remove this_account_unclean_currency = csv_header_info[5] this_account_currency = re.search( '\xe2\x82\xac', this_account_unclean_currency[1]) obp_logger.debug("set this_account_currency") obp_logger.debug("set this_account_kind") this_account_kind = 'current' obp_logger.debug("set this_account_ni") this_account_ni = "" # ni = national_identifier obp_logger.debug("set this_account_bank_name") this_account_bank_name = 'Postbank' # Need to use row 4 if we're sending money, # and row 5 when we're getting money. obp_logger.debug("check that this_account_holder is not other_account_holder") if input_row[5].rstrip() != this_account_holder[1]: other_account_holder = input_row[5].rstrip() obp_logger.debug("set other_account_holder") else: other_account_holder = input_row[4].rstrip() obp_logger.debug("set other_account_holder") # Don't print out the JSON, to ensure no sensitive data gets displayed. obp_logger.debug("create json dump") obp_transaction_data = json.dumps([ { "obp_transaction": { "this_account": { "holder": this_account_holder[1], "number": this_account_number[1], "kind": this_account_kind, "bank": { "IBAN": this_account_IBAN[1], "national_identifier": this_account_ni, "name": this_account_bank_name } }, "other_account": { "holder": other_account_holder, "number": input_row[3].rstrip(), "kind": "", "bank": { "IBAN": "", "national_identifier": "", "name": "" } }, "details": { "type_en": "", "type_de": input_row[2], "posted": { "$dt": convert_date(input_row[0]) # Have to set to $dt so Scala can work with it. }, "completed": { "$dt": convert_date(input_row[1]) # Have to set to $dt so Scala can work with it. }, "new_balance":{ "currency": currency_sign_to_text(this_account_currency.group()), "amount": new_balance.group() }, "value": { "currency": currency_sign_to_text(this_account_currency.group()), "amount": amount.group() }, "other_data": input_row[5] } } }], sort_keys=False) obp_logger.debug("Done filling json, returning obp_transaction_data") return obp_transaction_data
def get_csv_with_selenium(path_to_save_csv, username, password): """Getting CSV file via Firefox, controlled by Selenium webdriver""" # TODO: When no username and password is set, use the demo login. # Clean up the OBP temp folder (delete all csv files there). # LINK: http://seleniumhq.org/docs/03_webdriver.html#getting-started-with-selenium-webdriver obp_logger.info("Setting csv_folder...") # Check for existing and empty tmp check_for_clean_folder(path_to_save_csv) csv_save_path = os.path.join(os.getcwd(), path_to_save_csv, TMP_CSV_SUFFIX) obp_logger.debug("csv_folder is: %s" % csv_save_path) check_for_clean_folder(csv_save_path) obp_logger.info("Start Selenium") obp_logger.debug("csv_save_path: %s" % csv_save_path) obp_logger.debug("username is set") obp_logger.debug("password is set") # Setting up a Profile for Firefox. # Proxy is disabled and download files without asking. obp_logger.info("Setup Firefox Profile") fp = webdriver.FirefoxProfile() obp_logger.debug("webdriver firefox") fp.set_preference("network.proxy.type", 0) obp_logger.debug("network.proxy.type 0") fp.set_preference("browser.download.folderList", 2) obp_logger.debug("rowser.download.fold 2") fp.set_preference("browser.download.manager.showWhenStarting", False) obp_logger.debug("rowser.download.manager.showWhenStarting False ") fp.set_preference("browser.download.dir", csv_save_path) obp_logger.debug("browser.download.dir %s" % csv_save_path) # Need to set CSV to saveToDisk, else it's unknown to FF and it will ask for it fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv") obp_logger.debug("browser.helperApps.neverAsk.saveToDisk text/csv") obp_logger.info("Start Firefox") browser = webdriver.Firefox( firefox_profile=fp) # Get local session of firefox obp_logger.debug("Open URL: %s" % postbank_main_url_login_page) browser.get(postbank_main_url_login_page) # Load page assert "Postbank Online-Banking" in browser.title #selenium.set_browser_log_level("error") # Here we will insert the username and password: # find the element that's name attribute is nutzername and kennwort obp_logger.info("Inserting Username and Password to Login") obp_logger.debug("searching for login box") inputElement_username = browser.find_element_by_name("nutzername") obp_logger.debug("searching for password box") inputElement_password = browser.find_element_by_name("kennwort") # send Username and Password obp_logger.debug("Inserting username into login box: %s " % username) inputElement_username.send_keys(username) obp_logger.debug("Inserting password into login box") inputElement_password.send_keys(password) # submit the Username and Password to Postbank. obp_logger.info("submitting login_data to login") inputElement_password.submit() # This opens the main page for accounts, and checks the name. # Call the Transaction Page obp_logger.debug("Open URL: %s" % postbank_main_url_value_page) browser.get(postbank_main_url_value_page) assert "Postbank Online-Banking" in browser.title # Call the CSV Link. # Warning! # The Postbank uses a :page counter, and when the URL doesn't have the right page counter it will return # an error message. obp_logger.debug("Open URL: %s" % postbank_main_url_value_download) result = browser.get(postbank_main_url_value_download) obp_logger.info("closing Firefox") browser.close() return csv_save_path