def parse_row_of_csv(csv_file_to_parse):
"""Gets rows from CSV file"""
obp_logger.info("start reading the csv file line by line")
# This is to set the separators in the CSV file
# TODO: This should be in the obp_config
delimiter = ';'
quote_char = '"'
obp_logger.debug("Set CSV delimiter to: %s" % delimiter)
obp_logger.debug("Set CSV quote_char to: %s" % quote_char)
# re : \d\d\.\d\d\.\d\d\d\d
# This will check if date is formatted like this: 23.01.2001
obp_logger.debug("Set regular expression to: \d\d\.\d\d\.\d\d")
data_expression = re.compile('\d\d\.\d\d\.\d\d\d\d')
obp_logger.debug("starting csv reader")
transaction_reader = csv.reader(
open(csv_file_to_parse, 'rb'),
delimiter=delimiter,
quotechar=quote_char)
obp_logger.debug("Start of for loop of transaction_reader")
for row in transaction_reader:
# The first valid entry always has a date: check for it.
# If it doesn't exist, add this row to the csv_header_info and then continue.
obp_logger.debug("checking for date in first row from csv")
if data_expression.match(row[0]) == None:
csv_header_info.append(row)
obp_logger.debug("append row to csv_header_info, row is: %s" % row)
continue
else:
# When we have a valid date, call get_info_from_row.
obp_transaction_dict = get_info_from_row(row)
obp_logger.debug("call get_info_from_row")
# This will create a hash and return it.
json_hash = create_hash(json_formatter(obp_transaction_dict))
obp_logger.debug("create json_hash from obp_transaction_dict")
# Some debug output. So that we may can see the content of the JSON
# and the hash.
obp_logger.info("The hash of the JSON is: %s" % json_hash)
print "%s:The hash of the JSON is: %s" % (date_now_formatted(), json_hash)
# Insert the hash into the cache. If it worked (the hash did not yet exist)
# send it to the API.
result = insert_hash_to_cache(json_hash, HASH_FILE)
if result == True:
result = insert_into_scala(
SCALA_HOST,
SCALA_PORT,
json_formatter(obp_transaction_dict))
obp_logger.debug("HTTP POST result is: %s" % result)
#obp_logger.debug("HTTP POST text from result is: %s" % result.text)
else:
obp_logger.info("Transaction is already in hash file, not inserting")
print "%s:Transaction is already in hash file, not inserting" % date_now_formatted()
def parse_row_of_csv(csv_file_to_parse):
"""Gets rows from CSV file"""
obp_logger.info("start reading the csv file line by line")
# This is to set the separators in the CSV file
# TODO: This should be in the obp_config
delimiter = ';'
quote_char = '"'
obp_logger.debug("Set CSV delimiter to: %s" % delimiter)
obp_logger.debug("Set CSV quote_char to: %s" % quote_char)
# re : \d\d\.\d\d\.\d\d\d\d
# This will check if date is formatted like this: 23.01.2001
obp_logger.debug("Set regular expression to: \d\d\.\d\d\.\d\d")
data_expression = re.compile('\d\d\.\d\d\.\d\d\d\d')
obp_logger.debug("starting csv reader")
transaction_reader = csv.reader(open(csv_file_to_parse, 'rb'),
delimiter=delimiter,
quotechar=quote_char)
obp_logger.debug("Start of for loop of transaction_reader")
for row in transaction_reader:
# The first valid entry always has a date: check for it.
# If it doesn't exist, add this row to the csv_header_info and then continue.
obp_logger.debug("checking for date in first row from csv")
if data_expression.match(row[0]) == None:
csv_header_info.append(row)
obp_logger.debug("append row to csv_header_info, row is: %s" % row)
continue
else:
# When we have a valid date, call get_info_from_row.
obp_transaction_dict = get_info_from_row(row)
obp_logger.debug("call get_info_from_row")
# This will create a hash and return it.
json_hash = create_hash(json_formatter(obp_transaction_dict))
obp_logger.debug("create json_hash from obp_transaction_dict")
# Some debug output. So that we may can see the content of the JSON
# and the hash.
obp_logger.info("The hash of the JSON is: %s" % json_hash)
print "%s:The hash of the JSON is: %s" % (date_now_formatted(),
json_hash)
# Insert the hash into the cache. If it worked (the hash did not yet exist)
# send it to the API.
result = insert_hash_to_cache(json_hash, HASH_FILE)
if result == True:
result = insert_into_scala(SCALA_HOST, SCALA_PORT,
json_formatter(obp_transaction_dict))
obp_logger.debug("HTTP POST result is: %s" % result)
#obp_logger.debug("HTTP POST text from result is: %s" % result.text)
else:
obp_logger.info(
"Transaction is already in hash file, not inserting")
print "%s:Transaction is already in hash file, not inserting" % date_now_formatted(
)
def parse_row_of_csv(csv_file_to_parse):
"""Get rows from CSV file"""
# This is for the speater in the CSV file
# TODO: This should be in the obp_config
delimiter = ';'
quote_char = '"'
# re : \d\d\.\d\d\.\d\d\d\d
# This will check of date formarted lile this: 23.01.2001
data_expression = re.compile('\d\d\.\d\d\.\d\d\d\d')
transactionReader = csv.reader(open(csv_file_to_parse, 'rb'), delimiter=delimiter, quotechar=quote_char)
for row in transactionReader:
# The first vaild entry has always a date, checking for it.
# When not, add this row to the csv_header_info and then continue.
if data_expression.match(row[0]) == None:
csv_header_info.append(row)
continue
else:
# When we have a valid date, call get_info_from_row.
obp_transaction_dict = get_info_from_row(row)
# This will create a hash and return it.
json_hash = create_hash(json_out_correter(obp_transaction_dict))
# Some debug output. So we may can see the content of the JSON and the Hash.
print "In the JSON is:\n%s" % json_out_correter(obp_transaction_dict)
print "The hash of the JSON is: %s" % json_hash
# Try to inserting the Hash, return inserting it when hash not already exist. Return then True,else False
# and this Hash was already in var/cache
result = inserting_hash(json_hash,HASH_FILE)
if result == True:
result = insert_into_scala(SCALA_HOST,SCALA_PORT,json_out_correter(obp_transaction_dict))
print output_with_date(),result
print output_with_date(), result.text
else:
print "%s:Transaction is already in hash file, will no inserting" % output_with_date()
