def iter_csv(csv_list):
"""
Iterating CSV files
:param csv_list: all csv file list in the folder
:return:
"""
for fl in csv_list:
logger.debug('Parsing a file %s', fl)
with open(fl) as csvfile:
read_csv = csv.reader(csvfile, delimiter=',')
count = 0
data = []
for row in read_csv:
# Ignore empty fields
data_line = list(filter(lambda elm: elm != '', row))
# Replace commas and convert String
data.append((str.join('|', data_line), basename(fl)))
count += 1
# Reset Inserting every 100 data inserting
if count % BATCH_SIZE == 0:
db_insert(data)
data.clear()
count = 0
if len(data) != 0:
db_insert(data)
def db_connect():
"""
Connecting to the MySQL database server
"""
# read connection parameters
params = config()
# connect to the MySQL server
logger.debug("Connecting to the MySQL database...")
conn = pymysql.connect(**params)
return conn
def main(cmd):
"""
Main entry point
:param cmd: command line passed args
:return: None
"""
print(cmd)
if len(cmd) == 2 and os.path.isdir(cmd[1]):
xpath = glob(cmd[1] + '*.csv', recursive=True)
if len(xpath) == 0:
logger.debug('No CSV files found...')
else:
iter_csv(xpath)
else:
logger.debug('Path should be a directory and allowed one input path')
def db_insert(sql_vals):
"""
Connecting to the MySQL database server
"""
conn = None
try:
# connect to the MySQL server
logger.debug("Connecting to the MySQL database...")
conn = db_connect()
# create a cursor
cur = conn.cursor()
# execute a statement
sql = 'INSERT INTO prop_land (line, file_name) VALUES (%s, %s)'
cur.executemany(sql, sql_vals)
# close the communication with the MySQL
conn.commit()
logger.debug('Data inserted in DB...')
cur.close()
except (Exception, pymysql.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
logger.error('Database connection closed.')
:return:
"""
for fl in csv_list:
logger.debug('Parsing a file %s', fl)
with open(fl) as csvfile:
read_csv = csv.reader(csvfile, delimiter=',')
count = 0
data = []
for row in read_csv:
# Ignore empty fields
data_line = list(filter(lambda elm: elm != '', row))
# Replace commas and convert String
data.append((str.join('|', data_line), basename(fl)))
count += 1
# Reset Inserting every 100 data inserting
if count % BATCH_SIZE == 0:
db_insert(data)
data.clear()
count = 0
if len(data) != 0:
db_insert(data)
if __name__ == "__main__":
print('LOG file is created in %s', LOG)
logger.debug('Creating a table structure "prop_land".')
db_create()
args = sys.argv
main(args)