def get_connec(tables): """Connect to database and calls build_tables""" try: create_db() cnx = mysql.connector.connect(user='******', password='******', database='cbpopdoxa') tables = check_existing_tables(tables, cnx.cursor()) if tables: bt.build_tables(tables, cnx.cursor()) cnx.commit() except mysql.connector.Error as err: print(err)
def get_connec(tables):
"""Connect to database and calls build_tables"""
try:
create_db()
cnx = mysql.connector.connect(user='******',
password='******',
database='cbpopdoxa')
tables = check_existing_tables(tables, cnx.cursor())
if tables:
bt.build_tables(tables, cnx.cursor())
cnx.commit()
except mysql.connector.Error as err:
print(err)
def test_load_with_copy(self):
"ensures data tables are loading correctly by loading these flat files"
build_flat_files.build_flat_files(self.temp_dir, self.out_dir)
#loads the data tables (INSERT method)
fn=os.path.join(self.out_dir, 'my_sub_dir_1_data.csv')
t_name='dir_1_data'
table=build_tables.build_tables(db=DB, pdir=self.out_dir, drop_old=True)
#grab the data and ensure it's right
curs.execute('END')
curs.execute("SELECT * FROM {}".format(t_name))
actual=curs.fetchall()
target=[(1.0, 1.0, 10011.0, 20011.0),
(1.0, 2.0, 10012.0, 20012.0),
(1.0, 3.0, 10013.0, 20013.0),
(1.0, 4.0, 10014.0, 20014.0),
(2.0, 1.0, 10021.0, 20021.0),
(2.0, 2.0, 10022.0, 20022.0),
(2.0, 3.0, 10023.0, 20023.0),
(2.0, 4.0, 10024.0, 20024.0),
(3.0, 1.0, 10031.0, 20031.0),
(3.0, 2.0, 10033.0, 20033.0),
(3.0, 3.0, 10033.0, 20033.0),
(3.0, 4.0, 10034.0, 20034.0),
(4.0, 1.0, 10041.0, 20041.0),
(4.0, 2.0, 10034.0, 20034.0),
(4.0, 3.0, 10043.0, 20043.0),
(4.0, 4.0, 10044.0, 20044.0)]
for t, a in zip(target, actual):
self.assertEqual(t, a, 'load_with_insert failed' )
def get_connec(tables, login_info): """Connect to database and calls build_tables""" try: create_db(login_info) cnx = mysql.connector.connect(user=login_info[1], password=login_info[2], database=login_info[3]) if tables == "all": bt.build_all(cnx.cursor()) else: for table in tables: if table != 'build': kill_table(table, cnx.cursor()) bt.build_tables(tables, cnx.cursor()) cnx.commit() except mysql.connector.Error as err: print(err)
h = logging.handlers.RotatingFileHandler(LOG_FILE,
'a',
maxBytes=10*1024*1024,
backupCount=5)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - '
'%(filename)s:%(lineno)s - %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
h.setFormatter(formatter)
logger.addHandler(h)
logger.setLevel(LOG_LEVEL)
if __name__=='__main__':
'main execution start'
#leave these statements here - logging info imported from settings
for ix, d in enumerate(dirs, start=1):
data_dir=d['data']
scratch_dir = d['scratch']
msg='Data loading from {} \n...to database {}. \n...Logging to {} \n'
print(msg.format(data_dir, DB, LOG_FILE))
#create the flat files for import
build_flat_files.build_flat_files(data_dir, scratch_dir, test_max_rows=500)
#create tables from header info in tables, then load data
build_tables.build_tables(db=DB, pdir = scratch_dir, drop_old=True)
if ix != len(dirs):
logger.info("*******************************")
logger.info("Beginning new scenario")
logger.info("*******************************")
describe_db.describe_db(db=DB)
from build_tables import build_tables
from validate import *
from load import *
if __name__ == '__main__':
validate_topics()
validate_pages()
build_tables()
load_topics()
load_pages()
