def get_connec(tables): """Connect to database and calls build_tables""" try: create_db() cnx = mysql.connector.connect(user='******', password='******', database='cbpopdoxa') tables = check_existing_tables(tables, cnx.cursor()) if tables: bt.build_tables(tables, cnx.cursor()) cnx.commit() except mysql.connector.Error as err: print(err)
def test_load_with_copy(self): "ensures data tables are loading correctly by loading these flat files" build_flat_files.build_flat_files(self.temp_dir, self.out_dir) #loads the data tables (INSERT method) fn=os.path.join(self.out_dir, 'my_sub_dir_1_data.csv') t_name='dir_1_data' table=build_tables.build_tables(db=DB, pdir=self.out_dir, drop_old=True) #grab the data and ensure it's right curs.execute('END') curs.execute("SELECT * FROM {}".format(t_name)) actual=curs.fetchall() target=[(1.0, 1.0, 10011.0, 20011.0), (1.0, 2.0, 10012.0, 20012.0), (1.0, 3.0, 10013.0, 20013.0), (1.0, 4.0, 10014.0, 20014.0), (2.0, 1.0, 10021.0, 20021.0), (2.0, 2.0, 10022.0, 20022.0), (2.0, 3.0, 10023.0, 20023.0), (2.0, 4.0, 10024.0, 20024.0), (3.0, 1.0, 10031.0, 20031.0), (3.0, 2.0, 10033.0, 20033.0), (3.0, 3.0, 10033.0, 20033.0), (3.0, 4.0, 10034.0, 20034.0), (4.0, 1.0, 10041.0, 20041.0), (4.0, 2.0, 10034.0, 20034.0), (4.0, 3.0, 10043.0, 20043.0), (4.0, 4.0, 10044.0, 20044.0)] for t, a in zip(target, actual): self.assertEqual(t, a, 'load_with_insert failed' )
def get_connec(tables, login_info): """Connect to database and calls build_tables""" try: create_db(login_info) cnx = mysql.connector.connect(user=login_info[1], password=login_info[2], database=login_info[3]) if tables == "all": bt.build_all(cnx.cursor()) else: for table in tables: if table != 'build': kill_table(table, cnx.cursor()) bt.build_tables(tables, cnx.cursor()) cnx.commit() except mysql.connector.Error as err: print(err)
h = logging.handlers.RotatingFileHandler(LOG_FILE, 'a', maxBytes=10*1024*1024, backupCount=5) formatter = logging.Formatter('%(asctime)s - %(levelname)s - ' '%(filename)s:%(lineno)s - %(message)s', datefmt="%Y-%m-%d %H:%M:%S") h.setFormatter(formatter) logger.addHandler(h) logger.setLevel(LOG_LEVEL) if __name__=='__main__': 'main execution start' #leave these statements here - logging info imported from settings for ix, d in enumerate(dirs, start=1): data_dir=d['data'] scratch_dir = d['scratch'] msg='Data loading from {} \n...to database {}. \n...Logging to {} \n' print(msg.format(data_dir, DB, LOG_FILE)) #create the flat files for import build_flat_files.build_flat_files(data_dir, scratch_dir, test_max_rows=500) #create tables from header info in tables, then load data build_tables.build_tables(db=DB, pdir = scratch_dir, drop_old=True) if ix != len(dirs): logger.info("*******************************") logger.info("Beginning new scenario") logger.info("*******************************") describe_db.describe_db(db=DB)
from build_tables import build_tables from validate import * from load import * if __name__ == '__main__': validate_topics() validate_pages() build_tables() load_topics() load_pages()