def helper_execute_select_into(query)->pd.DataFrame:
config=TstConfig()
db=str(config.get("DATABASE_LOCATION","SQLITE_DB_FOR_TESTING"))
table2=str(config.get("DATABASE_LOCATION","SQLITE_TABLE_FOR_TESTING2"))
classObj=SqliteDataWarehouse(LocalConfig())
retVal=classObj.select_into(query,db,table2)
return retVal
def helper_execute_import_csv()->bool:
config=TstConfig()
#get testing csv file
db=str(config.get("DATABASE_LOCATION","SQLITE_DB_FOR_TESTING"))
table=str(config.get("DATABASE_LOCATION","SQLITE_TABLE_FOR_TESTING1"))
csv=str(config.get("FLAT_FILE_LOCATION","CSV_DATA_FILE"))
classObj=SqliteDataWarehouse(LocalConfig())
retVal=classObj.import_csv(csv,db,table)
return retVal
def test_table_schema():
config=TstConfig()
table2=str(config.get("DATABASE_LOCATION","SQLITE_TABLE_FOR_TESTING2"))
db=str(config.get("DATABASE_LOCATION","SQLITE_DB_FOR_TESTING"))
populate_table_in_database(db,table2)
#populated test data in table2
classObj=SqliteDataWarehouse(LocalConfig())
retVal=classObj.table_schema(db, table2)
testDataColumns=[]
testDataColumns.append(SqlColumn("id", "int64", True))
testDataColumns.append(SqlColumn("title", "object", True))
testDataColumns.append(SqlColumn("author", "object", True))
testDataColumns.append(SqlColumn("price", "object", True))
testDataColumns.append(SqlColumn("year", "object", True))
expectedTable=SqlTable(table2,table2,testDataColumns)
# see if all column count match
assert expectedTable==retVal
def test_dry_run():
config=TstConfig()
localConfig=LocalConfig()
table2=str(config.get("DATABASE_LOCATION","SQLITE_TABLE_FOR_TESTING2"))
db=localConfig.default_sql_lite_db_file
populate_table_in_database(db,table2)
#populated test data in table2
classObj=SqliteDataWarehouse(LocalConfig())
retVal=classObj.dry_run(f"select * from {table2} ")
testDataColumns=[]
testDataColumns.append(SqlColumn("id", "int64", True))
testDataColumns.append(SqlColumn("title", "object", True))
testDataColumns.append(SqlColumn("author", "object", True))
testDataColumns.append(SqlColumn("price", "object", True))
testDataColumns.append(SqlColumn("year", "object", True))
#see if all column values match
for col in retVal:
assert col in testDataColumns
# see if all column count match
assert len(testDataColumns)==len(retVal)
def __init__(self):
self._config: LocalConfig = LocalConfig()
self._lake: LocalDataLake = LocalDataLake(self.config)
self._warehouse: SqliteDataWarehouse = SqliteDataWarehouse(self.config)
self._git: GitLabHost = GitLabHost(self.config)
def helper_dataframe_from_query(query: str) -> pd.DataFrame:
classObj=SqliteDataWarehouse(LocalConfig())
retVal=classObj.dataframe_from_query(query)
return retVal
def helper_dataframe_from_table( dbLocation: str, tableName: str)->pd.DataFrame:
classObj=SqliteDataWarehouse(LocalConfig())
retVal=classObj.dataframe_from_table(dbLocation,tableName)
return retVal