def preprocess(dbName, userName, passwd, host, port, dirName, startDate,
currentDate):
# fileName, wordsToBeReplaced, wordsToReplace
preprocessing_files = [
#[
#'create_longitudinal_features.sql',
#['moocdb'],
#[dbName]
#],
#[
#'populate_longitudinal_features.py',
#[],
#[]
#],
#[
#'create_models_table.sql',
#['moocdb'],
#[dbName]
#],
#[
#'create_experiments_table.sql',
#['moocdb'],
#[dbName]
#],
['create_user_longitudinal_feature_values.sql', ['moocdb'], [dbName]],
[
'users_populate_dropout_week.sql',
['START_DATE_PLACEHOLDER', 'moocdb'], [startDate, dbName]
]
]
conn = sql_functions.openSQLConnectionP(dbName, userName, passwd, host,
port)
for fileName, toBeReplaced, replaceBy in preprocessing_files:
if fileName[-2:] == 'py':
print "executing: ", fileName
sql_functions.runPythonFile(conn, conn, dirName, fileName[:-3],
dbName, startDate, currentDate)
else:
this_file = os.path.dirname(os.path.realpath(__file__))
fileLocation = dirName + '/' + fileName
fileLocation = this_file + '/' + fileLocation
newFile = sql_functions.replaceWordsInFile(fileLocation,
toBeReplaced, replaceBy)
print "executing: ", fileName
sql_functions.executeSQL(conn, newFile)
conn.commit()
sql_functions.closeSQLConnection(conn)
def run_sql_curation_files(dbName, userName, passwd, dbHost, dbPort,preprocessing_files):
conn = sql_functions.openSQLConnectionP(dbName, userName, passwd, dbHost,dbPort)
for fileName, toBeReplaced, replaceBy in preprocessing_files:
fileLocation = os.path.dirname(os.path.realpath(__file__))+'/'+ fileName
print fileLocation
newFile = sql_functions.replaceWordsInFile(fileLocation, toBeReplaced, replaceBy)
print "executing: ", fileName
sql_functions.executeSQL(conn, newFile)
conn.commit()
print "done"
sql_functions.closeSQLConnection(conn)
def run_sql_curation_files(dbName, userName, passwd, dbHost, dbPort,preprocessing_files):
conn = sql_functions.openSQLConnectionP(dbName, userName, passwd, dbHost,dbPort)
for fileName, toBeReplaced, replaceBy in preprocessing_files:
fileLocation = os.path.dirname(os.path.realpath(__file__))+'/'+ fileName
print fileLocation
newFile = sql_functions.replaceWordsInFile(fileLocation, toBeReplaced, replaceBy)
print "executing: ", fileName
sql_functions.executeSQL(conn, newFile)
conn.commit()
print "done"
sql_functions.closeSQLConnection(conn)
def preprocess(dbName, userName, passwd, host, port, dirName, startDate, currentDate):
# fileName, wordsToBeReplaced, wordsToReplace
preprocessing_files = [
#[
#'create_longitudinal_features.sql',
#['moocdb'],
#[dbName]
#],
#[
#'populate_longitudinal_features.py',
#[],
#[]
#],
#[
#'create_models_table.sql',
#['moocdb'],
#[dbName]
#],
#[
#'create_experiments_table.sql',
#['moocdb'],
#[dbName]
#],
[
'create_user_longitudinal_feature_values.sql',
['moocdb'],
[dbName]
],
[
'users_populate_dropout_week.sql',
['START_DATE_PLACEHOLDER','moocdb'],
[startDate,dbName]
]
]
conn = sql_functions.openSQLConnectionP(dbName, userName, passwd, host,port)
for fileName, toBeReplaced, replaceBy in preprocessing_files:
if fileName[-2:] == 'py':
print "executing: ", fileName
sql_functions.runPythonFile(conn,conn,dirName,
fileName[:-3],dbName,startDate, currentDate)
else:
this_file = os.path.dirname(os.path.realpath(__file__))
fileLocation = dirName+'/'+fileName
fileLocation = this_file+'/'+fileLocation
newFile = sql_functions.replaceWordsInFile(fileLocation, toBeReplaced, replaceBy)
print "executing: ", fileName
sql_functions.executeSQL(conn, newFile)
conn.commit()
sql_functions.closeSQLConnection(conn)