def cli_readIterator(lines, start_index, parent, gui):
i = start_index
it = MikenetIterator(gui, parent)
while '</iterator>' not in lines[i]:
if '<parameter>' in lines[i]:
i = readParameter(lines, i + 1, it, gui)
elif '<varying_parameter>' in lines[i]:
i = readVaryingParameter(lines, i + 1, it, gui)
elif '<iterator>' in lines[i]:
i = cli_readIterator(lines, i + 1, it, gui)
elif '<run>' in lines[i]:
i = cli_readRun(lines, i + 1, it, gui)
elif 'random_flag' in lines[i]:
l, r = guts.evaluateEq(lines[i])
it.setRandomFlag(guts.smartEval(r))
elif 'applied_paths' in lines[i]:
l, r = guts.evaluateEq(lines[i])
it.setAppliedPaths(guts.smartEval(r))
i += 1
# add newly created iterator to parent
parent.getChildren().append(it)
it.syncToRun()
return i
def cli_readIterator(lines,start_index,parent,gui):
i = start_index
it = MikenetIterator(gui,parent)
while '</iterator>' not in lines[i]:
if '<parameter>' in lines[i]:
i = readParameter(lines,i+1,it,gui)
elif '<varying_parameter>' in lines[i]:
i = readVaryingParameter(lines,i+1,it,gui)
elif '<iterator>' in lines[i]:
i = cli_readIterator(lines,i+1,it,gui)
elif '<run>' in lines[i]:
i = cli_readRun(lines,i+1,it,gui)
elif 'random_flag' in lines[i]:
l,r = guts.evaluateEq(lines[i])
it.setRandomFlag(guts.smartEval(r))
elif 'applied_paths' in lines[i]:
l,r = guts.evaluateEq(lines[i])
it.setAppliedPaths(guts.smartEval(r))
i += 1
# add newly created iterator to parent
parent.getChildren().append(it)
it.syncToRun()
return i
def cli_readPhaseItem(lines,start_index,parent,gui):
i = start_index
phase_item = MikenetPhaseItem(gui,parent)
#TODO read test sets
while '</phase_item>' not in lines[i]:
if '=' in lines[i]:
l,r = guts.evaluateEq(lines[i])
if l == 'my_profile':
phase_item.setProfile(r)
elif l == 'mode':
phase_item.setModeWithText(r)
elif l == 'recording_data':
phase_item.setRawRecordingData(guts.smartEval(r))
elif l == 'net_components':
phase_item.setRawNetComponents(guts.smartEval(r))
elif l == 'noise_data':
phase_item.setRawNoiseData(guts.smartEval(r))
elif l == 'test_profiles':
phase_item.test_profiles = guts.smartEval(r)
elif '<parameter>' in lines[i]:
i = readParameter(lines,i+1,phase_item,gui)
elif '<overrides>' in lines[i]:
i = readOverrides(lines,i+1,phase_item,gui)
i += 1
# add newly created phase item to phase
parent.children.append(phase_item)
return i
def cli_readPhaseItem(lines, start_index, parent, gui):
i = start_index
phase_item = MikenetPhaseItem(gui, parent)
#TODO read test sets
while '</phase_item>' not in lines[i]:
if '=' in lines[i]:
l, r = guts.evaluateEq(lines[i])
if l == 'my_profile':
phase_item.setProfile(r)
elif l == 'mode':
phase_item.setModeWithText(r)
elif l == 'recording_data':
phase_item.setRawRecordingData(guts.smartEval(r))
elif l == 'net_components':
phase_item.setRawNetComponents(guts.smartEval(r))
elif l == 'noise_data':
phase_item.setRawNoiseData(guts.smartEval(r))
elif l == 'test_profiles':
phase_item.test_profiles = guts.smartEval(r)
elif '<parameter>' in lines[i]:
i = readParameter(lines, i + 1, phase_item, gui)
elif '<overrides>' in lines[i]:
i = readOverrides(lines, i + 1, phase_item, gui)
i += 1
# add newly created phase item to phase
parent.children.append(phase_item)
return i
def initializeTables(path, db, driver):
# map python datatypes to sql datatypes
type_map = {}
if driver == 'QSQLITE':
type_map = {int: 'INTEGER', float: 'REAL', str: 'TEXT'}
elif driver == 'QMYSQL':
type_map = {int: 'INT', float: 'FLOAT', str: 'TEXT'}
# metadata
# get the field names from the metadata file in this directory
fields = ['id ' + type_map[int], 'run_id ' + type_map[int]]
f_list = glob.glob(os.path.join(path, '*.metadata'))
for f in f_list:
# open the metadata file
with open(f, 'rU') as src:
for line in src:
# skip empty lines
if line == '\n':
pass
elif 'GROUP\t' not in line:
data = line.split('\t')
data = [x for x in data if len(x) > 1]
for d in data:
l, r = guts.evaluateEq(d)
r = guts.smartEval(r)
fields.append(str(str(l) + ' ' + type_map[type(r)]))
fields = list(set(fields))
field_str = ','.join(fields)
q = QtSql.QSqlQuery(db)
q.exec_(str('create table if not exists metadata(' + field_str + ');'))
# groupdata
# group fields and datatypes are going to be more restricted
q = QtSql.QSqlQuery(db)
q.exec_(
str('create table if not exists groupdata(' + 'id ' + type_map[int] +
',run_id ' + type_map[int] + ',group ' + type_map[str] +
',units ' + type_map[int] + ',activation_type ' + type_map[str] +
',error_computation_type ' + type_map[str] + ');'))
# errordata
q = QtSql.QSqlQuery(db)
q.exec_(
str('create table if not exists errordata(' + 'id ' + type_map[int] +
',run_id ' + type_map[int] + ',run_trial ' + type_map[int] +
',trial ' + type_map[int] + ',error ' + type_map[float] + ');'))
# activationdata
# noisedata
q = QtSql.QSqlQuery(db)
q.exec_(
str('create table if not exists noisedata(' + 'id ' + type_map[int] +
',run_id ' + type_map[int] + ',noise_type ' + type_map[str] +
',noise_object ' + type_map[str] + ',noise_amount ' +
type_map[float] + ');'))
def readTrainingProfile(lines, start_index, parent, gui):
i = start_index
prof = MikenetTrainingProfile(gui, parent)
while '</training_profile>' not in lines[i]:
if 'category_labels' in lines[i]:
l, r = guts.evaluateEq(lines[i])
prof.setCategories(guts.smartEval(r))
elif '<parameter>' in lines[i]:
i = readParameter(lines, i + 1, prof, gui)
i += 1
# add the newly created profile to the script
parent.getChildren().append(prof)
return i
def readTrainingProfile(lines,start_index,parent,gui):
i = start_index
prof = MikenetTrainingProfile(gui,parent)
while '</training_profile>' not in lines[i]:
if 'category_labels' in lines[i]:
l,r = guts.evaluateEq(lines[i])
prof.setCategories(guts.smartEval(r))
elif '<parameter>' in lines[i]:
i = readParameter(lines,i+1,prof,gui)
i += 1
# add the newly created profile to the script
parent.getChildren().append(prof)
return i
def readParameterOverride(lines,start_index,o_list,gui):
i = start_index
paramDict = {}
while '</parameter>' not in lines[i]:
if '=' in lines[i]:
l,r = guts.evaluateEq(lines[i])
if r == ';':
# must have been an empty text field
paramDict[l] = ''
else:
r = r[:-1]
paramDict[l] = guts.smartEval(r)
i += 1
o_list.append(MikenetParameter(gui,**paramDict))
return i
def readParameterOverride(lines, start_index, o_list, gui):
i = start_index
paramDict = {}
while '</parameter>' not in lines[i]:
if '=' in lines[i]:
l, r = guts.evaluateEq(lines[i])
if r == ';':
# must have been an empty text field
paramDict[l] = ''
else:
r = r[:-1]
paramDict[l] = guts.smartEval(r)
i += 1
o_list.append(MikenetParameter(gui, **paramDict))
return i
def readVaryingParameter(lines,start_index,iterator,gui):
'''special method for iterator's "varying" parameter'''
i = start_index
paramDict = {}
while '</varying_parameter>' not in lines[i]:
if '=' in lines[i]:
l,r = guts.evaluateEq(lines[i])
if r == ';':
# must have been an empty text field
paramDict[l] = ''
else:
r = r[:-1]
paramDict[l] = guts.smartEval(r)
i += 1
iterator.setVarying(MikenetParameter(gui,**paramDict))
return i
def readVaryingParameter(lines, start_index, iterator, gui):
'''special method for iterator's "varying" parameter'''
i = start_index
paramDict = {}
while '</varying_parameter>' not in lines[i]:
if '=' in lines[i]:
l, r = guts.evaluateEq(lines[i])
if r == ';':
# must have been an empty text field
paramDict[l] = ''
else:
r = r[:-1]
paramDict[l] = guts.smartEval(r)
i += 1
iterator.setVarying(MikenetParameter(gui, **paramDict))
return i
def cli_readRun(lines, start_index, parent, gui):
i = start_index
run = MikenetRun(gui, parent)
while '</run>' not in lines[i]:
if '=' in lines[i]:
l, r = guts.evaluateEq(lines[i])
if l == 'groups':
run.setGroups(guts.smartEval(r))
elif l == 'matrix':
run.setMatrix(guts.smartEval(r))
elif '<phase>' in lines[i]:
i = cli_readPhase(lines, i + 1, run, gui)
elif '<parameter>' in lines[i]:
i = readParameter(lines, i + 1, run, gui)
i += 1
# add newly created run to parent
parent.children.append(run)
return i
def cli_readRun(lines,start_index,parent,gui):
i = start_index
run = MikenetRun(gui,parent)
while '</run>' not in lines[i]:
if '=' in lines[i]:
l,r = guts.evaluateEq(lines[i])
if l == 'groups':
run.setGroups(guts.smartEval(r))
elif l == 'matrix':
run.setMatrix(guts.smartEval(r))
elif '<phase>' in lines[i]:
i = cli_readPhase(lines,i+1,run,gui)
elif '<parameter>' in lines[i]:
i = readParameter(lines,i+1,run,gui)
i += 1
# add newly created run to parent
parent.children.append(run)
return i
def readParameter(lines,start_index,parent,gui):
i = start_index
paramDict = {}
while '</parameter>' not in lines[i]:
if '=' in lines[i]:
l,r = guts.evaluateEq(lines[i])
if r == ';':
# must have been an empty text field
paramDict[l] = ''
else:
r = r[:-1]
paramDict[l] = guts.smartEval(r)
i += 1
if type(parent.parameters) == dict:
name = paramDict['variable_name']
parent.parameters[name] = MikenetParameter(gui,**paramDict)
else:
parent.parameters.append(MikenetParameter(gui,**paramDict))
return i
def readParameter(lines, start_index, parent, gui):
i = start_index
paramDict = {}
while '</parameter>' not in lines[i]:
if '=' in lines[i]:
l, r = guts.evaluateEq(lines[i])
if r == ';':
# must have been an empty text field
paramDict[l] = ''
else:
r = r[:-1]
paramDict[l] = guts.smartEval(r)
i += 1
if type(parent.parameters) == dict:
name = paramDict['variable_name']
parent.parameters[name] = MikenetParameter(gui, **paramDict)
else:
parent.parameters.append(MikenetParameter(gui, **paramDict))
return i
def readScript(gui, fname):
script = MikenetScript(gui)
try:
with open(fname, 'r') as f:
lines = f.readlines()
# preprocess by removing empty lines and stripping whitespace
lines = [x.strip() for x in lines if x != '\n']
i = 0
while '</script>' not in lines[i]:
if '&default&' in lines[i]:
l, r = guts.evaluateEq(lines[i])
script.defaults.append(guts.smartEval(r))
elif '<run>' in lines[i]:
i = readRun(lines, i + 1, script, gui)
elif '<iterator>' in lines[i]:
i = readIterator(lines, i + 1, script, gui)
elif '<parameter>' in lines[i]:
i = readParameter(lines, i + 1, script, gui)
elif '<training_profile>' in lines[i]:
i = readTrainingProfile(lines, i + 1,
script.training_profiles, gui)
elif '<test_profile>' in lines[i]:
i = readTestProfile(lines, i + 1, script.test_profiles,
gui)
i += 1
script.createTab()
return script
except:
dialogs.showError(
gui, 'There was a problem reading file ' + fname +
'. Please check the file formatting and try again.', '')
traceback.print_exc(file=sys.stdout)
return None
def readRun(lines, start_index, parent, gui):
i = start_index
run = MikenetRun(gui, parent)
while '</run>' not in lines[i]:
if '=' in lines[i]:
l, r = guts.evaluateEq(lines[i])
if l == 'groups':
run.setGroups(guts.smartEval(r))
elif l == 'matrix':
run.setMatrix(guts.smartEval(r))
elif '<phase>' in lines[i]:
i = readPhase(lines, i + 1, run, gui)
elif '<parameter>' in lines[i]:
i = readParameter(lines, i + 1, run, gui)
i += 1
# add newly created run to parent
parent.children.append(run)
run.createTab()
# phases have special widgets instead of tabs
# need to create these AFTER the run tab is made
for phase in run.getChildren():
phase.createWidget()
return i
def readRun(lines,start_index,parent,gui):
i = start_index
run = MikenetRun(gui,parent)
while '</run>' not in lines[i]:
if '=' in lines[i]:
l,r = guts.evaluateEq(lines[i])
if l == 'groups':
run.setGroups(guts.smartEval(r))
elif l == 'matrix':
run.setMatrix(guts.smartEval(r))
elif '<phase>' in lines[i]:
i = readPhase(lines,i+1,run,gui)
elif '<parameter>' in lines[i]:
i = readParameter(lines,i+1,run,gui)
i += 1
# add newly created run to parent
parent.children.append(run)
run.createTab()
# phases have special widgets instead of tabs
# need to create these AFTER the run tab is made
for phase in run.getChildren():
phase.createWidget()
return i
def readScript(gui,fname):
script = MikenetScript(gui)
try:
with open(fname,'r') as f:
lines = f.readlines()
# preprocess by removing empty lines and stripping whitespace
lines = [x.strip() for x in lines if x != '\n']
i = 0
while '</script>' not in lines[i]:
if '&default&' in lines[i]:
l,r = guts.evaluateEq(lines[i])
script.defaults.append(guts.smartEval(r))
elif '<run>' in lines[i]:
i = readRun(lines,i+1,script,gui)
elif '<iterator>' in lines[i]:
i = readIterator(lines,i+1,script,gui)
elif '<parameter>' in lines[i]:
i = readParameter(lines,i+1,script,gui)
elif '<training_profile>' in lines[i]:
i = readTrainingProfile(lines,i+1,script.training_profiles,gui)
elif '<test_profile>' in lines[i]:
i = readTestProfile(lines,i+1,script.test_profiles,gui)
i += 1
script.createTab()
return script
except:
dialogs.showError(gui,'There was a problem reading file '+fname+
'. Please check the file formatting and try again.','')
traceback.print_exc(file=sys.stdout)
return None
def initializeTables(path,db,driver):
# map python datatypes to sql datatypes
type_map = {}
if driver == 'QSQLITE':
type_map = {int: 'INTEGER',
float: 'REAL',
str: 'TEXT'}
elif driver == 'QMYSQL':
type_map = {int: 'INT',
float: 'FLOAT',
str: 'TEXT'}
# metadata
# get the field names from the metadata file in this directory
fields = ['id '+type_map[int],'run_id '+type_map[int]]
f_list = glob.glob(os.path.join(path,'*.metadata'))
for f in f_list:
# open the metadata file
with open(f,'rU') as src:
for line in src:
# skip empty lines
if line == '\n':
pass
elif 'GROUP\t' not in line:
data = line.split('\t')
data = [x for x in data if len(x) > 1]
for d in data:
l,r = guts.evaluateEq(d)
r = guts.smartEval(r)
fields.append(str(str(l) + ' ' +
type_map[type(r)]))
fields = list(set(fields))
field_str = ','.join(fields)
q = QtSql.QSqlQuery(db)
q.exec_(str('create table if not exists metadata(' +
field_str + ');'))
# groupdata
# group fields and datatypes are going to be more restricted
q = QtSql.QSqlQuery(db)
q.exec_(str('create table if not exists groupdata(' +
'id ' + type_map[int] +
',run_id ' + type_map[int] +
',group ' + type_map[str] +
',units ' + type_map[int] +
',activation_type ' + type_map[str] +
',error_computation_type ' + type_map[str] +
');'))
# errordata
q = QtSql.QSqlQuery(db)
q.exec_(str('create table if not exists errordata(' +
'id ' + type_map[int] +
',run_id ' + type_map[int] +
',run_trial ' + type_map[int] +
',trial ' + type_map[int] +
',error ' + type_map[float] +
');'))
# activationdata
# noisedata
q = QtSql.QSqlQuery(db)
q.exec_(str('create table if not exists noisedata(' +
'id ' + type_map[int] +
',run_id ' + type_map[int] +
',noise_type ' + type_map[str] +
',noise_object ' + type_map[str] +
',noise_amount ' + type_map[float] +
');'))
def pushRunData(path, gui):
driver = decodeDriver(gui)
# connect to database
# to run as a standalone test, call with gui == None
if not gui:
db = testDB(driver)
else:
# need unique connection name. use name of folder, which is the run name
garbage, run_name = os.path.split(path)
db = connectToDB(gui, driver, run_name) # for real life
if not db:
# return path if there was a problem so it can be reported
return path
###############################################################
# create tables (if a new database)
initializeTables(path, db, driver)
test_fields = initializeTestTables(path, db, driver)
# get a unique index for this whole run
run_index = getRunIndex(db)
# create a dict to map phase item names to unique indices
phase_item_map = {}
# push metadata AND group data if metadata file exists
f_list = glob.glob(os.path.join(path, '*.metadata'))
for f in f_list:
# open the metadata file
with open(f, 'rU') as src:
for line in src:
# skip empty lines
if line == '\n':
pass
elif 'GROUP\t' in line:
# pushing group record
data = line.split('\t')
data = [x for x in data if len(x) > 1]
# get rid of 'GROUP' tag
data = data[1:]
column_names = []
values = []
for d in data:
l, r = guts.evaluateEq(d)
r = guts.smartEval(r)
if str(l) == 'phase_item':
if str(r) not in phase_item_map:
assignUnusedIndex(db, str(r), phase_item_map)
column_names.insert(0, 'run_id')
values.insert(0, run_index)
column_names.insert(0, 'id')
values.insert(0, phase_item_map[str(r)])
# don't actually write the phase item name.
# it's only needed to obtain the unique id
else:
# not phase item name, so write this entry
column_names.append(str(l))
values.append(str(r))
pushRecord(db, 'groupdata', column_names, values)
else:
# pushing metadata record
# push each column individually to be robust against
# changes in column order
data = line.split('\t')
data = [x for x in data if len(x) > 1]
column_names = []
values = []
for d in data:
l, r = guts.evaluateEq(d)
column_names.append(str(l))
values.append(str(r))
if str(l) == 'phase_item':
# in metadata, each row needs to be unique
# and needs to include phase item and run names
assignUnusedIndex(db, str(r), phase_item_map)
column_names.insert(0, 'run_id')
values.insert(0, run_index)
column_names.insert(0, 'id')
values.insert(0, phase_item_map[str(r)])
pushRecord(db, 'metadata', column_names, values)
# same for error and noise (both are taken from the log file)
f_list = glob.glob(os.path.join(path, '*.log'))
for f in f_list:
with open(f, 'rU') as src:
srclines = src.readlines()
srclines = [x.replace('\n', '') for x in srclines]
# error first
lines = [x for x in srclines if 'avgError:' in x]
for l in lines:
lsplit = l.split('\t')
# make sure the id is unique, get a new one if necessary
if lsplit[1] not in phase_item_map:
assignUnusedIndex(db, str(lsplit[1]), phase_item_map)
lsplit[0] = phase_item_map[str(lsplit[1])]
lsplit.insert(0, run_index)
# don't need phase item name in the actual record
lsplit.pop(2)
pushRecord(db, 'errordata',
['run_id', 'id', 'run_trial', 'trial', 'error'],
lsplit)
# then noise
lines = [x for x in srclines if 'noiseData:' in x]
for l in lines:
lsplit = l.split('\t')
# make sure the id is unique, get a new one if necessary
if lsplit[1] not in phase_item_map:
assignUnusedIndex(db, str(lsplit[1]), phase_item_map)
lsplit[0] = phase_item_map[str(lsplit[1])]
lsplit.insert(0, run_index)
# don't need phase item name in the actual record
lsplit.pop(2)
pushRecord(db, 'noisedata', [
'run_id', 'id', 'noise_type', 'noise_object',
'noise_amount'
], lsplit)
# same for activations
# same for test output
f_list = glob.glob(os.path.join(path, '*.test'))
for f in f_list:
trash, name_plus = os.path.split(f)
f_split = name_plus.split('_')
test_name = str(f_split[0])
run_trial = str(f_split[1])
with open(f, 'rU') as src:
for line in src:
# add run_id and run_trial to the data entry
data = line.split('\t')
data.insert(0, str(run_trial))
data.insert(0, str(run_index))
if data[-1] == '\n':
data = data[:-1]
pushRecord(db, test_name, test_fields[:len(data)], data)
db.close()
return None
def pushRunData(path,gui):
driver = decodeDriver(gui)
# connect to database
# to run as a standalone test, call with gui == None
if not gui:
db = testDB(driver)
else:
# need unique connection name. use name of folder, which is the run name
garbage,run_name = os.path.split(path)
db = connectToDB(gui,driver,run_name) # for real life
if not db:
# return path if there was a problem so it can be reported
return path
###############################################################
# create tables (if a new database)
initializeTables(path,db,driver)
test_fields = initializeTestTables(path,db,driver)
# get a unique index for this whole run
run_index = getRunIndex(db)
# create a dict to map phase item names to unique indices
phase_item_map = {}
# push metadata AND group data if metadata file exists
f_list = glob.glob(os.path.join(path,'*.metadata'))
for f in f_list:
# open the metadata file
with open(f,'rU') as src:
for line in src:
# skip empty lines
if line == '\n':
pass
elif 'GROUP\t' in line:
# pushing group record
data = line.split('\t')
data = [x for x in data if len(x) > 1]
# get rid of 'GROUP' tag
data = data[1:]
column_names = []
values = []
for d in data:
l,r = guts.evaluateEq(d)
r = guts.smartEval(r)
if str(l) == 'phase_item':
if str(r) not in phase_item_map:
assignUnusedIndex(db,str(r),phase_item_map)
column_names.insert(0,'run_id')
values.insert(0,run_index)
column_names.insert(0,'id')
values.insert(0,phase_item_map[str(r)])
# don't actually write the phase item name.
# it's only needed to obtain the unique id
else:
# not phase item name, so write this entry
column_names.append(str(l))
values.append(str(r))
pushRecord(db,'groupdata',column_names,values)
else:
# pushing metadata record
# push each column individually to be robust against
# changes in column order
data = line.split('\t')
data = [x for x in data if len(x) > 1]
column_names = []
values = []
for d in data:
l,r = guts.evaluateEq(d)
column_names.append(str(l))
values.append(str(r))
if str(l) == 'phase_item':
# in metadata, each row needs to be unique
# and needs to include phase item and run names
assignUnusedIndex(db,str(r),phase_item_map)
column_names.insert(0,'run_id')
values.insert(0,run_index)
column_names.insert(0,'id')
values.insert(0,phase_item_map[str(r)])
pushRecord(db,'metadata',column_names,values)
# same for error and noise (both are taken from the log file)
f_list = glob.glob(os.path.join(path,'*.log'))
for f in f_list:
with open(f,'rU') as src:
srclines = src.readlines()
srclines = [x.replace('\n','') for x in srclines]
# error first
lines = [x for x in srclines if 'avgError:' in x]
for l in lines:
lsplit = l.split('\t')
# make sure the id is unique, get a new one if necessary
if lsplit[1] not in phase_item_map:
assignUnusedIndex(db,str(lsplit[1]),phase_item_map)
lsplit[0] = phase_item_map[str(lsplit[1])]
lsplit.insert(0,run_index)
# don't need phase item name in the actual record
lsplit.pop(2)
pushRecord(db,'errordata',
['run_id','id','run_trial','trial','error'],
lsplit)
# then noise
lines = [x for x in srclines if 'noiseData:' in x]
for l in lines:
lsplit = l.split('\t')
# make sure the id is unique, get a new one if necessary
if lsplit[1] not in phase_item_map:
assignUnusedIndex(db,str(lsplit[1]),phase_item_map)
lsplit[0] = phase_item_map[str(lsplit[1])]
lsplit.insert(0,run_index)
# don't need phase item name in the actual record
lsplit.pop(2)
pushRecord(db,'noisedata',
['run_id','id','noise_type','noise_object','noise_amount'],
lsplit)
# same for activations
# same for test output
f_list = glob.glob(os.path.join(path,'*.test'))
for f in f_list:
trash,name_plus = os.path.split(f)
f_split = name_plus.split('_')
test_name = str(f_split[0])
run_trial = str(f_split[1])
with open(f,'rU') as src:
for line in src:
# add run_id and run_trial to the data entry
data = line.split('\t')
data.insert(0,str(run_trial))
data.insert(0,str(run_index))
if data[-1] == '\n':
data = data[:-1]
pushRecord(db,test_name,test_fields[:len(data)],data)
db.close()
return None