def main(args=sys.argv[1:]):
if '--suffix' in args:
ind = args.index('--suffix')
suffix = args[ind + 1]
del args[ind]
del args[ind]
else:
suffix = ''
if '--monitor' in args:
args.remove('--monitor')
return monitorCondor(args, suffix)
optionsNoSuffix, _ = parseArgs(addArgs(args))
args = addArgs(args, suffix)
options, _ = parseArgs(args)
if os.path.exists(options.saveFile):
return
if not (os.path.exists(addSuffix(options.saveFile))) and os.path.exists(
addSuffix(optionsNoSuffix.saveFile)):
import shutil
shutil.copy(addSuffix(optionsNoSuffix.saveFile),
addSuffix(options.saveFile))
def repl(x):
numStudentsToAdd = 1
filename = ''
saveFile = addSuffix(options.saveFile)
if os.path.exists(saveFile):
filename = options.saveFile
x = x.replace(
'$(EVAL_PATH)',
'data/dt/studentsNew29-unperturbed-%i/weighted/only-$(EVAL_STUDENT).weka'
% options.numSource)
x = x.replace(
'$(SOURCE_DATA_PATH)',
'data/dt/studentsNew29-unperturbed-%i/train/$(DATA_STUDENT).arff' %
options.numSource)
x = x.replace('$(TARGET_STUDENT)', options.student)
x = x.replace('$(NUM_STUDENTS_TO_ADD)', str(numStudentsToAdd))
x = x.replace('$(FILENAME)', filename)
return x
trainClassifierMain(options, repl)
def monitorCondor(args, suffix):
numStudents = len(getUniqueStudents())
jobs = [-1 for i in range(numStudents)]
unfinished = range(numStudents)
saveFiles = []
for studentInd in range(numStudents):
options, _ = parseArgs(addArgs(args, suffix, studentInd))
saveFiles.append(options.saveFile)
while len(unfinished) > 0:
needToRun = []
# check what's running and still needs to run
p = subprocess.Popen(['condor_q', 'sbarrett'], stdout=subprocess.PIPE)
out, _ = p.communicate()
for studentInd in list(unfinished):
if out.find(str(jobs[studentInd])) < 0:
if os.path.exists(saveFiles[studentInd]):
unfinished.remove(studentInd)
else:
needToRun.append(studentInd)
# submit new jobs as needed
for studentInd in needToRun:
jobs[studentInd] = submit([str(studentInd)] + args, suffix)
# good-night sweet prince
time.sleep(20)
def monitorCondor(args,suffix):
numStudents = len(getUniqueStudents())
jobs = [-1 for i in range(numStudents)]
unfinished = range(numStudents)
saveFiles = []
for studentInd in range(numStudents):
options,_ = parseArgs(addArgs(args,suffix,studentInd))
saveFiles.append(options.saveFile)
while len(unfinished) > 0:
needToRun = []
# check what's running and still needs to run
p = subprocess.Popen(['condor_q','sbarrett'],stdout=subprocess.PIPE)
out,_ = p.communicate()
for studentInd in list(unfinished):
if out.find(str(jobs[studentInd])) < 0:
if os.path.exists(saveFiles[studentInd]):
unfinished.remove(studentInd)
else:
needToRun.append(studentInd)
# submit new jobs as needed
for studentInd in needToRun:
jobs[studentInd] = submit([str(studentInd)] + args,suffix)
# good-night sweet prince
time.sleep(20)
def main(args=sys.argv[1:]):
args = ['twostagetradaboost-partial'] + args
combine = False
if '--combine' in args:
args.remove('--combine')
combine = True
args = args + ['--ignorePartialMax']
options, _ = parseArgs(args)
directory = os.path.join('configs/learners/saved/twostage-partial',
options.baseLearner, options.student)
pathBase = os.path.join(directory, '%i.txt')
if combine:
print options.student, options.partialMax
bestT = None
bestError = numpy.inf
for t in range(options.partialMax):
path = pathBase % t
with open(path, 'r') as f:
error = float(f.read().strip())
if error < bestError:
bestError = error
bestT = t
options.partialInd = bestT
print 'bestT:', bestT
print 'bestError:', bestError
trainClassifierMain(options)
else:
args = args + ['--no-save', '--catchOutput']
options, _ = parseArgs(args)
output, error = trainClassifierMain(options)
res = re.findall('BEST T: (\d+)\nBEST ERROR: ([.0-9]+)', output)
bestT = int(res[0][0])
bestError = float(res[0][1])
try:
os.makedirs(directory)
except:
pass
filename = pathBase % bestT
with open(filename, 'w') as f:
f.write('%f\n' % bestError)
def main(args=sys.argv[1:]):
args = ['twostagetradaboost-partial'] + args
combine = False
if '--combine' in args:
args.remove('--combine')
combine = True
args = args + ['--ignorePartialMax']
options,_ = parseArgs(args)
directory = os.path.join('configs/learners/saved/twostage-partial',options.baseLearner,options.student)
pathBase = os.path.join(directory,'%i.txt')
if combine:
print options.student,options.partialMax
bestT = None
bestError = numpy.inf
for t in range(options.partialMax):
path = pathBase % t
with open(path,'r') as f:
error = float(f.read().strip())
if error < bestError:
bestError = error
bestT = t
options.partialInd = bestT
print 'bestT:',bestT
print 'bestError:',bestError
trainClassifierMain(options)
else:
args = args + ['--no-save','--catchOutput']
options,_ = parseArgs(args)
output,error = trainClassifierMain(options)
res = re.findall('BEST T: (\d+)\nBEST ERROR: ([.0-9]+)',output)
bestT = int(res[0][0])
bestError = float(res[0][1])
try:
os.makedirs(directory)
except:
pass
filename = pathBase % bestT
with open(filename,'w') as f:
f.write('%f\n'%bestError)
def main(args=sys.argv[1:]):
if '--suffix' in args:
ind = args.index('--suffix')
suffix = args[ind+1]
del args[ind]
del args[ind]
else:
suffix = ''
if '--monitor' in args:
args.remove('--monitor')
return monitorCondor(args,suffix)
optionsNoSuffix,_ = parseArgs(addArgs(args))
args = addArgs(args,suffix)
options,_ = parseArgs(args)
if os.path.exists(options.saveFile):
return
if not(os.path.exists(addSuffix(options.saveFile))) and os.path.exists(addSuffix(optionsNoSuffix.saveFile)):
import shutil
shutil.copy(addSuffix(optionsNoSuffix.saveFile),addSuffix(options.saveFile))
def repl(x):
numStudentsToAdd = 1
filename = ''
saveFile = addSuffix(options.saveFile)
if os.path.exists(saveFile):
filename = options.saveFile
x = x.replace('$(EVAL_PATH)','data/dt/studentsNew29-unperturbed-%i/weighted/only-$(EVAL_STUDENT).weka' % options.numSource)
x = x.replace('$(SOURCE_DATA_PATH)','data/dt/studentsNew29-unperturbed-%i/train/$(DATA_STUDENT).arff' % options.numSource)
x = x.replace('$(TARGET_STUDENT)',options.student)
x = x.replace('$(NUM_STUDENTS_TO_ADD)',str(numStudentsToAdd))
x = x.replace('$(FILENAME)',filename)
return x
trainClassifierMain(options,repl)
def submit(args,suffix):
options,_ = parseArgs(addArgs(args,suffix))
base = 'condor/createTwoStageTransfer'
orig = os.path.join(base,'base.condor')
path = os.path.join(base,'jobs/%i%s.condor' % (options.studentInd,suffix))
with open(orig,'r') as f:
contents = f.read()
contents = contents.replace('$(Process)','%i%s' % (options.studentInd,suffix))
argStr = ' '.join(args)
if suffix.strip() != '':
argStr += ' --suffix %s' % suffix
contents = contents.replace('$(ARGS)',argStr)
with open(path,'w') as f:
f.write(contents)
print 'SUBMITTING ',options.studentInd
p = subprocess.Popen(['condor_submit',path],stdout=subprocess.PIPE)
out,_ = p.communicate()
jobNum = int(re.findall('cluster (\d+)',out)[0])
return jobNum
def submit(args, suffix):
options, _ = parseArgs(addArgs(args, suffix))
base = 'condor/createTwoStageTransfer'
orig = os.path.join(base, 'base.condor')
path = os.path.join(base,
'jobs/%i%s.condor' % (options.studentInd, suffix))
with open(orig, 'r') as f:
contents = f.read()
contents = contents.replace('$(Process)',
'%i%s' % (options.studentInd, suffix))
argStr = ' '.join(args)
if suffix.strip() != '':
argStr += ' --suffix %s' % suffix
contents = contents.replace('$(ARGS)', argStr)
with open(path, 'w') as f:
f.write(contents)
print 'SUBMITTING ', options.studentInd
p = subprocess.Popen(['condor_submit', path], stdout=subprocess.PIPE)
out, _ = p.communicate()
jobNum = int(re.findall('cluster (\d+)', out)[0])
return jobNum
