def fineTune(self,
minibatchStream,
epochs,
mbPerEpoch,
loss=None,
progressBar=True,
useDropout=False):
for ep in range(epochs):
totalCases = 0
sumErr = 0
sumLoss = 0
if self.nesterov:
step = self.stepNesterov
else:
step = self.step
prog = Progress(mbPerEpoch) if progressBar else DummyProgBar()
for i in range(mbPerEpoch):
if isinstance(self.outputActFunct, LinearMasked):
inpMB, targMB, targMaskMB = minibatchStream.next()
err, outMB = step(inpMB, targMB, self.learnRates,
self.momentum, self.L2Costs, useDropout,
targMaskMB)
else:
inpMB, targMB = minibatchStream.next()
err, outMB = step(inpMB, targMB, self.learnRates,
self.momentum, self.L2Costs, useDropout)
sumErr += err
if loss != None:
sumLoss += loss(targMB, outMB)
totalCases += inpMB.shape[0]
prog.tick()
prog.done()
yield sumErr / float(totalCases), sumLoss / float(totalCases)
def fineTune(self,
minibatchStream,
trainInps,
epochs,
mbPerEpoch,
loss=None,
validSet=False,
progressBar=True,
useDropout=False):
for ep in xrange(epochs):
print
print 'learnRates:', self.learnRates
totalCases = 0
sumErr = 0
sumLoss = 0
if self.nesterov:
step = self.stepNesterov
else:
step = self.step
prog = Progress(mbPerEpoch) if progressBar else DummyProgBar()
for i in range(mbPerEpoch):
# print 'Epoch:', ep, 'minibatch', i
(inpMB, targMB, mbgraph) = minibatchStream.next()
if len(targMB.shape
) != 3: # Convert to a cubic matrix (3d matrix)
targMB = targMB.reshape(-1, 1, targMB.shape[1])
# Each dimensions of inpMB (3d), refers to a pivot vector. Now, we want to select
# training samples that falls in the neighborhood of this guy, and store in the
# corresponding dimension of xsl (x_selected).
xsl = np.zeros((mbgraph.indx.shape[0], mbgraph.indx.shape[1],
trainInps.shape[1]))
for j in xrange(mbgraph.indx.shape[0]):
xsl[j] = trainInps[mbgraph.indx[j] - 1]
# -1 because I need to covert the indices from matlab format to python
#distribute graph.vals to 3d
vals_select = mbgraph.vals #It has been converted to 3d inside manifold.py
del mbgraph
err = step(xsl, vals_select, inpMB, targMB, self.learnRates,
self.momentum, self.L2Costs, useDropout)
# gnp.free_reuse_cache()
sumErr += err
# print err, sumErr
totalCases += inpMB.shape[0]
prog.tick()
prog.done()
self.learnRates = [
y * self.learnRatesMultiplier for y in self.learnRates
]
# If validation set is given
if validSet:
val_outputActs = self.fprop_xf(validSet['trainInps'])
val_error = self.outputActFunct.error(
gnp.garray(validSet['trainTargs']), self.state[-1],
val_outputActs)
yield sumErr / float(
totalCases), val_error / validSet['trainInps'].shape[0]
else:
yield sumErr / float(totalCases)
def run_batch_locally(scripts, language='python', paths=[], max_cpu=0.9, max_mem=0.9, submit_sleep=1, job_check_sleep=30, \
verbose=True, max_files_open=100, max_running_jobs=10, single_thread=True):
'''
Receives a list of python scripts to run
Assumes the code has an output file that will be managed by this function
Returns a list of local file names where the code has presumably stored output
'''
# Define some code constants
#### Do we need to set paths explicitly?
#### This will be deprecated in future MATLAB - hopefully the -singleCompThread command is sufficient
matlab_single_thread = '''
maxNumCompThreads(1);
'''
python_path_code = '''
import sys
sys.path.append('%s')
'''
matlab_path_code = '''
addpath(genpath('%s'))
'''
python_completion_code = '''
print 'Writing completion flag'
with open('%(flag_file)s', 'w') as f:
f.write('Goodbye, World')
print "Goodbye, World"
quit()
'''
#### TODO - Is this completely stable
matlab_completion_code = '''
fprintf('\\nWriting completion flag\\n');
ID = fopen('%(flag_file)s', 'w');
fprintf(ID, 'Goodbye, world');
fclose(ID);
fprintf('\\nGoodbye, World\\n');
quit()
'''
# Initialise lists of file locations job ids
shell_files = [None] * len(scripts)
script_files = [None] * len(scripts)
output_files = [None] * len(scripts)
stdout_files = [None] * len(scripts)
stdout_file_handles = [None] * len(scripts)
flag_files = [None] * len(scripts)
processes = [None] * len(scripts)
fear_finished = False
job_finished = [False] * len(scripts)
files_open = 0
# Loop through jobs, submitting jobs whenever CPU usage low enough, re-submitting failed jobs
if not verbose:
prog = Progress(len(scripts))
while not fear_finished:
should_sleep = True
for (i, code) in enumerate(scripts):
if (not job_finished[i]) and (processes[i] is None) and (
files_open <= max_files_open) and (len([
1 for p in processes if not p is None
]) < max_running_jobs):
# This script has not been run - check CPU and potentially run
#### FIXME - Merge if statements
if (psutil.cpu_percent() < max_cpu * 100) and (
psutil.virtual_memory().percent < max_mem * 100):
# Jobs can run
should_sleep = False
# Get the job ready
if LOCATION == 'local':
temp_dir = LOCAL_TEMP_PATH
else:
temp_dir = HOME_TEMP_PATH
if language == 'python':
script_files[i] = (mkstemp_safe(temp_dir, '.py'))
elif language == 'matlab':
script_files[i] = (mkstemp_safe(temp_dir, '.m'))
# Create necessary files in local path
shell_files[i] = (mkstemp_safe(temp_dir, '.sh'))
output_files[i] = (mkstemp_safe(temp_dir, '.out'))
stdout_files[i] = (mkstemp_safe(temp_dir, '.o'))
flag_files[i] = (mkstemp_safe(temp_dir, '.flg'))
# Customise code
#### TODO - make path and output_transfer optional
if language == 'python':
code = code + python_completion_code
for path in paths:
code = (python_path_code % path) + code
elif language == 'matlab':
code = code + matlab_completion_code
for path in paths:
code = (matlab_path_code % path) + code
code = code % {
'output_file': output_files[i],
'flag_file': flag_files[i]
}
# Write code and shell file
with open(script_files[i], 'w') as f:
f.write(code)
with open(shell_files[i], 'w') as f:
#### TODO - is os.path.join always correct - what happens if this program is being run on windows?
if language == 'python':
f.write('python ' + script_files[i] + '\n')
elif language == 'matlab':
if LOCATION == 'home':
matlab_path = HOME_MATLAB
else:
matlab_path = LOCAL_MATLAB
if single_thread:
f.write('cd ' + os.path.split(script_files[i])[0] + ';\n' + matlab_path + ' -nosplash -nojvm -nodisplay -singleCompThread -r ' + \
os.path.split(script_files[i])[-1].split('.')[0] + '\n')
else:
f.write('cd ' + os.path.split(script_files[i])[0] + ';\n' + matlab_path + ' -nosplash -nojvm -nodisplay -r ' + \
os.path.split(script_files[i])[-1].split('.')[0] + '\n')
# Start running the job
if verbose:
print 'Submitting job %d of %d' % (i + 1, len(scripts))
stdout_file_handles[i] = open(stdout_files[i], 'w')
files_open = files_open + 1
processes[i] = subprocess.Popen(
['sh', shell_files[i]], stdout=stdout_file_handles[i])
# Sleep for a bit so the process can kick in (prevents 100s of jobs being sent to processor)
time.sleep(submit_sleep)
elif (not job_finished[i]) and (not processes[i] is None):
# Ask the process how its doing
processes[i].poll()
# Check to see if the process has completed
if not processes[i].returncode is None:
if os.path.isfile(flag_files[i]):
job_finished[i] = True
if verbose:
print 'Job %d of %d has completed' % (i + 1,
len(scripts))
else:
prog.tick()
else:
if verbose:
print 'Job %d has failed - will try again later' % i + 1
processes[i] = None
# Tidy up temp files
os.remove(script_files[i])
os.remove(shell_files[i])
stdout_file_handles[i].close()
files_open = files_open - 1
os.remove(stdout_files[i])
os.remove(flag_files[i])
processes[i] = None
# Something useful happened
should_sleep = False
if all(job_finished):
fear_finished = True
if not verbose:
prog.done()
elif should_sleep:
# Count how many jobs are queued
n_queued = 0
# Count how many jobs are running
n_running = 0
if verbose:
# print '%d jobs running' % n_running
# print '%d jobs queued' % n_queued
print 'Sleeping for %d seconds' % job_check_sleep
time.sleep(job_check_sleep)
#### TODO - return job output and error files as applicable (e.g. there may be multiple error files associated with one script)
return output_files
