def fineTune(self, minibatchStream, epochs, mbPerEpoch, loss=None, progressBar=True, useDropout=False): for ep in range(epochs): totalCases = 0 sumErr = 0 sumLoss = 0 if self.nesterov: step = self.stepNesterov else: step = self.step prog = Progress(mbPerEpoch) if progressBar else DummyProgBar() for i in range(mbPerEpoch): if isinstance(self.outputActFunct, LinearMasked): inpMB, targMB, targMaskMB = minibatchStream.next() err, outMB = step(inpMB, targMB, self.learnRates, self.momentum, self.L2Costs, useDropout, targMaskMB) else: inpMB, targMB = minibatchStream.next() err, outMB = step(inpMB, targMB, self.learnRates, self.momentum, self.L2Costs, useDropout) sumErr += err if loss != None: sumLoss += loss(targMB, outMB) totalCases += inpMB.shape[0] prog.tick() prog.done() yield sumErr / float(totalCases), sumLoss / float(totalCases)
def fineTune(self, minibatchStream, trainInps, epochs, mbPerEpoch, loss=None, validSet=False, progressBar=True, useDropout=False): for ep in xrange(epochs): print print 'learnRates:', self.learnRates totalCases = 0 sumErr = 0 sumLoss = 0 if self.nesterov: step = self.stepNesterov else: step = self.step prog = Progress(mbPerEpoch) if progressBar else DummyProgBar() for i in range(mbPerEpoch): # print 'Epoch:', ep, 'minibatch', i (inpMB, targMB, mbgraph) = minibatchStream.next() if len(targMB.shape ) != 3: # Convert to a cubic matrix (3d matrix) targMB = targMB.reshape(-1, 1, targMB.shape[1]) # Each dimensions of inpMB (3d), refers to a pivot vector. Now, we want to select # training samples that falls in the neighborhood of this guy, and store in the # corresponding dimension of xsl (x_selected). xsl = np.zeros((mbgraph.indx.shape[0], mbgraph.indx.shape[1], trainInps.shape[1])) for j in xrange(mbgraph.indx.shape[0]): xsl[j] = trainInps[mbgraph.indx[j] - 1] # -1 because I need to covert the indices from matlab format to python #distribute graph.vals to 3d vals_select = mbgraph.vals #It has been converted to 3d inside manifold.py del mbgraph err = step(xsl, vals_select, inpMB, targMB, self.learnRates, self.momentum, self.L2Costs, useDropout) # gnp.free_reuse_cache() sumErr += err # print err, sumErr totalCases += inpMB.shape[0] prog.tick() prog.done() self.learnRates = [ y * self.learnRatesMultiplier for y in self.learnRates ] # If validation set is given if validSet: val_outputActs = self.fprop_xf(validSet['trainInps']) val_error = self.outputActFunct.error( gnp.garray(validSet['trainTargs']), self.state[-1], val_outputActs) yield sumErr / float( totalCases), val_error / validSet['trainInps'].shape[0] else: yield sumErr / float(totalCases)
def run_batch_locally(scripts, language='python', paths=[], max_cpu=0.9, max_mem=0.9, submit_sleep=1, job_check_sleep=30, \ verbose=True, max_files_open=100, max_running_jobs=10, single_thread=True): ''' Receives a list of python scripts to run Assumes the code has an output file that will be managed by this function Returns a list of local file names where the code has presumably stored output ''' # Define some code constants #### Do we need to set paths explicitly? #### This will be deprecated in future MATLAB - hopefully the -singleCompThread command is sufficient matlab_single_thread = ''' maxNumCompThreads(1); ''' python_path_code = ''' import sys sys.path.append('%s') ''' matlab_path_code = ''' addpath(genpath('%s')) ''' python_completion_code = ''' print 'Writing completion flag' with open('%(flag_file)s', 'w') as f: f.write('Goodbye, World') print "Goodbye, World" quit() ''' #### TODO - Is this completely stable matlab_completion_code = ''' fprintf('\\nWriting completion flag\\n'); ID = fopen('%(flag_file)s', 'w'); fprintf(ID, 'Goodbye, world'); fclose(ID); fprintf('\\nGoodbye, World\\n'); quit() ''' # Initialise lists of file locations job ids shell_files = [None] * len(scripts) script_files = [None] * len(scripts) output_files = [None] * len(scripts) stdout_files = [None] * len(scripts) stdout_file_handles = [None] * len(scripts) flag_files = [None] * len(scripts) processes = [None] * len(scripts) fear_finished = False job_finished = [False] * len(scripts) files_open = 0 # Loop through jobs, submitting jobs whenever CPU usage low enough, re-submitting failed jobs if not verbose: prog = Progress(len(scripts)) while not fear_finished: should_sleep = True for (i, code) in enumerate(scripts): if (not job_finished[i]) and (processes[i] is None) and ( files_open <= max_files_open) and (len([ 1 for p in processes if not p is None ]) < max_running_jobs): # This script has not been run - check CPU and potentially run #### FIXME - Merge if statements if (psutil.cpu_percent() < max_cpu * 100) and ( psutil.virtual_memory().percent < max_mem * 100): # Jobs can run should_sleep = False # Get the job ready if LOCATION == 'local': temp_dir = LOCAL_TEMP_PATH else: temp_dir = HOME_TEMP_PATH if language == 'python': script_files[i] = (mkstemp_safe(temp_dir, '.py')) elif language == 'matlab': script_files[i] = (mkstemp_safe(temp_dir, '.m')) # Create necessary files in local path shell_files[i] = (mkstemp_safe(temp_dir, '.sh')) output_files[i] = (mkstemp_safe(temp_dir, '.out')) stdout_files[i] = (mkstemp_safe(temp_dir, '.o')) flag_files[i] = (mkstemp_safe(temp_dir, '.flg')) # Customise code #### TODO - make path and output_transfer optional if language == 'python': code = code + python_completion_code for path in paths: code = (python_path_code % path) + code elif language == 'matlab': code = code + matlab_completion_code for path in paths: code = (matlab_path_code % path) + code code = code % { 'output_file': output_files[i], 'flag_file': flag_files[i] } # Write code and shell file with open(script_files[i], 'w') as f: f.write(code) with open(shell_files[i], 'w') as f: #### TODO - is os.path.join always correct - what happens if this program is being run on windows? if language == 'python': f.write('python ' + script_files[i] + '\n') elif language == 'matlab': if LOCATION == 'home': matlab_path = HOME_MATLAB else: matlab_path = LOCAL_MATLAB if single_thread: f.write('cd ' + os.path.split(script_files[i])[0] + ';\n' + matlab_path + ' -nosplash -nojvm -nodisplay -singleCompThread -r ' + \ os.path.split(script_files[i])[-1].split('.')[0] + '\n') else: f.write('cd ' + os.path.split(script_files[i])[0] + ';\n' + matlab_path + ' -nosplash -nojvm -nodisplay -r ' + \ os.path.split(script_files[i])[-1].split('.')[0] + '\n') # Start running the job if verbose: print 'Submitting job %d of %d' % (i + 1, len(scripts)) stdout_file_handles[i] = open(stdout_files[i], 'w') files_open = files_open + 1 processes[i] = subprocess.Popen( ['sh', shell_files[i]], stdout=stdout_file_handles[i]) # Sleep for a bit so the process can kick in (prevents 100s of jobs being sent to processor) time.sleep(submit_sleep) elif (not job_finished[i]) and (not processes[i] is None): # Ask the process how its doing processes[i].poll() # Check to see if the process has completed if not processes[i].returncode is None: if os.path.isfile(flag_files[i]): job_finished[i] = True if verbose: print 'Job %d of %d has completed' % (i + 1, len(scripts)) else: prog.tick() else: if verbose: print 'Job %d has failed - will try again later' % i + 1 processes[i] = None # Tidy up temp files os.remove(script_files[i]) os.remove(shell_files[i]) stdout_file_handles[i].close() files_open = files_open - 1 os.remove(stdout_files[i]) os.remove(flag_files[i]) processes[i] = None # Something useful happened should_sleep = False if all(job_finished): fear_finished = True if not verbose: prog.done() elif should_sleep: # Count how many jobs are queued n_queued = 0 # Count how many jobs are running n_running = 0 if verbose: # print '%d jobs running' % n_running # print '%d jobs queued' % n_queued print 'Sleeping for %d seconds' % job_check_sleep time.sleep(job_check_sleep) #### TODO - return job output and error files as applicable (e.g. there may be multiple error files associated with one script) return output_files