def qsub_matlab_code(code,
verbose=True,
local_dir='../temp/',
remote_dir='./temp/',
fear=None):
# Write to a temp script
script_file = mkstemp_safe(local_dir, '.m')
shell_file = mkstemp_safe(local_dir, '.sh')
f = open(script_file, 'w')
f.write(code)
f.close()
#### Local file reference without extension - MATLAB fails silently otherwise
f = open(shell_file, 'w')
f.write(
'/usr/local/apps/matlab/matlabR2011b/bin/matlab -nosplash -nojvm -nodisplay -singleCompThread -r '
+ script_file.split('/')[-1].split('.')[0] + '\n')
f.close()
# Copy this to fear
copy_to_fear(script_file, remote_dir + script_file.split('/')[-1], fear)
copy_to_fear(shell_file, remote_dir + shell_file.split('/')[-1], fear)
# Create fear call
#### WARNING - hardcoded path 'temp'
fear_string = ' '.join([
'. /usr/local/grid/divf2/common/settings.sh;',
'cd temp;'
'chmod +x %s;' % shell_file.split('/')[-1], 'qsub -l lr=0',
shell_file.split('/')[-1] + ';', 'cd ..'
])
if verbose:
print 'Submitting : %s' % fear_string
# Send this command to fear
fear_command(fear_string, fear)
# Tell the caller where the script file was written
return script_file, shell_file
def qsub_matlab_code(code, verbose=True, local_dir ='../temp/', remote_dir ='./temp/', fear=None):
# Write to a temp script
script_file = mkstemp_safe(local_dir, '.m')
shell_file = mkstemp_safe(local_dir, '.sh')
f = open(script_file, 'w')
f.write(code)
f.close()
#### Local file reference without extension - MATLAB fails silently otherwise
f = open(shell_file, 'w')
f.write('/usr/local/apps/matlab/matlabR2011b/bin/matlab -nosplash -nojvm -nodisplay -singleCompThread -r ' + script_file.split('/')[-1].split('.')[0] + '\n')
f.close()
# Copy this to fear
copy_to_fear(script_file, remote_dir + script_file.split('/')[-1], fear)
copy_to_fear(shell_file, remote_dir + shell_file.split('/')[-1], fear)
# Create fear call
#### WARNING - hardcoded path 'temp'
fear_string = ' '.join(['. /usr/local/grid/divf2/common/settings.sh;',
'cd temp;'
'chmod +x %s;' % shell_file.split('/')[-1],
'qsub -l lr=0',
shell_file.split('/')[-1] + ';',
'cd ..'])
if verbose:
print 'Submitting : %s' % fear_string
# Send this command to fear
fear_command(fear_string, fear)
# Tell the caller where the script file was written
return script_file, shell_file
def fear_run_experiments(kernels, X, y, return_all=False, verbose=True, noise=None, iters=300, local_dir ='../temp/', remote_dir ='./temp/', \
sleep_time=10, n_sleep_timeout=6, re_submit_wait=60):
'''
Sends jobs to fear, waits for them, returns the results
'''
# Not sure what this is for
if X.ndim == 1:
X = X[:, nax]
if y.ndim == 1:
y = y[:, nax]
if noise is None:
noise = np.log(np.var(y) / 10) #### Just a heuristic.
data = {'X': X, 'y': y}
# Setup the connection to fear
fear = fear_connect()
# Submit all the jobs and remember where we put them
data_files = []
write_files = []
script_files = []
shell_files = []
for kernel in kernels:
# Create data file and results file
data_files.append(mkstemp_safe(local_dir, '.mat'))
write_files.append(mkstemp_safe(local_dir, '.mat'))
# Save data
scipy.io.savemat(data_files[-1], data)
# Copy files to fear
copy_to_fear(data_files[-1],
remote_dir + data_files[-1].split('/')[-1], fear)
# copy_to_fear(write_files[-1], remote_dir + write_files[-1].split('/')[-1])
# Create MATLAB code
code = OPTIMIZE_KERNEL_CODE % {
'datafile':
data_files[-1].split('/')[-1],
'writefile':
write_files[-1].split('/')[-1],
'gpml_path':
config.FEAR_GPML_PATH,
'kernel_family':
kernel.gpml_kernel_expression(),
'kernel_params':
'[ %s ]' % ' '.join(str(p) for p in kernel.param_vector()),
'noise':
str(noise),
'iters':
str(iters)
}
# Submit this to fear and save the file names
script_file, shell_file = qsub_matlab_code(code=code,
verbose=verbose,
local_dir=local_dir,
remote_dir=remote_dir,
fear=fear)
script_files.append(script_file)
shell_files.append(shell_file)
# Let the scripts run
# if verbose:
# print 'Giving the jobs some time to run'
# time.sleep(re_submit_wait)
# Wait for and read in results
fear_finished = False
job_finished = [False] * len(write_files)
results = [None] * len(write_files)
sleep_count = 0
while not fear_finished:
for (i, write_file) in enumerate(write_files):
if not job_finished[i]:
if fear_file_exists(remote_dir + write_file.split('/')[-1],
fear):
# Another job has finished
job_finished[i] = True
sleep_count = 0
# Copy files
os.remove(write_file) # Not sure if necessary
copy_from_fear(remote_dir + write_file.split('/')[-1],
write_file, fear)
# Read results ##### THIS WILL CHANGE IF RUNNING DIFFERENT TYPE OF EXPERIMENT
gpml_result = scipy.io.loadmat(write_file)
optimized_hypers = gpml_result['hyp_opt']
nll = gpml_result['best_nll'][0, 0]
# nlls = gpml_result['nlls'].ravel()
laplace_nle = gpml_result['laplace_nle'][0, 0]
kernel_hypers = optimized_hypers['cov'][0, 0].ravel()
k_opt = kernels[i].family().from_param_vector(
kernel_hypers)
BIC = 2 * nll + len(kernel_hypers) * np.log(y.shape[0])
results[i] = (k_opt, nll, laplace_nle, BIC)
# Tidy up
fear_rm(remote_dir + data_files[i].split('/')[-1], fear)
fear_rm(remote_dir + write_files[i].split('/')[-1], fear)
fear_rm(remote_dir + script_files[i].split('/')[-1], fear)
fear_rm(remote_dir + shell_files[i].split('/')[-1], fear)
fear_rm(remote_dir + shell_files[i].split('/')[-1] + '*',
fear)
os.remove(data_files[i])
os.remove(write_files[i])
os.remove(script_files[i])
os.remove(shell_files[i])
# Tell the world
if verbose:
print '%d / %d jobs complete' % (sum(job_finished),
len(job_finished))
if sum(job_finished) == len(job_finished):
fear_finished = True
if not fear_finished:
if verbose:
print 'Sleeping'
sleep_count += 1
if sleep_count < n_sleep_timeout:
time.sleep(sleep_time)
else:
# Jobs taking too long - assume failure - resubmit
fear_qdel_all(fear)
for (i, shell_file) in enumerate(shell_files):
if not job_finished[i]:
re_qsub(shell_file, verbose=verbose, fear=fear)
if verbose:
print 'Giving the jobs some time to run'
time.sleep(re_submit_wait)
sleep_count = 0
fear.close()
return results
def fear_run_experiments(kernels, X, y, return_all=False, verbose=True, noise=None, iters=300, local_dir ='../temp/', remote_dir ='./temp/', \
sleep_time=10, n_sleep_timeout=6, re_submit_wait=60):
'''
Sends jobs to fear, waits for them, returns the results
'''
# Not sure what this is for
if X.ndim == 1:
X = X[:, nax]
if y.ndim == 1:
y = y[:, nax]
if noise is None:
noise = np.log(np.var(y)/10) #### Just a heuristic.
data = {'X': X, 'y': y}
# Setup the connection to fear
fear = fear_connect()
# Submit all the jobs and remember where we put them
data_files = []
write_files = []
script_files = []
shell_files = []
for kernel in kernels:
# Create data file and results file
data_files.append(mkstemp_safe(local_dir, '.mat'))
write_files.append(mkstemp_safe(local_dir, '.mat'))
# Save data
scipy.io.savemat(data_files[-1], data)
# Copy files to fear
copy_to_fear(data_files[-1], remote_dir + data_files[-1].split('/')[-1], fear)
# copy_to_fear(write_files[-1], remote_dir + write_files[-1].split('/')[-1])
# Create MATLAB code
code = OPTIMIZE_KERNEL_CODE % {'datafile': data_files[-1].split('/')[-1],
'writefile': write_files[-1].split('/')[-1],
'gpml_path': config.FEAR_GPML_PATH,
'kernel_family': kernel.gpml_kernel_expression(),
'kernel_params': '[ %s ]' % ' '.join(str(p) for p in kernel.param_vector()),
'noise': str(noise),
'iters': str(iters)}
# Submit this to fear and save the file names
script_file, shell_file = qsub_matlab_code(code=code, verbose=verbose, local_dir=local_dir, remote_dir=remote_dir, fear=fear)
script_files.append(script_file)
shell_files.append(shell_file)
# Let the scripts run
# if verbose:
# print 'Giving the jobs some time to run'
# time.sleep(re_submit_wait)
# Wait for and read in results
fear_finished = False
job_finished = [False] * len(write_files)
results = [None] * len(write_files)
sleep_count = 0
while not fear_finished:
for (i, write_file) in enumerate(write_files):
if not job_finished[i]:
if fear_file_exists(remote_dir + write_file.split('/')[-1], fear):
# Another job has finished
job_finished[i] = True
sleep_count = 0
# Copy files
os.remove(write_file) # Not sure if necessary
copy_from_fear(remote_dir + write_file.split('/')[-1], write_file, fear)
# Read results ##### THIS WILL CHANGE IF RUNNING DIFFERENT TYPE OF EXPERIMENT
gpml_result = scipy.io.loadmat(write_file)
optimized_hypers = gpml_result['hyp_opt']
nll = gpml_result['best_nll'][0, 0]
# nlls = gpml_result['nlls'].ravel()
laplace_nle = gpml_result['laplace_nle'][0, 0]
kernel_hypers = optimized_hypers['cov'][0, 0].ravel()
k_opt = kernels[i].family().from_param_vector(kernel_hypers)
BIC = 2 * nll + len(kernel_hypers) * np.log(y.shape[0])
results[i] = (k_opt, nll, laplace_nle, BIC)
# Tidy up
fear_rm(remote_dir + data_files[i].split('/')[-1], fear)
fear_rm(remote_dir + write_files[i].split('/')[-1], fear)
fear_rm(remote_dir + script_files[i].split('/')[-1], fear)
fear_rm(remote_dir + shell_files[i].split('/')[-1], fear)
fear_rm(remote_dir + shell_files[i].split('/')[-1] + '*', fear)
os.remove(data_files[i])
os.remove(write_files[i])
os.remove(script_files[i])
os.remove(shell_files[i])
# Tell the world
if verbose:
print '%d / %d jobs complete' % (sum(job_finished), len(job_finished))
if sum(job_finished) == len(job_finished):
fear_finished = True
if not fear_finished:
if verbose:
print 'Sleeping'
sleep_count += 1
if sleep_count < n_sleep_timeout:
time.sleep(sleep_time)
else:
# Jobs taking too long - assume failure - resubmit
fear_qdel_all(fear)
for (i, shell_file) in enumerate(shell_files):
if not job_finished[i]:
re_qsub(shell_file, verbose=verbose, fear=fear)
if verbose:
print 'Giving the jobs some time to run'
time.sleep(re_submit_wait)
sleep_count = 0
fear.close()
return results
from cblparallel.util import mkstemp_safe
import os
import shutil
import scipy.io
import numpy as np
# Move data file to fear after saving to temporary location
for file_name in os.listdir('data'):
if file_name[-4:] == '.mat':
data_file = os.path.join('data', file_name)
break
temp_data_file_name = mkstemp_safe(LOCAL_TEMP_PATH, '.mat')
shutil.move(data_file, temp_data_file_name)
cblparallel.copy_to_remote(temp_data_file_name)
# Load scripts
scripts = []
script_names = sorted(os.listdir('scripts'))
used_script_names = []
for file_name in script_names:
if file_name[-2:] == '.m':
# print 'Reading %s' % file_name
used_script_names.append(file_name)
benchmarks = f.readlines()
if len(benchmarks) > 500:
benchmarks = [benchmarks[0]] + benchmarks[-499:]
# Append to results
benchmarks.append('%s,%f,%s\n' % (the_time, t, name))
with open('../data/bench.csv', 'w') as f:
f.writelines(benchmarks)
# Now time how long it takes to send, unzip and remove 50 files on fear
temp_dir = LOCAL_TEMP_PATH
temp_files = [mkstemp_safe(temp_dir, '.txt') for i in range(50)]
for temp_file in temp_files:
with open(temp_file, 'w') as f:
f.write('test text')
print 'Zipping files'
zip_file_name = mkstemp_safe(temp_dir, '.zip')
zf = zipfile.ZipFile(zip_file_name, mode='w')
for temp_file in temp_files:
zf.write(temp_file, arcname=(os.path.split(temp_file)[-1]), compress_type=zipfile.ZIP_DEFLATED)
zf.close()
with pyfear.fear(via_gate=(LOCATION=='home')) as fear:
# Start timing
t1 = time.time()
fear.copy_to_temp(zip_file_name)
