def slurm_prep(log_dir, niter=10000, partition_name='debug',
this_dir=os.getcwd()):
"""
Prep the SLRUM runs.
@param log_dir: The directory to store the results in.
@param niter: The number of iterations to perform.
@param partition_name: The partition name of the cluster to use.
@param this_dir: The full path to the directory where this file is located.
"""
# Get the configuration details
static_config, dynamic_config = parallel_params(log_dir, niter)
# Create the runs
for i in xrange(1, niter + 1):
# Build the initial params
params = {k:v.rvs() for k, v in sorted(dynamic_config.items())}
for k, v in static_config.items():
params[k] = v
# Create the base directory
dir = params['log_dir']
splits = os.path.basename(dir).split('-')
dir = os.path.join(os.path.dirname(dir),
'-'.join(s for s in splits[:-1]))
try:
os.makedirs(dir)
except OSError:
pass
# Dump the params as JSON
s = json.dumps(params, sort_keys=True, indent=4,
separators=(',', ': ')).replace('},', '},\n')
with open(os.path.join(dir, 'config.json'), 'wb') as f:
f.write(s)
# Create the runner
mnist_runner_path = os.path.join(this_dir,
'mnist_novelty_detection.py')
command = 'python "{0}" "{1}"'.format(mnist_runner_path, dir)
runner_path = os.path.join(dir, 'runner.sh')
job_name = str(i)
stdio_path = os.path.join(dir, 'stdio.txt')
stderr_path = os.path.join(dir, 'stderr.txt')
create_runner(command=command, runner_path=runner_path,
job_name=job_name, partition_name=partition_name,
stdio_path=stdio_path, stderr_path=stderr_path,
time_limit='00-00:45:00', memory_limit=512)
# Execute the runner
execute_runner(runner_path)
def launch_top_runs(top_paths,
bp,
command,
auto_pupdate=False,
partition_name='debug',
time_limit='04-00:00:00',
memory_limit=2048):
"""
Launch the top runs.
@param top_paths: The full path to the base directory containing the top
results.
@param bp: The new base directory.
@param command: The base command to execute in the runner. Two additional
arguments will be passed - the base directory and the fold index.
@param auto_pupdate: If True the permanence increment and decrement amounts
will automatically be computed by the runner. If False, the ones specified
in the config file will be used.
@param partition_name: The partition name to use.
@param time_limit: The maximum time limit.
@param memory_limit: The maximum memory requirements in MB.
"""
for p in top_paths:
# Path where the run should occur
job_name = os.path.basename(p)
p2 = os.path.join(bp, job_name)
try:
os.makedirs(p2)
except OSError:
pass # Overwrite the files
# Create the runner
runner_path = os.path.join(p2, 'runner.sh')
command_new = '{0} "{1}" "{2}" {3}'.format(command, p, p2,
int(auto_pupdate))
stdio_path = os.path.join(p2, 'stdio.txt')
stderr_path = os.path.join(p2, 'stderr.txt')
create_runner(command=command_new,
runner_path=runner_path,
job_name=job_name,
partition_name=partition_name,
stdio_path=stdio_path,
stderr_path=stderr_path,
time_limit=time_limit,
memory_limit=memory_limit)
# Execute the runner
execute_runner(runner_path)
def launch_missing(missing,
command,
partition_name='debug',
time_limit='00-04:00:00',
memory_limit=512):
"""Launch the missing results on the cluster.
Assumes that the convention <run_instance>-<fold_instance> for the
directories was utilized.
@param missing: The missing items.
@param command: The base command to execute in the runner. Two additional
arguments will be passed - the base directory and the fold index.
@param partition_name: The partition name to use.
@param time_limit: The maximum time limit.
@param memory_limit: The maximum memory requirements in MB.
"""
# Execute each missing item
for p in missing:
# Build the SP kargs for the proper path
bn, ix = os.path.basename(p).split('-')
bp = os.path.join(os.path.dirname(p), bn)
with open(os.path.join(bp, 'config.json'), 'r') as f:
kargs = json.load(f)
kargs['log_dir'] = p
# Dump the arguments to a new file
s = json.dumps(kargs, sort_keys=True, indent=4,
separators=(',', ': ')).replace('},', '},\n')
with open(os.path.join(bp, 'config-{0}.json'.format(ix)), 'w') as f:
f.write(s)
# Create the runner
runner_path = os.path.join(bp, 'runner-{0}.sh'.format(ix))
job_name = os.path.basename(p)
command_new = '{0} "{1}" {2}'.format(command, bp, ix)
stdio_path = os.path.join(bp, 'stdio-{0}.txt'.format(ix))
stderr_path = os.path.join(bp, 'stderr-{0}.txt'.format(ix))
create_runner(command=command_new,
runner_path=runner_path,
job_name=job_name,
partition_name=partition_name,
stdio_path=stdio_path,
stderr_path=stderr_path,
time_limit=time_limit,
memory_limit=memory_limit)
# Execute the runner
execute_runner(runner_path)
def slurm_prep(log_dir, partition_name='debug', this_dir=os.getcwd()):
"""
Prep the SLRUM runs.
@param log_dir: The directory to store the results in.
@param partition_name: The partition name of the cluster to use.
@param this_dir: The full path to the directory where this file is located.
"""
# Create the runs
i = 1
for noise in np.linspace(0, 1, 101):
for overlap in np.arange(0, 41):
dir = os.path.join(log_dir, '{0}-{1}'.format(noise, overlap))
# Create the base directory
try:
os.makedirs(dir)
except OSError:
pass
# Dump the params as JSON
s = json.dumps(create_base_config(dir),
sort_keys=True,
indent=4,
separators=(',', ': ')).replace('},', '},\n')
with open(os.path.join(dir, 'config.json'), 'wb') as f:
f.write(s)
# Create the runner
mnist_runner_path = os.path.join(this_dir,
'novelty_detection_slurm.py')
command = 'python "{0}" "{1}" "{2}" "{3}"'.format(
mnist_runner_path, dir, noise, overlap)
runner_path = os.path.join(dir, 'runner.sh')
job_name = str(i)
stdio_path = os.path.join(dir, 'stdio.txt')
stderr_path = os.path.join(dir, 'stderr.txt')
create_runner(command=command,
runner_path=runner_path,
job_name=job_name,
partition_name=partition_name,
stdio_path=stdio_path,
stderr_path=stderr_path,
time_limit='00-00:10:00',
memory_limit=128)
# Execute the runner
execute_runner(runner_path)
i += 1
def slurm_prep(log_dir, partition_name='debug', this_dir=os.getcwd()):
"""
Prep the SLRUM runs.
@param log_dir: The directory to store the results in.
@param partition_name: The partition name of the cluster to use.
@param this_dir: The full path to the directory where this file is located.
"""
# Create the runs
i = 1
for noise in np.linspace(0, 1, 101):
for overlap in np.arange(0, 41):
dir = os.path.join(log_dir, '{0}-{1}'.format(noise, overlap))
# Create the base directory
try:
os.makedirs(dir)
except OSError:
pass
# Dump the params as JSON
s = json.dumps(create_base_config(dir), sort_keys=True,
indent=4, separators=(',', ': ')).replace('},', '},\n')
with open(os.path.join(dir, 'config.json'), 'wb') as f:
f.write(s)
# Create the runner
mnist_runner_path = os.path.join(this_dir,
'novelty_detection_slurm.py')
command = 'python "{0}" "{1}" "{2}" "{3}"'.format(
mnist_runner_path, dir, noise, overlap)
runner_path = os.path.join(dir, 'runner.sh')
job_name = str(i)
stdio_path = os.path.join(dir, 'stdio.txt')
stderr_path = os.path.join(dir, 'stderr.txt')
create_runner(command=command, runner_path=runner_path,
job_name=job_name, partition_name=partition_name,
stdio_path=stdio_path, stderr_path=stderr_path,
time_limit='00-00:10:00', memory_limit=128)
# Execute the runner
execute_runner(runner_path)
i += 1
def launch_missing(missing, command, partition_name='debug',
time_limit='00-04:00:00', memory_limit=512):
"""
Launch the missing results on the cluster. It assumes that the convention
<run_instance>-<fold_instance> for the directories was utilized.
@param missing: The missing items.
@param command: The base command to execute in the runner. Two additional
arguments will be passed - the base directory and the fold index.
@param partition_name: The partition name to use.
@param time_limit: The maximum time limit.
@param memory_limit: The maximum memory requirements in MB.
"""
# Execute each missing item
for p in missing:
# Build the SP kargs for the proper path
bn, ix = os.path.basename(p).split('-')
bp = os.path.join(os.path.dirname(p), bn)
with open(os.path.join(bp, 'config.json'), 'rb') as f:
kargs = json.load(f)
kargs['log_dir'] = p
# Dump the arguments to a new file
s = json.dumps(kargs, sort_keys=True, indent=4,
separators=(',', ': ')).replace('},', '},\n')
with open(os.path.join(bp, 'config-{0}.json'.format(ix)), 'wb') as f:
f.write(s)
# Create the runner
runner_path = os.path.join(bp, 'runner-{0}.sh'.format(ix))
job_name = os.path.basename(p)
command_new = '{0} "{1}" {2}'.format(command, bp, ix)
stdio_path = os.path.join(bp, 'stdio-{0}.txt'.format(ix))
stderr_path = os.path.join(bp, 'stderr-{0}.txt'.format(ix))
create_runner(command=command_new, runner_path=runner_path,
job_name=job_name, partition_name=partition_name,
stdio_path=stdio_path, stderr_path=stderr_path,
time_limit=time_limit, memory_limit=memory_limit)
# Execute the runner
execute_runner(runner_path)
def launch_top_runs(top_paths, bp, command, auto_pupdate=False,
partition_name='debug', time_limit='04-00:00:00', memory_limit=2048):
"""
Launch the top runs.
@param top_paths: The full path to the base directory containing the top
results.
@param bp: The new base directory.
@param command: The base command to execute in the runner. Two additional
arguments will be passed - the base directory and the fold index.
@param auto_pupdate: If True the permanence increment and decrement amounts
will automatically be computed by the runner. If False, the ones specified
in the config file will be used.
@param partition_name: The partition name to use.
@param time_limit: The maximum time limit.
@param memory_limit: The maximum memory requirements in MB.
"""
for p in top_paths:
# Path where the run should occur
job_name = os.path.basename(p)
p2 = os.path.join(bp, job_name)
try:
os.makedirs(p2)
except OSError:
pass # Overwrite the files
# Create the runner
runner_path = os.path.join(p2, 'runner.sh')
command_new = '{0} "{1}" "{2}" {3}'.format(command, p, p2,
int(auto_pupdate))
stdio_path = os.path.join(p2, 'stdio.txt')
stderr_path = os.path.join(p2, 'stderr.txt')
create_runner(command=command_new, runner_path=runner_path,
job_name=job_name, partition_name=partition_name,
stdio_path=stdio_path, stderr_path=stderr_path,
time_limit=time_limit, memory_limit=memory_limit)
# Execute the runner
execute_runner(runner_path)
def main_slurm(log_dir,
ntrain=800,
ntest=200,
niter=5,
nsplits=3,
global_inhibition=True,
partition_name='debug',
seed=None):
"""
Perform CV on a subset of the MNIST dataset, using SLRUM. Iterations will
be run in complete parallel. Splits within an iteration will be run
sequentially.
@param log_dir: The directory to store the results in.
@param ntrain: The number of training samples to use.
@param ntest: The number of testing samples to use.
@param niter: The number of parameter iterations to use.
@param nsplits: The number of splits of the data to use.
@param global_inhibition: If True use global inhibition; otherwise, use
local inhibition.
@param partition_name: The partition name of the cluster to use.
@param seed: The seed for the random number generators.
"""
# Run the initialization
x, y, kargs, params, cv = main(log_dir, ntrain, ntest, niter, nsplits,
global_inhibition, seed)
# Create the runs
for i in xrange(1, niter + 1):
# Build the initial params
param = {k: v.rvs() for k, v in sorted(params.items())}
# Create the base directory
dir = param['log_dir']
splits = os.path.basename(dir).split('-')
dir = os.path.join(os.path.dirname(dir),
'-'.join(s for s in splits[:-1]))
try:
os.makedirs(dir)
except OSError:
pass
# Dump the CV data
with open(os.path.join(dir, 'cv.pkl'), 'wb') as f:
cPickle.dump(list(cv), f, cPickle.HIGHEST_PROTOCOL)
# Build the full params
for k, v in kargs.items():
if k != 'clf': # Add the classifier later
param[k] = v
# Dump the params as JSON
s = json.dumps(param, sort_keys=True, indent=4,
separators=(',', ': ')).replace('},', '},\n')
with open(os.path.join(dir, 'config.json'), 'wb') as f:
f.write(s)
# Create the runner
mnist_runner_path = os.path.join(
pkgutil.get_loader('mHTM.examples').filename, 'mnist_runner.py')
command = 'python "{0}" "{1}"'.format(mnist_runner_path, dir)
runner_path = os.path.join(dir, 'runner.sh')
job_name = str(i)
stdio_path = os.path.join(dir, 'stdio.txt')
stderr_path = os.path.join(dir, 'stderr.txt')
create_runner(command=command,
runner_path=runner_path,
job_name=job_name,
partition_name=partition_name,
stdio_path=stdio_path,
stderr_path=stderr_path)
# Execute the runner
execute_runner(runner_path)
def run_experiment(experiments,
base_dir,
nsamples=500,
nbits=100,
pct_active=0.4,
pct_noise=0.15,
seed=123456789,
ntrials=10,
partition_name='debug',
this_dir=os.getcwd()):
"""Run an experiment for the SP.
This experiment is used to vary various sets of parameters on the SP
dataset. This function uses SLURM to conduct the experiments.
@param experiments: A list containing the experiments details. Refer to one
of the examples in this module for more details.
@param base_dir: The base directory to use for logging.
@param nsamples: The number of samples to add to the dataset.
@param nbits: The number of bits each sample should have.
@param pct_active: The percentage of bits that will be active in the base
class SDR.
@param pct_noise: The percentage of noise to add to the data.
@param seed: The seed used to initialize the random number generator.
@param ntrials: The number of parameter trials to use. Each iteration will
be used to initialize the SP in a different manner.
@param partition_name: The partition name of the cluster to use.
@param this_dir: The full path to the directory where this file is located.
"""
# Create the dataset
data = SPDataset(nsamples, nbits, pct_active, pct_noise, seed).data
# Metrics
metrics = SPMetrics()
# Get the metrics for the dataset
uniqueness_data = metrics.compute_uniqueness(data)
overlap_data = metrics.compute_overlap(data)
correlation_data = 1 - metrics.compute_distance(data)
# Prep each experiment for execution
for experiment_name, time_limit, memory_limit, params in experiments:
# Iterate through each type of inhibition type
for i, global_inhibition in enumerate((True, False)):
# Get base configuration
base_config = create_base_config(base_dir, experiment_name,
global_inhibition)
# Add the parameters
for param_name, param_value in params:
base_config[param_name] = param_value
config_gen = ConfigGenerator(base_config, ntrials)
# Make the configurations
for config in config_gen.get_config():
# Make the base directory
dir = config['log_dir']
splits = os.path.basename(dir).split('-')
base_name = '-'.join(s for s in splits[:-1])
dir = os.path.join(os.path.dirname(dir), base_name)
try:
os.makedirs(dir)
except OSError:
pass
# Dump the config as JSON
s = json.dumps(config,
sort_keys=True,
indent=4,
separators=(',', ': ')).replace('},', '},\n')
with open(os.path.join(dir, 'config.json'), 'w') as f:
f.write(s)
# Dump the dataset and the metrics
with open(os.path.join(dir, 'dataset.pkl'), 'wb') as f:
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
pickle.dump(
(uniqueness_data, overlap_data, correlation_data), f,
pickle.HIGHEST_PROTOCOL)
# Create the runner
this_path = os.path.join(this_dir, 'parameter_exploration.py')
command = 'python "{0}" "{1}" {2} {3}'.format(
this_path, dir, ntrials, seed)
runner_path = os.path.join(dir, 'runner.sh')
job_name = '{0}_{1}{2}'.format(
experiment_name, 'G' if global_inhibition else 'L',
base_name)
stdio_path = os.path.join(dir, 'stdio.txt')
stderr_path = os.path.join(dir, 'stderr.txt')
create_runner(command=command,
runner_path=runner_path,
job_name=job_name,
partition_name=partition_name,
stdio_path=stdio_path,
stderr_path=stderr_path,
time_limit=time_limit[i],
memory_limit=memory_limit)
# Execute the runner
execute_runner(runner_path)
def main_slurm(log_dir, ntrain=800, ntest=200, niter=5, nsplits=3,
global_inhibition=True, partition_name='debug', seed=None):
"""
Perform CV on a subset of the MNIST dataset, using SLRUM. Iterations will
be run in complete parallel. Splits within an iteration will be run
sequentially.
@param log_dir: The directory to store the results in.
@param ntrain: The number of training samples to use.
@param ntest: The number of testing samples to use.
@param niter: The number of parameter iterations to use.
@param nsplits: The number of splits of the data to use.
@param global_inhibition: If True use global inhibition; otherwise, use
local inhibition.
@param partition_name: The partition name of the cluster to use.
@param seed: The seed for the random number generators.
"""
# Run the initialization
x, y, kargs, params, cv = main(log_dir, ntrain, ntest, niter, nsplits,
global_inhibition, seed)
# Create the runs
for i in xrange(1, niter + 1):
# Build the initial params
param = {k:v.rvs() for k, v in sorted(params.items())}
# Create the base directory
dir = param['log_dir']
splits = os.path.basename(dir).split('-')
dir = os.path.join(os.path.dirname(dir),
'-'.join(s for s in splits[:-1]))
try:
os.makedirs(dir)
except OSError:
pass
# Dump the CV data
with open(os.path.join(dir, 'cv.pkl'), 'wb') as f:
cPickle.dump(list(cv), f, cPickle.HIGHEST_PROTOCOL)
# Build the full params
for k, v in kargs.items():
if k != 'clf': # Add the classifier later
param[k] = v
# Dump the params as JSON
s = json.dumps(param, sort_keys=True, indent=4,
separators=(',', ': ')).replace('},', '},\n')
with open(os.path.join(dir, 'config.json'), 'wb') as f:
f.write(s)
# Create the runner
mnist_runner_path = os.path.join(pkgutil.get_loader('mHTM.examples').
filename, 'mnist_runner.py')
command = 'python "{0}" "{1}"'.format(mnist_runner_path, dir)
runner_path = os.path.join(dir, 'runner.sh')
job_name = str(i)
stdio_path = os.path.join(dir, 'stdio.txt')
stderr_path = os.path.join(dir, 'stderr.txt')
create_runner(command=command, runner_path=runner_path,
job_name=job_name, partition_name=partition_name,
stdio_path=stdio_path, stderr_path=stderr_path)
# Execute the runner
execute_runner(runner_path)
def run_experiment(experiments, base_dir, nsamples=500, nbits=100,
pct_active=0.4, pct_noise=0.15, seed=123456789, ntrials=10,
partition_name='debug', this_dir=os.getcwd()):
"""
Run an experiment for the SP. This experiment is used to vary various sets
of parameters on the SP dataset. This function uses SLURM to conduct the
experiments.
@param experiments: A list containing the experiments details. Refer to one
of the examples in this module for more details.
@param base_dir: The base directory to use for logging.
@param nsamples: The number of samples to add to the dataset.
@param nbits: The number of bits each sample should have.
@param pct_active: The percentage of bits that will be active in the base
class SDR.
@param pct_noise: The percentage of noise to add to the data.
@param seed: The seed used to initialize the random number generator.
@param ntrials: The number of parameter trials to use. Each iteration will
be used to initialize the SP in a different manner.
@param partition_name: The partition name of the cluster to use.
@param this_dir: The full path to the directory where this file is located.
"""
# Create the dataset
data = SPDataset(nsamples, nbits, pct_active, pct_noise, seed).data
# Metrics
metrics = SPMetrics()
# Get the metrics for the dataset
uniqueness_data = metrics.compute_uniqueness(data)
overlap_data = metrics.compute_overlap(data)
correlation_data = 1 - metrics.compute_distance(data)
# Prep each experiment for execution
for experiment_name, time_limit, memory_limit, params in experiments:
# Iterate through each type of inhibition type
for i, global_inhibition in enumerate((True, False)):
# Get base configuration
base_config = create_base_config(base_dir, experiment_name,
global_inhibition)
# Add the parameters
for param_name, param_value in params:
base_config[param_name] = param_value
config_gen = ConfigGenerator(base_config, ntrials)
# Make the configurations
for config in config_gen.get_config():
# Make the base directory
dir = config['log_dir']
splits = os.path.basename(dir).split('-')
base_name = '-'.join(s for s in splits[:-1])
dir = os.path.join(os.path.dirname(dir), base_name)
try:
os.makedirs(dir)
except OSError:
pass
# Dump the config as JSON
s = json.dumps(config, sort_keys=True, indent=4,
separators=(',', ': ')).replace('},', '},\n')
with open(os.path.join(dir, 'config.json'), 'wb') as f:
f.write(s)
# Dump the dataset and the metrics
with open(os.path.join(dir, 'dataset.pkl'), 'wb') as f:
cPickle.dump(data, f, cPickle.HIGHEST_PROTOCOL)
cPickle.dump((uniqueness_data, overlap_data,
correlation_data), f, cPickle.HIGHEST_PROTOCOL)
# Create the runner
this_path = os.path.join(this_dir, 'parameter_exploration.py')
command = 'python "{0}" "{1}" {2} {3}'.format(this_path, dir,
ntrials, seed)
runner_path = os.path.join(dir, 'runner.sh')
job_name = '{0}_{1}{2}'.format(experiment_name, 'G' if
global_inhibition else 'L', base_name)
stdio_path = os.path.join(dir, 'stdio.txt')
stderr_path = os.path.join(dir, 'stderr.txt')
create_runner(command=command, runner_path=runner_path,
job_name=job_name, partition_name=partition_name,
stdio_path=stdio_path, stderr_path=stderr_path,
time_limit=time_limit[i], memory_limit=memory_limit)
# Execute the runner
execute_runner(runner_path)