def recurse_add(d):
print "Exploring " + d
files = os.listdir(d)
for f in files:
if f.startswith('.'):
continue
if f == 'LOGS':
continue
full_path = d + '/' + f
if os.path.isdir(full_path):
recurse_add(full_path)
elif any(f.endswith(suffix) for suffix in suffixes):
run_shell_command("git add " + full_path)
def run_pyflakes(no_warnings = False):
""" Returns a dictionary mapping pylearn2 .py filepaths
to outputs from pyflakes.
Omits files for which there was no output.
If no_warnings = True, omits pyflakes outputs that don't
correspond to actual errors.
"""
files = list_files(".py")
rval = {}
for filepath in files:
output, rc = run_shell_command('pyflakes ' + filepath)
if 'pyflakes: not found' in output:
# The return code alone does not make it possible to detect
# if pyflakes is present or not. When pyflakes is not present,
# the return code seems to always be 127, but 127 can also be
# the result of finding 127 warnings in a file.
# Therefore, we examine the output instead.
raise RuntimeError("Couldn't run 'pyflakes " + filepath + "'. "
"Error code returned:" + str(rc)\
+ " Output was: " + output)
output = _filter(output, no_warnings)
if output is not None:
rval[filepath] = output
return rval
def run_pyflakes(no_warnings=False):
""" Returns a dictionary mapping pylearn2 .py filepaths
to outputs from pyflakes.
Omits files for which there was no output.
If no_warnings = True, omits pyflakes outputs that don't
correspond to actual errors.
"""
files = list_files(".py")
rval = {}
for filepath in files:
output, rc = run_shell_command('pyflakes ' + filepath)
if 'pyflakes: not found' in output:
# The return code alone does not make it possible to detect
# if pyflakes is present or not. When pyflakes is not present,
# the return code seems to always be 127, but 127 can also be
# the result of finding 127 warnings in a file.
# Therefore, we examine the output instead.
raise RuntimeError("Couldn't run 'pyflakes " + filepath + "'. "
"Error code returned:" + str(rc)\
+ " Output was: " + output)
output = _filter(output, no_warnings)
if output is not None:
rval[filepath] = output
return rval
def run_pyflakes(no_warnings = False):
""" Returns a dictionary mapping pylearn2 .py filepaths
to outputs from pyflakes.
Omits files for which there was no output.
If no_warnings = True, omits pyflakes outputs that don't
correspond to actual errors.
"""
files = list_files(".py")
rval = {}
for filepath in files:
output, rc = run_shell_command('pyflakes '+filepath)
if rc not in [0,1]:
#pyflakes will return 1 if you give it an invalid file or if
#the file contains errors, so it's not clear how to detect if
#pyflakes failed
#however, if pyflakes just plain isn't installed we should get 127
raise RuntimeError("Couldn't run 'pyflakes "+filepath+"'."\
+ "Output was: "+output)
output = _filter(output, no_warnings)
if output is not None:
rval[filepath] = output
return rval
def run_pyflakes(no_warnings=False):
""" Returns a dictionary mapping pylearn2 .py filepaths
to outputs from pyflakes.
Omits files for which there was no output.
If no_warnings = True, omits pyflakes outputs that don't
correspond to actual errors.
"""
files = list_files(".py")
rval = {}
for filepath in files:
output, rc = run_shell_command('pyflakes ' + filepath)
if rc not in [0, 1]:
#pyflakes will return 1 if you give it an invalid file or if
#the file contains errors, so it's not clear how to detect if
#pyflakes failed
#however, if pyflakes just plain isn't installed we should get 127
raise RuntimeError("Couldn't run 'pyflakes "+filepath+"'."\
+ "Output was: "+output)
output = _filter(output, no_warnings)
if output is not None:
rval[filepath] = output
return rval
def run_pyflakes(no_warnings=False):
"""
Return a description of all errors pyflakes finds in Pylearn2.
Parameters
----------
no_warnings : bool
If True, omits pyflakes outputs that don't correspond to actual
errors.
Returns
-------
rval : dict
Keys are pylearn2 .py filepaths
Values are outputs from pyflakes
"""
files = list_files(".py")
rval = {}
for filepath in files:
output, rc = run_shell_command('pyflakes ' + filepath)
output = output.decode(sys.getdefaultencoding())
if u'pyflakes: not found' in output:
# The return code alone does not make it possible to detect
# if pyflakes is present or not. When pyflakes is not present,
# the return code seems to always be 127, but 127 can also be
# the result of finding 127 warnings in a file.
# Therefore, we examine the output instead.
raise RuntimeError("Couldn't run 'pyflakes " + filepath + "'. "
"Error code returned:" + str(rc) +
" Output was: " + output)
output = _filter(output, no_warnings)
if output is not None:
rval[filepath] = output
return rval
},
},
cost : !obj:galatea.dbm.inpaint.super_dbm.SuperDBM_ConditionalNLL {
},
termination_criterion: !obj:pylearn2.termination_criteria.MonitorBased
{
channel_name: "valid_err",
prop_decrease: .000,
N : 10
}
},
extensions: [
!obj:pylearn2.train_extensions.best_params.MonitorBasedSaveBest {
channel_name: "valid_err",
save_path: "${PYLEARN2_TRAIN_FILE_FULL_STEM}_best.pkl"
}
],
save_path: "${PYLEARN2_TRAIN_FILE_FULL_STEM}.pkl",
save_freq : 1
}
""" % locals())
f.close()
print 'run train yourself, for some reason the subprocess was claiming a second gpu'
quit()
from pylearn2.utils.shell import run_shell_command
print 'shell will hide output until done, yay python'
out, rc = run_shell_command('train.py '+outpath)
print 'return code ',rc
print out
import os
import shutil
from pylearn2.config import yaml_parse
from pylearn2.utils import serial
from pylearn2.utils import shell
status, rc = shell.run_shell_command("qstat -u goodfell -t @hades")
assert rc == 0
results = open("results.dat", "r")
lines = results.readlines()
results.close()
params = yaml_parse.load_path('params.yaml')
added = 0
print 'Experiment numbers reported by this script start at 0.'
print 'Keep in mind that vim will refer to the first line of results.dat as line 1'
for expnum, line in enumerate(lines):
elems = line.split(' ')
assert elems[-1] == '\n'
obj = elems[0]
if obj == 'P':
# print expnum, 'pending according to results.dat'
expdir = '/RQexec/goodfell/experiment_7/%d' % expnum
if not os.path.exists(expdir):
print 'Experiment not yet configured for experiment',expnum
continue
cluster_info = expdir + '/cluster_info.txt'
if not os.path.exists(cluster_info):
import os
from pylearn2.utils.shell import run_shell_command
myDir = "/data/lisatmp/ift6266h13/mouretge/transfer/"
myYamls = myDir + "yaml_files/"
names = os.listdir(myYamls)
dirs = [myYamls + name for name in names]
command = """jobdispatch --condor --env=THEANO_FLAGS=floatX=float32 --duree=48:00:00 --whitespace python %(dir)strain_model.py """ % {"dir":myDir}
command += """\"{{"""
command += ','.join(dirs)
command += """}}\""""
run_shell_command(command)
print command
import os
import shutil
from pylearn2.config import yaml_parse
from pylearn2.utils import serial
from pylearn2.utils import shell
status, rc = shell.run_shell_command("qstat -u goodfell -t @hades")
assert rc == 0
results = open("results.dat", "r")
lines = results.readlines()
results.close()
params = yaml_parse.load_path('params.yaml')
added = 0
print 'Experiment numbers reported by this script start at 0.'
print 'Keep in mind that vim will refer to the first line of results.dat as line 1'
for expnum, line in enumerate(lines):
elems = line.split(' ')
assert elems[-1] == '\n'
obj = elems[0]
if obj == 'P':
# print expnum, 'pending according to results.dat'
expdir = '/RQexec/goodfell/experiment_7/%d' % expnum
if not os.path.exists(expdir):
print 'Experiment not yet configured for experiment', expnum
continue
cluster_info = expdir + '/cluster_info.txt'
if not os.path.exists(cluster_info):
print 'Experiment not yet launched for experiment', expnum
#run with no args
import os
from pylearn2.utils.shell import run_shell_command
expdir = '/RQexec/goodfell/experiment_6'
names = os.listdir(expdir)
dirs = [expdir + '/' + name for name in names]
dirs = [d for d in dirs if not os.path.exists(d + '/cluster_info.txt')]
command = 'jobdispatch --torque --env=THEANO_FLAGS=device=gpu,floatX=float32,force_device=True' + \
' --duree=48:00:00 --whitespace ' + \
'--gpu bash /RQexec/goodfell/galatea/mlp/experiment_6/worker.sh '
command += ' "{{'
command += ', '.join(dirs)
command += '}}" '
print 'Running '+command
output, rc = run_shell_command(command)
print 'Output was'
print output
print 'Writing command output to all directories...'
for i, d in enumerate(dirs):
out = d + '/cluster_info.txt'
out = open(out, 'w')
out.write("job id: "+str(i)+'\n')
out.write(output)
out.close()
#run with no args
import os
from pylearn2.utils.shell import run_shell_command
expdir = '/RQexec/goodfell/experiment_6'
names = os.listdir(expdir)
dirs = [expdir + '/' + name for name in names]
dirs = [d for d in dirs if not os.path.exists(d + '/cluster_info.txt')]
command = 'jobdispatch --torque --env=THEANO_FLAGS=device=gpu,floatX=float32,force_device=True' + \
' --duree=48:00:00 --whitespace ' + \
'--gpu bash /RQexec/goodfell/galatea/mlp/experiment_6/worker.sh '
command += ' "{{'
command += ', '.join(dirs)
command += '}}" '
print 'Running ' + command
output, rc = run_shell_command(command)
print 'Output was'
print output
print 'Writing command output to all directories...'
for i, d in enumerate(dirs):
out = d + '/cluster_info.txt'
out = open(out, 'w')
out.write("job id: " + str(i) + '\n')
out.write(output)
out.close()
