def load(self):
cmd = 'tar -C %(workarea)s -xvf %(tarfile)s %(redir)s' % {
'workarea' : self.tempdir,
'tarfile' : self.result,
'redir' : '' if _logger.getEffectiveLevel() < ezlog.INFO else '>/dev/null'}
_logger.debug('Executing: %s' % cmd)
os.system(cmd)
pattern = os.path.join(self.tempdir, 'RUN*/CLONE*/GEN*.dat')
for datafile in glob.iglob(pattern):
run, clone, gen = dax.read_cannonical(datafile)
try:
results = np.loadtxt(datafile, delimiter=',', unpack=True, dtype=str)
except IOError, e:
r,c,g = dax.read_cannonical(datafile)
_logger.error('Could not load data from (%d,%d,%d): %s' % (r,c,g,e))
continue
data = results[-1]
frames = np.arange(0, len(data), step=1, dtype=int)
self.data.append((run, clone, gen, frames, data))
def _pyworker(self):
import dax
import codeop, inspect
import sys
import os
import shutil
import tempfile
import cPickle as pickle
RESULTS_NAME = 'results.tar.bz2'
EXITCODE = 0
infile = sys.argv[1]
outfile = sys.argv[2]
paramfiles = sys.argv[3:]
tempdir = tempfile.mkdtemp(prefix='max-wa')
with open(outfile, 'w') as fd_log:
fd_log.write('Using python version %s\n' % ' '.join(sys.version.split('\n')))
fd_log.write('Loading function and arguments\n')
with open(infile) as fd:
func = pickle.load(fd)
fd_log.write('Loaded function %s\n' % func)
for path in paramfiles:
run, clone, gen = dax.read_cannonical(path)
fd_log.write('Loading Location from %s\n' % path)
with dax.Location.from_file(path) as name:
fd_log.write('Processing: %s as %s\n' % (path, name))
fd_log.write('\tRUN %d CLONE %d GEN %d\n' % (run,clone,gen))
workarea = dax.cannonical_traj(run, clone)
workarea = os.path.join(tempdir, workarea)
target_name = 'GEN%04d.dat' % gen
target = os.path.join(workarea, target_name)
if not os.path.exists(workarea):
fd_log.write('\tCreating workarea %s\n' % workarea)
os.makedirs(workarea)
with open(target, 'w') as fd:
fd_log.write('\t Applying %s\n' % func)
try:
results = func(name)
except Exception, e:
fd_log.write('\tFailed to work on %s as %s: %s\n' % (path, name, e))
EXITCODE = 1
continue
fd_log.write('\tWriting results\n')
for frame, result in enumerate(results):
line = '%(run)d,%(clone)d,%(gen)d,%(frame)d,%(result)s' % {
'run' : run, 'clone' : clone, 'gen' : gen, 'frame' : frame,
'result' : result}
fd.write(line + '\n')
cwd = os.getcwd()
os.chdir(tempdir)
fd_log.write('In %s, contents: %s\n' % (tempdir, os.listdir('.')))
compress = 'tar cvf %s RUN*' % RESULTS_NAME
fd_log.write('Compressing results: %s\n' % compress)
fd_log.write('Results file: %s\n' % os.path.join(tempdir, RESULTS_NAME))
compress_success = os.system(compress)
if compress_success == 0:
fd_log.write('\tOK\n')
else:
fd_log.write('\tFail: %s\n' % compress_success)
os.chdir(cwd)
fd_log.write('Copying results file %s to WQ Worker workarea %s\n' % (RESULTS_NAME, cwd))
shutil.copyfile(os.path.join(tempdir, RESULTS_NAME),
os.path.join(cwd, RESULTS_NAME))
fd_log.write('Finished. Exiting with %s\n' % EXITCODE)
