def load(self): cmd = 'tar -C %(workarea)s -xvf %(tarfile)s %(redir)s' % { 'workarea' : self.tempdir, 'tarfile' : self.result, 'redir' : '' if _logger.getEffectiveLevel() < ezlog.INFO else '>/dev/null'} _logger.debug('Executing: %s' % cmd) os.system(cmd) pattern = os.path.join(self.tempdir, 'RUN*/CLONE*/GEN*.dat') for datafile in glob.iglob(pattern): run, clone, gen = dax.read_cannonical(datafile) try: results = np.loadtxt(datafile, delimiter=',', unpack=True, dtype=str) except IOError, e: r,c,g = dax.read_cannonical(datafile) _logger.error('Could not load data from (%d,%d,%d): %s' % (r,c,g,e)) continue data = results[-1] frames = np.arange(0, len(data), step=1, dtype=int) self.data.append((run, clone, gen, frames, data))
def _pyworker(self): import dax import codeop, inspect import sys import os import shutil import tempfile import cPickle as pickle RESULTS_NAME = 'results.tar.bz2' EXITCODE = 0 infile = sys.argv[1] outfile = sys.argv[2] paramfiles = sys.argv[3:] tempdir = tempfile.mkdtemp(prefix='max-wa') with open(outfile, 'w') as fd_log: fd_log.write('Using python version %s\n' % ' '.join(sys.version.split('\n'))) fd_log.write('Loading function and arguments\n') with open(infile) as fd: func = pickle.load(fd) fd_log.write('Loaded function %s\n' % func) for path in paramfiles: run, clone, gen = dax.read_cannonical(path) fd_log.write('Loading Location from %s\n' % path) with dax.Location.from_file(path) as name: fd_log.write('Processing: %s as %s\n' % (path, name)) fd_log.write('\tRUN %d CLONE %d GEN %d\n' % (run,clone,gen)) workarea = dax.cannonical_traj(run, clone) workarea = os.path.join(tempdir, workarea) target_name = 'GEN%04d.dat' % gen target = os.path.join(workarea, target_name) if not os.path.exists(workarea): fd_log.write('\tCreating workarea %s\n' % workarea) os.makedirs(workarea) with open(target, 'w') as fd: fd_log.write('\t Applying %s\n' % func) try: results = func(name) except Exception, e: fd_log.write('\tFailed to work on %s as %s: %s\n' % (path, name, e)) EXITCODE = 1 continue fd_log.write('\tWriting results\n') for frame, result in enumerate(results): line = '%(run)d,%(clone)d,%(gen)d,%(frame)d,%(result)s' % { 'run' : run, 'clone' : clone, 'gen' : gen, 'frame' : frame, 'result' : result} fd.write(line + '\n') cwd = os.getcwd() os.chdir(tempdir) fd_log.write('In %s, contents: %s\n' % (tempdir, os.listdir('.'))) compress = 'tar cvf %s RUN*' % RESULTS_NAME fd_log.write('Compressing results: %s\n' % compress) fd_log.write('Results file: %s\n' % os.path.join(tempdir, RESULTS_NAME)) compress_success = os.system(compress) if compress_success == 0: fd_log.write('\tOK\n') else: fd_log.write('\tFail: %s\n' % compress_success) os.chdir(cwd) fd_log.write('Copying results file %s to WQ Worker workarea %s\n' % (RESULTS_NAME, cwd)) shutil.copyfile(os.path.join(tempdir, RESULTS_NAME), os.path.join(cwd, RESULTS_NAME)) fd_log.write('Finished. Exiting with %s\n' % EXITCODE)