def run(parameter_set_name, filenames, grp_pattern, systemsize): print 'submitting ...', parameter_set_name print dicttoinfo.dicttoinfo(getattr(parameterSetsO2, parameter_set_name)) print 'for files', filenames dirs = set() for fn in filenames: with h5py.File(fn, 'r') as f: d = myutils.walkh5(f, grp_pattern) assert len(d), 'you f****d up, pattern "%s" not found in "%s"!' % ( grp_pattern, fn) dirs = set.union(dirs, d) print 'and resolved groups therein: %s' % ','.join(dirs) o2params = getattr(parameterSetsO2, parameter_set_name) o2params['name'] = parameter_set_name if callable(o2params): o2paramsList = o2params(len(filenames)) else: o2paramsList = itertools.repeat(o2params) for (o2params, fn) in zip(o2paramsList, filenames): o2params, num_threads = prepareParametersWithNumThreads( copy.deepcopy(o2params), systemsize) qsub.submit( qsub.func(worker_on_client, fn, grp_pattern, o2params), name='job_o2_' + parameter_set_name + '_' + basename(fn), num_cpus=num_threads, days= 5, # about one day per thread, the idea being that number of threads is proportional to systems size and runtime is mem='%iMB' % (2000 * num_threads), change_cwd=True)
def DoIt(inputFileNames, pattern, options): for this_enlargement_factor in options.enlarge_factor: inFiles = [h5files.open(fn, 'r') for fn in inputFileNames] inGroups = list( itertools.chain.from_iterable( myutils.walkh5(f, pattern, return_h5objects=True) for f in inFiles)) if len(inGroups) <= 0: print 'no matching groups in hdf file(s)' sys.exit(0) for in_file in inputFileNames: bloodflowparams = krebsutils.pyDictFromParamGroup( h5files.open(in_file, 'r')['parameters/calcflow']) #enlarge_vessels(float(this_enlargement_factor),in_file, bloodflowparams) qsub.submit( qsub.func(enlarge_vessels, float(this_enlargement_factor), in_file, bloodflowparams), name='job_modify_enlarge_' + str(this_enlargement_factor) + '_', num_cpus=6, days= 5, # about one day per thread, the idea being that number of threads is proportional to systems size and runtime is mem='%iMB' % (1000), change_cwd=True)
def run(vessel_fn, name, paramSet, mem, days): name, c = krebsjobs.PrepareConfigurationForSubmission(vessel_fn, name, 'fakeTum', paramSet) configstr = dicttoinfo(c) config_file_name = '%s.info' % c['fn_out'] with open(config_file_name, 'w') as f: f.write(configstr) qsub.submit(qsub.func(krebs.tumors.run_faketum, config_file_name), name = 'job_'+name, mem = mem, days = days, num_cpus = c['num_threads'], change_cwd = True)
def rerun(fn_of_previous_run, job_name, mem, days): #name, c = krebsjobs.PrepareConfigurationForSubmission(vessel_fn, name, 'fakeTum', paramSet) #configstr = dicttoinfo(c) #config_file_name = '%s.info' % c['fn_out'] #with open(config_file_name, 'w') as f: # f.write(configstr) qsub.submit(qsub.func(krebs.tumors.rerun_faketum, fn_of_previous_run), name = 'job_'+ job_name, mem = mem, days = days, #num_cpus = c['num_threads'], change_cwd = True)
def run_with_vessels(vessel_fn, name, config_, mem, days): name, config_ = PrepareConfigurationForSubmission(vessel_fn, name, 'tumBulk', config_) config_ = krebs.tumors.set_lattice_size(config_, vessel_fn) sx, sy, sz = config_['lattice_size'] print('cont size: %i, %i, %i' % (sx, sy, sz)) #c = PrepConfig_new_python_dict(c) configstr = dicttoinfo(config_) qsub.submit(qsub.func(krebs.tumors.run_bulktissue_w_vessels, configstr), name='job_' + name, mem=mem, days=days, num_cpus=config_['num_threads'], change_cwd=True)
def run_simple(name, config): if 1: days = 10. if config['ift'] else 0.5 qsub.submit( qsub.func(runs_on_client, name, config), name='job_iff' + name, num_cpus=config['num_threads'], days=days, change_cwd=True, ) else: fn = splitext(config['fn_out'])[0] import json with open(fn + '.json', 'w') as f: f.write(json.dumps(config, indent=4))
def run2(parameter_set, filenames, grp_pattern): print 'submitting ...', parameter_set['name'] num_threads = 1 if 'num_threads' in parameter_set: num_threads = parameter_set['num_threads'] for fn in filenames: qsub.submit(qsub.func(worker_on_client, fn, grp_pattern, parameter_set, num_threads), name = 'job_adaption_'+parameter_set['name']+'_'+basename(fn), num_cpus = num_threads, days = 4., mem = '3500MB', change_cwd = True)
def run(vessel_fn, name, paramSet, mem, days): name, paramSet = PrepareConfigurationForSubmission(vessel_fn, name, 'fakeTumMTS', paramSet) configstr = dicttoinfo(paramSet) config_file_name = '%s.info' % paramSet['fn_out'] with open(config_file_name, 'w') as f: f.write(configstr) #o2params = getattr(parameterSetsO2, "breastv3") qsub.submit(qsub.func(krebs.tumors.run_faketum_mts, config_file_name), name = 'job_'+name, mem = mem, days = days, num_cpus = paramSet['num_threads'], change_cwd = True)
def run_config_samples(configfactory, num_samples, client_worker_function): vdcopy = configfactory(0) vdcopy.roots = [] print vdcopy.generate_info_string() for sample_num in num_samples: vd = configfactory(sample_num) vd.outfilename += '-sample%02i' % sample_num vd.num_threads = num_threads print 'submitting %s, estimated runtime %f h, %i iters' % ( vd.outfilename, runtime_sec(vd) / 60 / 60, vd.num_iter) fn = vd.outfilename + '.info' s = vd.generate_info_string() with open(fn, 'w') as f: f.write(s) qsub.submit(qsub.func(client_worker_function, fn, os.getcwd(), vd.outfilename + '.h5'), name='job_' + basename(vd.outfilename), days=(runtime_sec(vd) / 60 / 60) / 24., mem="2000MB", num_cpus=num_threads)
def main(dry=False): n_jobs = 0 for threshold in np.arange(0.64,0.5,-0.05): for support in np.arange(20,0.4,-0.5): for dname, dpath in MATRIX_PATHS.items(): local_dir = "%s_%.2f_%.2f" % (dname, support, threshold) work_dir = os.path.join(WORK_ROOT, local_dir) cmd = CMD_PTN % { 'npy_fname': dpath, 'support': support, 'threshold': threshold, 'work_dir': work_dir, } jobname = local_dir + "_MBC_gse15745_gpl6104_gpl8490" qsub_txt = qsub.fill_template(jobname=jobname, n_ppn=2, walltime="84:00:00", script=cmd) print qsub_txt if not dry: qsub.submit(qsub_txt) n_jobs += 1 if dry: print "(DRY RUN): no actual jobs submitted" print "Submitted %d jobs." % n_jobs
if not 'lattice_size' in config_.keys(): raise AssertionError('No lattice_size found in configuration %s' % getattr(config_)) if type(config_['lattice_size']) == str: if config_['lattice_size'] == 'set me to match the vessel domain': raise AssertionError('Find better lattice size') except Exception, e: print e.message sys.exit(-1) sx, sy, sz = config_['lattice_size'] print('cont size: %i, %i, %i' % (sx, sy, sz)) #c = PrepConfig_new_python_dict(c) configstr = dicttoinfo(config_) qsub.submit(qsub.func(krebs.tumors.run_bulktissue_no_vessels, configstr), name='job_' + name, mem=mem, days=days, num_cpus=config_['num_threads'], change_cwd=True) if not qsub.is_client and __name__ == '__main__': #parser = argparse.ArgumentParser(parents=[general_group]) #subparsers=parser.add_subparsers(dest='action') #subparsers.add_parser('Restart',parents=[general_group, second_group]) #subparsers.add_parser('Start', parents=[general_group]) import argparse parent_parser = argparse.ArgumentParser( add_help=False, description='Compute BulkTissue tumor. Either with or without vessels') parent_parser.add_argument( '--no_vessel',
boxcount_func = boxcountTumorVessels # for fn in goodArguments.files: # if goodArguments.q_local: # boxcount_func(fn.name, goodArguments) # else: # qsub.submit(qsub.Func(boxcount_func, fn.name, goodArguments), # num_cpus = goodArguments.num_threads, # days = 1, # name = 'job_'+basename(fn.name), # change_cwd = True # ) for fn in goodArguments.files: qsub.submit(qsub.Func(boxcount_func, fn, goodArguments), num_cpus=goodArguments.num_threads, days=1, name='job_' + basename(fn), change_cwd=True) else: # plotting try: for fn in goodArguments.files: with h5py.File(fn, 'r') as f: if not f.attrs.get('tumorcode_file_type' ) == 'fractaldim_analyze_bulktum': raise AssertionError( 'This in not the right filetype %s!' % fn) except Exception, e: print e.message sys.exit(-1)
tumorParameterName) if not o2ParameterName in dir(krebsjobs.parameters.parameterSetsO2): raise AssertionError('Unknown o2 parameter set %s!' % o2ParameterName) for fn in filenames: if not os.path.isfile(fn): raise AssertionError('The file %s is not present!' % fn) except Exception, e: print e.message sys.exit(-1) #tumor_parameter_sets = [('defaultconfig', tum_only_vessels.defaultconfig)] #o2_parameter_set_name = 'breastv2' tumorParameterSet = getattr( krebsjobs.parameters.parameterSetsBulkTissueTumor, tumorParameterName) o2ParameterSet = getattr(krebsjobs.parameters.parameterSetsO2, o2ParameterName) for vessel_fn in filenames: #for name, tumor_parameters in tumor_parameter_sets: job_name, tumorParameterSet = krebsjobs.submitFakeTum.PrepareConfigurationForSubmission( vessel_fn, tumorParameterName, 'fakeTumDetailed', tumorParameterSet) qsub.submit(qsub.func(worker_on_client, vessel_fn, tumorParameterSet, o2ParameterName, num_threads), name=('job_%s_%s' % (tumorParameterName, o2ParameterName)), num_cpus=num_threads, days=4., mem='2500MB', change_cwd=True)
You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. ''' if __name__ == '__main__': import os.path, sys sys.path.append( os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) import os, sys from os.path import basename import qsub def runs_on_client(filename): import krebs.plotIff import myutils import h5py dataman = myutils.DataManager(100, krebs.plotIff.GetDataManDataInstances()) with h5py.File(filename, 'r+') as iff_file: krebs.plotIff.ComputeIfpVsIffCorrelationDataLocal(dataman, iff_file) if not qsub.is_client: filenames = sys.argv[1:] for i, filename in enumerate(filenames): qsub.submit(qsub.func(runs_on_client, filename), name='job_' + basename(filename), num_cpus=1, days=0.1, change_cwd=True)
import qsub def ClientWorker(*args, **kwargs): print("hello, i'm running on the cluster. my args are %s and %s" % (str(args), str(kwargs))) print("my cwd is: %s" % os.getcwd()) if not qsub.is_client and __name__ == '__main__': argv = qsub.parse_args(sys.argv) qsub.submit(qsub.func(ClientWorker, 'Test Argument', a_kw_arg=9001), name='qsub-script-python-test', num_cpus=1, days=0, hours=0.1, mem='100MB', change_cwd=True) test_program_name = 'client-test-script.sh' with open(test_program_name, 'w') as f: f.write('''#! /bin/sh\n''') f.write('''echo "hello, i'm running on the cluster"\n''') f.write('''echo "my cwd is:"\n''') f.write('''pwd\n''') os.chmod(test_program_name, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) qsub.submit(qsub.exe(['ulimit -c unlimited'], [join('.', test_program_name)]), name='qsub-script-executable-submission-test',
parser.add_argument('-v', '--movie', default=False, action='store_true') goodArguments, otherArguments = parser.parse_known_args() qsub.parse_args(otherArguments) dbg_fn = goodArguments.debug_filename.name options = getattr(krebs.povrayRenderSettings, 'dbg_vessels') f = h5py.File(dbg_fn, 'r') if not goodArguments.noPov: jobs = [] a = jobs.append #for fn in filenames: with h5files.open(dbg_fn, 'r') as f: paths = myutils.walkh5(f, '*/vessels') for path in paths: j = RenderJob(f, path, '', options) a(j) for job in jobs: t, m = job.runtime_and_mem print('submit %s, %i mb, %f h' % (job.imageFilename, m, t)) qsub.submit(qsub.func(clientfunc, job, os.getcwd()), name='job_render_' + basename(job.imageFilename), num_cpus=job.params['num_threads'], mem=('%iMB' % m), days=t / 24.) if goodArguments.movie: time.sleep(30) #wait 30 seconds for the queing system to finish create_movie(goodArguments)
filenames = [] for fn in goodArguments.vesselFileNames: filenames.append(fn.name) if goodArguments.analyze: filenames = goodArguments.vesselFileNames grp_pattern = goodArguments.grp_pattern try: systemsize = int(goodArguments.systemsize) except: print 'no valid --systemsize given, using 2 as default' systemsize = 2 qsub.submit( qsub.func(worker_plots_for_paper, filenames, grp_pattern), name='job_o2_analysis', num_cpus= 1, # single threaded only, but scale memory usage with num_threads days=2, mem='%iMB' % (1000 * systemsize), change_cwd=True) if goodArguments.render: #filenames = parseResult.args[:-1] grp_pattern = goodArguments.grp_pattern try: systemsize = int(goodArguments.systemsize) except: print 'no valid --systemsize given, using 2 as default' systemsize = 2 for fn in filenames: qsub.submit(qsub.func(worker_render_images, fn, grp_pattern, systemsize), name='job_o2_render',