def process_resource(resconf, preprocs, plugins, config, opts): with outputter.factory(config, resconf, dry_run=opts["dry_run"]) as m: if resconf['batch_system'] == "XDMoD": dbif = XDMoDAcct(resconf['resource_id'], config) else: dbif = DbAcct(resconf['resource_id'], config) for job in get_jobs(opts, dbif): try: summarize_start = time.time() res = summarizejob(job, config, resconf, plugins, preprocs, opts) if res is None: continue # Extract-only mode s, mdata, success, s_err = res summarize_time = time.time() - summarize_start summary_dict = s.get() except Exception as e: logging.error( "Failure for summarization of job %s %s. Error: %s %s", job.job_id, job.jobdir, str(e), traceback.format_exc()) clean_jobdir(opts, job) if opts["fail_fast"]: raise else: continue process_summary(m, dbif, opts, job, summarize_time, (summary_dict, mdata, success, s_err)) clean_jobdir(opts, job)
def processjobs(config, opts, procid): """ main function that does the work. One run of this function per process """ preprocs = loadpreprocessors() logging.debug("Loaded %s preprocessors", len(preprocs)) plugins = loadplugins() logging.debug("Loaded %s plugins", len(plugins)) for r, resconf in config.resourceconfigs(): if opts['resource'] == None or opts['resource'] == r or opts['resource'] == str(resconf['resource_id']): logging.info("Processing resource %s", r) else: continue resconf = override_defaults(resconf, opts) with outputter.factory(config, resconf) as m: if resconf['batch_system'] == "XDMoD": dbif = XDMoDAcct(resconf['resource_id'], config, opts['threads'], procid) else: dbif = DbAcct(resconf['resource_id'], config, opts['threads'], procid) if opts['mode'] == "single": for job in dbif.getbylocaljobid(opts['local_job_id']): summarizejob(job, config, resconf, plugins, preprocs, m, dbif, opts) elif opts['mode'] == "timerange": for job in dbif.getbytimerange(opts['start'], opts['end']): summarizejob(job, config, resconf, plugins, preprocs, m, dbif, opts) else: for job in dbif.get(None, None): summarizejob(job, config, resconf, plugins, preprocs, m, dbif, opts)
def processjobs(config, opts, procid): """ main function that does the work. One run of this function per process """ preprocs = loadpreprocessors() logging.debug("Loaded %s preprocessors", len(preprocs)) plugins = loadplugins() logging.debug("Loaded %s plugins", len(plugins)) for r, resconf in config.resourceconfigs(): if opts['resource'] == None or opts['resource'] == r or opts['resource'] == str(resconf['resource_id']): logging.info("Processing resource %s", r) else: continue resconf = override_defaults(resconf, opts) with outputter.factory(config, resconf) as m: if resconf['batch_system'] == "XDMoD": dbif = XDMoDAcct(resconf['resource_id'], config, opts['threads'], procid) else: dbif = DbAcct(resconf['resource_id'], config, opts['threads'], procid) if opts['mode'] == "single": for job in dbif.getbylocaljobid(opts['local_job_id']): summarizejob(job, config, resconf, plugins, preprocs, m, dbif, opts) elif opts['mode'] == "timerange": for job in dbif.getbytimerange(opts['start'], opts['end']): summarizejob(job, config, resconf, plugins, preprocs, m, dbif, opts) else: for job in dbif.get(None, None): summarizejob(job, config, resconf, plugins, preprocs, m, dbif, opts)
def process_resource_multiprocessing(resconf, preprocs, plugins, config, opts, pool): with outputter.factory(config, resconf, dry_run=opts['dry_run']) as m: if resconf['batch_system'] == "XDMoD": dbif = XDMoDAcct(resconf['resource_id'], config) else: dbif = DbAcct(resconf['resource_id'], config) jobs = get_jobs(opts, dbif) it = iter_jobs(jobs, config, resconf, plugins, preprocs, opts) pool_iter = pool.imap_unordered(do_summarize, it) while True: try: job, result, summarize_time = pool_iter.next(timeout=600) except StopIteration: break if result is not None: process_summary(m, dbif, opts, job, summarize_time, result) clean_jobdir(opts, job) else: clean_jobdir(opts, job)
def processjobs(config, opts, procid, comm): """ main function that does the work. One run of this function per process """ allpreprocs = loadpreprocessors() logging.debug("Loaded %s preprocessors", len(allpreprocs)) allplugins = loadplugins() logging.debug("Loaded %s plugins", len(allplugins)) for r, resconf in config.resourceconfigs(): if opts['resource'] == None or opts['resource'] == r or opts['resource'] == str(resconf['resource_id']): logging.info("Processing resource %s", r) else: continue resconf = override_defaults(resconf, opts) preprocs, plugins = filter_plugins(resconf, allpreprocs, allplugins) logging.debug("Using %s preprocessors", len(preprocs)) logging.debug("Using %s plugins", len(plugins)) with outputter.factory(config, resconf, dry_run=opts["dry_run"]) as m: if resconf['batch_system'] == "XDMoD": dbif = XDMoDAcct(resconf['resource_id'], config) else: dbif = DbAcct(resconf['resource_id'], config) list_procs = 0 # Master if procid == 0: getjobs = {} if opts['mode'] == "single": getjobs['cmd'] = dbif.getbylocaljobid getjobs['opts'] = [opts['local_job_id'],] elif opts['mode'] == "timerange": getjobs['cmd'] = dbif.getbytimerange getjobs['opts'] = [opts['start'], opts['end'], opts] else: getjobs['cmd'] = dbif.get getjobs['opts'] = [None, None] logging.debug("MASTER STARTING") numworkers = opts['threads']-1 numsent = 0 numreceived = 0 for job in getjobs['cmd'](*(getjobs['opts'])): if numsent >= numworkers: list_procs += 1 if opts['dump_proclist'] and (list_procs == 1 or list_procs == 1000): # Once all ranks are going, dump the process list for debugging logging.info("Dumping process list") allpinfo = {} for proc in psutil.process_iter(): try: pinfo = proc.as_dict() except psutil.NoSuchProcess: pass else: allpinfo[pinfo['pid']] = pinfo with open("rank-{}_{}.proclist".format(procid, list_procs), 'w') as outfile: json.dump(allpinfo, outfile, indent=2) # Wait for a worker to be done and then send more work process = comm.recv(source=MPI.ANY_SOURCE, tag=1) numreceived += 1 comm.send(job, dest=process, tag=1) numsent += 1 logging.debug("Sent new job: %d sent, %d received", numsent, numreceived) else: # Initial batch comm.send(job, dest=numsent+1, tag=1) numsent += 1 logging.debug("Initial Batch: %d sent, %d received", numsent, numreceived) logging.info("After all jobs sent: %d sent, %d received", numsent, numreceived) # Get leftover results while numsent > numreceived: comm.recv(source=MPI.ANY_SOURCE, tag=1) numreceived += 1 logging.debug("Getting leftovers. %d sent, %d received", numsent, numreceived) # Shut them down for worker in xrange(numworkers): logging.debug("Shutting down: %d", worker+1) comm.send(None, dest=worker+1, tag=1) # Worker else: sendtime = time.time() midtime = time.time() recvtime = time.time() logging.debug("WORKER %d STARTING", procid) while True: recvtries = 0 while not comm.Iprobe(source=0, tag=1): if recvtries < 1000: recvtries += 1 continue # Sleep so we can instrument how efficient we are # Otherwise, workers spin on exit at the hidden mpi_finalize call. # If you care about maximum performance and don't care about wasted cycles, remove the Iprobe/sleep loop # Empirically, a tight loop with time.sleep(0.001) uses ~1% CPU time.sleep(0.001) job = comm.recv(source=0, tag=1) recvtime = time.time() mpisendtime = midtime-sendtime mpirecvtime = recvtime-midtime if (mpisendtime+mpirecvtime) > 2: logging.warning("MPI send/recv took %s/%s", mpisendtime, mpirecvtime) if job != None: logging.debug("Rank: %s, Starting: %s", procid, job.job_id) process_job(config, dbif, job, m, opts, plugins, preprocs, resconf) logging.debug("Rank: %s, Finished: %s", procid, job.job_id) sendtime = time.time() comm.send(procid, dest=0, tag=1) midtime = time.time() list_procs += 1 if opts['dump_proclist'] and (list_procs == 1 or list_procs == 10): # Once all ranks are going, dump the process list for debugging logging.info("Dumping process list") allpinfo = {} for proc in psutil.process_iter(): try: pinfo = proc.as_dict() except psutil.NoSuchProcess: pass else: allpinfo[pinfo['pid']] = pinfo with open("rank-{}_{}.proclist".format(procid, list_procs), 'w') as outfile: json.dump(allpinfo, outfile, indent=2) else: # Got shutdown message break