def merge(pres_result, iparams, txt_out_prefix, output_prefix, avg_mode): #pre-merge task if rank == 0: pres_results = sum(pres_result, []) print "Scaling/post-refinement is done on %d cores for %d frames" % ( size, len(pres_results)) master((pres_results, avg_mode), iparams, "pre_merge") premerge_result = [] else: pres_results = None premerge_result = client() premerge_result = comm.gather(premerge_result, root=0) comm.Barrier() #merge task if rank == 0: print "Pre-merge is done on %d cores" % (len(premerge_result)) master((premerge_result, avg_mode), iparams, "merge") merge_result = [] else: merge_result = client() #finalize merge merge_result = comm.gather(merge_result, root=0) comm.Barrier() if rank == 0: print "Merge completed on %d cores" % (len(merge_result)) merge_results = sum(merge_result, []) mdh = merge_data_handler() txt_out_rejection = "" for _mdh, _txt_out_rejection in merge_results: mdh.extend(_mdh) txt_out_rejection += _txt_out_rejection #selet only indices with non-Inf non-Nan stats selections = flex.bool([ False if (math.isnan(r0) or math.isinf(r0) or math.isnan(r1) or math.isinf(r1)) else True for r0, r1 in zip(mdh.r_meas_div, mdh.r_meas_divisor) ]) mdh.reduce_by_selection(selections) its = intensities_scaler() mdh, txt_merge_mean_table = its.write_output( mdh, iparams, os.path.join(iparams.run_no, output_prefix), avg_mode) print txt_merge_mean_table #write log output with open(os.path.join(iparams.run_no, 'log.txt'), 'a') as f: f.write(txt_out_prefix + txt_merge_mean_table) with open(os.path.join(iparams.run_no, 'rejections.txt'), 'a') as f: f.write(txt_out_rejection) else: merge_results = None mdh = None return pres_results, mdh
def run(argv): comm.Barrier() start_time = MPI.Wtime() # broadcast parameters if rank == 0: iparams, txt_out_input = process_input(argv) iparams.flag_volume_correction = False iparams.flag_hush = True print(txt_out_input) frame_files = read_pickles(iparams.data) else: iparams = None frame_files = None comm.Barrier() # assign scaling task if rank == 0: master(frame_files, iparams, "scale") result = [] else: result = client() result = comm.gather(result, root=0) comm.Barrier() # pre-merge task if rank == 0: results = sum(result, []) print("Scaling is done on %d cores for %d frames" % (size, len(results))) master(results, iparams, "pre_merge") result = [] else: result = client() result = comm.gather(result, root=0) comm.Barrier() # merge task if rank == 0: print("Pre-merge is done on %d cores" % (len(result))) master(result, iparams, "merge") result = [] else: result = client() # finalize merge result = comm.gather(result, root=0) comm.Barrier() if rank == 0: print("Merge completed on %d cores" % (len(result))) results = sum(result, []) mdh = merge_data_handler() txt_out_rejection = "" for _mdh, _txt_out_rejection in results: mdh.extend(_mdh) txt_out_rejection += _txt_out_rejection # selet only indices with non-Inf non-Nan stats selections = flex.bool([ False if (math.isnan(r0) or math.isinf(r0) or math.isnan(r1) or math.isinf(r1)) else True for r0, r1 in zip(mdh.r_meas_div, mdh.r_meas_divisor) ]) mdh.reduce_by_selection(selections) its = intensities_scaler() mdh, txt_merge_mean_table = its.write_output(mdh, iparams, "test", "average") print(txt_merge_mean_table) # collect time profile comm.Barrier() end_time = MPI.Wtime() txt_time = "Elapsed Time (s):%10.2f\n" % (end_time - start_time) # write log output if rank == 0: print(txt_time) with open(os.path.join(iparams.run_no, "log.txt"), "w") as f: f.write(txt_out_input + txt_merge_mean_table + txt_time) with open(os.path.join(iparams.run_no, "rejections.txt"), "w") as f: f.write(txt_out_rejection) MPI.Finalize()