def big_main(args): # Setup timer = timedict() scale = args.scale NP = args.procs par_name = 'Tree'+str(NP) flat_name = 'Flat'+str(NP) timer.tic('spawning pool') pool = multiprocessing.Pool(processes=NP) timer.toc('spawning pool') #SEQ = random.random(2**scale) SEQ = random.random_integers(0,10,2**scale) print(SEQ) # Testing timer.tic(0) count = serial_main(SEQ) timer.toc(0) print('serial') if args.tree: timer.tic(par_name) tree_count = tree_main(pool, SEQ, NP)[0] timer.toc(par_name) print('tree') timer.tic(flat_name) np_count = flat_main(pool, SEQ, NP) timer.toc(flat_name) print('flat') # Reporting print('sum:%s,%s' % (count, np_count)) eps = .00001 assert count-np_count<eps, "we got the wrong answer %f != %f" %(count, np_count) if args.tree: print('sum:%s,%s' % (count, tree_count)) assert count == tree_count print(repr(timer.ends)) if args.tree: print('Tree speedup: %f' % (timer.ends[0]/timer.ends[par_name])) print('Flat speedup: %f' % (timer.ends[0]/timer.ends[flat_name])) pool.close() pool.join() return (scale, NP, timer[0], timer[flat_name])
def big_main(args): """Does the setup and teardown to test both methods """ # Setup timer = timedict() NP = args.procs scale = args.scale if args.verbose: if args.tree: print('no tree method for inner_product because they are too slow') flat_name = 'Flat'+str(NP) timer.tic('spawning pool') pool = multiprocessing.Pool(processes=NP) timer.toc('spawning pool') SEQ = np.arange(0,2**scale) mask = random.random_integers(0,1,2**scale) if args.verbose: vprint("SEQ", SEQ) vprint("mask", mask) # Testing timer.tic(0) arr = serial_main(SEQ, mask) timer.toc(0) if args.verbose: print('serial done') timer.tic(flat_name) par_arr = flat_main(pool,SEQ, mask, NP) timer.toc(flat_name) #print('flat') # Reporting assert (SEQ*mask).sum() == arr, "we did not get the right answer" #print('Same result: %s' % (par_arr == arr)) #print(repr(timer.ends)) flat_speedup = timer.ends[0]/timer.ends[flat_name] #print('Flat speedup: %f' % # (flat_speedup)) pool.close() pool.join() ans = (scale, NP, timer[0], timer[flat_name]) return ans
def big_main(args): """Does the setup and teardown to test both methods """ # Setup timer = timedict() scale = args.scale NP = args.procs if args.verbose: if args.tree: print('no tree method for scans because they are too slow') flat_name = 'Flat'+str(NP) timer.tic('spawning pool') pool = multiprocessing.Pool(processes=NP) timer.toc('spawning pool') SEQ = np.arange(0,2**scale) mask = random.random_integers(0,1,2**scale) if args.verbose: print(SEQ) # Testing timer.tic(0) arr = serial_main(SEQ, mask) timer.toc(0) print('serial') timer.tic(flat_name) seqs = flat_main(pool,SEQ, mask, NP) par_arr = parutils.serial_concat(seqs) timer.toc(flat_name) print('flat') # Reporting #print('sum:%s,%s,%s' % (count, tree_count, np_count)) print('Same result: %s' % (par_arr == arr)) print(repr(timer.ends)) flat_speedup = timer.ends[0]/timer.ends[flat_name] print('Flat speedup: %f' % (flat_speedup)) pool.close() pool.join() return (scale, NP, timer[0], timer[flat_name])
def big_main(args): timer = timedict() scale = args.scale NP = args.procs if args.verbose: if args.tree: print('no tree method for scans because they are too slow') flat_name = 'Flat'+str(NP) timer.tic('spawning pool') pool = multiprocessing.Pool(processes=NP) timer.toc('spawning pool') SEQ = random.random_integers(0,1,2**scale) if args.verbose: print(SEQ) # Testing timer.tic(0) sans = serial_main(SEQ) timer.toc(0) print('serial') timer.tic(flat_name) pans = flat_main(pool, SEQ, NP) timer.toc(flat_name) pans = unpartition(pans) print('flat') # Reporting #print('sum:%s,%s,%s' % (count, tree_count, np_count)) assert all([sa==pa for sa, pa in zip(sans, pans)]), 'we got the wrong answer' print('Same result: %s' % (sans == pans)) print(repr(timer.ends)) print('Flat speedup: %f' % (timer.ends[0]/timer.ends[flat_name])) pool.close() pool.join() return (scale,NP, timer[0], timer[flat_name])
TIMEINDEX = pd.Index(np.arange(INIT_SAMPLE, END_SAMPLE, STRIDE)) NTARGETS = 8 #number of series for the plots BINCOUNT = 50 #how many bins for histograms TARGETSV = [] '''TARGETSV = [688773, 756680, 984640, 1067645, 3030528, 3035516, ]''' #[3784858, 2357671, 2975930, 359724, 2124973, 3732925,] #vertices that I picked by hand KERNEL_NAME = "betweenness_centrality" KERNEL_NAMES = ['bc', 'communities', 'components', 'triangles'] timer = td.timedict() #run_lognormal_analysis(DATA_DIR, NSAMPLES, KERNEL_NAME) timer.tic('loading data') # figure out where data should be found df, names = get_me_data() print('data loaded') timer.toc('loading data') t = 98 lf = np.log1p(df) if args.summary: timer.tic('summary') print("summarizing vertices") whitf = ka.summarize_vertices(lf, pos_filter=True, whiten=True) whitf = whitf.join(names)
#answers = np.concatenate(answers, axis=0) answers = packed_reduction(pool, answers, num_procs) return answers def serial_main(): """Does the main without any parallel overhead :returns: the sum of the matvec """ #answers = np.sqrt((SEQ * SEQ)) answers = MAT.dot(SEQ) count = np.sum(answers) return count if __name__ == '__main__': timer = timedict() scale = 14 SEQ = np.random.random(2**scale) MAT = np.random.rand(2**scale, 2**scale) #print(MAT) print(SEQ) MAX_PROCS = 4 NP = 4 pool = multiprocessing.Pool(processes=NP) par_name = 'MP'+str(MAX_PROCS) npname = 'NumPy'+str(MAX_PROCS) timer.tic(0) count = serial_main() timer.toc(0) print('serial')
TIMEINDEX = pd.Index(np.arange(INIT_SAMPLE,END_SAMPLE,STRIDE)) NTARGETS = 8 #number of series for the plots BINCOUNT = 50 #how many bins for histograms TARGETSV = [] '''TARGETSV = [688773, 756680, 984640, 1067645, 3030528, 3035516, ]''' #[3784858, 2357671, 2975930, 359724, 2124973, 3732925,] #vertices that I picked by hand KERNEL_NAME = "betweenness_centrality" KERNEL_NAMES = ['bc', 'communities', 'components', 'triangles'] timer = td.timedict() #run_lognormal_analysis(DATA_DIR, NSAMPLES, KERNEL_NAME) timer.tic('loading data') # figure out where data should be found df, names = get_me_data() print('data loaded') timer.toc('loading data') t = 98 lf = np.log1p(df) if args.summary: timer.tic('summary') print("summarizing vertices") whitf = ka.summarize_vertices(lf, pos_filter=True, whiten=True) whitf = whitf.join(names)