def fit_dec(data, start, end, debug=False): lst = np.load("data/report/3C144_03-28-2014_001926.npz")["lst"][start:end] data = data[(len(data) - len(lst)) / 2 : -(len(data) - len(lst)) / 2] generate_plot(data, dft=True) source = ephem.FixedBody() obs = ephem.Observer() obs.lat = ephem.degrees(37.8732) obs.long = ephem.degrees(-122.2573) obs.date = ephem.date("2014/3/28 0:19:26") source._ra = ephem.hours("5:34:31.95") source._dec = ephem.degrees("22:00:51.1") source._epoch = ephem.J2000 source.compute(obs) hs = np.array(ephem.hours(source.ra) - lst) # 'Brute force' least squares lmbda = 2.5 # Wavelength lambda in cm YBAR = [] By = 1000 cos_dec = np.linspace(0.01, 1, 500) A = [] S_SQ = [] # Iterate over baseline lengths in cm for dec in cos_dec: C = 2 * np.pi * By / lmbda * dec x1 = np.cos(C * np.sin(hs)) x2 = np.sin(C * np.sin(hs)) X = np.matrix([x1, x2]) ybar, a, s_sq = least_squares(data, X) S_SQ.append(s_sq) ybar = np.array(ybar[:, 0]) YBAR.append(ybar) A.append(a) Dopt = np.argmin(S_SQ) if debug: plt.figure() plt.plot(cos_dec, S_SQ / max(S_SQ)) plt.title("Normalized Residuals for Declination Fit") plt.savefig("img/crab/residual_dec.png") plt.figure() plt.plot(data) plt.plot(YBAR[Dopt]) plt.title("Least Squares Fit for Declination") plt.legend(["Filtered", "Fit"]) plt.savefig("img/crab/fit_dec.png") print "By = " + str(By) print "delta (deg)= " + str(np.arccos(cos_dec[Dopt]) * 180 / np.pi) print "A = " + str(A[Dopt]) return YBAR[Dopt], A[Dopt], np.arccos(cos_dec[Dopt]) * 180 / np.pi
def fit_dec(data, start, end, debug=False): lst = np.load('data/report/3C144_03-28-2014_001926.npz')['lst'][start:end] data = data[(len(data)-len(lst))/2:-(len(data)-len(lst))/2] generate_plot(data, dft=True) source = ephem.FixedBody() obs = ephem.Observer() obs.lat = ephem.degrees(37.8732) obs.long = ephem.degrees(-122.2573) obs.date = ephem.date('2014/3/28 0:19:26') source._ra = ephem.hours("5:34:31.95") source._dec = ephem.degrees("22:00:51.1") source._epoch = ephem.J2000 source.compute(obs) hs = np.array(ephem.hours(source.ra) - lst) # 'Brute force' least squares lmbda = 2.5 # Wavelength lambda in cm YBAR = [] By = 1000 cos_dec = np.linspace(0.01,1,500) A = [] S_SQ = [] # Iterate over baseline lengths in cm for dec in cos_dec: C = 2 * np.pi * By/lmbda * dec x1 = np.cos( C * np.sin(hs) ) x2 = np.sin( C * np.sin(hs) ) X = np.matrix([x1, x2]) ybar, a, s_sq = least_squares(data, X) S_SQ.append(s_sq) ybar = np.array(ybar[:,0]) YBAR.append(ybar) A.append(a) Dopt = np.argmin(S_SQ) if debug: plt.figure() plt.plot(cos_dec, S_SQ/max(S_SQ)) plt.title('Normalized Residuals for Declination Fit') plt.savefig('img/crab/residual_dec.png') plt.figure() plt.plot(data) plt.plot(YBAR[Dopt]) plt.title('Least Squares Fit for Declination') plt.legend(['Filtered', 'Fit']) plt.savefig('img/crab/fit_dec.png') print 'By = '+str(By) print 'delta (deg)= '+str(np.arccos(cos_dec[Dopt])*180/np.pi) print 'A = '+str(A[Dopt]) return YBAR[Dopt], A[Dopt], np.arccos(cos_dec[Dopt])*180/np.pi
def main(): base_title = "Sun on 04-02-2014 at 15:31:05\n" data = np.load('data/report/sun_04-02-2014_153105.npz')['volts'] data -= np.average(data) generate_plot(data, dft=True, title=base_title, outfile='img/sun/raw') START = 3800 END = 6500 print radius(5400, 1) #print radius(1888,1) #print radius(3000,1) #print radius(5000,1) #print radius(5000,2) #print radius(5400, 1) data = np.load('data/report/sun_04-02-2014_153105.npz')['volts'][START:END] data -= np.average(data) data /= np.max(data) MFtheory, fR = gen_MFtheory(10000) MFtheory /= max(MFtheory) # Plot Bessel function plt.figure() plt.plot(fR, MFtheory) plt.xlabel('Frequency x Radians') plt.title('Bessel Function') plt.savefig('img/sun/bessel.png') # Plot MFtheory over segment of data plt.figure() plt.plot(data) plt.plot(MFtheory[START:END]) plt.legend(['Measured', 'Bessel']) plt.title('Comparison of Measurements and Theoretical Bessel Function') plt.savefig('img/sun/compare_bessel.png') filtered = filter_sun(data) envelope = envelope_med(data, 301) YBAR, S_SQ, min_index = sun(data, START, END, base_title + "Fit", MFtheory, "data") print "Data" radius(min_index + START, 1, debug=True) YBAR, S_SQ, min_index = sun(filtered, START, END, base_title + "Fit of Filtered", MFtheory, "filtered") radius(min_index + START, 1, debug=True) YBAR, S_SQ, min_index = sun(envelope, START, END, base_title + "Fit of Envelope", MFtheory, "envelope") radius(min_index + START, 1, debug=True)
def main(): base_title = "Sun on 04-02-2014 at 15:31:05\n" data = np.load('data/report/sun_04-02-2014_153105.npz')['volts'] data -= np.average(data) generate_plot(data, dft=True, title=base_title, outfile='img/sun/raw') START = 3800 END = 6500 print radius(5400,1) #print radius(1888,1) #print radius(3000,1) #print radius(5000,1) #print radius(5000,2) #print radius(5400, 1) data = np.load('data/report/sun_04-02-2014_153105.npz')['volts'][START:END] data -= np.average(data) data /= np.max(data) MFtheory, fR = gen_MFtheory(10000) MFtheory /= max(MFtheory) # Plot Bessel function plt.figure() plt.plot(fR, MFtheory) plt.xlabel('Frequency x Radians') plt.title('Bessel Function') plt.savefig('img/sun/bessel.png') # Plot MFtheory over segment of data plt.figure() plt.plot(data) plt.plot(MFtheory[START:END]) plt.legend(['Measured', 'Bessel']) plt.title('Comparison of Measurements and Theoretical Bessel Function') plt.savefig('img/sun/compare_bessel.png') filtered = filter_sun(data) envelope = envelope_med(data, 301) YBAR, S_SQ, min_index = sun(data, START, END, base_title+"Fit", MFtheory, "data") print "Data" radius(min_index+START, 1, debug=True) YBAR, S_SQ, min_index = sun(filtered, START, END, base_title+"Fit of Filtered", MFtheory, "filtered") radius(min_index+START, 1, debug=True) YBAR, S_SQ, min_index = sun(envelope, START, END, base_title+"Fit of Envelope", MFtheory, "envelope") radius(min_index+START, 1, debug=True)
def main(): base_title = "Moon on 04-06-2014 beginning at 03:34:44\n" data = np.load('data/report/moon_04-06-2014_033444.npz')['volts'] data -= np.average(data) generate_plot(data, dft=True, title=base_title, outfile='img/moon/raw') START = 1000 #3800 END = 10000 data = np.load( 'data/report/moon_04-06-2014_033444.npz')['volts'][START:END] data -= np.average(data) data /= np.max(data) filtered = filter_moon(data) generate_plot(filtered, dft=True, outfile='img/moon/filtered') MFtheory, fR = gen_MFtheory(10000, start=0, end=2) MFtheory /= max(MFtheory) # Plot Bessel function plt.figure() plt.plot(fR, MFtheory) plt.xlabel('Frequency \times Radians') plt.title('Bessel Function') plt.savefig('img/moon/bessel.png') # Plot MFtheory over segment of data plt.figure() plt.plot(filtered) plt.plot(MFtheory[START:END]) plt.legend(['Measured', 'Bessel']) plt.title('Comparison of Measurements and Theoretical Bessel Function') plt.savefig('img/moon/compare_bessel.png') envelope = envelope_med(data, 301) YBAR, S_SQ, min_index = moon(filtered, START, END, base_title + "Fit", MFtheory, "data") print "Data" print min_index radius(min_index + START, 1, debug=True)
def main(): base_title = "Moon on 04-06-2014 beginning at 03:34:44\n" data = np.load('data/report/moon_04-06-2014_033444.npz')['volts'] data -= np.average(data) generate_plot(data, dft=True, title=base_title, outfile='img/moon/raw') START = 1000 #3800 END = 10000 data = np.load('data/report/moon_04-06-2014_033444.npz')['volts'][START:END] data -= np.average(data) data /= np.max(data) filtered = filter_moon(data) generate_plot(filtered, dft=True, outfile='img/moon/filtered') MFtheory, fR = gen_MFtheory(10000, start=0, end=2) MFtheory /= max(MFtheory) # Plot Bessel function plt.figure() plt.plot(fR, MFtheory) plt.xlabel('Frequency \times Radians') plt.title('Bessel Function') plt.savefig('img/moon/bessel.png') # Plot MFtheory over segment of data plt.figure() plt.plot(filtered) plt.plot(MFtheory[START:END]) plt.legend(['Measured', 'Bessel']) plt.title('Comparison of Measurements and Theoretical Bessel Function') plt.savefig('img/moon/compare_bessel.png') envelope = envelope_med(data, 301) YBAR, S_SQ, min_index = moon(filtered, START, END, base_title+"Fit", MFtheory, "data") print "Data" print min_index radius(min_index+START, 1, debug=True)
'SRR554369_1', 'SRR327342_1', 'MH0001_081026_clean_1', 'SRR1284073_1', 'SRR870667_1', 'ERR174310_1', 'ERP001775' ] else: tools = [ 'pigz', 'pbzip2', 'samtools', 'picard', 'sambamba', 'cramtools', 'scramble', 'scramble-noref', 'scramble-bzip2', 'deez', 'deez-qual', 'tsc', 'quip', 'quip-ref' ] samples = [ 'MiSeq_Ecoli_DH10B_110721_PF', '9827_2#49', 'sample-2-10_sorted', 'K562_cytosol_LID8465_TopHat_v2', 'dm3PacBio', 'NA12878.pacbio.bwa-sw.20140202', 'HCC1954.mix1.n80t20', 'NA12878_S1' ] plot.generate_plot(data, tools, samples, filetype) exit(0) if mode == 'paired': samples = samples[~samples.str.contains('_') & ~samples.str.contains('ERP') & ~samples.str.contains('SRR1')] # tools = ['raw','pigz', 'pbzip2', 'dsrc', 'dsrc-m2', 'fqzcomp', 'fqzcomp-extra', 'fastqz','slimfastq', 'fqc', 'scalce', 'lw-fqzip', 'quip', 'leon', 'kic', 'mince'] # samples = ['SRR554369', 'SRR327342', 'MH0001_081026_clean', 'SRR870667', 'ERR174310'] columns_top = ['size', 'seq'] columns = ['walltime_ratio', 'd_walltime_ratio'] print 'Tool', '\t', for sample in samples:
def run_test_single_dataset(net, test_dataloader, params, data_config, train_index, start_time, use_wandb=True, wandb_suffix=''): net.eval() ponder_weight = params['loss']['ponder_weight'] loss_fn = get_loss_fn(params['loss']) fs = data_config['fs'] total_losses = [] ponder_losses = [] enhance_losses = [] sdrs = [] per_db_results = {} test_index = 0 audio_objs = [] for (clean, noise, mix, file_db) in tqdm.tqdm(test_dataloader): clean, mix = clean.cuda(), mix.cuda() db = file_db[0][:-4] # Train pred, ponder = net(mix) loss_enhance, loss_ponder = loss_fn(clean, pred, ponder) total_loss = loss_enhance + ponder_weight * loss_ponder if db not in per_db_results: per_db_results[db] = {'enhance': [], 'ponder': []} per_db_results[db]['enhance'].append(loss_enhance.item()) per_db_results[db]['ponder'].append(loss_ponder.item()) total_losses.append(total_loss.item()) ponder_losses.append(loss_ponder.item()) enhance_losses.append(loss_enhance.item()) sdr = test_sisdr(pred, clean) sdrs.append(sdr.item()) if test_index < params['log']['num_audio_save'] and use_wandb: audio_objs += make_wandb_audio(clean, 'clean_{}'.format(test_index), fs) audio_objs += make_wandb_audio(mix, 'mix_{}'.format(test_index), fs, is_multi=True) audio_objs += make_wandb_audio(pred, 'pred_{}'.format(test_index), fs) test_index += 1 if use_wandb: wandb.log({'Test Outputs' + wandb_suffix: audio_objs}) wandb.log({'Total Test Loss' + wandb_suffix: np.array(total_losses).mean()}) wandb.log({'Ponder Test Loss' + wandb_suffix: np.array(ponder_losses).mean()}) wandb.log({'Enhance Test Loss' + wandb_suffix: np.array(enhance_losses).mean()}) wandb.log({'Test SDR' + wandb_suffix: np.array(sdrs).mean()}) fig_ponder, _, ponder_stats = generate_plot( per_db_results, train_index) wandb.log({'per_db_ponder' + wandb_suffix: wandb.Image(fig_ponder)}) fig_enhance, _, enhance_stats = generate_plot(per_db_results, train_index, metric='enhance') wandb.log({'per_db_enhance' + wandb_suffix: wandb.Image(fig_enhance)}) save_dict(per_db_results, 'per_db_data' + wandb_suffix, train_index, start_time, params) save_dict(ponder_stats, 'per_db_ponder_stats' + wandb_suffix, train_index, start_time, params) save_dict(enhance_stats, 'per_db_enhance_stats' + wandb_suffix, train_index, start_time, params) net.train() return np.array(total_losses).mean(), np.array(ponder_losses).mean()
def main(): """ """ START = 2000 END = 8000 chunks = False baseline = True dec = True show = True base_title = "3C144 on 03-28-2014 beginning at 00:19:26\n" D = True S = False data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"][START:END] if chunks: data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"] normalized_chunks = normalize_chunks(data, 1000) filtered_chunks = filter_baseline(normalized_chunks, debug=D) generate_plot(normalized, dft=True) # fit(normalized_chunks) if baseline: START = 0 END = 14400 data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"][START:END] generate_plot(data - np.average(data), dft=True, title=base_title, outfile="img/crab/raw") filtered = filter_baseline(data, debug=D) normalized = normalize(filtered) generate_plot( normalized, dft=True, title=base_title + "Filtered and Normalized", outfile="img/crab/baseline_filtered" ) fit_baseline(normalized, START, END, debug=D) START = 2000 END = 8000 data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"][START:END] data -= np.average(data) generate_plot( data - np.average(data), dft=True, title=base_title + "Segment of Original", outfile="img/crab/seg" ) filtered = filter_dec(data, debug=False) normalized = normalize(filtered) generate_plot( normalized, dft=True, title=base_title + "Filtered and Normalized Segment", outfile="img/crab/baseline_filtered_seg", ) normalized = normalize(filtered) YBAR, A, By, S_SQ = fit_baseline(normalized, START, END, debug=False) plt.figure() plt.plot(normalized) plt.plot(YBAR) plt.title("Least Squares Fit for Baseline of Segment") plt.legend(["Filtered", "Fit"]) plt.savefig("img/crab/fit_baseline_seg.png") plt.figure() plt.plot(By, S_SQ / max(S_SQ)) plt.title("Normalized Residuals for Baseline Fit of Segment") plt.savefig("img/crab/residual_baseline_seg.png") plt.show() if dec: START = 2000 END = 8000 data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"][START:END] filtered = filter_dec(data, debug=D) normalized = normalize(filtered) generate_plot( normalized, dft=True, title=base_title + "Filtered and Normalized", outfile="img/crab/dec_filtered" ) fit_dec(normalized, START, END, debug=D) if S: plt.show()
def main(): """ """ START = 2000 END = 8000 chunks = False baseline = True dec = True show = True base_title = "3C144 on 03-28-2014 beginning at 00:19:26\n" D = True S = False data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts'][START:END] if chunks: data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts'] normalized_chunks = normalize_chunks(data, 1000) filtered_chunks = filter_baseline(normalized_chunks, debug=D) generate_plot(normalized, dft=True) #fit(normalized_chunks) if baseline: START = 0 END = 14400 data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts'][START:END] generate_plot(data-np.average(data), dft=True, title=base_title, outfile="img/crab/raw") filtered = filter_baseline(data, debug=D) normalized = normalize(filtered) generate_plot(normalized, dft=True, title=base_title+"Filtered and Normalized", outfile="img/crab/baseline_filtered") fit_baseline(normalized, START, END, debug=D) START = 2000 END = 8000 data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts'][START:END] data -= np.average(data) generate_plot(data-np.average(data), dft=True, title=base_title+"Segment of Original", outfile="img/crab/seg") filtered = filter_dec(data, debug=False) normalized = normalize(filtered) generate_plot(normalized, dft=True, title=base_title+"Filtered and Normalized Segment", outfile="img/crab/baseline_filtered_seg") normalized = normalize(filtered) YBAR, A, By, S_SQ = fit_baseline(normalized, START, END, debug=False) plt.figure() plt.plot(normalized) plt.plot(YBAR) plt.title('Least Squares Fit for Baseline of Segment') plt.legend(['Filtered', 'Fit']) plt.savefig('img/crab/fit_baseline_seg.png') plt.figure() plt.plot(By, S_SQ/max(S_SQ)) plt.title('Normalized Residuals for Baseline Fit of Segment') plt.savefig('img/crab/residual_baseline_seg.png') plt.show() if dec: START = 2000 END = 8000 data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts'][START:END] filtered = filter_dec(data, debug=D) normalized = normalize(filtered) generate_plot(normalized, dft=True, title=base_title+"Filtered and Normalized", outfile="img/crab/dec_filtered") fit_dec(normalized, START, END, debug=D) if S: plt.show()
if __name__ == "__main__": players = loader.loadPlayers() teams = loader.loadTeam() matches = loader.loadMatch() bets = loader.loadBet() pond = loader.loadPond() players_stat = copy.deepcopy(players) for a_player in players_stat: players_stat[a_player] = {"win": 0.0, "loss": 0.0} # reports = dict() for a_match in matches: core.process(players, teams, a_match, bets[a_match["ID"]], pond, players_stat) # report = dict() # pond_report = {"sum": 0} # for player_name in players: # report[player_name] = 0 # core.process(report, teams, a_match, bets[a_match["ID"]], pond_report) # report["ID"] = a_match["ID"] # report["pond"] = pond_report["sum"] # reports["ID"] = report # print "result", players # print "report", report try: plot.generate_plot(players, pond, matches, teams, bets) except: print "Error: cannot draw a plot!" generate_markdown(players, players_stat, pond, matches, teams, bets) print "all done."