def fit_dec(data, start, end, debug=False):
lst = np.load("data/report/3C144_03-28-2014_001926.npz")["lst"][start:end]
data = data[(len(data) - len(lst)) / 2 : -(len(data) - len(lst)) / 2]
generate_plot(data, dft=True)
source = ephem.FixedBody()
obs = ephem.Observer()
obs.lat = ephem.degrees(37.8732)
obs.long = ephem.degrees(-122.2573)
obs.date = ephem.date("2014/3/28 0:19:26")
source._ra = ephem.hours("5:34:31.95")
source._dec = ephem.degrees("22:00:51.1")
source._epoch = ephem.J2000
source.compute(obs)
hs = np.array(ephem.hours(source.ra) - lst)
# 'Brute force' least squares
lmbda = 2.5 # Wavelength lambda in cm
YBAR = []
By = 1000
cos_dec = np.linspace(0.01, 1, 500)
A = []
S_SQ = []
# Iterate over baseline lengths in cm
for dec in cos_dec:
C = 2 * np.pi * By / lmbda * dec
x1 = np.cos(C * np.sin(hs))
x2 = np.sin(C * np.sin(hs))
X = np.matrix([x1, x2])
ybar, a, s_sq = least_squares(data, X)
S_SQ.append(s_sq)
ybar = np.array(ybar[:, 0])
YBAR.append(ybar)
A.append(a)
Dopt = np.argmin(S_SQ)
if debug:
plt.figure()
plt.plot(cos_dec, S_SQ / max(S_SQ))
plt.title("Normalized Residuals for Declination Fit")
plt.savefig("img/crab/residual_dec.png")
plt.figure()
plt.plot(data)
plt.plot(YBAR[Dopt])
plt.title("Least Squares Fit for Declination")
plt.legend(["Filtered", "Fit"])
plt.savefig("img/crab/fit_dec.png")
print "By = " + str(By)
print "delta (deg)= " + str(np.arccos(cos_dec[Dopt]) * 180 / np.pi)
print "A = " + str(A[Dopt])
return YBAR[Dopt], A[Dopt], np.arccos(cos_dec[Dopt]) * 180 / np.pi
def fit_dec(data, start, end, debug=False):
lst = np.load('data/report/3C144_03-28-2014_001926.npz')['lst'][start:end]
data = data[(len(data)-len(lst))/2:-(len(data)-len(lst))/2]
generate_plot(data, dft=True)
source = ephem.FixedBody()
obs = ephem.Observer()
obs.lat = ephem.degrees(37.8732)
obs.long = ephem.degrees(-122.2573)
obs.date = ephem.date('2014/3/28 0:19:26')
source._ra = ephem.hours("5:34:31.95")
source._dec = ephem.degrees("22:00:51.1")
source._epoch = ephem.J2000
source.compute(obs)
hs = np.array(ephem.hours(source.ra) - lst)
# 'Brute force' least squares
lmbda = 2.5 # Wavelength lambda in cm
YBAR = []
By = 1000
cos_dec = np.linspace(0.01,1,500)
A = []
S_SQ = []
# Iterate over baseline lengths in cm
for dec in cos_dec:
C = 2 * np.pi * By/lmbda * dec
x1 = np.cos( C * np.sin(hs) )
x2 = np.sin( C * np.sin(hs) )
X = np.matrix([x1, x2])
ybar, a, s_sq = least_squares(data, X)
S_SQ.append(s_sq)
ybar = np.array(ybar[:,0])
YBAR.append(ybar)
A.append(a)
Dopt = np.argmin(S_SQ)
if debug:
plt.figure()
plt.plot(cos_dec, S_SQ/max(S_SQ))
plt.title('Normalized Residuals for Declination Fit')
plt.savefig('img/crab/residual_dec.png')
plt.figure()
plt.plot(data)
plt.plot(YBAR[Dopt])
plt.title('Least Squares Fit for Declination')
plt.legend(['Filtered', 'Fit'])
plt.savefig('img/crab/fit_dec.png')
print 'By = '+str(By)
print 'delta (deg)= '+str(np.arccos(cos_dec[Dopt])*180/np.pi)
print 'A = '+str(A[Dopt])
return YBAR[Dopt], A[Dopt], np.arccos(cos_dec[Dopt])*180/np.pi
def main():
base_title = "Sun on 04-02-2014 at 15:31:05\n"
data = np.load('data/report/sun_04-02-2014_153105.npz')['volts']
data -= np.average(data)
generate_plot(data, dft=True, title=base_title, outfile='img/sun/raw')
START = 3800
END = 6500
print radius(5400, 1)
#print radius(1888,1)
#print radius(3000,1)
#print radius(5000,1)
#print radius(5000,2)
#print radius(5400, 1)
data = np.load('data/report/sun_04-02-2014_153105.npz')['volts'][START:END]
data -= np.average(data)
data /= np.max(data)
MFtheory, fR = gen_MFtheory(10000)
MFtheory /= max(MFtheory)
# Plot Bessel function
plt.figure()
plt.plot(fR, MFtheory)
plt.xlabel('Frequency x Radians')
plt.title('Bessel Function')
plt.savefig('img/sun/bessel.png')
# Plot MFtheory over segment of data
plt.figure()
plt.plot(data)
plt.plot(MFtheory[START:END])
plt.legend(['Measured', 'Bessel'])
plt.title('Comparison of Measurements and Theoretical Bessel Function')
plt.savefig('img/sun/compare_bessel.png')
filtered = filter_sun(data)
envelope = envelope_med(data, 301)
YBAR, S_SQ, min_index = sun(data, START, END, base_title + "Fit", MFtheory,
"data")
print "Data"
radius(min_index + START, 1, debug=True)
YBAR, S_SQ, min_index = sun(filtered, START, END,
base_title + "Fit of Filtered", MFtheory,
"filtered")
radius(min_index + START, 1, debug=True)
YBAR, S_SQ, min_index = sun(envelope, START, END,
base_title + "Fit of Envelope", MFtheory,
"envelope")
radius(min_index + START, 1, debug=True)
def main():
base_title = "Sun on 04-02-2014 at 15:31:05\n"
data = np.load('data/report/sun_04-02-2014_153105.npz')['volts']
data -= np.average(data)
generate_plot(data, dft=True, title=base_title, outfile='img/sun/raw')
START = 3800
END = 6500
print radius(5400,1)
#print radius(1888,1)
#print radius(3000,1)
#print radius(5000,1)
#print radius(5000,2)
#print radius(5400, 1)
data = np.load('data/report/sun_04-02-2014_153105.npz')['volts'][START:END]
data -= np.average(data)
data /= np.max(data)
MFtheory, fR = gen_MFtheory(10000)
MFtheory /= max(MFtheory)
# Plot Bessel function
plt.figure()
plt.plot(fR, MFtheory)
plt.xlabel('Frequency x Radians')
plt.title('Bessel Function')
plt.savefig('img/sun/bessel.png')
# Plot MFtheory over segment of data
plt.figure()
plt.plot(data)
plt.plot(MFtheory[START:END])
plt.legend(['Measured', 'Bessel'])
plt.title('Comparison of Measurements and Theoretical Bessel Function')
plt.savefig('img/sun/compare_bessel.png')
filtered = filter_sun(data)
envelope = envelope_med(data, 301)
YBAR, S_SQ, min_index = sun(data, START, END, base_title+"Fit", MFtheory, "data")
print "Data"
radius(min_index+START, 1, debug=True)
YBAR, S_SQ, min_index = sun(filtered, START, END, base_title+"Fit of Filtered", MFtheory, "filtered")
radius(min_index+START, 1, debug=True)
YBAR, S_SQ, min_index = sun(envelope, START, END, base_title+"Fit of Envelope", MFtheory, "envelope")
radius(min_index+START, 1, debug=True)
def main():
base_title = "Moon on 04-06-2014 beginning at 03:34:44\n"
data = np.load('data/report/moon_04-06-2014_033444.npz')['volts']
data -= np.average(data)
generate_plot(data, dft=True, title=base_title, outfile='img/moon/raw')
START = 1000 #3800
END = 10000
data = np.load(
'data/report/moon_04-06-2014_033444.npz')['volts'][START:END]
data -= np.average(data)
data /= np.max(data)
filtered = filter_moon(data)
generate_plot(filtered, dft=True, outfile='img/moon/filtered')
MFtheory, fR = gen_MFtheory(10000, start=0, end=2)
MFtheory /= max(MFtheory)
# Plot Bessel function
plt.figure()
plt.plot(fR, MFtheory)
plt.xlabel('Frequency \times Radians')
plt.title('Bessel Function')
plt.savefig('img/moon/bessel.png')
# Plot MFtheory over segment of data
plt.figure()
plt.plot(filtered)
plt.plot(MFtheory[START:END])
plt.legend(['Measured', 'Bessel'])
plt.title('Comparison of Measurements and Theoretical Bessel Function')
plt.savefig('img/moon/compare_bessel.png')
envelope = envelope_med(data, 301)
YBAR, S_SQ, min_index = moon(filtered, START, END, base_title + "Fit",
MFtheory, "data")
print "Data"
print min_index
radius(min_index + START, 1, debug=True)
def main():
base_title = "Moon on 04-06-2014 beginning at 03:34:44\n"
data = np.load('data/report/moon_04-06-2014_033444.npz')['volts']
data -= np.average(data)
generate_plot(data, dft=True, title=base_title, outfile='img/moon/raw')
START = 1000 #3800
END = 10000
data = np.load('data/report/moon_04-06-2014_033444.npz')['volts'][START:END]
data -= np.average(data)
data /= np.max(data)
filtered = filter_moon(data)
generate_plot(filtered, dft=True, outfile='img/moon/filtered')
MFtheory, fR = gen_MFtheory(10000, start=0, end=2)
MFtheory /= max(MFtheory)
# Plot Bessel function
plt.figure()
plt.plot(fR, MFtheory)
plt.xlabel('Frequency \times Radians')
plt.title('Bessel Function')
plt.savefig('img/moon/bessel.png')
# Plot MFtheory over segment of data
plt.figure()
plt.plot(filtered)
plt.plot(MFtheory[START:END])
plt.legend(['Measured', 'Bessel'])
plt.title('Comparison of Measurements and Theoretical Bessel Function')
plt.savefig('img/moon/compare_bessel.png')
envelope = envelope_med(data, 301)
YBAR, S_SQ, min_index = moon(filtered, START, END, base_title+"Fit", MFtheory, "data")
print "Data"
print min_index
radius(min_index+START, 1, debug=True)
'SRR554369_1', 'SRR327342_1', 'MH0001_081026_clean_1',
'SRR1284073_1', 'SRR870667_1', 'ERR174310_1', 'ERP001775'
]
else:
tools = [
'pigz', 'pbzip2', 'samtools', 'picard', 'sambamba',
'cramtools', 'scramble', 'scramble-noref', 'scramble-bzip2',
'deez', 'deez-qual', 'tsc', 'quip', 'quip-ref'
]
samples = [
'MiSeq_Ecoli_DH10B_110721_PF', '9827_2#49',
'sample-2-10_sorted', 'K562_cytosol_LID8465_TopHat_v2',
'dm3PacBio', 'NA12878.pacbio.bwa-sw.20140202',
'HCC1954.mix1.n80t20', 'NA12878_S1'
]
plot.generate_plot(data, tools, samples, filetype)
exit(0)
if mode == 'paired':
samples = samples[~samples.str.contains('_')
& ~samples.str.contains('ERP')
& ~samples.str.contains('SRR1')]
# tools = ['raw','pigz', 'pbzip2', 'dsrc', 'dsrc-m2', 'fqzcomp', 'fqzcomp-extra', 'fastqz','slimfastq', 'fqc', 'scalce', 'lw-fqzip', 'quip', 'leon', 'kic', 'mince']
# samples = ['SRR554369', 'SRR327342', 'MH0001_081026_clean', 'SRR870667', 'ERR174310']
columns_top = ['size', 'seq']
columns = ['walltime_ratio', 'd_walltime_ratio']
print 'Tool', '\t',
for sample in samples:
def run_test_single_dataset(net, test_dataloader, params, data_config, train_index, start_time,
use_wandb=True, wandb_suffix=''):
net.eval()
ponder_weight = params['loss']['ponder_weight']
loss_fn = get_loss_fn(params['loss'])
fs = data_config['fs']
total_losses = []
ponder_losses = []
enhance_losses = []
sdrs = []
per_db_results = {}
test_index = 0
audio_objs = []
for (clean, noise, mix, file_db) in tqdm.tqdm(test_dataloader):
clean, mix = clean.cuda(), mix.cuda()
db = file_db[0][:-4]
# Train
pred, ponder = net(mix)
loss_enhance, loss_ponder = loss_fn(clean, pred, ponder)
total_loss = loss_enhance + ponder_weight * loss_ponder
if db not in per_db_results:
per_db_results[db] = {'enhance': [], 'ponder': []}
per_db_results[db]['enhance'].append(loss_enhance.item())
per_db_results[db]['ponder'].append(loss_ponder.item())
total_losses.append(total_loss.item())
ponder_losses.append(loss_ponder.item())
enhance_losses.append(loss_enhance.item())
sdr = test_sisdr(pred, clean)
sdrs.append(sdr.item())
if test_index < params['log']['num_audio_save'] and use_wandb:
audio_objs += make_wandb_audio(clean,
'clean_{}'.format(test_index), fs)
audio_objs += make_wandb_audio(mix,
'mix_{}'.format(test_index), fs, is_multi=True)
audio_objs += make_wandb_audio(pred,
'pred_{}'.format(test_index), fs)
test_index += 1
if use_wandb:
wandb.log({'Test Outputs' + wandb_suffix: audio_objs})
wandb.log({'Total Test Loss' + wandb_suffix: np.array(total_losses).mean()})
wandb.log({'Ponder Test Loss' + wandb_suffix: np.array(ponder_losses).mean()})
wandb.log({'Enhance Test Loss' + wandb_suffix: np.array(enhance_losses).mean()})
wandb.log({'Test SDR' + wandb_suffix: np.array(sdrs).mean()})
fig_ponder, _, ponder_stats = generate_plot(
per_db_results, train_index)
wandb.log({'per_db_ponder' + wandb_suffix: wandb.Image(fig_ponder)})
fig_enhance, _, enhance_stats = generate_plot(per_db_results, train_index,
metric='enhance')
wandb.log({'per_db_enhance' + wandb_suffix: wandb.Image(fig_enhance)})
save_dict(per_db_results, 'per_db_data' + wandb_suffix, train_index, start_time, params)
save_dict(ponder_stats, 'per_db_ponder_stats' + wandb_suffix, train_index, start_time, params)
save_dict(enhance_stats, 'per_db_enhance_stats' + wandb_suffix, train_index, start_time, params)
net.train()
return np.array(total_losses).mean(), np.array(ponder_losses).mean()
def main():
"""
"""
START = 2000
END = 8000
chunks = False
baseline = True
dec = True
show = True
base_title = "3C144 on 03-28-2014 beginning at 00:19:26\n"
D = True
S = False
data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"][START:END]
if chunks:
data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"]
normalized_chunks = normalize_chunks(data, 1000)
filtered_chunks = filter_baseline(normalized_chunks, debug=D)
generate_plot(normalized, dft=True)
# fit(normalized_chunks)
if baseline:
START = 0
END = 14400
data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"][START:END]
generate_plot(data - np.average(data), dft=True, title=base_title, outfile="img/crab/raw")
filtered = filter_baseline(data, debug=D)
normalized = normalize(filtered)
generate_plot(
normalized, dft=True, title=base_title + "Filtered and Normalized", outfile="img/crab/baseline_filtered"
)
fit_baseline(normalized, START, END, debug=D)
START = 2000
END = 8000
data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"][START:END]
data -= np.average(data)
generate_plot(
data - np.average(data), dft=True, title=base_title + "Segment of Original", outfile="img/crab/seg"
)
filtered = filter_dec(data, debug=False)
normalized = normalize(filtered)
generate_plot(
normalized,
dft=True,
title=base_title + "Filtered and Normalized Segment",
outfile="img/crab/baseline_filtered_seg",
)
normalized = normalize(filtered)
YBAR, A, By, S_SQ = fit_baseline(normalized, START, END, debug=False)
plt.figure()
plt.plot(normalized)
plt.plot(YBAR)
plt.title("Least Squares Fit for Baseline of Segment")
plt.legend(["Filtered", "Fit"])
plt.savefig("img/crab/fit_baseline_seg.png")
plt.figure()
plt.plot(By, S_SQ / max(S_SQ))
plt.title("Normalized Residuals for Baseline Fit of Segment")
plt.savefig("img/crab/residual_baseline_seg.png")
plt.show()
if dec:
START = 2000
END = 8000
data = np.load("data/report/3C144_03-28-2014_001926.npz")["volts"][START:END]
filtered = filter_dec(data, debug=D)
normalized = normalize(filtered)
generate_plot(
normalized, dft=True, title=base_title + "Filtered and Normalized", outfile="img/crab/dec_filtered"
)
fit_dec(normalized, START, END, debug=D)
if S:
plt.show()
def main():
"""
"""
START = 2000
END = 8000
chunks = False
baseline = True
dec = True
show = True
base_title = "3C144 on 03-28-2014 beginning at 00:19:26\n"
D = True
S = False
data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts'][START:END]
if chunks:
data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts']
normalized_chunks = normalize_chunks(data, 1000)
filtered_chunks = filter_baseline(normalized_chunks, debug=D)
generate_plot(normalized, dft=True)
#fit(normalized_chunks)
if baseline:
START = 0
END = 14400
data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts'][START:END]
generate_plot(data-np.average(data), dft=True, title=base_title, outfile="img/crab/raw")
filtered = filter_baseline(data, debug=D)
normalized = normalize(filtered)
generate_plot(normalized, dft=True, title=base_title+"Filtered and Normalized", outfile="img/crab/baseline_filtered")
fit_baseline(normalized, START, END, debug=D)
START = 2000
END = 8000
data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts'][START:END]
data -= np.average(data)
generate_plot(data-np.average(data), dft=True, title=base_title+"Segment of Original", outfile="img/crab/seg")
filtered = filter_dec(data, debug=False)
normalized = normalize(filtered)
generate_plot(normalized, dft=True, title=base_title+"Filtered and Normalized Segment", outfile="img/crab/baseline_filtered_seg")
normalized = normalize(filtered)
YBAR, A, By, S_SQ = fit_baseline(normalized, START, END, debug=False)
plt.figure()
plt.plot(normalized)
plt.plot(YBAR)
plt.title('Least Squares Fit for Baseline of Segment')
plt.legend(['Filtered', 'Fit'])
plt.savefig('img/crab/fit_baseline_seg.png')
plt.figure()
plt.plot(By, S_SQ/max(S_SQ))
plt.title('Normalized Residuals for Baseline Fit of Segment')
plt.savefig('img/crab/residual_baseline_seg.png')
plt.show()
if dec:
START = 2000
END = 8000
data = np.load('data/report/3C144_03-28-2014_001926.npz')['volts'][START:END]
filtered = filter_dec(data, debug=D)
normalized = normalize(filtered)
generate_plot(normalized, dft=True, title=base_title+"Filtered and Normalized", outfile="img/crab/dec_filtered")
fit_dec(normalized, START, END, debug=D)
if S:
plt.show()
if __name__ == "__main__":
players = loader.loadPlayers()
teams = loader.loadTeam()
matches = loader.loadMatch()
bets = loader.loadBet()
pond = loader.loadPond()
players_stat = copy.deepcopy(players)
for a_player in players_stat:
players_stat[a_player] = {"win": 0.0, "loss": 0.0}
# reports = dict()
for a_match in matches:
core.process(players, teams, a_match, bets[a_match["ID"]], pond, players_stat)
# report = dict()
# pond_report = {"sum": 0}
# for player_name in players:
# report[player_name] = 0
# core.process(report, teams, a_match, bets[a_match["ID"]], pond_report)
# report["ID"] = a_match["ID"]
# report["pond"] = pond_report["sum"]
# reports["ID"] = report
# print "result", players
# print "report", report
try:
plot.generate_plot(players, pond, matches, teams, bets)
except:
print "Error: cannot draw a plot!"
generate_markdown(players, players_stat, pond, matches, teams, bets)
print "all done."
