def get_stats_info(basename, stats):
notes = adjust_decays.notes_removing(basename)
text = "%s\t%i\t%i\t%i\t%i\t%i\t%i\t%i\t%i\t%i\t%s" % (
tables.anonymize_name(basename),
stats.num_harms_original,
stats.num_harms_max_no_above_noise,
stats.num_harms_num_no_above_noise,
stats.num_harms_no_drop,
stats.num_harms_no_variance,
stats.num_harms_no_fit,
stats.num_harms_no_rsquared,
stats.num_harms_end,
stats.highest_harm,
notes,
)
return text
def get_info(basename, nf, nb, stats):
notes = adjust_decays.notes_decays(basename)
text = "%s\t%.3g\t%.3g\t%.2f\t%s\t%.1f\t%.1f\t%.1f\t%.1f\t%.1f\t%.1f" % (
tables.anonymize_name(basename),
nf,
nb,
stats.rsquared,
notes,
stats.alpha_1,
stats.alpha_2,
stats.alpha_3,
stats.alpha_4,
stats.alpha_5,
stats.alpha_10,
)
return text
def handle_decays(decayss, basename, f0, B, T, L, harmonic_stats,
plot=False, recalc=False):
#plot_losses(decayss)
consider = '-'.join( basename.split('-')[0:3] )
print consider
global B3
if "violin-e" in consider:
B3 = 2.0
else:
B3 = 3.0
#B3 = 2.0
if not defs.MODES_FIXED_EXPONENT:
if not "glasgow" in consider:
B3 = 2.0
#print "Error: can only safely evaluate the Glasgow recordings"
#print "without a fixed exponent"
#exit(1)
nss = numpy.array([d.n for d in decayss])
#Q = numpy.array([d.Q for d in decayss])
alpha = numpy.array([d.alpha for d in decayss])
weights = numpy.array([ 1. / d.variance for d in decayss])
if defs.MODES_HACK_LIMIT_WEIGHTS_MIN and defs.MODES_HACK_LIMIT_WEIGHTS_MAX:
weights = weights.clip(min=defs.MODES_HACK_LIMIT_WEIGHTS_MIN,
max=defs.MODES_HACK_LIMIT_WEIGHTS_MAX)
alpha_list = split_into_list(nss, alpha)
weights_list = split_into_list(nss, weights)
nss_use, alpha_use, weights_use, nss_exclude, alpha_exclude, weights_exclude = filter_Q(
nss, alpha, weights)
blank_triple_cutoff = 40
missing = []
for a in range(5,max(nss_use)):
if a not in nss_use:
missing.append(a)
if len(missing) > 2:
for a in range(5,max(nss_use)):
if a in missing:
if a+1 in missing:
if a+2 in missing:
blank_triple_cutoff = a
break
#print nss_use
#print blank_triple_cutoff
nss_use_act = []
alpha_use_act = []
weights_use_act = []
nss_exclude = list(nss_exclude)
alpha_exclude = list(alpha_exclude)
weights_exclude = list(weights_exclude)
for n, a, w in zip(nss_use, alpha_use, weights_use):
#print n, a
if n < blank_triple_cutoff:
nss_use_act.append(n)
alpha_use_act.append(a)
weights_use_act.append(w)
else:
nss_exclude.append(n)
alpha_exclude.append(a)
weights_exclude.append(w)
nss_use = numpy.array(nss_use_act)
alpha_use = numpy.array(alpha_use_act)
weights_use = numpy.array(weights_use_act)
#wns_use = 2*numpy.pi*partials.mode_B2freq(f0, nss_use, B)
p = fit_modes(nss_use, alpha_use, weights_use)
nss_eval = numpy.arange(1, defs.MODES_FIT_TO+1)
pred = mode_func(p, nss_eval)
#wns_eval = 2*numpy.pi*partials.mode_B2freq(f0, nss_eval, B)
min_dots = harmonic_stats.num_files/2
extra_n = []
extra = []
for n in range(1, int(max(pred))):
try:
consider = alpha_list[n]
con_weight = weights_list[n]
except:
continue
#std = scipy.std(consider)
#mean = scipy.mean(consider)
#median = scipy.median(consider)
#print "%i\t%i\t%.3f\t%.3f\t%.3f\t%.3f" % (
# n, len(consider), std, mean, median, weighted_average)
if len(consider) >= min_dots:
weighted_average = scipy.average(consider, weights=con_weight)
extra_n.append(n)
extra.append(weighted_average)
if defs.DECAYS_PLOT_USED:
pylab.figure()
pylab.plot(nss_use, alpha_use, '.')
pylab.plot(nss_exclude, alpha_exclude, 'x', color="red")
pylab.plot(nss_eval, pred, color="green")
pylab.xlabel("mode")
pylab.ylabel("Q")
pylab.title(basename)
pylab.show()
if defs.DECAYS_PLOT_Q:
pylab.figure()
for d in decayss:
#print "%i, %.2e, %.2e" %(d.n, 1./d.Q, d.rsquared)
wei = 1./d.variance
if wei > 1:
wei = 1.
dot, color, markersize = dot_color(wei)
#pylab.plot(d.n, 1./d.Q,
pylab.plot(d.n, d.Q,
dot, color=color,
markersize=markersize,
linewidth=0,
)
pylab.plot(nss_eval, pred, color="red")
pylab.xlabel("mode")
pylab.ylabel("Q")
pylab.show()
#if False:
if defs.DECAYS_PLOT_DECAYS:
pylab.figure()
for d in decayss:
#print "%i, %.2e, %.2e" %(d.n, 1./d.Q, d.rsquared)
#print d.variance
dot, color, markersize = dot_color(d.variance)
pylab.plot(d.n, d.alpha,
#pylab.plot(d.w, d.alpha,
dot, color=color,
markersize=markersize,
linewidth=0,
)
pylab.xlabel("mode")
pylab.ylabel("alpha")
for i in range(len(extra)):
pylab.plot(extra_n[i], extra[i],
'o',
color="orange")
#pylab.plot(nss_eval, wns_eval / (2*pred), color="red")
pylab.plot(nss_eval, pred, color="red")
pylab.show()
predicted = mode_func(p, nss_use)
#print "fit:", p
ys = alpha_use
#ys = Q_use
alpha_use = ys
ss_err = ((ys - predicted)**2).sum()
ss_tot = ((ys-ys.mean())**2).sum()
rsquared = 1. - (ss_err / ss_tot)
pred_combo = []
for i in range(1, defs.MODES_FIT_TO+1):
try:
ind = extra_n.index(i)
alpha = extra[ind]
except:
alpha = pred[i-1]
pred_combo.append(alpha)
pred_combo = numpy.array(pred_combo)
stats = ModePredict_stats(basename, len(nss_use), max(nss_use), rsquared)
stats.alpha_1 = pred_combo[0]
stats.alpha_2 = pred_combo[1]
stats.alpha_3 = pred_combo[2]
stats.alpha_4 = pred_combo[3]
stats.alpha_5 = pred_combo[4]
stats.alpha_10 = pred_combo[9]
#stats.alpha_40 = pred_combo[39]
basename = tables.anonymize_name(basename)
filename = os.path.join('out', basename+".predicted.modes.txt")
#ns_deted = numpy.arange(1, highest_safe)
ns_deted = numpy.arange(1, defs.MODES_FIT_TO + 1)
pred_deted = mode_func(p, ns_deted)
data = numpy.vstack((ns_deted, pred_deted)).transpose()
numpy.savetxt(filename, data)
filename = os.path.join('out', basename+".detected.modes.txt")
data = numpy.vstack((nss_use, alpha_use)).transpose()
numpy.savetxt(filename, data)
filename = os.path.join('out', basename+".model.modes.txt")
data = numpy.vstack(pred_combo)
numpy.savetxt(filename, data)
filename = os.path.join('out', basename+".detected.omit.txt")
data = numpy.vstack((nss_exclude, alpha_exclude)).transpose()
numpy.savetxt(filename, data)
#write_h(basename, pred_combo)
#write_pickle(basename, pred_combo)
#write_h(basename, pred)
write_pickle(basename, pred_combo)
#print
#print
#print stats.rsquared
#print
#print
return p[0], p[1], stats
value += 5
elif at[2] == 'wilf':
value += 5
elif at[2] == 'wcams':
value += 6
elif at[2] == 'old':
value += 7
elif at[2] == 'melissa':
value += 4
else:
print "Don't recognize owner:", at[2]
return value
data = sorted(data, key=sort_inst_names)
prev_consider = ''
out_filename = os.path.join("out", "string-Bs.txt")
out = open(out_filename, 'w')
out.write(estimate_f0_B.get_header().replace("\t", "&") + "\n")
for datum in data:
inst, dat = datum
consider = '-'.join( get_split_filename(inst)[0:2] )
if consider != prev_consider:
out.write("\n")
prev_consider = consider
detected_freqs, adjusted_freqs, stats, final = dat
text = tables.anonymize_name(inst) + "&" + estimate_f0_B.get_info(detected_freqs, adjusted_freqs, stats, final)
out.write(text.replace("\t", "&") + "\n")
out.close()
def handle_decays(decayss,
basename,
f0,
B,
T,
L,
harmonic_stats,
plot=False,
recalc=False):
#plot_losses(decayss)
consider = '-'.join(basename.split('-')[0:3])
print consider
global B3
if "violin-e" in consider:
B3 = 2.0
else:
B3 = 3.0
#B3 = 2.0
if not defs.MODES_FIXED_EXPONENT:
if not "glasgow" in consider:
B3 = 2.0
#print "Error: can only safely evaluate the Glasgow recordings"
#print "without a fixed exponent"
#exit(1)
nss = numpy.array([d.n for d in decayss])
#Q = numpy.array([d.Q for d in decayss])
alpha = numpy.array([d.alpha for d in decayss])
weights = numpy.array([1. / d.variance for d in decayss])
if defs.MODES_HACK_LIMIT_WEIGHTS_MIN and defs.MODES_HACK_LIMIT_WEIGHTS_MAX:
weights = weights.clip(min=defs.MODES_HACK_LIMIT_WEIGHTS_MIN,
max=defs.MODES_HACK_LIMIT_WEIGHTS_MAX)
alpha_list = split_into_list(nss, alpha)
weights_list = split_into_list(nss, weights)
nss_use, alpha_use, weights_use, nss_exclude, alpha_exclude, weights_exclude = filter_Q(
nss, alpha, weights)
blank_triple_cutoff = 40
missing = []
for a in range(5, max(nss_use)):
if a not in nss_use:
missing.append(a)
if len(missing) > 2:
for a in range(5, max(nss_use)):
if a in missing:
if a + 1 in missing:
if a + 2 in missing:
blank_triple_cutoff = a
break
#print nss_use
#print blank_triple_cutoff
nss_use_act = []
alpha_use_act = []
weights_use_act = []
nss_exclude = list(nss_exclude)
alpha_exclude = list(alpha_exclude)
weights_exclude = list(weights_exclude)
for n, a, w in zip(nss_use, alpha_use, weights_use):
#print n, a
if n < blank_triple_cutoff:
nss_use_act.append(n)
alpha_use_act.append(a)
weights_use_act.append(w)
else:
nss_exclude.append(n)
alpha_exclude.append(a)
weights_exclude.append(w)
nss_use = numpy.array(nss_use_act)
alpha_use = numpy.array(alpha_use_act)
weights_use = numpy.array(weights_use_act)
#wns_use = 2*numpy.pi*partials.mode_B2freq(f0, nss_use, B)
p = fit_modes(nss_use, alpha_use, weights_use)
nss_eval = numpy.arange(1, defs.MODES_FIT_TO + 1)
pred = mode_func(p, nss_eval)
#wns_eval = 2*numpy.pi*partials.mode_B2freq(f0, nss_eval, B)
min_dots = harmonic_stats.num_files / 2
extra_n = []
extra = []
for n in range(1, int(max(pred))):
try:
consider = alpha_list[n]
con_weight = weights_list[n]
except:
continue
#std = scipy.std(consider)
#mean = scipy.mean(consider)
#median = scipy.median(consider)
#print "%i\t%i\t%.3f\t%.3f\t%.3f\t%.3f" % (
# n, len(consider), std, mean, median, weighted_average)
if len(consider) >= min_dots:
weighted_average = scipy.average(consider, weights=con_weight)
extra_n.append(n)
extra.append(weighted_average)
if defs.DECAYS_PLOT_USED:
pylab.figure()
pylab.plot(nss_use, alpha_use, '.')
pylab.plot(nss_exclude, alpha_exclude, 'x', color="red")
pylab.plot(nss_eval, pred, color="green")
pylab.xlabel("mode")
pylab.ylabel("Q")
pylab.title(basename)
pylab.show()
if defs.DECAYS_PLOT_Q:
pylab.figure()
for d in decayss:
#print "%i, %.2e, %.2e" %(d.n, 1./d.Q, d.rsquared)
wei = 1. / d.variance
if wei > 1:
wei = 1.
dot, color, markersize = dot_color(wei)
#pylab.plot(d.n, 1./d.Q,
pylab.plot(
d.n,
d.Q,
dot,
color=color,
markersize=markersize,
linewidth=0,
)
pylab.plot(nss_eval, pred, color="red")
pylab.xlabel("mode")
pylab.ylabel("Q")
pylab.show()
#if False:
if defs.DECAYS_PLOT_DECAYS:
pylab.figure()
for d in decayss:
#print "%i, %.2e, %.2e" %(d.n, 1./d.Q, d.rsquared)
#print d.variance
dot, color, markersize = dot_color(d.variance)
pylab.plot(
d.n,
d.alpha,
#pylab.plot(d.w, d.alpha,
dot,
color=color,
markersize=markersize,
linewidth=0,
)
pylab.xlabel("mode")
pylab.ylabel("alpha")
for i in range(len(extra)):
pylab.plot(extra_n[i], extra[i], 'o', color="orange")
#pylab.plot(nss_eval, wns_eval / (2*pred), color="red")
pylab.plot(nss_eval, pred, color="red")
pylab.show()
predicted = mode_func(p, nss_use)
#print "fit:", p
ys = alpha_use
#ys = Q_use
alpha_use = ys
ss_err = ((ys - predicted)**2).sum()
ss_tot = ((ys - ys.mean())**2).sum()
rsquared = 1. - (ss_err / ss_tot)
pred_combo = []
for i in range(1, defs.MODES_FIT_TO + 1):
try:
ind = extra_n.index(i)
alpha = extra[ind]
except:
alpha = pred[i - 1]
pred_combo.append(alpha)
pred_combo = numpy.array(pred_combo)
stats = ModePredict_stats(basename, len(nss_use), max(nss_use), rsquared)
stats.alpha_1 = pred_combo[0]
stats.alpha_2 = pred_combo[1]
stats.alpha_3 = pred_combo[2]
stats.alpha_4 = pred_combo[3]
stats.alpha_5 = pred_combo[4]
stats.alpha_10 = pred_combo[9]
#stats.alpha_40 = pred_combo[39]
basename = tables.anonymize_name(basename)
filename = os.path.join('out', basename + ".predicted.modes.txt")
#ns_deted = numpy.arange(1, highest_safe)
ns_deted = numpy.arange(1, defs.MODES_FIT_TO + 1)
pred_deted = mode_func(p, ns_deted)
data = numpy.vstack((ns_deted, pred_deted)).transpose()
numpy.savetxt(filename, data)
filename = os.path.join('out', basename + ".detected.modes.txt")
data = numpy.vstack((nss_use, alpha_use)).transpose()
numpy.savetxt(filename, data)
filename = os.path.join('out', basename + ".model.modes.txt")
data = numpy.vstack(pred_combo)
numpy.savetxt(filename, data)
filename = os.path.join('out', basename + ".detected.omit.txt")
data = numpy.vstack((nss_exclude, alpha_exclude)).transpose()
numpy.savetxt(filename, data)
#write_h(basename, pred_combo)
#write_pickle(basename, pred_combo)
#write_h(basename, pred)
write_pickle(basename, pred_combo)
#print
#print
#print stats.rsquared
#print
#print
return p[0], p[1], stats
