def get_stats_info(basename, stats): notes = adjust_decays.notes_removing(basename) text = "%s\t%i\t%i\t%i\t%i\t%i\t%i\t%i\t%i\t%i\t%s" % ( tables.anonymize_name(basename), stats.num_harms_original, stats.num_harms_max_no_above_noise, stats.num_harms_num_no_above_noise, stats.num_harms_no_drop, stats.num_harms_no_variance, stats.num_harms_no_fit, stats.num_harms_no_rsquared, stats.num_harms_end, stats.highest_harm, notes, ) return text
def get_info(basename, nf, nb, stats): notes = adjust_decays.notes_decays(basename) text = "%s\t%.3g\t%.3g\t%.2f\t%s\t%.1f\t%.1f\t%.1f\t%.1f\t%.1f\t%.1f" % ( tables.anonymize_name(basename), nf, nb, stats.rsquared, notes, stats.alpha_1, stats.alpha_2, stats.alpha_3, stats.alpha_4, stats.alpha_5, stats.alpha_10, ) return text
def handle_decays(decayss, basename, f0, B, T, L, harmonic_stats, plot=False, recalc=False): #plot_losses(decayss) consider = '-'.join( basename.split('-')[0:3] ) print consider global B3 if "violin-e" in consider: B3 = 2.0 else: B3 = 3.0 #B3 = 2.0 if not defs.MODES_FIXED_EXPONENT: if not "glasgow" in consider: B3 = 2.0 #print "Error: can only safely evaluate the Glasgow recordings" #print "without a fixed exponent" #exit(1) nss = numpy.array([d.n for d in decayss]) #Q = numpy.array([d.Q for d in decayss]) alpha = numpy.array([d.alpha for d in decayss]) weights = numpy.array([ 1. / d.variance for d in decayss]) if defs.MODES_HACK_LIMIT_WEIGHTS_MIN and defs.MODES_HACK_LIMIT_WEIGHTS_MAX: weights = weights.clip(min=defs.MODES_HACK_LIMIT_WEIGHTS_MIN, max=defs.MODES_HACK_LIMIT_WEIGHTS_MAX) alpha_list = split_into_list(nss, alpha) weights_list = split_into_list(nss, weights) nss_use, alpha_use, weights_use, nss_exclude, alpha_exclude, weights_exclude = filter_Q( nss, alpha, weights) blank_triple_cutoff = 40 missing = [] for a in range(5,max(nss_use)): if a not in nss_use: missing.append(a) if len(missing) > 2: for a in range(5,max(nss_use)): if a in missing: if a+1 in missing: if a+2 in missing: blank_triple_cutoff = a break #print nss_use #print blank_triple_cutoff nss_use_act = [] alpha_use_act = [] weights_use_act = [] nss_exclude = list(nss_exclude) alpha_exclude = list(alpha_exclude) weights_exclude = list(weights_exclude) for n, a, w in zip(nss_use, alpha_use, weights_use): #print n, a if n < blank_triple_cutoff: nss_use_act.append(n) alpha_use_act.append(a) weights_use_act.append(w) else: nss_exclude.append(n) alpha_exclude.append(a) weights_exclude.append(w) nss_use = numpy.array(nss_use_act) alpha_use = numpy.array(alpha_use_act) weights_use = numpy.array(weights_use_act) #wns_use = 2*numpy.pi*partials.mode_B2freq(f0, nss_use, B) p = fit_modes(nss_use, alpha_use, weights_use) nss_eval = numpy.arange(1, defs.MODES_FIT_TO+1) pred = mode_func(p, nss_eval) #wns_eval = 2*numpy.pi*partials.mode_B2freq(f0, nss_eval, B) min_dots = harmonic_stats.num_files/2 extra_n = [] extra = [] for n in range(1, int(max(pred))): try: consider = alpha_list[n] con_weight = weights_list[n] except: continue #std = scipy.std(consider) #mean = scipy.mean(consider) #median = scipy.median(consider) #print "%i\t%i\t%.3f\t%.3f\t%.3f\t%.3f" % ( # n, len(consider), std, mean, median, weighted_average) if len(consider) >= min_dots: weighted_average = scipy.average(consider, weights=con_weight) extra_n.append(n) extra.append(weighted_average) if defs.DECAYS_PLOT_USED: pylab.figure() pylab.plot(nss_use, alpha_use, '.') pylab.plot(nss_exclude, alpha_exclude, 'x', color="red") pylab.plot(nss_eval, pred, color="green") pylab.xlabel("mode") pylab.ylabel("Q") pylab.title(basename) pylab.show() if defs.DECAYS_PLOT_Q: pylab.figure() for d in decayss: #print "%i, %.2e, %.2e" %(d.n, 1./d.Q, d.rsquared) wei = 1./d.variance if wei > 1: wei = 1. dot, color, markersize = dot_color(wei) #pylab.plot(d.n, 1./d.Q, pylab.plot(d.n, d.Q, dot, color=color, markersize=markersize, linewidth=0, ) pylab.plot(nss_eval, pred, color="red") pylab.xlabel("mode") pylab.ylabel("Q") pylab.show() #if False: if defs.DECAYS_PLOT_DECAYS: pylab.figure() for d in decayss: #print "%i, %.2e, %.2e" %(d.n, 1./d.Q, d.rsquared) #print d.variance dot, color, markersize = dot_color(d.variance) pylab.plot(d.n, d.alpha, #pylab.plot(d.w, d.alpha, dot, color=color, markersize=markersize, linewidth=0, ) pylab.xlabel("mode") pylab.ylabel("alpha") for i in range(len(extra)): pylab.plot(extra_n[i], extra[i], 'o', color="orange") #pylab.plot(nss_eval, wns_eval / (2*pred), color="red") pylab.plot(nss_eval, pred, color="red") pylab.show() predicted = mode_func(p, nss_use) #print "fit:", p ys = alpha_use #ys = Q_use alpha_use = ys ss_err = ((ys - predicted)**2).sum() ss_tot = ((ys-ys.mean())**2).sum() rsquared = 1. - (ss_err / ss_tot) pred_combo = [] for i in range(1, defs.MODES_FIT_TO+1): try: ind = extra_n.index(i) alpha = extra[ind] except: alpha = pred[i-1] pred_combo.append(alpha) pred_combo = numpy.array(pred_combo) stats = ModePredict_stats(basename, len(nss_use), max(nss_use), rsquared) stats.alpha_1 = pred_combo[0] stats.alpha_2 = pred_combo[1] stats.alpha_3 = pred_combo[2] stats.alpha_4 = pred_combo[3] stats.alpha_5 = pred_combo[4] stats.alpha_10 = pred_combo[9] #stats.alpha_40 = pred_combo[39] basename = tables.anonymize_name(basename) filename = os.path.join('out', basename+".predicted.modes.txt") #ns_deted = numpy.arange(1, highest_safe) ns_deted = numpy.arange(1, defs.MODES_FIT_TO + 1) pred_deted = mode_func(p, ns_deted) data = numpy.vstack((ns_deted, pred_deted)).transpose() numpy.savetxt(filename, data) filename = os.path.join('out', basename+".detected.modes.txt") data = numpy.vstack((nss_use, alpha_use)).transpose() numpy.savetxt(filename, data) filename = os.path.join('out', basename+".model.modes.txt") data = numpy.vstack(pred_combo) numpy.savetxt(filename, data) filename = os.path.join('out', basename+".detected.omit.txt") data = numpy.vstack((nss_exclude, alpha_exclude)).transpose() numpy.savetxt(filename, data) #write_h(basename, pred_combo) #write_pickle(basename, pred_combo) #write_h(basename, pred) write_pickle(basename, pred_combo) #print #print #print stats.rsquared #print #print return p[0], p[1], stats
value += 5 elif at[2] == 'wilf': value += 5 elif at[2] == 'wcams': value += 6 elif at[2] == 'old': value += 7 elif at[2] == 'melissa': value += 4 else: print "Don't recognize owner:", at[2] return value data = sorted(data, key=sort_inst_names) prev_consider = '' out_filename = os.path.join("out", "string-Bs.txt") out = open(out_filename, 'w') out.write(estimate_f0_B.get_header().replace("\t", "&") + "\n") for datum in data: inst, dat = datum consider = '-'.join( get_split_filename(inst)[0:2] ) if consider != prev_consider: out.write("\n") prev_consider = consider detected_freqs, adjusted_freqs, stats, final = dat text = tables.anonymize_name(inst) + "&" + estimate_f0_B.get_info(detected_freqs, adjusted_freqs, stats, final) out.write(text.replace("\t", "&") + "\n") out.close()
def handle_decays(decayss, basename, f0, B, T, L, harmonic_stats, plot=False, recalc=False): #plot_losses(decayss) consider = '-'.join(basename.split('-')[0:3]) print consider global B3 if "violin-e" in consider: B3 = 2.0 else: B3 = 3.0 #B3 = 2.0 if not defs.MODES_FIXED_EXPONENT: if not "glasgow" in consider: B3 = 2.0 #print "Error: can only safely evaluate the Glasgow recordings" #print "without a fixed exponent" #exit(1) nss = numpy.array([d.n for d in decayss]) #Q = numpy.array([d.Q for d in decayss]) alpha = numpy.array([d.alpha for d in decayss]) weights = numpy.array([1. / d.variance for d in decayss]) if defs.MODES_HACK_LIMIT_WEIGHTS_MIN and defs.MODES_HACK_LIMIT_WEIGHTS_MAX: weights = weights.clip(min=defs.MODES_HACK_LIMIT_WEIGHTS_MIN, max=defs.MODES_HACK_LIMIT_WEIGHTS_MAX) alpha_list = split_into_list(nss, alpha) weights_list = split_into_list(nss, weights) nss_use, alpha_use, weights_use, nss_exclude, alpha_exclude, weights_exclude = filter_Q( nss, alpha, weights) blank_triple_cutoff = 40 missing = [] for a in range(5, max(nss_use)): if a not in nss_use: missing.append(a) if len(missing) > 2: for a in range(5, max(nss_use)): if a in missing: if a + 1 in missing: if a + 2 in missing: blank_triple_cutoff = a break #print nss_use #print blank_triple_cutoff nss_use_act = [] alpha_use_act = [] weights_use_act = [] nss_exclude = list(nss_exclude) alpha_exclude = list(alpha_exclude) weights_exclude = list(weights_exclude) for n, a, w in zip(nss_use, alpha_use, weights_use): #print n, a if n < blank_triple_cutoff: nss_use_act.append(n) alpha_use_act.append(a) weights_use_act.append(w) else: nss_exclude.append(n) alpha_exclude.append(a) weights_exclude.append(w) nss_use = numpy.array(nss_use_act) alpha_use = numpy.array(alpha_use_act) weights_use = numpy.array(weights_use_act) #wns_use = 2*numpy.pi*partials.mode_B2freq(f0, nss_use, B) p = fit_modes(nss_use, alpha_use, weights_use) nss_eval = numpy.arange(1, defs.MODES_FIT_TO + 1) pred = mode_func(p, nss_eval) #wns_eval = 2*numpy.pi*partials.mode_B2freq(f0, nss_eval, B) min_dots = harmonic_stats.num_files / 2 extra_n = [] extra = [] for n in range(1, int(max(pred))): try: consider = alpha_list[n] con_weight = weights_list[n] except: continue #std = scipy.std(consider) #mean = scipy.mean(consider) #median = scipy.median(consider) #print "%i\t%i\t%.3f\t%.3f\t%.3f\t%.3f" % ( # n, len(consider), std, mean, median, weighted_average) if len(consider) >= min_dots: weighted_average = scipy.average(consider, weights=con_weight) extra_n.append(n) extra.append(weighted_average) if defs.DECAYS_PLOT_USED: pylab.figure() pylab.plot(nss_use, alpha_use, '.') pylab.plot(nss_exclude, alpha_exclude, 'x', color="red") pylab.plot(nss_eval, pred, color="green") pylab.xlabel("mode") pylab.ylabel("Q") pylab.title(basename) pylab.show() if defs.DECAYS_PLOT_Q: pylab.figure() for d in decayss: #print "%i, %.2e, %.2e" %(d.n, 1./d.Q, d.rsquared) wei = 1. / d.variance if wei > 1: wei = 1. dot, color, markersize = dot_color(wei) #pylab.plot(d.n, 1./d.Q, pylab.plot( d.n, d.Q, dot, color=color, markersize=markersize, linewidth=0, ) pylab.plot(nss_eval, pred, color="red") pylab.xlabel("mode") pylab.ylabel("Q") pylab.show() #if False: if defs.DECAYS_PLOT_DECAYS: pylab.figure() for d in decayss: #print "%i, %.2e, %.2e" %(d.n, 1./d.Q, d.rsquared) #print d.variance dot, color, markersize = dot_color(d.variance) pylab.plot( d.n, d.alpha, #pylab.plot(d.w, d.alpha, dot, color=color, markersize=markersize, linewidth=0, ) pylab.xlabel("mode") pylab.ylabel("alpha") for i in range(len(extra)): pylab.plot(extra_n[i], extra[i], 'o', color="orange") #pylab.plot(nss_eval, wns_eval / (2*pred), color="red") pylab.plot(nss_eval, pred, color="red") pylab.show() predicted = mode_func(p, nss_use) #print "fit:", p ys = alpha_use #ys = Q_use alpha_use = ys ss_err = ((ys - predicted)**2).sum() ss_tot = ((ys - ys.mean())**2).sum() rsquared = 1. - (ss_err / ss_tot) pred_combo = [] for i in range(1, defs.MODES_FIT_TO + 1): try: ind = extra_n.index(i) alpha = extra[ind] except: alpha = pred[i - 1] pred_combo.append(alpha) pred_combo = numpy.array(pred_combo) stats = ModePredict_stats(basename, len(nss_use), max(nss_use), rsquared) stats.alpha_1 = pred_combo[0] stats.alpha_2 = pred_combo[1] stats.alpha_3 = pred_combo[2] stats.alpha_4 = pred_combo[3] stats.alpha_5 = pred_combo[4] stats.alpha_10 = pred_combo[9] #stats.alpha_40 = pred_combo[39] basename = tables.anonymize_name(basename) filename = os.path.join('out', basename + ".predicted.modes.txt") #ns_deted = numpy.arange(1, highest_safe) ns_deted = numpy.arange(1, defs.MODES_FIT_TO + 1) pred_deted = mode_func(p, ns_deted) data = numpy.vstack((ns_deted, pred_deted)).transpose() numpy.savetxt(filename, data) filename = os.path.join('out', basename + ".detected.modes.txt") data = numpy.vstack((nss_use, alpha_use)).transpose() numpy.savetxt(filename, data) filename = os.path.join('out', basename + ".model.modes.txt") data = numpy.vstack(pred_combo) numpy.savetxt(filename, data) filename = os.path.join('out', basename + ".detected.omit.txt") data = numpy.vstack((nss_exclude, alpha_exclude)).transpose() numpy.savetxt(filename, data) #write_h(basename, pred_combo) #write_pickle(basename, pred_combo) #write_h(basename, pred) write_pickle(basename, pred_combo) #print #print #print stats.rsquared #print #print return p[0], p[1], stats