def filter_matfile(fname, outstem): stack = read_mat(fname) for pos in range(stack.shape[0]): print("File: %s, pos %d" % (fname, pos)) sys.stdout.flush() first = stack[pos,0,...] aver = mean_stack(stack[pos,...]) filt, inds = chifilter_points(stack[pos,...]) outname = "%s.p%02d.fil.ydat" % (outstem, pos) write_filtered(filt, first, aver, inds, outname, \ os.path.basename(fname), pos) print(outname)
def filter_matfile(fname, outstem, p_reject=0.001, plot=1): stack = read_mat(fname) md5 = md5_file(fname) print("Rejection probability: %0.3g" % p_reject) N = np.sum(np.logical_not(np.isnan(stack[0,0,1,:]))) print("Number of valid channels: %d" % N) threshold = chi2.ppf(1.0 - p_reject, N) / N print("Chisq rejection threshold: %0.3g" % threshold) for pos in range(stack.shape[0]): reps = stack[pos,...] incinds, cdm = filter_outliers(reps, threshold=threshold, plot=plot) ms = mean_stack(reps[incinds,...]) disinds = range(reps.shape[0]) for i in incinds: disinds.remove(i) print("Pos %d, discarded: %s" % (pos, str(disinds))) ad = { 'chi2cutoff' : float(threshold), 'rejection_prob' : float(p_reject), 'incinds' : map(int, list(incinds)), 'disinds' : map(int, list(disinds)), 'chi2matrix' : map(float, list(cdm)), 'method' : "filter_outliers", 'inputfile' : [ fname, md5 ], 'inputposition' : int(pos), 'q~unit' : '1/nm', 'I~unit' : 'arb.', 'Ierr~unit' : 'arb.', 'I_first~unit' : 'arb.', 'Ierr_first~unit' : 'arb.', 'I_all~unit' : 'arb.', 'Ierr_all~unit' : 'arb.', } outarr = np.zeros((7, ms.shape[1])) outarr[0:3,:] = ms outarr[3:5,:] = reps[0,1:3,:] outarr[5:7,:] = mean_stack(reps)[1:3,:] outname = "%s.p%02d.out.ydat" % (outstem, pos) print(outname) write_ydat(outarr, outname, addict=ad, cols=['q','I','Ierr','I_first','Ierr_first','I_all','Ierr_all'], attributes=['~unit'])