def add_zero_markers(o_ts, r_ts, flux_floor): new_ts = r_ts r_times = r_ts.times r_values = r_ts.values num_steps = o_ts.times.size if (o_ts.values[0] <= 0): zero_ind = np.where(o_ts.values > 0)[0][0] - 1 if not np.any(r_times == o_ts.times[zero_ind]): r_times, r_values = insert_point(r_times, r_values, o_ts.times[zero_ind], o_ts.values[zero_ind]) #new_ts = TimeSeries(r_times,r_values,None,None) if o_ts.times[0] != r_times[0]: r_times, r_values = insert_point(r_times, r_values, o_ts.times[0], o_ts.values[0]) #new_ts = TimeSeries(r_times,r_values,None,None) zero_ind = 0 if (o_ts.values[-1] <= 0): zero_ind = np.where(o_ts.values > 0)[0][-1] + 1 if zero_ind < num_steps and not np.any( r_times == o_ts.times[zero_ind]): r_times, r_values = insert_point(r_times, r_values, o_ts.times[zero_ind], o_ts.values[zero_ind]) #new_ts = TimeSeries(r_times,r_values,None,None) if o_ts.times[-1] != r_times[-1]: r_times, r_values = insert_point(r_times, r_values, o_ts.times[-1], o_ts.values[-1]) #new_ts = TimeSeries(r_times,r_values,None,None) new_ts = TimeSeries(r_times, r_values, None, None) return ReductionResult(flux=o_ts, mass=o_ts.integrate(), reduced_flux=new_ts, reduced_mass=new_ts.integrate())
def build_segments(rr, peaks, pneg, inflection_area): inf_pts = get_inflection_points(rr.flux, peaks, pneg, inflection_area) #segments,t_mass = build_segments(inf_pts,rr,peaks,pneg) x = rr.flux.times y = rr.flux.values r_x = rr.reduced_flux.times r_y = rr.reduced_flux.values segments = [] segs_total_mass = 0 for years in inf_pts: s_year = x[years[0]] e_year = x[years[1]] if (e_year - s_year) > 4: if not np.any(r_x == x[years[0]]): r_x, r_y = insert_point(r_x, r_y, x[years[0]], y[years[0]]) if not np.any(r_x == x[years[0] + 1]): r_x, r_y = insert_point(r_x, r_y, x[years[0] + 1], y[years[0] + 1]) if not np.any(r_x == x[years[1]]): r_x, r_y = insert_point(r_x, r_y, x[years[1]], y[years[1]]) if not np.any(r_x == x[years[1] - 1]): r_x, r_y = insert_point(r_x, r_y, x[years[1] - 1], y[years[1] - 1]) r_seg = np.where((r_x >= s_year) & (r_x <= e_year)) seg_x = r_x[r_seg] seg_y = r_y[r_seg] timeseries = TimeSeries(seg_x, seg_y, None, None) segs_total_mass += timeseries.integrate().values[-1] #timeseries = TimeSeries(r_x[r_seg],r_y[r_seg],None,None) #timeseries = TimeSeries(r_x[r_start:r_end],r_y[r_start:r_end],None,None) segments.append(timeseries) return segments, segs_total_mass
def adjust_flux(data, error): total_mass = float(0.0) #sum cumulative mass of all segments for seg in data: if len(seg) > 0: ts = TimeSeries(seg.times, seg.values, None, None) temp_series = ts.integrate() total_mass += temp_series.values[-1] adjusted = {} #total_error_perc = float(0.0) #mass_used = float(0.0) #figure precentage to adjust each point by # flux_diff = (total_mass+error)/total_mass for seg in data: #if segment has atleast 3 points (mid points are adjusted) if seg.times.size > 2: x = seg.times #[1:-1] y = seg.values #[1:-1] #ts = TimeSeries(x,y,None,None) mass = seg.integrate().values[-1] #get Percent mass current segment is of the total mass p_mass = mass / total_mass p_error = error / total_mass flux_diff = p_mass * p_error #get find equivalent percentage of total_error #e_mass = error * p_mass #divide reduced total error by total mass of segment #flux_diff = e_mass / mass # if dif is greater than 10% reduce it to 10% if abs(flux_diff) > .1: flux_diff = abs(flux_diff) / flux_diff * 0.1 # if flux_diff >0: # flux_diff = .1 # else: # flux_diff = -.1 #if abs(flux_diff) < 0.001: # flux_diff = abs(flux_diff)/flux_diff *0.001 adjusted[x[0]] = y[0] max_flux = max(y) #for each value (except first and last values) adjust value by percent (flux_diff) for i in range(1, x.size - 1): new_val = y[i] + (y[i] * flux_diff) if new_val > max_flux: new_val = y[i] + ((max_flux - y[i]) * .1) #should not happen but just in case negative numbers not allowed if new_val < 0: new_val = float(0.0) adjusted[x[i]] = new_val return adjusted
def find_inflection(x, y, s_ind, e_ind): #Calculate mass ts1 = TimeSeries(x[s_ind:e_ind], y[s_ind:e_ind], None, None) mass = ts1.integrate().values[-1] half_mass = mass / 2 #build loop criteria process from peak to valley #if value at s_ind is < than value at e_ind then e_ind is the peak. # therefore process in reverse loop = range(e_ind, s_ind, -1) reverse = True #start_ind = s_ind #last_ind = e_ind #if value at s_ind is > than value at e_ind then s_ind is the peak. #therefor process in sequence if y[s_ind] > y[e_ind]: loop = range(s_ind, e_ind) reverse = False #starting at peak find the total mass between valley and i. stop when <= half_mass for i in loop: #s_ind is the peak start = s_ind end = i #s_ind is the valley if reverse == False: start = i end = e_ind #calculate mass ts2 = TimeSeries(x[start:end], y[start:end], None, None) mass = ts2.integrate().values[-1] #check if mass <= half mass if mass <= half_mass: return i #should never get to here. if reverse: return e_ind else: return s_ind
def reduce_dataset(years, values, flux_floor=0, max_tm_error=0, min_reduction_steps=200): """ takes times and values and then reduces it returns reduced_times and reduced_values if all elements are zero, it returns False flux_floor > flux == 0 max_tm_error > total mass error """ non_zero_ind, min_retained_zero_years = remove_begin_end_zero_flux( years, values, flux_floor, min_reduction_steps) years_mod = years[non_zero_ind] values_mod = values[non_zero_ind] if years_mod.size < 3: years_mod = years values_mod = values values_mod = 0 else: #makes ure you have not removed more than 1% of the mass when removing 0 or flux floor rates o_mass = TimeSeries(years, values, None, None).integrate().values[-1] r_mass = TimeSeries(years_mod, values_mod, None, None).integrate().values[-1] if abs((o_mass - r_mass) / o_mass) * 100 > 1: years_mod = years values_mod = values timeseries = TimeSeries(years_mod, values_mod, None, None) mass = timeseries.integrate() #normalize Values maxval = np.max(values_mod) values_mod = values_mod / maxval o_timeseries = TimeSeries(years, values / maxval, None, None) o_mass = o_timeseries.integrate() timeseries = TimeSeries(years_mod, values_mod, None, None) mass = timeseries.integrate() mx = np.argmax(timeseries.values) points = [0, mx, len(timeseries)] x = timeseries.times ythresh = 100 * np.mean(timeseries.values) out_error = 1 out_error_last = out_error OUT_ERROR_THRESHOLD = 1e-2 UPPER_N = 200 LOWER_N = 50 last_result = None MAX_ITERATIONS = 80 solve_type = SMOOTH simple_peaks = False last_result, ix = reduct_iter(timeseries, flux_floor, ythresh, out_error, out_error_last, OUT_ERROR_THRESHOLD, UPPER_N, LOWER_N, last_result, MAX_ITERATIONS) last_result = retain_min_years(last_result.reduced_flux, o_timeseries, o_mass, min_retained_zero_years) #if there are less points than the min_reduction_steps then use the remaining #points to rebalance the segments with the largest mass errors. play_points = min_reduction_steps - last_result.num_reduced_points bef = last_result.reduced_flux.times.size if play_points > 0: last_result = red_flux.rebalance_extra_points(last_result, play_points) rr = last_result #find peaks for data rebalance and reporting peaks, _ = sig.find_peaks(rr.reduced_flux.values, width=3, rel_height=1) if peaks.size == 0: peaks, _ = sig.find_peaks(rr.reduced_flux.values, width=2, rel_height=1) if peaks.size == 0: peaks, _ = sig.find_peaks(rr.reduced_flux.values, width=1, rel_height=1) pneg, _ = sig.find_peaks(-rr.reduced_flux.values, width=3, rel_height=1) if pneg.size == 0: pneg, _ = sig.find_peaks(-rr.reduced_flux.values, width=2, rel_height=1) if pneg.size == 0: pneg, _ = sig.find_peaks(-rr.reduced_flux.values, width=1, rel_height=1) peaks = rr.reduced_flux.times[peaks] pneg = rr.reduced_flux.times[pneg] peaks = np.isin(o_timeseries.times, peaks) pneg = np.isin(o_timeseries.times, pneg) peaks = np.where(peaks) pneg = np.where(pneg) peaks = peaks[0] pneg = pneg[0] iter = 0 while iter < 100 and ( abs(last_result.total_mass_error * maxval) > max_tm_error or abs(last_result.total_mass_error / last_result.mass.values[-1]) * 100 > .001): rr = red_flux.rebalance_valleys(rr, peaks, pneg) #keep the lowest total_mass_error if abs(rr.total_mass_error) < abs(last_result.total_mass_error): last_result = rr else: break iter += 1 out_times = last_result.reduced_flux.times out_values = last_result.reduced_flux.values #return the reduced data, undo normalize of the values (*maxval) return out_times, out_values * maxval, -(last_result.total_mass_error * maxval), peaks.size, iter