def add_zero_markers(o_ts,r_ts,flux_floor):
new_ts = r_ts
r_times = r_ts.times
r_values = r_ts.values
num_steps = o_ts.times.size
if(o_ts.values[0] <= 0):
zero_ind = np.where(o_ts.values > 0)[0][0]-1
if not np.any(r_times == o_ts.times[zero_ind]):
r_times,r_values = insert_point(r_times,r_values,o_ts.times[zero_ind],o_ts.values[zero_ind])
#new_ts = TimeSeries(r_times,r_values,None,None)
if o_ts.times[0] != r_times[0]:
r_times,r_values = insert_point(r_times,r_values,o_ts.times[0],o_ts.values[0])
#new_ts = TimeSeries(r_times,r_values,None,None)
zero_ind = 0
if(o_ts.values[-1] <= 0):
zero_ind = np.where(o_ts.values > 0)[0][-1]+1
if zero_ind < num_steps and not np.any(r_times == o_ts.times[zero_ind]):
r_times,r_values = insert_point(r_times,r_values,o_ts.times[zero_ind],o_ts.values[zero_ind])
#new_ts = TimeSeries(r_times,r_values,None,None)
if o_ts.times[-1] != r_times[-1]:
r_times,r_values = insert_point(r_times,r_values,o_ts.times[-1],o_ts.values[-1])
#new_ts = TimeSeries(r_times,r_values,None,None)
new_ts = TimeSeries(r_times,r_values,None,None)
return ReductionResult(
flux=o_ts,
mass=o_ts.integrate(),
reduced_flux=new_ts,
reduced_mass=new_ts.integrate())
def build_segments(rr, peaks, pneg, inflection_area):
inf_pts = get_inflection_points(rr.flux, peaks, pneg, inflection_area)
#segments,t_mass = build_segments(inf_pts,rr,peaks,pneg)
x = rr.flux.times
y = rr.flux.values
r_x = rr.reduced_flux.times
r_y = rr.reduced_flux.values
segments = []
segs_total_mass = 0
for years in inf_pts:
s_year = x[years[0]]
e_year = x[years[1]]
if (e_year - s_year) > 4:
if not np.any(r_x == x[years[0]]):
r_x, r_y = insert_point(r_x, r_y, x[years[0]], y[years[0]])
if not np.any(r_x == x[years[0] + 1]):
r_x, r_y = insert_point(r_x, r_y, x[years[0] + 1],
y[years[0] + 1])
if not np.any(r_x == x[years[1]]):
r_x, r_y = insert_point(r_x, r_y, x[years[1]], y[years[1]])
if not np.any(r_x == x[years[1] - 1]):
r_x, r_y = insert_point(r_x, r_y, x[years[1] - 1],
y[years[1] - 1])
r_seg = np.where((r_x >= s_year) & (r_x <= e_year))
seg_x = r_x[r_seg]
seg_y = r_y[r_seg]
timeseries = TimeSeries(seg_x, seg_y, None, None)
segs_total_mass += timeseries.integrate().values[-1]
#timeseries = TimeSeries(r_x[r_seg],r_y[r_seg],None,None)
#timeseries = TimeSeries(r_x[r_start:r_end],r_y[r_start:r_end],None,None)
segments.append(timeseries)
return segments, segs_total_mass
def adjust_flux(data, error):
total_mass = float(0.0)
#sum cumulative mass of all segments
for seg in data:
if len(seg) > 0:
ts = TimeSeries(seg.times, seg.values, None, None)
temp_series = ts.integrate()
total_mass += temp_series.values[-1]
adjusted = {}
#total_error_perc = float(0.0)
#mass_used = float(0.0)
#figure precentage to adjust each point by
# flux_diff = (total_mass+error)/total_mass
for seg in data:
#if segment has atleast 3 points (mid points are adjusted)
if seg.times.size > 2:
x = seg.times #[1:-1]
y = seg.values #[1:-1]
#ts = TimeSeries(x,y,None,None)
mass = seg.integrate().values[-1]
#get Percent mass current segment is of the total mass
p_mass = mass / total_mass
p_error = error / total_mass
flux_diff = p_mass * p_error
#get find equivalent percentage of total_error
#e_mass = error * p_mass
#divide reduced total error by total mass of segment
#flux_diff = e_mass / mass
# if dif is greater than 10% reduce it to 10%
if abs(flux_diff) > .1:
flux_diff = abs(flux_diff) / flux_diff * 0.1
# if flux_diff >0:
# flux_diff = .1
# else:
# flux_diff = -.1
#if abs(flux_diff) < 0.001:
# flux_diff = abs(flux_diff)/flux_diff *0.001
adjusted[x[0]] = y[0]
max_flux = max(y)
#for each value (except first and last values) adjust value by percent (flux_diff)
for i in range(1, x.size - 1):
new_val = y[i] + (y[i] * flux_diff)
if new_val > max_flux:
new_val = y[i] + ((max_flux - y[i]) * .1)
#should not happen but just in case negative numbers not allowed
if new_val < 0:
new_val = float(0.0)
adjusted[x[i]] = new_val
return adjusted
def find_inflection(x, y, s_ind, e_ind):
#Calculate mass
ts1 = TimeSeries(x[s_ind:e_ind], y[s_ind:e_ind], None, None)
mass = ts1.integrate().values[-1]
half_mass = mass / 2
#build loop criteria process from peak to valley
#if value at s_ind is < than value at e_ind then e_ind is the peak.
# therefore process in reverse
loop = range(e_ind, s_ind, -1)
reverse = True
#start_ind = s_ind
#last_ind = e_ind
#if value at s_ind is > than value at e_ind then s_ind is the peak.
#therefor process in sequence
if y[s_ind] > y[e_ind]:
loop = range(s_ind, e_ind)
reverse = False
#starting at peak find the total mass between valley and i. stop when <= half_mass
for i in loop:
#s_ind is the peak
start = s_ind
end = i
#s_ind is the valley
if reverse == False:
start = i
end = e_ind
#calculate mass
ts2 = TimeSeries(x[start:end], y[start:end], None, None)
mass = ts2.integrate().values[-1]
#check if mass <= half mass
if mass <= half_mass:
return i
#should never get to here.
if reverse:
return e_ind
else:
return s_ind
def adjust_flux(data, error):
total_mass = float(0.0)
#sum cumulative mass of all segments
for seg in data:
if len(seg) > 0:
ts = TimeSeries(seg.times, seg.values, None, None)
temp_series = ts.integrate()
total_mass += temp_series.values[-1]
adjusted = {}
#figure the amount of flux to adjust each timestep by
# flux_diff = ((error) * mass/total_mass)/(number of timesteps to distributed over)
for seg in data:
#if segment has atleast 3 points (mid points are adjusted)
if seg.times.size > 2:
years = seg.times #[1:-1]
fluxes = seg.values #[1:-1]
mass = seg.integrate().values[-1]
#get percent mass current segment is of the total mass
p_mass = mass / total_mass
#get find equivalent percentage of total_error
p_error = error / total_mass
#multiply p_error * p_mass to distribute correction
#based on mass percentage of segmenmt
flux_diff = p_mass * p_error
if abs(flux_diff) > 0.1:
flux_diff = abs(flux_diff) / flux_diff * 0.1
adjusted[years[0]] = fluxes[0]
max_flux = max(fluxes)
#for each value (except first and last values(they never change)) adjust value by flux_diff
for i in range(1, years.size - 1):
new_val = fluxes[i] + (fluxes[i] * flux_diff)
if new_val > max_flux:
new_val = fluxes[i] + ((max_flux - fluxes[i]) * .1)
#should not happen but just in case negative numbers not allowed
if new_val < 0:
new_val = float(0.0)
adjusted[years[i]] = new_val
return adjusted
def reduce_dataset(years, values,flux_floor=0,max_tm_error=0,min_reduction_steps=200):
""" takes times and values and then reduces it
returns reduced_times and reduced_values
if all elements are zero, it returns False
flux_floor > flux == 0
max_tm_error > total mass error
"""
non_zero_ind, min_retained_zero_years = remove_begin_end_zero_flux(years,values,flux_floor,min_reduction_steps)
years_mod = years[non_zero_ind]
values_mod = values[non_zero_ind]
if years_mod.size <3:
years_mod = years
values_mod = values
values_mod = 0
else:
#makes ure you have not removed more than 1% of the mass when removing 0 or flux floor rates
o_mass = TimeSeries(years,values,None,None).integrate().values[-1]
r_mass = TimeSeries(years_mod, values_mod, None, None).integrate().values[-1]
if abs((o_mass-r_mass)/o_mass)*100 > 1:
years_mod = years
values_mod = values
timeseries = TimeSeries(years_mod, values_mod, None, None)
mass = timeseries.integrate()
#normalize Values
maxval = np.max(values_mod)
values_mod = values_mod/maxval
o_timeseries = TimeSeries(years,values/maxval,None,None)
o_mass = o_timeseries.integrate()
timeseries = TimeSeries(years_mod, values_mod, None, None)
mass = timeseries.integrate()
mx = np.argmax(timeseries.values)
points = [0, mx, len(timeseries)]
x = timeseries.times
ythresh = 100*np.mean(timeseries.values)
out_error = 1
out_error_last = out_error
OUT_ERROR_THRESHOLD = 1e-2
UPPER_N = 200
LOWER_N = 50
last_result = None
MAX_ITERATIONS = 80
solve_type = SMOOTH
simple_peaks = False
last_result,ix = reduct_iter(timeseries,flux_floor,ythresh,out_error,out_error_last,OUT_ERROR_THRESHOLD,UPPER_N,LOWER_N,last_result,MAX_ITERATIONS)
last_result = retain_min_years(last_result.reduced_flux,o_timeseries,o_mass,min_retained_zero_years)
#if there are less points than the min_reduction_steps then use the remaining
#points to rebalance the segments with the largest mass errors.
play_points = min_reduction_steps - last_result.num_reduced_points
bef = last_result.reduced_flux.times.size
if play_points > 0:
last_result = red_flux.rebalance_extra_points(last_result,play_points)
rr = last_result
#find peaks for data rebalance and reporting
peaks, _ = sig.find_peaks(rr.reduced_flux.values,width=3,rel_height=1)
if peaks.size == 0 :
peaks, _ = sig.find_peaks(rr.reduced_flux.values,width=2,rel_height=1)
if peaks.size == 0:
peaks, _ = sig.find_peaks(rr.reduced_flux.values,width=1,rel_height=1)
pneg, _ = sig.find_peaks(-rr.reduced_flux.values,width=3,rel_height=1)
if pneg.size == 0:
pneg, _ = sig.find_peaks(-rr.reduced_flux.values,width=2,rel_height=1)
if pneg.size == 0:
pneg, _ = sig.find_peaks(-rr.reduced_flux.values,width=1,rel_height=1)
peaks = rr.reduced_flux.times[peaks]
pneg = rr.reduced_flux.times[pneg]
peaks = np.isin(o_timeseries.times,peaks)
pneg = np.isin(o_timeseries.times,pneg)
peaks = np.where(peaks)
pneg = np.where(pneg)
peaks = peaks[0]
pneg = pneg[0]
iter = 0
while iter < 100 and (abs(last_result.total_mass_error*maxval) > max_tm_error or abs(last_result.total_mass_error/last_result.mass.values[-1])*100 > .001) :
rr = red_flux.rebalance_valleys(rr,peaks,pneg)
#keep the lowest total_mass_error
if abs(rr.total_mass_error) < abs(last_result.total_mass_error):
last_result = rr
else:
break
iter += 1
out_times = last_result.reduced_flux.times
out_values = last_result.reduced_flux.values
#return the reduced data, undo normalize of the values (*maxval)
return out_times, out_values*maxval,-(last_result.total_mass_error * maxval),peaks.size,iter