def smooth_target(df, window): window = window signal_df = pd.DataFrame(index=df.index) signal_df['signal'] = 0.0 signal_df['Smooth'] = smooth(df['Close'].values, 2 * window + 1, 3) # cubic signal_df['Smooth'] = smooth(signal_df['Smooth'].values, window + 1, 1) # linear df['Smooth'] = signal_df['Smooth'] max_list = extrema(df['Smooth'].values, np.greater)[0].tolist() min_list = extrema(df['Smooth'].values, np.less)[0].tolist() for x in min_list: t = df.index[x] signal_df.loc[t, 'signal'] = 1 for x in max_list: t = df.index[x] signal_df.loc[t, 'signal'] = -1 signal_df['positions'] = signal_df['signal'].diff() df['SMOOTH_TARGET'] = signal_df['positions'] df['SMOOTH_TARGET'] = df['SMOOTH_TARGET'].replace( 0, np.nan).interpolate(method='slinear').ffill().bfill() df['SMOOTH_TARGET'] = (df['SMOOTH_TARGET'] >= 0.5).astype(np.uint8) return df, signal_df
def get_supports_and_resistances(ltp: np.array, n: int) -> (list, list): """ This function takes a numpy array of last traded price and returns a list of support and resistance levels respectively. :param ltp: :param n: is the number of entries to be scanned. :return: A tuple of lists of points. First item is the support list, second is the resistance list. """ from scipy.signal import savgol_filter as smooth # converting n to a nearest even number if n % 2 != 0: n += 1 highs = ltp['high'] lows = ltp['low'] n_ltp = ltp.shape[0] # smoothening the curve highs_s = smooth(highs, (n + 1), 2) lows_s = smooth(lows, (n + 1), 2) # taking a simple derivative highs_d = np.zeros(n_ltp) highs_d[1:] = np.subtract(highs_s[1:], highs_s[:-1]) lows_d = np.zeros(n_ltp) lows_d[1:] = np.subtract(lows_s[1:], lows_s[:-1]) resistance = [] support = [] for i in range(n_ltp - n): lows_sl = lows_d[i:int(i + n)] lows_first = lows_sl[:int(n / 2)] # first half lows_last = lows_sl[int(n / 2):] # second half highs_sl = highs_d[i:int(i + n)] highs_first = highs_sl[:int(n / 2)] # first half highs_last = highs_sl[int(n / 2):] # second half r_1 = np.sum(highs_first > 0) r_2 = np.sum(highs_last < 0) s_1 = np.sum(lows_first < 0) s_2 = np.sum(lows_last > 0) # local maxima detection if (r_1 == (n / 2)) and (r_2 == (n / 2)): resistance.append(i + (int(n / 2) - 1)) # local minima detection if (s_1 == (n / 2)) and (s_2 == (n / 2)): support.append(i + (int(n / 2) - 1)) return support, resistance
def basic_features(df, short, long): # Bundle of baseline features that were initially applied. Short and Long are windows given in days def slope(y): x = np.array(range(len(y))) m, b = np.polyfit(x, y, 1) return m def acc(y): x = np.array(range(len(y))) A, v, x0 = np.polyfit(x, y, 2) g = 0.5 * A return g # Slopes based on short and long windows df['slope_s'] = df['Close'].rolling(short).apply(slope, raw=True) df['slope_l'] = df['Close'].rolling(long).apply(slope, raw=True) # Acceleration value based on short and long windows df['acc_s'] = df['Close'].rolling(short).apply(acc, raw=True) df['acc_l'] = df['Close'].rolling(long).apply(acc, raw=True) # A doubly smoothed curve derived using the short window df['Smooth'] = smooth(df['Close'].values, 2 * short + 1, 3) # cubic df['Smooth'] = smooth(df['Smooth'].values, short + 1, 1) # linear # Short and long moving averages df['MA_s'] = df['Close'].rolling(window=short, min_periods=1, center=False).mean() df['MA_l'] = df['Close'].rolling(window=long, min_periods=1, center=False).mean() # Exponential moving average for short and long windows ema_df = df[['Close']].copy() df['EMA_s'] = ema_df.ewm(span=short, adjust=False).mean() df['EMA_l'] = ema_df.ewm(span=long, adjust=False).mean() # Feature calculating the 'distance' between the actual close price and the moving averages df['dist_c2ma_s'] = df['Close'] - df['MA_s'] df['dist_c2ma_l'] = df['Close'] - df['MA_l'] df['dist_ma2ma'] = df['MA_s'] - df['MA_l'] # Lag feature df['lag_s'] = df['Close'].shift(short) df['lag_l'] = df['Close'].shift(long) # 'Distance' between lag price and actual close price df['dist_lag2ma_s'] = df['Close'].diff(short) df['dist_lag2ma_l'] = df['Close'].diff(long) # Features based on date df['year'] = df['date'].dt.year df['month'] = df['date'].dt.month df['week'] = df['date'].dt.week df['day'] = df['date'].dt.day df['dow'] = df['date'].dt.dayofweek df['date'] = df['date'].apply(date2num) # Percent change between daily values of close price df['PctChange'] = df['Close'].pct_change() # Spread of the High/Low and Open/Close prices df['Spread_HL'] = df['High'] - df['Low'] df['Spread_OC'] = df['Close'] - df['Open'] return df
def mask(pointx, pointy, wave, wave_temp, flux, fluxerror, flux_temp, continuum, chebfitval, linewidth=0.2, exclude_width=20, sigma_mask=3, lower_mask_bound=0): """Mask areas where signal is present""" from scipy.signal import medfilt as smooth difference = smooth(abs(chebfitval - flux), 9) ind_err = np.array( [ i for i, j in enumerate(fluxerror) if difference[i] < sigma_mask * j and flux[i] >= lower_mask_bound ] ) # and 6*fluxerror[i] < flux[i]]) #and np.average(self.difference[i-100:i+100]) <= j] ) b = exclude_width ind_err = np.array([ j for i, j in enumerate(ind_err[:-b]) if j + b == ind_err[i + b] and j - b == ind_err[i - b] ]) wave_temp = wave[ind_err] flux_temp = flux[ind_err] return wave_temp, flux_temp
def getBB(phase, wave, flux, name, startWave=4200, endWave=9000, winConst=6, constant=1, ax=None): newWave = np.arange(5000, 30000, 50) allFlux = [] for p in range(len(phase)): if phase[p] > 3 or phase[p] < -3: allFlux.append(np.zeros(len(newWave))) else: flux2 = flux[p][wave < endWave][ wave[wave < endWave] > startWave] * constant wave2 = wave[wave < endWave][ wave[wave < endWave] > startWave] * u.AA win = int( len(flux2) / winConst) if int(len(flux2) / winConst) % 2 == 1 else int( len(flux2) / winConst) + 1 res = minimize(_bbChi, np.array([6000, 1]), args=(wave2, smooth(flux2, win, 2)), bounds=((1000, 20000), (1, None))) temp, const = res.x const /= constant bbFlux = fluxFromBB(temp, const, newWave) allFlux.append(bbFlux) return (newWave, np.array(allFlux))
def max_min(ltp, n, apply_smooth=True, poly_order=3): assert n % 2 == 1, "n should be an odd number greater than 3" if apply_smooth: ltp_sm = smooth(ltp['Close'], n, poly_order) else: ltp_sm = ltp['Close'] max_points = [] min_points = [] ltp_dif = np.zeros(ltp_sm.shape[0] - 1) ltp_dif = ltp_sm[1:] - ltp_sm[:-1] center = int((n - 1) / 2) for i in range(ltp_dif.shape[0] - n + 2): window = ltp_dif[i:i + n - 1] front = window[:center] back = window[center:] s_first = np.sum(front < 0) s_back = np.sum(back > 0) r_first = np.sum(front > 0) r_back = np.sum(back < 0) if (r_first == center and r_back == center): # max_point = ltp['Close'].iloc[i+center] # max_value_left = ltp['Close'].iloc[i:i+center]-max_point # max_value_right = ltp['Close'].iloc[i+center+1:i+2*center+1]-max_point # if np.min(max_value_left)<max_point*0.01 and np.min(max_value_right)<rpar: max_points.append((ltp['Date_Time'].iloc[i + center], ltp['Close'].iloc[i + center])) if (s_first == center and s_back == center): # min_value_left = ltp['Close'].iloc[i:i+center]-ltp['Close'].iloc[i+center] # min_value_right = ltp['Close'].iloc[i+center+1:i+2*center+1]-ltp['Close'].iloc[i+center] # if np.max(min_value_left)>spar and np.max(min_value_right)>spar: min_points.append((ltp['Date_Time'].iloc[i + center], ltp['Close'].iloc[i + center])) return max_points, min_points
def read_data(csv_file, window): """This function reads price data downloaded from datahub.io adds log-transformed and leveled column as well as smoothed values with window = 2*window + 1""" #load weekly data df = pd.read_csv(csv_file, index_col='Date', parse_dates=True) df.reset_index(inplace=True) #time range: print('Time range: from', df.Date.min(), 'to', df.Date.max()) #df.head() #log transform and level the data #log transform ref = df.Price.iloc[0] df['log_price'] = np.log(df.Price / ref) #remove trend by fitting m, b = ols(df['log_price']) #leveled_log_price df['y'] = df['log_price'] - (b + df.index * m) #add column with smoothed original price data df[f'y_smooth_w{window}'] = smooth(df['y'].values, 2 * window + 1, 3) df.drop(columns=['log_price'], inplace=True) return df, m, b, ref
def supres(ltp, n): from scipy.signal import savgol_filter as smooth if n % 2 != 0: n += 1 n_ltp = ltp.shape[0] ltp_s = smooth(ltp, (n + 1), 3) ltp_d = np.zeros(n_ltp) ltp_d[1:] = np.subtract(ltp_s[1:], ltp_s[:-1]) resistance = [] support = [] for i in xrange(n_ltp - n): arr_sl = ltp_d[i:(i + n)] first = arr_sl[:(n / 2)] last = arr_sl[(n / 2):] r_1 = np.sum(first > 0) r_2 = np.sum(last < 0) s_1 = np.sum(first < 0) s_2 = np.sum(last > 0) if (r_1 == (n / 2)) and (r_2 == (n / 2)): resistance.append(ltp[i + ((n / 2) - 1)]) if (s_1 == (n / 2)) and (s_2 == (n / 2)): support.append(ltp[i + ((n / 2) - 1)]) return support, resistance
def max_min(ltp,n): assert n%2==1, "n should be an odd number greater than 3" ltp_sm = smooth(ltp['Close'],n,3) max_points = [] min_points = [] ltp_dif = np.zeros(ltp_sm.shape[0]-1) ltp_dif = ltp_sm[1:]-ltp_sm[:-1] center = int((n-1)/2) for i in range(ltp_dif.shape[0]-n+2): window = ltp_dif[i:i+n-1] front = window[:center] back = window[center:] s_first = np.sum(front<0) s_back = np.sum(back>0) r_first = np.sum(front>0) r_back = np.sum(back<0) if(r_first == center and r_back == center): max_value_left = ltp['Close'].iloc[i:i+center]-ltp['Close'].iloc[i+center] max_value_right = ltp['Close'].iloc[i+center+1:i+2*center+1]-ltp['Close'].iloc[i+center] if np.min(max_value_left)<-0.3 and np.min(max_value_right)<-0.3: max_points.append((ltp['Date_Time'].iloc[i+center],ltp['Close'].iloc[i+center])) # max_points.append((ltp['Date_Time'].iloc[i+center],ltp['Close'].iloc[i+center])) if(s_first== center and s_back== center): min_value_left = ltp['Close'].iloc[i:i+center]-ltp['Close'].iloc[i+center] min_value_right = ltp['Close'].iloc[i+center+1:i+2*center+1]-ltp['Close'].iloc[i+center] if np.max(min_value_left)>0.27 and np.max(min_value_right)>0.27: min_points.append((ltp['Date_Time'].iloc[i+center],ltp['Close'].iloc[i+center])) # min_points.append((ltp['Date_Time'].iloc[i+center],ltp['Close'].iloc[i+center])) return max_points,min_points
def supres(ltp, n): """ This function takes a numpy array of last traded price and returns a list of support and resistance levels respectively. n is the number of entries to be scanned. """ from scipy.signal import savgol_filter as smooth # converting n to a nearest even number if n % 2 != 0: n += 1 n_ltp = ltp.shape[0] # smoothening the curve ltp_s = smooth(ltp, (n + 1), 3) # taking a simple derivative ltp_d = np.zeros(n_ltp) ltp_d[1:] = np.subtract(ltp_s[1:], ltp_s[:-1]) resistance = [] support = [] for i in range(n_ltp - n): arr_sl = ltp_d[i:(i + n)] first = arr_sl[:(n // 2)] # first half last = arr_sl[(n // 2):] # second half r_1 = np.sum(first > 0) r_2 = np.sum(last < 0) s_1 = np.sum(first < 0) s_2 = np.sum(last > 0) # local maxima detection if (r_1 == (n / 2)) and (r_2 == (n / 2)): resistance.append(ltp[i + ((n // 2) - 1)]) # local minima detection if (s_1 == (n / 2)) and (s_2 == (n / 2)): support.append(ltp[i + ((n // 2) - 1)]) return support, resistance
def diff_find_resonance(frec, diffS11, margin=31, doplots=False): """Returns the resonance frequency from maximum of the derivative of the phase Not currently in use. Stripped down version of fit """ #Smooth data for initial guesses sReS11 = np.array(smooth(diffS11.real, margin, 3)) sImS11 = np.array(smooth(diffS11.imag, margin, 3)) sS11 = np.array([x + 1j * y for x, y in zip(sReS11, sImS11)]) #Make smoothed phase vector removing 2pi jumps sArgS11 = np.angle(sS11) sArgS11 = np.unwrap(sArgS11) sdiffang = np.diff(sArgS11) #Get resonance index from maximum of the derivative of the phase avgph = np.average(sdiffang) errvec = [np.power(x - avgph, 2.) for x in sdiffang] ires = np.argmax(errvec[margin:-margin]) + margin f0i = frec[ires] print("Max index: ", ires, " Max frequency: ", f0i) if doplots: plt.title('Original signal (Re,Im)') plt.plot(frec, diffS11.real) plt.plot(frec, diffS11.imag) plt.plot(frec, np.abs(diffS11)) plt.show() plt.plot(np.angle(sS11)) plt.title('Smoothed Phase') plt.axis('auto') plt.show() plt.plot(sdiffang[margin:-margin]) plt.title('Diff of Smoothed Phase') plt.show() plt.title('Smoothed (ph-phavg)\^2') plt.plot(errvec) plt.plot([ires], [errvec[ires]], 'ro') plt.show() return f0i
def diff_find_resonance(frec,diffS11,margin=31,doplots=False): """Returns the resonance frequency from maximum of the derivative of the phase Not currently in use. Stripped down version of fit """ #Smooth data for initial guesses sReS11 = np.array(smooth(diffS11.real,margin,3)) sImS11 = np.array(smooth(diffS11.imag,margin,3)) sS11 = np.array( [x+1j*y for x,y in zip(sReS11,sImS11) ] ) #Make smoothed phase vector removing 2pi jumps sArgS11 = np.angle(sS11) sArgS11 = np.unwrap(sArgS11) sdiffang = np.diff(sArgS11) #Get resonance index from maximum of the derivative of the phase avgph = np.average(sdiffang) errvec = [np.power(x-avgph,2.) for x in sdiffang] ires = np.argmax(errvec[margin:-margin])+margin f0i=frec[ires] print("Max index: ",ires," Max frequency: ",f0i) if doplots: plt.title('Original signal (Re,Im)') plt.plot(frec,diffS11.real) plt.plot(frec,diffS11.imag) plt.plot(frec,np.abs(diffS11)) plt.show() plt.plot(np.angle(sS11)) plt.title('Smoothed Phase') plt.axis('auto') plt.show() plt.plot(sdiffang[margin:-margin]) plt.title('Diff of Smoothed Phase') plt.show() plt.title('Smoothed (ph-phavg)\^2') plt.plot(errvec) plt.plot([ires],[errvec[ires]],'ro') plt.show() return f0i
def plot(*lines, smooth_days=0, degree=0): ''' plots average curve, possibility to smooth NOT used in figure 2 ''' if smooth_days > 0: if smooth_days > 2: redraw = lines lines = [] for line in redraw: lines.append(smooth(line, smooth_days, degree)) lines = [np.array(line) / max(lines[0]) for line in lines] for line in lines: lab.plot(line) lab.show()
def find_resistance_line(cls, candles: List[Candle]) -> Optional[PriceLine]: """ Returns a resistance line, if one can be found. """ if len(candles) < cls.n * 3: raise ValueError( f'Cannot find resistance line with fewer than {cls.n * 3} candles' ) prices = np.array([(candle.high + candle.open) / 2.0 for candle in candles]) # Smoothen the curve prices_smooth = smooth(prices, (cls.n + 1), 3) # Take the derivative as the difference of consecutive prices prices_dydx = np.zeros(len(candles)) prices_dydx[1:] = np.subtract(prices_smooth[1:], prices_smooth[:-1]) highest_moments = [candles[0].moment, candles[3].moment] highest_prices = [(candles[0].high + candles[0].open) / 2.0, (candles[0].high + candles[0].open) / 2.0] for i in range(len(candles) - cls.n): midpoint: int = int(cls.n / 2) arr_sl = prices_dydx[i:(i + cls.n)] first_half = arr_sl[:midpoint] last_half = arr_sl[midpoint:] r_1 = np.sum(first_half > 0) r_2 = np.sum(last_half < 0) candle = candles[i + (midpoint - 1)] price = (candle.high + candle.open) / 2.0 # Detect a local maximum if r_1 == cls.n / 2 == r_2: if price < min(highest_prices): # Insert the lowest value first, drop the third (and smallest) value highest_moments.insert(0, candle.moment) highest_moments = highest_moments[:2] highest_prices.insert(0, price) highest_prices = highest_prices[:2] return PriceLine(point_1_moment=highest_moments[0], point_1_price=highest_prices[0], point_2_moment=highest_moments[1], point_2_price=highest_prices[1], first_moment=candles[0].moment, last_moment=candles[-1].moment)
def predict(epoch,model,alpha,option): model.eval() test_x,vel_loc= Reader(). get_real_gaussian_map() vel_pred_total=[] for i in range(len(test_x)): # dispersion axis input = torch.Tensor(test_x[i]) input = input.view([1,input.size(0),input.size(1),input.size(2)]) # compute output output = model(input) # output[batchsize,H,W] output = output.view([output.size(1)]).data.numpy() vel_pred = smooth(output,5,3) vel_pred_total.append(vel_pred) return vel_pred_total,vel_loc
def support_resistance(self, ltp, n): """ This function takes a numpy array of last traded price (ltp) and returns a list of support and resistance levels respectively. n is the number of entries to be scanned, aka the lookback. A higher n value will cause more smoothing and vice versa. """ from scipy.signal import savgol_filter as smooth ltp = np.asarray(ltp, dtype=np.double) # Converting n to a nearest even number if n % 2 != 0: n += 1 n_ltp = ltp.shape[0] # Smooth the curve ltp_s = smooth(ltp, (n + 1), 3) # Taking a simple derivative ltp_d = np.zeros(n_ltp) ltp_d[1:] = np.subtract(ltp_s[1:], ltp_s[:-1]) resistance = [] support = [] for i in xrange(n_ltp - n): arr_sl = ltp_d[i:(i + n)] first = arr_sl[:(n / 2)] # First half last = arr_sl[(n / 2):] # Second half r_1 = np.sum(first > 0) r_2 = np.sum(last < 0) s_1 = np.sum(first < 0) s_2 = np.sum(last > 0) # Local maxima detection if (r_1 == (n / 2)) and (r_2 == (n / 2)): resistance.append(ltp[i + ((n / 2) - 1)]) # Local minima detection if (s_1 == (n / 2)) and (s_2 == (n / 2)): support.append(ltp[i + ((n / 2) - 1)]) return support, resistance
def supres(ltp, n): """ This function takes a numpy array of last traded price and returns a list of support and resistance levels respectively. n is the number of entries to be scanned. """ #converting n to a nearest even number if n % 2 != 0: n += 1 n_ltp = ltp.shape[0] # smoothening the curve ltp_s = smooth(ltp, (n + 1), 3) #taking a simple derivative ltp_d = np.zeros(n_ltp) ltp_d[1:] = np.subtract(ltp_s[1:], ltp_s[:-1]) resistance = [] support = [] for i in range(n_ltp - n): arr_sl = ltp_d[i:(i + n)] half = int(n / 2) # asr chagned n/2 to half all below to getrid of error first = arr_sl[:half] #first half last = arr_sl[half:] #second half r_1 = np.sum(first > 0) r_2 = np.sum(last < 0) s_1 = np.sum(first < 0) s_2 = np.sum(last > 0) #local maxima detection if (r_1 == (n / 2)) and (r_2 == half): resistance.append(ltp[i + (half - 1)]) #local minima detection if (s_1 == (n / 2)) and (s_2 == (n / 2)): support.append(ltp[i + ((n / 2) - 1)]) print(support, resistance) return support, resistance
def gen_smoothed(serie, win_len=20, charts=True): """ Generate smoothed value for a serie """ from scipy.signal import savgol_filter as smooth POLYORDER = 3 smoothed_cls = pd.Series(smooth(serie, (win_len + 1), POLYORDER), serie.index) if charts: import matplotlib.pyplot as plt plt.figure(figsize=(18, 7)) plt.plot(smoothed_cls, label='smoothed close', c='r') plt.plot(serie, label='close', c='b') plt.grid(True) plt.legend(loc='best') return smoothed_cls
def getBB(phase, wave, flux): #fluxes=[] newWave = np.arange(5000, 30000, 50) allFlux = [] for p in range(len(phase)): if phase[p] > 3 or phase[p] < -3: allFlux.append(np.zeros(len(newWave))) else: flux2 = flux[p][wave < 9000][wave[wave < 9000] > 4200] wave2 = wave[wave < 9000][wave[wave < 9000] > 4200] * u.AA res = minimize(_bbChi, np.array([6000, 1]), args=(wave2, smooth(flux2, len(flux2) / 3, 2)), bounds=((0, None), (0, None))) temp, const = res.x bbFlux = fluxFromBB(temp, const, newWave) allFlux.append(bbFlux) return (newWave, np.array(allFlux))
def cal_SL_RL_latest(ticker, df): #START_DATE = datetime.date.today() - timedelta(days=180) #df = si.get_data(ticker, start_date = START_DATE) past100Days = list(df['close'].tail(100)) if (len(past100Days) >= 99): ltp = np.array(past100Days) #print(ltp) current_price = ltp[99] #print(current_rpice) #ltp = np.ravel(ltp) #print(ltp) n = 4 """ This function takes a numpy array of last traded price and returns a list of support and resistance levels respectively. n is the number of entries to be scanned. """ #converting n to a nearest even number if n % 2 != 0: n += 1 n_ltp = ltp.shape[0] #print(n_ltp) # smoothening the curve ltp_s = smooth(ltp, (n + 1), 3) #print(ltp_s) #taking a simple derivative ltp_d = np.zeros(n_ltp) #print(ltp_d) ltp_d[1:] = np.subtract(ltp_s[1:], ltp_s[:-1]) #print(ltp_d[1:]) resistance = [] support = [] for i in range(n_ltp - n): arr_sl = ltp_d[i:(i + n)] #print(len(arr_sl)) #print(arr_sl) first = arr_sl[:len(arr_sl) // 2] #first half A[:len(A)//2] #print(first) last = arr_sl[len(arr_sl) // 2:] #second half #print(last) r_1 = np.sum(first > 0) #print(r_1) r_2 = np.sum(last < 0) #print(r_2) s_1 = np.sum(first < 0) s_2 = np.sum(last > 0) #local maxima detection if (r_1 == (n / 2)) and (r_2 == (n / 2)): res = ltp[i + int((n / 2) - 1)] if (res >= current_price): resistance.append(res) #local minima detection if (s_1 == (n / 2)) and (s_2 == (n / 2)): sup = ltp[i + int((n / 2) - 1)] if (sup < current_price): support.append(sup) support.sort(reverse=True) resistance.sort() return {'support': support[0:3], 'resistance': resistance[0:3]}
def cal_SL_RL(ticker, location): #START_DATE = datetime.date.today() - timedelta(days=180) #df = si.get_data(ticker, start_date = START_DATE) if (location == 'LSE'): filename = ticker.replace('.L', '_' + location) elif (location == 'NSE'): filename = ticker.replace('.NS', '_' + location) elif (location == 'BSE'): filename = ticker.replace('.BO', '_' + location) #print(filename) df = pd.read_csv("/Users/rajubingi/Equity_Data/Data/" + filename + ".csv", sep=',') past100Days = list(df['close'].tail(150)) if (len(past100Days) == 150): ltp = np.array(past100Days) #if ticker == 'FCIT.L' : # print(ltp) current_price = ltp[149] #print(current_rpice) #ltp = np.ravel(ltp) #print(ltp) n = 4 """ This function takes a numpy array of last traded price and returns a list of support and resistance levels respectively. n is the number of entries to be scanned. """ #converting n to a nearest even number if n % 2 != 0: n += 1 n_ltp = ltp.shape[0] #print(n_ltp) # smoothening the curve ltp_s = smooth(ltp, (n + 1), 3) #print(ltp_s) #taking a simple derivative ltp_d = np.zeros(n_ltp) #print(ltp_d) ltp_d[1:] = np.subtract(ltp_s[1:], ltp_s[:-1]) #print(ltp_d[1:]) resistance = [] support = [] for i in range(n_ltp - n): arr_sl = ltp_d[i:(i + n)] #print(len(arr_sl)) #print(arr_sl) first = arr_sl[:len(arr_sl) // 2] #first half A[:len(A)//2] #print(first) last = arr_sl[len(arr_sl) // 2:] #second half #print(last) r_1 = np.sum(first > 0) #print(r_1) r_2 = np.sum(last < 0) #print(r_2) s_1 = np.sum(first < 0) s_2 = np.sum(last > 0) #local maxima detection if (r_1 == (n / 2)) and (r_2 == (n / 2)): res = ltp[i + int((n / 2) - 1)] if (res >= current_price): resistance.append(res) #local minima detection if (s_1 == (n / 2)) and (s_2 == (n / 2)): sup = ltp[i + int((n / 2) - 1)] if (sup < current_price): support.append(sup) support.sort(reverse=True) resistance.sort() return {'support': support[0:3], 'resistance': resistance[0:3]}
def sr_levels(bars, n=8, t=0.02, s=3, f=3): """ Find support and resistance levels using smoothed close price. Args: bars: OHLCV dataframe. n: bar window size. t: tolerance, % variance between min/maxima to be considered a level. s: smoothing factor. lower is more sensitive. f: number of filter passes. Returns: support: list of support levels resistance: list of resistance levels Raises: None. """ from scipy.signal import savgol_filter as smooth # Convert n to next even number. if n % 2 != 0: n += 1 # Find number of bars. n_ltp = bars.close.values.shape[0] # Smooth close data. ltp_smoothed = smooth(bars.close.values, (n + 1), s) # Find delta (difference in adjacent prices). ltp_delta = np.zeros(n_ltp) ltp_delta[1:] = np.subtract(ltp_smoothed[1:], ltp_smoothed[:-1]) resistance = [] support = [] # Identify initial levels. for i in range(n_ltp - n): # Get window for current bar. window = ltp_delta[i:(i + n)] # Split window in half. first = window[:int((n / 2))] # first half last = window[int((n / 2)):] # second half # Find highs and lows for both halves of window. # First/last being higher or lower indicates asc/desc price. r_1 = np.sum(first > 0) r_2 = np.sum(last < 0) s_1 = np.sum(first < 0) s_2 = np.sum(last > 0) # Detect local maxima. If two points match, its a level. if r_1 == (n / 2) and r_2 == (n / 2): try: resistance.append(bars.close.values[i + (int((n / 2)) - 1)]) # Catch empty list error if no levels are present. except Exception as ex: pass # Detect local minima. If two points match, its a level. if s_1 == (n / 2) and s_2 == (n / 2): try: support.append(bars.close.values[i + (int((n / 2)) - 1)]) # Catch empty list error if no levels are present. except Exception as ex: pass # Filter levels f times. levels = np.sort(np.append(support, resistance)) filtered_levels = cluster_filter(levels, t, multipass=True) for i in range(f - 1): filtered_levels = cluster_filter(filtered_levels, t, multipass=True) return filtered_levels
def fit(frec, S11, ftype='A', fitbackground=True, trimwidth=5., doplots=False, margin=51, oldpars=None, refitback=True, reusefitpars=False, fitwidth=None): """**MAIN FIT ROUTINE** Fits complex data S11 vs frecuency to one of 4 models adjusting for a multiplicative complex background It fits the data in three steps. Firstly it fits the background signal removing a certain window around the detected peak position. Then it fits the model times the background to the full data set keeping the background parameters fixed at the fitted values. Finally it refits all background and model parameters once more starting from the previously fitted values. The fit model is: .. math:: \Gamma'(\omega)=(a+b\omega+c\omega^2)\exp(j(a'+b'\omega))\cdot \Gamma(\omega,Q_\\textrm{int},Q_\\textrm{ext},\\theta), Parameters ---------- frec : array_like Array of X values (typically frequency) S11 : array_like Complex array of Z values (typically S11 data) ftype : {'A','B','-A','-B', 'X'}, optional Fit model function (A,B,-A,-B, see S11theo for formulas) fitbackground : bool, optional If "True" will attempt to fit and remove background. If "False", will use a constant background equal to 1 and fit only model function to data. trimwidth : float, optional Number of linewidths around resonance (estimated pre-fit) to remove for background only fit. doplots : bool, optional If "True", shows debugging and intermediate plots margin : float, optional Smoothing window to apply to signal for initial guess procedures (the fit uses unsmoothed data) oldpars : lmfit.Parameters, optional Parameter data from previous fit (expects lmfit Parameter object). Used when "refitback" is "False" or "reusefitpars" is "True". refitback : bool, optional If set to False, does not fit the background but uses parameters provided in "oldpars". If set to "True", fits background normally reusefitpars : bool, optional If set to True, uses parameters provided in "oldpars" as initial guess for fit parameters in main model fit (ignored by background fit) fitwidth : float, optional If set to a numerical value, will trim the signal to a certain number of widths around the resonance for all the fit Returns ------- params : lmfit.Parameters Fitted parameter values freq : numpy.ndarray Array of frequency values within the fitted range S11: numpy.ndarray Array of complex signal values within the fitted range finalresult : lmfit.MinimizerResult The full minimizer result object (see lmfit documentation for details) """ # Convert frequency and S11 into arrays frec = np.array(frec) S11 = np.array(S11) #Smooth data for initial guesses if margin == None or margin == 0: #If no smooting desired, pass None as margin margin = 0 sReS11 = S11.real sImS11 = S11.imag sS11 = np.array([x + 1j * y for x, y in zip(sReS11, sImS11)]) elif type(margin) != int or margin % 2 == 0 or margin <= 3: raise ValueError( 'margin has to be either None, 0, or an odd integer larger than 3') else: sReS11 = np.array(smooth(S11.real, margin, 3)) sImS11 = np.array(smooth(S11.imag, margin, 3)) sS11 = np.array([x + 1j * y for x, y in zip(sReS11, sImS11)]) #Make smoothed phase vector removing 2pi jumps sArgS11 = np.angle(sS11) sArgS11 = np.unwrap(sArgS11) sdiffang = np.diff(sArgS11) #sdiffang = [ x if np.abs(x)<pi else -pi for x in np.diff(np.angle(sS11)) ] #Get resonance index from maximum of the derivative of the imaginary part #ires = np.argmax(np.abs(np.diff(sImS11))) #f0i=frec[ires] #Get resonance index from maximum of the derivative of the phase avgph = np.average(sdiffang) errvec = [np.power(x - avgph, 2.) for x in sdiffang] #ires = np.argmax(np.abs(sdiffang[margin:-margin]))+margin if margin == 0: ires = np.argmax(errvec[0:-1]) + 0 else: ires = np.argmax(errvec[margin:-margin]) + margin f0i = frec[ires] print("Max index: ", ires, " Max frequency: ", f0i) if doplots: plt.clf() plt.title('Original signal (Re,Im)') plt.plot(frec, S11.real) plt.plot(frec, S11.imag) plt.axis('auto') plt.show() plt.plot(np.angle(sS11)) plt.title('Smoothed Phase') plt.axis('auto') plt.show() if margin == 0: plt.plot(sdiffang[0:-1]) else: plt.plot(sdiffang[margin:-margin]) plt.plot(sdiffang) plt.title('Diff of Smoothed Phase') plt.show() #Get peak width by finding width of spike in diffphase plot (imin, imax) = getwidth_phase(ires, errvec, margin) di = imax - imin print("Peak limits: ", imin, imax) print("Lower edge: ", frec[imin], " Center: ", frec[ires], " Upper edge: ", frec[imax], " Points in width: ", di) if doplots: plt.title('Smoothed (ph-phavg)\^2') plt.plot(errvec) plt.plot([imin], [errvec[imin]], 'ro') plt.plot([imax], [errvec[imax]], 'ro') plt.show() if not fitwidth == None: i1 = max(int(ires - di * fitwidth), 0) i2 = min(int(ires + di * fitwidth), len(frec)) frec = frec[i1:i2] S11 = S11[i1:i2] ires = ires - i1 imin = imin - i1 imax = imax - i1 #Trim peak from data (trimwidth times the width) (backfrec, backsig) = trim(frec, S11, ires - trimwidth * di, ires + trimwidth * di) if doplots: plt.title('Trimmed signal for background fit (Re,Im)') plt.plot(backfrec, backsig.real, backfrec, backsig.imag) plt.plot(frec, S11.real, frec, S11.imag) plt.show() plt.title('Trimmed signal for background fit (Abs)') plt.plot(backfrec, np.abs(backsig)) plt.plot(frec, np.abs(S11)) plt.show() plt.title('Trimmed signal for background fit (Phase)') plt.plot(backfrec, np.angle(backsig)) plt.plot(frec, np.angle(S11)) plt.show() if fitbackground: #Make initial background guesses b0 = (np.abs(sS11)[-1] - np.abs(sS11)[0]) / (frec[-1] - frec[0]) # a0 = np.abs(sS11)[0] - b0*frec[0] a0 = np.average(np.abs(sS11)) - b0 * backfrec[0] # a0 = np.abs(sS11)[0] - b0*backfrec[0] c0 = 0. # bp0 = ( np.angle(sS11[di])-np.angle(sS11[0]) )/(frec[di]-frec[0]) xx = [] for i in range(0, len(backfrec) - 1): df = backfrec[i + 1] - backfrec[i] dtheta = np.angle(backsig[i + 1]) - np.angle(backsig[i]) if (np.abs(dtheta) > pi): continue xx.append(dtheta / df) #Remove infinite values in xx (from repeated frequency points for example) xx = np.array(xx) idx = np.isfinite(xx) xx = xx[idx] # bp0 = np.average([ x if np.abs(x)<pi else 0 for x in np.diff(np.angle(backsig))] )/(frec[1]-frec[0]) bp0 = np.average(xx) # ap0 = np.angle(sS11[0]) - bp0*frec[0] # ap0 = np.average(np.unwrap(np.angle(backsig))) - bp0*backfrec[0] ap0 = np.unwrap(np.angle(backsig))[0] - bp0 * backfrec[0] cp0 = 0. print(a0, b0, ap0, bp0) else: a0 = 0 b0 = 0 c0 = 0 ap0 = 0 bp0 = 0 cp0 = 0 params = Parameters() myvary = True params.add('a', value=a0, vary=myvary) params.add('b', value=b0, vary=myvary) params.add('c', value=c0, vary=myvary) params.add('ap', value=ap0, vary=myvary) params.add('bp', value=bp0, vary=myvary) params.add('cp', value=cp0, vary=myvary) if not fitbackground: if ftype == 'A' or ftype == 'B': params['a'].set(value=-1, vary=False) elif ftype == '-A' or ftype == '-B' or ftype == 'X': params['a'].set(value=1, vary=False) params['b'].set(value=0, vary=False) params['c'].set(value=0, vary=False) params['ap'].set(value=0, vary=False) params['bp'].set(value=0, vary=False) params['cp'].set(value=0, vary=False) elif not refitback and oldpars != None: params['a'].set(value=oldpars['a'].value, vary=False) params['b'].set(value=oldpars['b'].value, vary=False) params['c'].set(value=oldpars['c'].value, vary=False) params['ap'].set(value=oldpars['ap'].value, vary=False) params['bp'].set(value=oldpars['bp'].value, vary=False) params['cp'].set(value=oldpars['cp'].value, vary=False) # do background fit params['cp'].set(value=0., vary=False) result = minimize(background2min, params, args=(backfrec, backsig)) ''' params = result.params params['a'].set(vary=False) params['b'].set(vary=False) params['c'].set(vary=False) params['ap'].set(vary=False) params['bp'].set(vary=False) params['cp'].set(vary=True) result = minimize(background2min, params, args=(backfrec, backsig)) params = result.params params['a'].set(vary=True) params['b'].set(vary=True) params['c'].set(vary=True) params['ap'].set(vary=True) params['bp'].set(vary=True) params['cp'].set(vary=True) result = minimize(background2min, params, args=(backfrec, backsig)) ''' # write error report report_fit(result.params) #calculate final background and remove background from original data complexresidual = un_realimag(result.residual) backgroundfit = backsig + complexresidual fullbackground = np.array([backmodel(xx, result.params) for xx in frec]) S11corr = -S11 / fullbackground if ftype == '-A' or ftype == '-B': S11corr = -S11corr if doplots: plt.title('Signal and fitted background (Re,Im)') plt.plot(frec, S11.real) plt.plot(frec, S11.imag) plt.plot(frec, fullbackground.real) plt.plot(frec, fullbackground.imag) plt.show() plt.title('Signal and fitted background (Phase)') plt.plot(frec, np.angle(S11)) plt.plot(frec, np.angle(fullbackground)) plt.show() plt.title('Signal and fitted background (Polar)') plt.plot(S11.real, S11.imag) plt.plot(fullbackground.real, fullbackground.imag) plt.show() plt.title('Signal with background removed (Re,Im)') plt.plot(frec, S11corr.real) plt.plot(frec, S11corr.imag) plt.show() plt.title('Signal with background removed (Phase)') ph = np.unwrap(np.angle(S11corr)) plt.plot(frec, ph) plt.show() plt.title('Signal with background removed (Polar)') plt.plot(S11corr.real, S11corr.imag) plt.show() #Make initial guesses for peak fit # ires = np.argmax(S11corr.real) # f0i=frec[ires] # imin = np.argmax(S11corr.imag) # imax = np.argmin(S11corr.imag) ktot = np.abs(frec[imax] - frec[imin]) if ftype == 'A': Tres = np.abs(S11corr[ires] + 1) kext0 = ktot * Tres / 2. elif ftype == '-A': Tres = np.abs(1 - S11corr[ires]) kext0 = ktot * Tres / 2. elif ftype == '-B': Tres = np.abs(S11corr[ires]) kext0 = (1 - Tres) * ktot elif ftype == 'B': Tres = np.abs(S11corr[ires]) kext0 = (1 + Tres) * ktot elif ftype == 'X': Tres = np.abs(S11corr[ires]) kext0 = (1 - Tres) * ktot kint0 = ktot - kext0 if kint0 <= 0.: kint0 = kext0 Qint0 = f0i / kint0 Qext0 = f0i / kext0 #Make new parameter object (includes previous fitted background values) params = result.params if reusefitpars and oldpars != None: params.add('Qint', value=oldpars['Qint'].value, vary=True, min=0) params.add('Qext', value=oldpars['Qext'].value, vary=True, min=0) params.add('f0', value=oldpars['f0'].value, vary=True) params.add('theta', value=oldpars['theta'].value, vary=True) else: params.add('Qint', value=Qint0, vary=True, min=0) params.add('Qext', value=Qext0, vary=True, min=0) params.add('f0', value=f0i, vary=True) params.add('theta', value=0, vary=True) params['a'].set(vary=False) params['b'].set(vary=False) params['c'].set(vary=False) params['ap'].set(vary=False) params['bp'].set(vary=False) params['cp'].set(vary=False) #Do final fit finalresult = minimize(S11residual, params, args=(frec, S11, ftype)) # write error report report_fit(finalresult.params) params = finalresult.params try: print('QLoaded = ', 1 / (1. / params['Qint'].value + 1. / params['Qext'].value)) except ZeroDivisionError: print('QLoaded = ', 0.) if doplots: plt.title('Pre-Final signal and fit (Re,Im)') plt.plot(frec, S11corr.real) plt.plot(frec, S11corr.imag) plt.plot(frec, S11theo(frec, params, ftype).real) plt.plot(frec, S11theo(frec, params, ftype).imag) plt.show() plt.title('Pre-Final signal and fit (Polar)') plt.plot(S11.real, S11.imag) plt.plot( S11full(frec, params, ftype).real, S11full(frec, params, ftype).imag) plt.axes().set_aspect('equal', 'datalim') plt.show() #REDO final fit varying all parameters if refitback and fitbackground: params['a'].set(vary=True) params['b'].set(vary=True) params['c'].set(vary=True) params['ap'].set(vary=True) params['bp'].set(vary=True) params['cp'].set(vary=False) finalresult = minimize(S11residual, params, args=(frec, S11, ftype)) # write error report report_fit(finalresult.params) params = finalresult.params try: print('QLoaded = ', 1 / (1. / params['Qint'].value + 1. / params['Qext'].value)) except ZeroDivisionError: print('QLoaded = ', 0.) #calculate final result and background complexresidual = un_realimag(finalresult.residual) finalfit = S11 + complexresidual newbackground = np.array( [backmodel(xx, finalresult.params) for xx in frec]) if doplots: plt.title('Final signal and fit (Re,Im)') plt.plot(frec, S11.real) plt.plot(frec, S11.imag) plt.plot(frec, finalfit.real) plt.plot(frec, finalfit.imag) plt.show() plt.title('Final signal and fit (Polar)') plt.plot(S11.real, S11.imag) plt.plot(finalfit.real, finalfit.imag) plt.axes().set_aspect('equal', 'datalim') plt.show() plt.title('Final signal and fit (Abs)') plt.plot(frec, np.abs(S11)) plt.plot(frec, np.abs(finalfit)) plt.show() chi2 = finalresult.chisqr return params, frec, S11, finalresult
def fit(frec,S11,ftype='A',fitbackground=True,trimwidth=5.,doplots=False,margin = 51, oldpars=None, refitback = True, reusefitpars = False, fitwidth=None): #Smooth data for initial guesses if margin == None or margin == 0: #If no smooting desired, pass None as margin margin=0 sReS11 = S11.real sImS11 = S11.imag sS11 = np.array( [x+1j*y for x,y in zip(sReS11,sImS11) ] ) else: sReS11 = np.array(smooth(S11.real,margin,3)) sImS11 = np.array(smooth(S11.imag,margin,3)) sS11 = np.array( [x+1j*y for x,y in zip(sReS11,sImS11) ] ) #Make smoothed phase vector removing 2pi jumps sArgS11 = np.angle(sS11) sArgS11 = np.unwrap(sArgS11) sdiffang = np.diff(sArgS11) #sdiffang = [ x if np.abs(x)<pi else -pi for x in np.diff(np.angle(sS11)) ] #Get resonance index from maximum of the derivative of the imaginary part #ires = np.argmax(np.abs(np.diff(sImS11))) #f0i=frec[ires] #Get resonance index from maximum of the derivative of the phase avgph = np.average(sdiffang) errvec = [np.power(x-avgph,2.) for x in sdiffang] #ires = np.argmax(np.abs(sdiffang[margin:-margin]))+margin if margin == 0: ires = np.argmax(errvec[0:-1])+0 else: ires = np.argmax(errvec[margin:-margin])+margin f0i=frec[ires] print("Max index: ",ires," Max frequency: ",f0i) if doplots: plt.clf() plt.title('Original signal (Re,Im)') plt.plot(frec,S11.real) plt.plot(frec,S11.imag) plt.axis('auto') plt.show() plt.plot(np.angle(sS11)) plt.title('Smoothed Phase') plt.axis('auto') plt.show() if margin == 0: plt.plot(sdiffang[0:-1]) else: plt.plot(sdiffang[margin:-margin]) plt.plot(sdiffang) plt.title('Diff of Smoothed Phase') plt.show() #Get peak width by finding width of spike in diffphase plot (imin,imax) = getwidth_phase(ires,errvec,margin) di = imax-imin print("Peak limits: ",imin,imax) print("Lower edge: ",frec[imin]," Center: ",frec[ires]," Upper edge: ",frec[imax]," Points in width: ", di) if doplots: plt.title('Smoothed (ph-phavg)\^2') plt.plot(errvec) plt.plot([imin],[errvec[imin]],'ro') plt.plot([imax],[errvec[imax]],'ro') plt.show() if not fitwidth==None: i1 = int(ires-di*fitwidth/2) i2 = int(ires+di*fitwidth/2) frec = frec[i1:i2] S11 = S11[i1:i2] ires = ires - i1 imin = imin - i1 imax = imax - i1 #Trim peak from data (trimwidth times the width) (backfrec, backsig) = trim(frec,S11,ires-trimwidth*di,ires+trimwidth*di) if doplots: plt.title('Trimmed signal for background fit (Re,Im)') plt.plot(backfrec,backsig.real,backfrec,backsig.imag) plt.plot(frec,S11.real,frec,S11.imag) plt.show() plt.title('Trimmed signal for background fit (Abs)') plt.plot(backfrec,np.abs(backsig)) plt.plot(frec,np.abs(S11)) plt.show() plt.title('Trimmed signal for background fit (Phase)') plt.plot(backfrec,np.angle(backsig)) plt.plot(frec,np.angle(S11)) plt.show() #Make initial background guesses b0 = (np.abs(sS11)[-1] - np.abs(sS11)[0])/(frec[-1]-frec[0]) # a0 = np.abs(sS11)[0] - b0*frec[0] a0 = np.average(np.abs(sS11)) - b0*backfrec[0] # a0 = np.abs(sS11)[0] - b0*backfrec[0] c0 = 0. # bp0 = ( np.angle(sS11[di])-np.angle(sS11[0]) )/(frec[di]-frec[0]) xx = [] for i in range(0,len(backfrec)-1): df = backfrec[i+1]-backfrec[i] dtheta = np.angle(backsig[i+1])-np.angle(backsig[i]) if (np.abs(dtheta)>pi): continue xx.append(dtheta/df) # bp0 = np.average([ x if np.abs(x)<pi else 0 for x in np.diff(np.angle(backsig))] )/(frec[1]-frec[0]) bp0 = np.average(xx) # ap0 = np.angle(sS11[0]) - bp0*frec[0] # ap0 = np.average(np.unwrap(np.angle(backsig))) - bp0*backfrec[0] ap0 = np.unwrap(np.angle(backsig))[0] - bp0*backfrec[0] cp0 = 0. print(a0,b0,ap0,bp0) params = Parameters() myvary = True params.add('a', value= a0, vary=myvary) params.add('b', value= b0, vary=myvary) params.add('c', value= c0, vary=myvary) params.add('ap', value= ap0, vary=myvary) params.add('bp', value= bp0, vary=myvary) params.add('cp', value= cp0, vary=myvary) if not fitbackground: if ftype == 'A' or ftype == 'B': params['a'].set(value=-1, vary=False) elif ftype == '-A' or ftype == '-B': params['a'].set(value=1, vary=False) params['b'].set(value=0, vary=False) params['c'].set(value=0, vary=False) params['ap'].set(value=0, vary=False) params['bp'].set(value=0, vary=False) params['cp'].set(value=0, vary=False) elif not refitback and oldpars != None: params['a'].set(value=oldpars['a'].value, vary=False) params['b'].set(value=oldpars['b'].value, vary=False) params['c'].set(value=oldpars['c'].value, vary=False) params['ap'].set(value=oldpars['ap'].value, vary=False) params['bp'].set(value=oldpars['bp'].value, vary=False) params['cp'].set(value=oldpars['cp'].value, vary=False) # do background fit params['cp'].set(value=0.,vary=False) result = minimize(background2min, params, args=(backfrec, backsig)) ''' params = result.params params['a'].set(vary=False) params['b'].set(vary=False) params['c'].set(vary=False) params['ap'].set(vary=False) params['bp'].set(vary=False) params['cp'].set(vary=True) result = minimize(background2min, params, args=(backfrec, backsig)) params = result.params params['a'].set(vary=True) params['b'].set(vary=True) params['c'].set(vary=True) params['ap'].set(vary=True) params['bp'].set(vary=True) params['cp'].set(vary=True) result = minimize(background2min, params, args=(backfrec, backsig)) ''' # write error report report_fit(result.params) #calculate final background and remove background from original data complexresidual = un_realimag(result.residual) backgroundfit = backsig + complexresidual fullbackground = np.array([backmodel(xx,result.params) for xx in frec]) S11corr = -S11 / fullbackground if ftype == '-A' or ftype == '-B': S11corr = -S11corr if doplots: plt.title('Signal and fitted background (Re,Im)') plt.plot(frec,S11.real) plt.plot(frec,S11.imag) plt.plot(frec,fullbackground.real) plt.plot(frec,fullbackground.imag) plt.show() plt.title('Signal and fitted background (Phase)') plt.plot(frec,np.angle(S11)) plt.plot(frec,np.angle(fullbackground)) plt.show() plt.title('Signal and fitted background (Polar)') plt.plot(S11.real,S11.imag) plt.plot(fullbackground.real,fullbackground.imag) plt.show() plt.title('Signal with background removed (Re,Im)') plt.plot(frec,S11corr.real) plt.plot(frec,S11corr.imag) plt.show() plt.title('Signal with background removed (Phase)') ph = np.unwrap(np.angle(S11corr)) plt.plot(frec,ph) plt.show() plt.title('Signal with background removed (Polar)') plt.plot(S11corr.real,S11corr.imag) plt.show() #Make initial guesses for peak fit # ires = np.argmax(S11corr.real) # f0i=frec[ires] # imin = np.argmax(S11corr.imag) # imax = np.argmin(S11corr.imag) ktot = np.abs(frec[imax]-frec[imin]) if ftype == 'A': Tres = np.abs(S11corr[ires]+1) kext0 = ktot*Tres/2. elif ftype == '-A': Tres = np.abs(1-S11corr[ires]) kext0 = ktot*Tres/2. elif ftype == '-B': Tres = np.abs(S11corr[ires]) kext0 = (1-Tres)*ktot elif ftype == 'B': Tres = np.abs(S11corr[ires]) kext0 = (1+Tres)*ktot kint0 = ktot-kext0 Qint0 = f0i/kint0 Qext0 = f0i/kext0 #Make new parameter object (includes previous fitted background values) params = result.params if reusefitpars and oldpars != None: params.add('Qint', value=oldpars['Qint'].value,vary=True,min = 0) params.add('Qext', value=oldpars['Qext'].value,vary=True,min = 0) params.add('f0', value=oldpars['f0'].value,vary=True) params.add('theta', value=oldpars['theta'].value,vary=True) else: params.add('Qint', value=Qint0,vary=True,min = 0) params.add('Qext', value=Qext0,vary=True,min = 0) params.add('f0', value=f0i,vary=True) params.add('theta', value=0,vary=True) params['a'].set(vary=False) params['b'].set(vary=False) params['c'].set(vary=False) params['ap'].set(vary=False) params['bp'].set(vary=False) params['cp'].set(vary=False) #Do final fit finalresult = minimize(S11residual, params, args=(frec, S11, ftype)) # write error report report_fit(finalresult.params) params = finalresult.params print('QLoaded = ', 1/(1./params['Qint'].value+1./params['Qext'].value)) if doplots: plt.title('Pre-Final signal and fit (Re,Im)') plt.plot(frec,S11corr.real) plt.plot(frec,S11corr.imag) plt.plot(frec,S11theo(frec,params,ftype).real) plt.plot(frec,S11theo(frec,params,ftype).imag) plt.show() plt.title('Pre-Final signal and fit (Polar)') plt.plot(S11.real,S11.imag) plt.plot(S11full(frec,params,ftype).real,S11full(frec,params,ftype).imag) plt.axes().set_aspect('equal', 'datalim') plt.show() #REDO final fit varying all parameters if refitback: params['a'].set(vary=True) params['b'].set(vary=True) params['c'].set(vary=True) params['ap'].set(vary=True) params['bp'].set(vary=True) params['cp'].set(vary=False) finalresult = minimize(S11residual, params, args=(frec, S11, ftype)) # write error report report_fit(finalresult.params) params = finalresult.params print('QLoaded = ', 1/(1./params['Qint'].value+1./params['Qext'].value)) #calculate final result and background complexresidual = un_realimag(finalresult.residual) finalfit = S11 + complexresidual newbackground = np.array([backmodel(xx,finalresult.params) for xx in frec]) if doplots: plt.title('Final signal and fit (Re,Im)') plt.plot(frec,S11.real) plt.plot(frec,S11.imag) plt.plot(frec,finalfit.real) plt.plot(frec,finalfit.imag) plt.show() plt.title('Final signal and fit (Polar)') plt.plot(S11.real,S11.imag) plt.plot(finalfit.real,finalfit.imag) plt.axes().set_aspect('equal', 'datalim') plt.show() plt.title('Final signal and fit (Polar)') plt.plot(frec,np.abs(S11)) plt.plot(frec,np.abs(finalfit)) plt.show() return params
def main(): parser = argparse.ArgumentParser() parser.add_argument('--batch_size', type=int) parser.add_argument('--lr', type=float) parser.add_argument('--epochs', type=int, default=20) parser.add_argument('--eval_every', type=int, default=10) args = parser.parse_args() ###### train_loader, val_loader = load_data(args.batch_size) model, loss_fnc, optimizer = load_model(args.lr) t_loss, v_loss, t_acc, v_acc = [], [], [], [] for epoch in range(args.epochs): n = args.eval_every printI = 1 lastN = [] for datum, label in train_loader: # set up optimizer with blank gradients optimizer.zero_grad() # predict with the current weights predict = model(datum.float()) # evaluate the loss for the predictions loss = loss_fnc(input=predict.squeeze(), target=label.float()) # calculate the gradients per the loss function we chose loss.backward() # changes the weights one step in the right direction (for the batch) optimizer.step() if len(lastN) < n: lastN += [[datum, label]] else: lastN = lastN[1:] + [[datum, label]] if printI == n: t_loss += [loss.item()] predict = model(X_test.float()) v_loss += [ loss_fnc(input=predict.squeeze(), target=Y_test.float()).item() ] t_acc_this = 0 for every in range(n): t_acc_this += evaluate( model, DataLoader( AdultDataset(lastN[every][0], lastN[every][1]))) t_acc += [t_acc_this / n] v_acc += [ evaluate(model, DataLoader(AdultDataset(X_test, Y_test))) ] printI = 1 print("Status update: currently on epoch", epoch, "and Loss is", loss) print("Number of correct predictions is ", t_acc[-1] * len(Y_train)) print("Validation accuracy is currently: ", v_acc[-1]) printI += 1 if plotting: # Plot losses plt.figure() x = [i for i in range(len(t_loss))] plt.plot(x, t_loss, label='Training Loss') plt.plot(x, v_loss, label='Validation Loss') plt.title("Losses as Function of Gradient Step") plt.ylabel("Loss") plt.xlabel("Gradient Steps") plt.legend() plt.show() # Plot accuracies x = [i for i in range(len(t_acc))] plt.figure() plt.plot(x, t_acc, label='Training Accuracy') plt.plot(x, v_acc, label='Validation Accuracy') plt.ylim(0, 1) plt.title("Accuracy as Function of Gradient Step") plt.ylabel("Accuracy") plt.xlabel("Gradient Steps") plt.legend() plt.show() # Plot losses smoothed plt.figure() x = [i for i in range(len(t_loss))] plt.plot(x, smooth(t_loss, 151, 3), label='Training Loss Smoothed') plt.plot(x, smooth(v_loss, 101, 4), label='Validation Loss') plt.title("Smoothed Losses as Function of Gradient Step") plt.ylabel("Loss") plt.xlabel("Gradient Steps") plt.legend() plt.show() # Plot accuracies smoothed plt.figure() plt.plot(x, smooth(t_acc, 151, 5), label='Training Accuracy Smoothed') plt.plot(x, smooth(v_acc, 101, 4), label='Validation Accuracy Smoothed') plt.ylim(0, 1) plt.title("Smoothed Accuracy as Function of Gradient Step") plt.ylabel("Accuracy") plt.xlabel("Gradient Steps") plt.legend() plt.show()
# dispersion axis temp= disp_pg_real[i,:,:] phase=temp[:,1];phase_un=temp[:,2] group=temp[:,3];group_un=temp[:,4] #convert map for j in range(num_perturbation): new_phase=phase+phase_un*(np.random.rand(17)-0.5)*2 new_group=group+group_un*(np.random.rand(17)-0.5)*2 if j==0: new_phase=phase[:] new_group=group[:] vel_map_p=get_gaussian_map(new_phase,vel_axis,radius=r) vel_map_g=get_gaussian_map(new_group,vel_axis,radius=r) input=torch.Tensor([vel_map_p,vel_map_g]) input = input.view([1,input.size(0),input.size(1),input.size(2)]) # compute output output = model(input) # output[batchsize,H,W] output = output.view([output.size(1)]).data.numpy() vel_pred = smooth(output,5,3) vel_count.append(vel_pred) vel=np.array(vel_count).T vel_loc= "{:.2f}_{:.2f}".format(locKeys[i,0],locKeys[i,1]) # print(str(i)+" "+vel_loc) prefixname = vel_loc mean=np.expand_dims(np.mean(vel,axis=1),axis=1) std=np.expand_dims(np.std(vel,axis=1),axis=1) vel1=np.expand_dims(vel[:,0],axis=1) #print(vel.shape,depth.shape,np.std(vel,axis=1)) output=np.hstack((depth,mean,std,vel)) np.savetxt("./vs_out/"+prefixname+'.txt',output,fmt='%10.5f')
def fit(frec,S11,ftype='A',fitbackground=True,trimwidth=5.,doplots=False,margin = 51, oldpars=None, refitback = True, reusefitpars = False, fitwidth=None): """**MAIN FIT ROUTINE** Fits complex data S11 vs frecuency to one of 4 models adjusting for a multiplicative complex background It fits the data in three steps. Firstly it fits the background signal removing a certain window around the detected peak position. Then it fits the model times the background to the full data set keeping the background parameters fixed at the fitted values. Finally it refits all background and model parameters once more starting from the previously fitted values. The fit model is: .. math:: \Gamma'(\omega)=(a+b\omega+c\omega^2)\exp(j(a'+b'\omega))\cdot \Gamma(\omega,Q_\\textrm{int},Q_\\textrm{ext},\\theta), Parameters ---------- frec : array_like Array of X values (typically frequency) S11 : array_like Complex array of Z values (typically S11 data) ftype : {'A','B','-A','-B', 'X'}, optional Fit model function (A,B,-A,-B, see S11theo for formulas) fitbackground : bool, optional If "True" will attempt to fit and remove background. If "False", will use a constant background equal to 1 and fit only model function to data. trimwidth : float, optional Number of linewidths around resonance (estimated pre-fit) to remove for background only fit. doplots : bool, optional If "True", shows debugging and intermediate plots margin : float, optional Smoothing window to apply to signal for initial guess procedures (the fit uses unsmoothed data) oldpars : lmfit.Parameters, optional Parameter data from previous fit (expects lmfit Parameter object). Used when "refitback" is "False" or "reusefitpars" is "True". refitback : bool, optional If set to False, does not fit the background but uses parameters provided in "oldpars". If set to "True", fits background normally reusefitpars : bool, optional If set to True, uses parameters provided in "oldpars" as initial guess for fit parameters in main model fit (ignored by background fit) fitwidth : float, optional If set to a numerical value, will trim the signal to a certain number of widths around the resonance for all the fit Returns ------- params : lmfit.Parameters Fitted parameter values freq : numpy.ndarray Array of frequency values within the fitted range S11: numpy.ndarray Array of complex signal values within the fitted range finalresult : lmfit.MinimizerResult The full minimizer result object (see lmfit documentation for details) """ # Convert frequency and S11 into arrays frec = np.array(frec) S11 = np.array(S11) #Smooth data for initial guesses if margin == None or margin == 0: #If no smooting desired, pass None as margin margin=0 sReS11 = S11.real sImS11 = S11.imag sS11 = np.array( [x+1j*y for x,y in zip(sReS11,sImS11) ] ) elif type(margin) != int or margin % 2 == 0 or margin <= 3: raise ValueError('margin has to be either None, 0, or an odd integer larger than 3') else: sReS11 = np.array(smooth(S11.real,margin,3)) sImS11 = np.array(smooth(S11.imag,margin,3)) sS11 = np.array( [x+1j*y for x,y in zip(sReS11,sImS11) ] ) #Make smoothed phase vector removing 2pi jumps sArgS11 = np.angle(sS11) sArgS11 = np.unwrap(sArgS11) sdiffang = np.diff(sArgS11) #sdiffang = [ x if np.abs(x)<pi else -pi for x in np.diff(np.angle(sS11)) ] #Get resonance index from maximum of the derivative of the imaginary part #ires = np.argmax(np.abs(np.diff(sImS11))) #f0i=frec[ires] #Get resonance index from maximum of the derivative of the phase avgph = np.average(sdiffang) errvec = [np.power(x-avgph,2.) for x in sdiffang] #ires = np.argmax(np.abs(sdiffang[margin:-margin]))+margin if margin == 0: ires = np.argmax(errvec[0:-1])+0 else: ires = np.argmax(errvec[margin:-margin])+margin f0i=frec[ires] print("Max index: ",ires," Max frequency: ",f0i) if doplots: plt.clf() plt.title('Original signal (Re,Im)') plt.plot(frec,S11.real) plt.plot(frec,S11.imag) plt.axis('auto') plt.show() plt.plot(np.angle(sS11)) plt.title('Smoothed Phase') plt.axis('auto') plt.show() if margin == 0: plt.plot(sdiffang[0:-1]) else: plt.plot(sdiffang[margin:-margin]) plt.plot(sdiffang) plt.title('Diff of Smoothed Phase') plt.show() #Get peak width by finding width of spike in diffphase plot (imin,imax) = getwidth_phase(ires,errvec,margin) di = imax-imin print("Peak limits: ",imin,imax) print("Lower edge: ",frec[imin]," Center: ",frec[ires]," Upper edge: ",frec[imax]," Points in width: ", di) if doplots: plt.title('Smoothed (ph-phavg)\^2') plt.plot(errvec) plt.plot([imin],[errvec[imin]],'ro') plt.plot([imax],[errvec[imax]],'ro') plt.show() if not fitwidth==None: i1 = max(int(ires-di*fitwidth),0) i2 = min(int(ires+di*fitwidth),len(frec)) frec = frec[i1:i2] S11 = S11[i1:i2] ires = ires - i1 imin = imin - i1 imax = imax - i1 #Trim peak from data (trimwidth times the width) (backfrec, backsig) = trim(frec,S11,ires-trimwidth*di,ires+trimwidth*di) if doplots: plt.title('Trimmed signal for background fit (Re,Im)') plt.plot(backfrec,backsig.real,backfrec,backsig.imag) plt.plot(frec,S11.real,frec,S11.imag) plt.show() plt.title('Trimmed signal for background fit (Abs)') plt.plot(backfrec,np.abs(backsig)) plt.plot(frec,np.abs(S11)) plt.show() plt.title('Trimmed signal for background fit (Phase)') plt.plot(backfrec,np.angle(backsig)) plt.plot(frec,np.angle(S11)) plt.show() if fitbackground: #Make initial background guesses b0 = (np.abs(sS11)[-1] - np.abs(sS11)[0])/(frec[-1]-frec[0]) # a0 = np.abs(sS11)[0] - b0*frec[0] a0 = np.average(np.abs(sS11)) - b0*backfrec[0] # a0 = np.abs(sS11)[0] - b0*backfrec[0] c0 = 0. # bp0 = ( np.angle(sS11[di])-np.angle(sS11[0]) )/(frec[di]-frec[0]) xx = [] for i in range(0,len(backfrec)-1): df = backfrec[i+1]-backfrec[i] dtheta = np.angle(backsig[i+1])-np.angle(backsig[i]) if (np.abs(dtheta)>pi): continue xx.append(dtheta/df) #Remove infinite values in xx (from repeated frequency points for example) xx = np.array(xx) idx = np.isfinite(xx) xx = xx[idx] # bp0 = np.average([ x if np.abs(x)<pi else 0 for x in np.diff(np.angle(backsig))] )/(frec[1]-frec[0]) bp0 = np.average(xx) # ap0 = np.angle(sS11[0]) - bp0*frec[0] # ap0 = np.average(np.unwrap(np.angle(backsig))) - bp0*backfrec[0] ap0 = np.unwrap(np.angle(backsig))[0] - bp0*backfrec[0] cp0 = 0. print(a0,b0,ap0,bp0) else: a0=0 b0=0 c0=0 ap0=0 bp0=0 cp0=0 params = Parameters() myvary = True params.add('a', value= a0, vary=myvary) params.add('b', value= b0, vary=myvary) params.add('c', value= c0, vary=myvary) params.add('ap', value= ap0, vary=myvary) params.add('bp', value= bp0, vary=myvary) params.add('cp', value= cp0, vary=myvary) if not fitbackground: if ftype == 'A' or ftype == 'B': params['a'].set(value=-1, vary=False) elif ftype == '-A' or ftype == '-B' or ftype == 'X': params['a'].set(value=1, vary=False) params['b'].set(value=0, vary=False) params['c'].set(value=0, vary=False) params['ap'].set(value=0, vary=False) params['bp'].set(value=0, vary=False) params['cp'].set(value=0, vary=False) elif not refitback and oldpars != None: params['a'].set(value=oldpars['a'].value, vary=False) params['b'].set(value=oldpars['b'].value, vary=False) params['c'].set(value=oldpars['c'].value, vary=False) params['ap'].set(value=oldpars['ap'].value, vary=False) params['bp'].set(value=oldpars['bp'].value, vary=False) params['cp'].set(value=oldpars['cp'].value, vary=False) # do background fit params['cp'].set(value=0.,vary=False) result = minimize(background2min, params, args=(backfrec, backsig)) ''' params = result.params params['a'].set(vary=False) params['b'].set(vary=False) params['c'].set(vary=False) params['ap'].set(vary=False) params['bp'].set(vary=False) params['cp'].set(vary=True) result = minimize(background2min, params, args=(backfrec, backsig)) params = result.params params['a'].set(vary=True) params['b'].set(vary=True) params['c'].set(vary=True) params['ap'].set(vary=True) params['bp'].set(vary=True) params['cp'].set(vary=True) result = minimize(background2min, params, args=(backfrec, backsig)) ''' # write error report report_fit(result.params) #calculate final background and remove background from original data complexresidual = un_realimag(result.residual) backgroundfit = backsig + complexresidual fullbackground = np.array([backmodel(xx,result.params) for xx in frec]) S11corr = -S11 / fullbackground if ftype == '-A' or ftype == '-B': S11corr = -S11corr if doplots: plt.title('Signal and fitted background (Re,Im)') plt.plot(frec,S11.real) plt.plot(frec,S11.imag) plt.plot(frec,fullbackground.real) plt.plot(frec,fullbackground.imag) plt.show() plt.title('Signal and fitted background (Phase)') plt.plot(frec,np.angle(S11)) plt.plot(frec,np.angle(fullbackground)) plt.show() plt.title('Signal and fitted background (Polar)') plt.plot(S11.real,S11.imag) plt.plot(fullbackground.real,fullbackground.imag) plt.show() plt.title('Signal with background removed (Re,Im)') plt.plot(frec,S11corr.real) plt.plot(frec,S11corr.imag) plt.show() plt.title('Signal with background removed (Phase)') ph = np.unwrap(np.angle(S11corr)) plt.plot(frec,ph) plt.show() plt.title('Signal with background removed (Polar)') plt.plot(S11corr.real,S11corr.imag) plt.show() #Make initial guesses for peak fit # ires = np.argmax(S11corr.real) # f0i=frec[ires] # imin = np.argmax(S11corr.imag) # imax = np.argmin(S11corr.imag) ktot = np.abs(frec[imax]-frec[imin]) if ftype == 'A': Tres = np.abs(S11corr[ires]+1) kext0 = ktot*Tres/2. elif ftype == '-A': Tres = np.abs(1-S11corr[ires]) kext0 = ktot*Tres/2. elif ftype == '-B': Tres = np.abs(S11corr[ires]) kext0 = (1-Tres)*ktot elif ftype == 'B': Tres = np.abs(S11corr[ires]) kext0 = (1+Tres)*ktot elif ftype == 'X': Tres = np.abs(S11corr[ires]) kext0 = (1-Tres)*ktot kint0 = ktot-kext0 if kint0<= 0.: kint0 = kext0 Qint0 = f0i/kint0 Qext0 = f0i/kext0 #Make new parameter object (includes previous fitted background values) params = result.params if reusefitpars and oldpars != None: params.add('Qint', value=oldpars['Qint'].value,vary=True,min = 0) params.add('Qext', value=oldpars['Qext'].value,vary=True,min = 0) params.add('f0', value=oldpars['f0'].value,vary=True) params.add('theta', value=oldpars['theta'].value,vary=True) else: params.add('Qint', value=Qint0,vary=True,min = 0) params.add('Qext', value=Qext0,vary=True,min = 0) params.add('f0', value=f0i,vary=True) params.add('theta', value=0,vary=True) params['a'].set(vary=False) params['b'].set(vary=False) params['c'].set(vary=False) params['ap'].set(vary=False) params['bp'].set(vary=False) params['cp'].set(vary=False) #Do final fit finalresult = minimize(S11residual, params, args=(frec, S11, ftype)) # write error report report_fit(finalresult.params) params = finalresult.params try: print('QLoaded = ', 1/(1./params['Qint'].value+1./params['Qext'].value)) except ZeroDivisionError: print('QLoaded = ', 0.) if doplots: plt.title('Pre-Final signal and fit (Re,Im)') plt.plot(frec,S11corr.real) plt.plot(frec,S11corr.imag) plt.plot(frec,S11theo(frec,params,ftype).real) plt.plot(frec,S11theo(frec,params,ftype).imag) plt.show() plt.title('Pre-Final signal and fit (Polar)') plt.plot(S11.real,S11.imag) plt.plot(S11full(frec,params,ftype).real,S11full(frec,params,ftype).imag) plt.axes().set_aspect('equal', 'datalim') plt.show() #REDO final fit varying all parameters if refitback and fitbackground: params['a'].set(vary=True) params['b'].set(vary=True) params['c'].set(vary=True) params['ap'].set(vary=True) params['bp'].set(vary=True) params['cp'].set(vary=False) finalresult = minimize(S11residual, params, args=(frec, S11, ftype)) # write error report report_fit(finalresult.params) params = finalresult.params try: print('QLoaded = ', 1/(1./params['Qint'].value+1./params['Qext'].value)) except ZeroDivisionError: print('QLoaded = ', 0.) #calculate final result and background complexresidual = un_realimag(finalresult.residual) finalfit = S11 + complexresidual newbackground = np.array([backmodel(xx,finalresult.params) for xx in frec]) if doplots: plt.title('Final signal and fit (Re,Im)') plt.plot(frec,S11.real) plt.plot(frec,S11.imag) plt.plot(frec,finalfit.real) plt.plot(frec,finalfit.imag) plt.show() plt.title('Final signal and fit (Polar)') plt.plot(S11.real,S11.imag) plt.plot(finalfit.real,finalfit.imag) plt.axes().set_aspect('equal', 'datalim') plt.show() plt.title('Final signal and fit (Abs)') plt.plot(frec,np.abs(S11)) plt.plot(frec,np.abs(finalfit)) plt.show() chi2 = finalresult.chisqr return params,frec,S11,finalresult