def init(self): self.windows = self.param['window'] self.cols = self.param['col'] self.types = self.param['type'] self.translation_cols = self.param.get('translation') self.scale_cols = self.param.get('scale') self.move_window_mapping = { "mean": lambda c, s, t, w: bn.move_mean(c, w) * s + t, "std": lambda c, s, t, w: bn.move_std(c, w) * s, "var": lambda c, s, t, w: bn.move_var(c, w) * s * s, "min": lambda c, s, t, w: bn.move_min(c, w) * s + t, "max": lambda c, s, t, w: bn.move_max(c, w) * s + t, "rank": lambda c, s, t, w: bn.move_rank(c, w), "sum": lambda c, s, t, w: bn.move_sum(c, w) * s + t * w, "ema": lambda c, s, t, w: F. ema(c, 2.0 / (w + 1), start_indices=self.base.start_indices) * s + t, "rsi": lambda c, s, t, w: F.rsi( c, w, start_indices=self.base.start_indices), "psy": lambda c, s, t, w: F.psy( c, w, start_indices=self.base.start_indices), "bias": lambda c, s, t, w: F.bias( c, w, start_indices=self.base.start_indices) }
def test_move_std_sqrt(): "Test move_std for neg sqrt." a = [ 0.0011448196318903589, 0.00028718669878572767, 0.00028718669878572767, 0.00028718669878572767, 0.00028718669878572767 ] err_msg = "Square root of negative number. ndim = %d" b = bn.move_std(a, window=3) assert_true(np.isfinite(b[2:]).all(), err_msg % 1) a2 = np.array([a, a]) b = bn.move_std(a2, window=3, axis=1) assert_true(np.isfinite(b[:, 2:]).all(), err_msg % 2) a3 = np.array([[a, a], [a, a]]) b = bn.move_std(a3, window=3, axis=2) assert_true(np.isfinite(b[:, :, 2:]).all(), err_msg % 3)
def test_move_std_sqrt(): "Test move_std for neg sqrt." a = [0.0011448196318903589, 0.00028718669878572767, 0.00028718669878572767, 0.00028718669878572767, 0.00028718669878572767] err_msg = "Square root of negative number. ndim = %d" b = bn.move_std(a, window=3) assert_true(np.isfinite(b[2:]).all(), err_msg % 1) a2 = np.array([a, a]) b = bn.move_std(a2, window=3, axis=1) assert_true(np.isfinite(b[:, 2:]).all(), err_msg % 2) a3 = np.array([[a, a], [a, a]]) b = bn.move_std(a3, window=3, axis=2) assert_true(np.isfinite(b[:, :, 2:]).all(), err_msg % 3)
def Stddev(A, n): ''' window天的移动标准差 window >= 2 ''' if n < 2: #print ("计算stddev,n不能小于2,返回输入") return A result = bk.move_std(A, n, min_count=2, axis=0, ddof=1) result = fillna(result) #利用每一天所有股票的均值来填充空值,根据broadcast的原理,需要转置后再填充 result[np.isnan(A)] = np.nan return result
def numpy_normxcorr(templates, stream, pads, *args, **kwargs): """ Compute the normalized cross-correlation using numpy and bottleneck. :param templates: 2D Array of templates :type templates: np.ndarray :param stream: 1D array of continuous data :type stream: np.ndarray :param pads: List of ints of pad lengths in the same order as templates :type pads: list :return: np.ndarray of cross-correlations :return: np.ndarray channels used """ import bottleneck # Generate a template mask used_chans = ~np.isnan(templates).any(axis=1) # Currently have to use float64 as bottleneck runs into issues with other # types: https://github.com/kwgoodman/bottleneck/issues/164 stream = stream.astype(np.float64) templates = templates.astype(np.float64) template_length = templates.shape[1] stream_length = len(stream) assert stream_length > template_length, "Template must be shorter than " \ "stream" fftshape = next_fast_len(template_length + stream_length - 1) # Set up normalizers stream_mean_array = bottleneck.move_mean( stream, template_length)[template_length - 1:] stream_std_array = bottleneck.move_std( stream, template_length)[template_length - 1:] # because stream_std_array is in denominator or res, nan all 0s stream_std_array[stream_std_array == 0] = np.nan # Normalize and flip the templates norm = ((templates - templates.mean(axis=-1, keepdims=True)) / (templates.std(axis=-1, keepdims=True) * template_length)) norm_sum = norm.sum(axis=-1, keepdims=True) stream_fft = np.fft.rfft(stream, fftshape) template_fft = np.fft.rfft(np.flip(norm, axis=-1), fftshape, axis=-1) res = np.fft.irfft(template_fft * stream_fft, fftshape)[:, 0:template_length + stream_length - 1] res = ((_centered(res, (templates.shape[0], stream_length - template_length + 1))) - norm_sum * stream_mean_array) / stream_std_array res[np.isnan(res)] = 0.0 for i, pad in enumerate(pads): res[i] = np.append(res[i], np.zeros(pad))[pad:] return res.astype(np.float32), used_chans
def moving_std(x: np.ndarray, n: int) -> np.ndarray: if bottleneck_found: return bn.move_std(x, n)[n - 1:] sums = np.empty(x.size - n + 1) sqrs = np.empty(x.size - n + 1) tab = np.cumsum(x) / n sums[0] = tab[n - 1] sums[1:] = tab[n:] - tab[:-n] tab = np.cumsum(x * x) / n sqrs[0] = tab[n - 1] sqrs[1:] = tab[n:] - tab[:-n] return np.sqrt(sqrs - sums * sums)
def get_mean_hierarchy_assignment(assignments, params_full): steps = assignments.shape[0] assign = _get_MPEAR(assignments) clusters = np.unique(assign) params = np.zeros((clusters.size, params_full.shape[2])) for i, cluster in enumerate(clusters): cells_cl_idx = assign == cluster cells = np.nonzero(cells_cl_idx)[0] other = np.nonzero(~cells_cl_idx)[0] # Paper - section 2.3: first criteria if cells.size == 1: same_cluster = np.ones(steps).astype(bool) else: same_cluster = 0 == bn.nansum(bn.move_std(assignments[:, cells], 2, axis=1), axis=1) # Paper - section 2.3: second criteria cl_ids = assignments[:, cells[0]] other_cl_id = assignments[:, other] no_others = [cl_ids[j] not in other_cl_id[j] for j in range(steps)] # At least criteria 1 fullfilled if any(same_cluster): # Both criteria fullfilled in at least 1 posterior sample if any(same_cluster & no_others): step_idx = np.argwhere(same_cluster & no_others).flatten() else: step_idx = np.argwhere(same_cluster).flatten() for step in step_idx: cl_id = np.argwhere(np.unique(assignments[step]) == cl_ids[step]) \ .flatten()[0] params[i] += params_full[step][cl_id] params[i] /= step_idx.size # If not, take parameters from all posterior samples else: for step, step_assign in enumerate(assignments): cl_id_all = np.unique(step_assign) cl_id, cnt = np.unique(step_assign[cells], return_counts=True) cl_id_new = np.argwhere(np.in1d(cl_id_all, cl_id)).flatten() params[i] += np.dot(cnt, params_full[step][cl_id_new]) params[i] /= steps * cells.size params_df = pd.DataFrame(params).T[assign] return assign, params_df
def _set_data(self): try: endcol1 = self.dl.dates_to_indices(self.identified_date) except KeyError: argwhere = np.argwhere(self.dl.dates > self.identified_date) if not len(argwhere): raise KeyError('It seems that {} is neither a valid date, nor a date when data is available'.format( self.identified_date)) else: endcol1 = argwhere[0][0] self.identified_date = self.dl.dates[endcol1] # endcol = min(endcol1 + self.MAX_OBS_DAYS + self.MAX_HLD_DAYS + 1, len(self.dl)) endcol = min(endcol1 + int((self.MAX_OBS_DAYS + self.MAX_HLD_DAYS) * 1.1), len(self.dl)) startcol = endcol1 - 252 assert startcol >= 0, '%s is too early to have enough data required for computation' % self.identified_date # according to R implementation, should minus 251, but here change it to 252 so the identified day can also be # trading trigger/activation day self._identified_date_id = endcol1 pair_prices = self.dl['PRCCD', self.pair][:, startcol:endcol] pair_wealth = self.dl['CUM_WEALTH', self.pair][:, startcol:endcol] pair_prices = pair_prices[:, :1] * pair_wealth / pair_wealth[:, :1] has_na = np.isnan(pair_prices[:, 252:]).any(axis=0) self._data_dict['has_na'] = has_na # start from identified self._data_dict['cum_na'] = has_na.cumsum() # start from identified # Note: actually no missing values were found during my experiment. # this block might be redundant pair_prices = foward_fillna_2darray(pair_prices) ratio = np.log(pair_prices[0] / pair_prices[1]) self._data_dict['ratio_history'] = ratio mean_mv = bn.move_mean(ratio, window=252, min_count=200)[251:] sd_mv = bn.move_std(ratio, window=252, min_count=200, ddof=1)[251:] # min_count is used to address the extreme case where the first 50 days are all missing data. # this is likely under the parameter settings of correlation computation ub_mv = mean_mv + 2. * sd_mv # start from identified - 1 lb_mv = mean_mv - 2. * sd_mv # start from identified - 1 ratio = ratio[251:] # start from identified - 1 self._data_dict['ratio'] = ratio[1:] # start from identified self._data_dict['above_upper'] = np.ediff1d(np.where(ratio >= ub_mv, 1, 0)) # start from identified self._data_dict['above_mean'] = np.ediff1d(np.where(ratio >= mean_mv, 1, 0)) self._data_dict['below_mean'] = np.ediff1d(np.where(ratio <= mean_mv, 1, 0)) self._data_dict['below_lower'] = np.ediff1d(np.where(ratio <= lb_mv, 1, 0)) self._data_dict['in_flag'] = bn.nansum(self.dl['IN_US_1', self.pair][:, endcol1:endcol], axis=0) == 2
def scipy_normxcorr(templates, stream, pads): """ Compute the normalized cross-correlation of multiple templates with data. :param templates: 2D Array of templates :type templates: np.ndarray :param stream: 1D array of continuous data :type stream: np.ndarray :param pads: List of ints of pad lengths in the same order as templates :type pads: list :return: np.ndarray of cross-correlations :return: np.ndarray channels used """ import bottleneck from scipy.signal.signaltools import _centered # Generate a template mask used_chans = ~np.isnan(templates).any(axis=1) # Currently have to use float64 as bottleneck runs into issues with other # types: https://github.com/kwgoodman/bottleneck/issues/164 stream = stream.astype(np.float64) templates = templates.astype(np.float64) template_length = templates.shape[1] stream_length = len(stream) fftshape = next_fast_len(template_length + stream_length - 1) # Set up normalizers stream_mean_array = bottleneck.move_mean( stream, template_length)[template_length - 1:] stream_std_array = bottleneck.move_std( stream, template_length)[template_length - 1:] # Normalize and flip the templates norm = ((templates - templates.mean(axis=-1, keepdims=True)) / (templates.std(axis=-1, keepdims=True) * template_length)) norm_sum = norm.sum(axis=-1, keepdims=True) stream_fft = np.fft.rfft(stream, fftshape) template_fft = np.fft.rfft(np.flip(norm, axis=-1), fftshape, axis=-1) res = np.fft.irfft(template_fft * stream_fft, fftshape)[:, 0:template_length + stream_length - 1] res = ((_centered(res, stream_length - template_length + 1)) - norm_sum * stream_mean_array) / stream_std_array res[np.isnan(res)] = 0.0 for i in range(len(pads)): res[i] = np.append(res[i], np.zeros(pads[i]))[pads[i]:] return res.astype(np.float32), used_chans
def genweight(datname, dpath, wpath): """ Combine time series with statistical weights calculated from scatter Arguments: - `datname`: Identifier of data file - `dpath` : Path to data file (time series). - `wpath` : Path to scatter file (with same time points!) """ # Pretty print print('Generating weights for {0} !'.format(dpath)) # Load data and weights t, d = np.loadtxt(dpath, unpack=True) tt, sig = np.loadtxt(wpath, unpack=True) # Check that times are indeed the same tdif = t - tt if tdif.any() != 0: print('Error! Not the same time points! Quitting!') exit() # Moving variance (Hans: M = 50 - 100) M = 70 movstd = bn.move_std(sig, M, min_count=1) movvar = np.square(movstd) # Remove first point x = 1 t = t[x:] d = d[x:] movvar = movvar[x:] # Calculate weights from scatter (1 / variance) w = np.divide(1.0, movvar) # Save outfile = star + '_with-weights.txt' np.savetxt(outfile, np.transpose([t, d, w]), fmt='%.15e', delimiter='\t') # Done! print('Done!\n')
def __init__(self, signal, time, dtS=0.0002): """ Parameters ---------- signal : ndarray signal to be analyzed time : ndarray time basis dtS : floating At the init we also compute a normalize signal where normalization is of the form (x-<x>)/std(x) where the mean and average is a rolling mean and standard deviation on a window of the time dtS/dt Dependences ----------- numpy scipy pycwt https://github.com/regeirk/pycwt.git astropy for better histogram function bottleneck (https://pypi.python.org/pypi/Bottleneck) for moving average """ self.sig = copy.deepcopy(signal) self.time = copy.deepcopy(time) self.dt = (self.time.max() - self.time.min()) / (self.time.size - 1) self.nsamp = self.time.size self.signorm = (self.sig - self.sig.mean()) / self.sig.std() # since the moments of the signal are # foundamental quantities we compute them # at the initial self.moments() _nPoint = int(dtS / self.dt) self.rmsnorm = ( self.sig - bottleneck.move_mean(self.sig, _nPoint, min_count=1)) / \ bottleneck.move_std(self.sig, _nPoint,min_count=1)
def _set_data_for_visualization(self, days_before_identification=21, days_after_close=10): startcol = max(self._identified_date_id - 252 - days_before_identification + 1, 0) endcol = min(self._identified_date_id + self.MAX_OBS_DAYS + self.MAX_HLD_DAYS + days_after_close + 1, len(self.dl)) pair_prices = self.dl['PRCCD', self.pair][:, startcol:endcol] pair_wealth = self.dl['CUM_WEALTH', self.pair][:, startcol:endcol] pair_prices = pair_prices[:, :1] * pair_wealth / pair_wealth[:, :1] pair_prices = foward_fillna_2darray(pair_prices) ratio = np.log(pair_prices[0] / pair_prices[1]) mean_mv = bn.move_mean(ratio, window=252, min_count=200)[251:] sd_mv = bn.move_std(ratio, window=252, min_count=200, ddof=1)[251:] ub_mv = mean_mv + 2. * sd_mv # start from identified - days_before_identification lb_mv = mean_mv - 2. * sd_mv # start from identified - days_before_identification ratio = ratio[251:] # start from identified - days_before_identification idtf_idx = self._identified_date_id - startcol - 251 open_idx = self.dl.dates_to_indices(self.open_date) - self._identified_date_id + idtf_idx close_idx = self.dl.dates_to_indices(self.close_date) - self._identified_date_id + idtf_idx end_idx = close_idx + days_after_close return {'ratio': ratio[:end_idx + 1], 'upper': ub_mv[:end_idx + 1], 'lower': lb_mv[:end_idx + 1], 'mean': mean_mv[:end_idx + 1], 'open_idx': open_idx, 'close_idx': close_idx, 'idtf_idx': idtf_idx}
p.xlabel(f1short) p.ylabel(f2short) p.axis('equal') #p.show() #sys.exit() ########################### ##### WINDOW DISTRIBUTION ########## windows = decSort2[decSort1.argsort()] - np.arange(decSort2.size) p.subplot(2, 3, 5) p.plot(windows, 'ro', alpha=0.5) p.title('Rank Differences with standard deviations') p.xlabel('Gene expression rank') p.ylabel(f1short + '-' + f2short) y = bn.move_std(windows, 10) p.plot(y, 'b') p.plot(-y, 'b') ########################### ##### VENN BUBBLES ########## thresh1 = data1['FPKM'] > 0 thresh2 = data2['FPKM'] > 0 set1 = set(data1[thresh1]['tracking_id']) set2 = set(data2[thresh2]['tracking_id']) aandb = len(set1.intersection(set2)) a = len(set1.difference(set2)) b = len(set2.difference(set1))
def height_plot_across_folders(folder_list, inputsuffix='allz2.dat', label='Mean Light Weighted Age [Gyr]', col=6, errcol=None, lowhigh=False, order=5, ylims=None, bigpoints=False, binz=True, combine_all=False, plot_std=False, exclude=[[],[],[],[],[],[]]): axlist = [] plist = [6,3,4,2,1,5] #color_list = ['blue','turquoise','chartreuse','yellow','tomato','red'] color_list = ['blue','seagreen','darkorange','crimson','dimgray','mediumorchid','lightblue'] style_list = ['-','-','-','-','-','-','-'] if not isinstance(col,list): col = [col] * len(folder_list) for i in range(6): pointing = plist[i] ax = plt.figure().add_subplot(111) ax.set_xlabel('|Height [kpc]|') ax.set_ylabel(label) ax.set_title('{}\nP{}'.format(time.asctime(),pointing)) for f, folder in enumerate(folder_list): color = color_list[f] style = style_list[f] dat = glob('{}/*P{}*{}'.format(folder, pointing, inputsuffix))[0] print dat loc = glob('{}/*P{}*locations.dat'.format(folder, pointing))[0] print loc print 'Excluding: ', exclude[pointing-1] if errcol == None: td = np.loadtxt(dat, usecols=(col[f],), unpack=True) else: if lowhigh: td, low, high = np.loadtxt(dat, usecols=(col[f],errcol,errcol+1), unpack=True) te = np.vstack((low,high)) else: td, te = np.loadtxt(dat, usecols=(col[f],errcol), unpack=True) r, tz = np.loadtxt(loc, usecols=(4,5), unpack=True) exarr = np.array(exclude[pointing-1])-1 #becuase aps are 1-indexed td = np.delete(td,exarr) r = np.delete(r,exarr) tz = np.delete(tz,exarr) if errcol != None: if lowhigh: te = np.delete(te,exarr,axis=1) else: te = np.delete(te,exarr) alpha=1.0 if combine_all and f == 0: bigD = np.zeros(td.size) alpha=0.3 if binz: z = np.array([]) d = np.array([]) e = np.array([]) while tz.size > 0: zi = tz[0] idx = np.where(np.abs(tz - zi) < 0.05) d = np.r_[d,np.mean(td[idx])] e = np.r_[e,np.std(td[idx])] z = np.r_[z,np.abs(zi)] tz = np.delete(tz, idx) td = np.delete(td, idx) else: z = tz d = td if errcol == None: e = np.zeros(tz.size) else: e = te if combine_all: bigD = np.vstack((bigD,d)) bigz = z gidx = d == d d = d[gidx] z = z[gidx] if lowhigh: e = e[:,gidx] else: e = e[gidx] sidx = np.argsort(z) dp = np.r_[d[sidx][order::-1],d[sidx]] zp = np.r_[z[sidx][order::-1],z[sidx]] mean = bn.move_mean(dp,order)[order+1:] std = bn.move_std(dp,order)[order+1:] spl = spi.UnivariateSpline(z[sidx],d[sidx]) mean = spl(z[sidx]) # mean = np.convolve(d[sidx],np.ones(order)/order,'same') # std = np.sqrt(np.convolve((d - mean)**2,np.ones(order)/order,'same')) # ax.plot(z[sidx],mean,color=color, ls=style, label=folder, alpha=alpha) # ax.fill_between(z[sidx],mean-std,mean+std, alpha=0.1, color=color) # print d.shape, np.sum(e,axis=0).shape # d = d/np.sum(e,axis=0) # e = np.diff(e,axis=0)[0] # print e.shape ax.errorbar(z, d, yerr=e, fmt='.', color=color,alpha=alpha,capsize=0, label=folder) ax.set_xlim(-0.1,2.6) if ylims is not None: ax.set_ylim(*ylims) ax.legend(loc=0,numpoints=1) if combine_all: sidx = np.argsort(bigz) bigD = bigD[1:] bigMean = bn.nanmean(bigD,axis=0) bigStd = bn.nanstd(bigD,axis=0) bigspl = spi.UnivariateSpline(bigz[sidx],bigMean[sidx]) bigFit = bigspl(bigz[sidx]) ax.plot(bigz[sidx], bigFit, 'k-', lw=2) ax.errorbar(bigz, bigMean, yerr=bigStd, fmt='.', color='k',capsize=0) axlist.append(ax) if combine_all and plot_std: ax2 = plt.figure().add_subplot(111) ax2.set_xlabel('|Height [kpc]|') ax2.set_ylabel('$\delta$'+label) ax2.set_title(ax.get_title()) ax2.plot(bigz, bigStd, 'k') axlist.append(ax2) return axlist
def plot_heights_with_err(inputsuffix,label=r'$\tau_{\mathrm{V,Balm}}$',basedir='.', col=1, errcol=2, lowhigh=False, order=5, bigorder=60, s=None, ylims=None, labelr=False, bigpoints=False, plotfit=True, exclude=exclude, printdate=True, printfit=True): zz = np.array([]) dd = np.array([]) if lowhigh: ee = np.array([[],[]]) else: ee = np.array([]) axlist = [] bigax = plt.figure().add_subplot(111) bigax.set_xlabel(r'$|z| \mathrm{\ [kpc]}$') bigax.set_ylabel(label) plist = [6,3,4,2,1,5] color_list = ['blue','seagreen','sienna','orange','yellowgreen','darkturquoise'] style_list = ['-','-','-','--','--','--'] for i in range(6): pointing = plist[i] color = color_list[i] style = style_list[i] dat = glob('{}/*P{}*{}'.format(basedir, pointing, inputsuffix))[0] print dat loc = glob('{}/*P{}*locations.dat'.format(basedir, pointing))[0] print loc print 'Excluding: ', exclude[pointing-1] if errcol is not None: if lowhigh: data, Lerr, Herr = np.loadtxt(dat, usecols=(col,errcol,errcol+1), unpack=True) err = np.vstack((Lerr,Herr)) else: data, err = np.loadtxt(dat, usecols=(col,errcol), unpack=True) else: data = np.loadtxt(dat, usecols=(col,), unpack=True) err = np.ones(data.size)*0.01 r, z = np.loadtxt(loc, usecols=(4,5), unpack=True) avgr = np.mean(r) ax = plt.figure().add_subplot(111) ax.set_xlabel('|Height [kpc]|') ax.set_ylabel(label) if labelr: ax.set_title('{:4.0f} kpc'.format(avgr)) linelabel = '{:4.0f} kpc'.format(avgr) else: ax.set_title('{}\nP{}'.format(time.asctime(),pointing)) linelabel = 'P{}'.format(pointing) exarr = np.array(exclude[pointing-1])-1 #becuase aps are 1-indexed data = np.delete(data,exarr) r = np.delete(r,exarr) z = np.delete(z,exarr) gidx = data == data data = data[gidx] z = z[gidx] if lowhigh: err = np.delete(err,exarr,axis=1) err = err[:,gidx] ee = np.hstack((ee,err)) else: err = np.delete(err,exarr) err = err[gidx] ee = np.r_[ee,err] zz = np.r_[zz,z] dd = np.r_[dd,data] sidx = np.argsort(z) data_pad = np.r_[data[sidx][order::-1],data[sidx]] z_pad = np.r_[z[sidx][order::-1],z[sidx]] # mean = bn.move_mean(data_pad,order)[order+1:] std = bn.move_std(data_pad,order)[order+1:] spl = spi.UnivariateSpline(z[sidx],data[sidx]) mean = spl(z[sidx]) # mean = np.convolve(d[sidx],np.ones(order)/order,'same') # std = np.sqrt(np.convolve((d - mean)**2,np.ones(order)/order,'same')) bigax.errorbar(z, data, yerr=err, fmt='.', label=linelabel, color=color, capsize=0) # ax.plot(z[sidx],mean,color=color, ls=style) # ax.fill_between(z[sidx],mean-std,mean+std, alpha=0.1, color=color) ax.errorbar(z, data, yerr=err, fmt='.', color=color, capsize=0) ax.set_xlim(-0.1,2.6) if ylims is not None: ax.set_ylim(*ylims) axlist.append(ax) if printdate: plot_title = time.asctime() else: plot_title = '' if plotfit: sidx = np.argsort(zz) big_data_pad = np.r_[dd[sidx][bigorder::-1],dd[sidx]] big_z_pad = np.r_[zz[sidx][bigorder::-1],zz[sidx]] big_e_pad = np.r_[ee[sidx][bigorder::-1],ee[sidx]] big_sum = bn.move_sum(big_data_pad/big_e_pad,bigorder)[bigorder+1:] big_weight = bn.move_sum(1./big_e_pad,bigorder)[bigorder+1:] big_mean = big_sum/big_weight # std = bn.move_std(data_pad,order)[order+1:] # big_spl = spi.UnivariateSpline(zz[sidx],dd[sidx],w = 1./ee[sidx]**2, k=k, s=s) # big_mean = big_spl(zz[sidx]) # big_pc = np.polyfit(zz[sidx], dd[sidx], polydeg, w=1./ee[sidx]**2) # big_poly = np.poly1d(big_pc) # big_mean = big_poly(zz[sidx]) p = np.poly1d(np.polyfit(zz[sidx],big_mean,1)) print p.coeffs # bigax.plot(zz[sidx],big_mean,'-k',lw=2) bigax.plot(zz[sidx],p(zz[sidx]),'--k',lw=2) if printdate: plot_title += '\n' if printfit: plot_title += label+'$={:4.2f}z{:+4.2f}$'.format(p.coeffs[0],p.coeffs[1]) bigax.set_title(plot_title) bigax.legend(loc=0, numpoints=1, scatterpoints=1) bigax.set_xlim(-0.1,2.6) print zz.size if ylims is not None: bigax.set_ylim(*ylims) axlist = [bigax] + axlist return axlist
def simple_plot(inputsuffix='allz2.dat', label='Mean Light Weighted Age [Gyr]', col=62, order=5, ylims=None, labelr=False, bigpoints=False, exclude=[[],[],[],[],[],[]]): zz = np.array([]) dd = np.array([]) axlist = [] bigax = plt.figure().add_subplot(111) bigax.set_xlabel('|Height [kpc]|') bigax.set_ylabel(label) plist = [6,3,4,2,1,5] #color_list = ['blue','turquoise','chartreuse','yellow','tomato','red'] color_list = ['blue','seagreen','sienna','sienna','seagreen','blue'] style_list = ['-','-','-','--','--','--'] for i in range(6): pointing = plist[i] color = color_list[i] style = style_list[i] dat = glob('*P{}*{}'.format(pointing, inputsuffix))[0] print dat loc = glob('*P{}*locations.dat'.format(pointing))[0] print loc print 'Excluding: ', exclude[pointing-1] td = np.loadtxt(dat, usecols=(col,), unpack=True) r, tz = np.loadtxt(loc, usecols=(4,5), unpack=True) avgr = np.mean(r) ax = plt.figure().add_subplot(111) ax.set_xlabel('|Height [kpc]|') ax.set_ylabel(label) if labelr: ax.set_title('{:4.0f} kpc'.format(avgr)) linelabel = '{:4.0f} kpc'.format(avgr) else: ax.set_title('{}\nP{}'.format(time.asctime(),pointing)) linelabel = 'P{}'.format(pointing) exarr = np.array(exclude[pointing-1])-1 #becuase aps are 1-indexed td = np.delete(td,exarr) t = np.delete(r,exarr) tz = np.delete(tz,exarr) z = np.array([]) d = np.array([]) e = np.array([]) while tz.size > 0: zi = tz[0] idx = np.where(np.abs(tz - zi) < 0.05) d = np.r_[d,np.mean(td[idx])] e = np.r_[e,np.std(td[idx])] z = np.r_[z,np.abs(zi)] tz = np.delete(tz, idx) td = np.delete(td, idx) gidx = d == d d = d[gidx] z = z[gidx] e = e[gidx] sidx = np.argsort(z) dp = np.r_[d[sidx][order::-1],d[sidx]] zp = np.r_[z[sidx][order::-1],z[sidx]] mean = bn.move_mean(dp,order)[order+1:] std = bn.move_std(dp,order)[order+1:] spl = spi.UnivariateSpline(z[sidx],d[sidx]) mean = spl(z[sidx]) # mean = np.convolve(d[sidx],np.ones(order)/order,'same') # std = np.sqrt(np.convolve((d - mean)**2,np.ones(order)/order,'same')) bigax.plot(z[sidx],mean, label=linelabel, color=color, ls=style) bigax.fill_between(z[sidx],mean-std,mean+std, alpha=0.1, color=color) if bigpoints: bigax.errorbar(z, d, yerr=e, fmt='.', color=color, alpha=0.6, capsize=0) ax.plot(z[sidx],mean,color=color, ls=style) ax.fill_between(z[sidx],mean-std,mean+std, alpha=0.1, color=color) ax.errorbar(z, d, yerr=e, fmt='.', color=color) ax.set_xlim(-0.1,2.6) if ylims is not None: ax.set_ylim(*ylims) axlist.append(ax) bigax.legend(loc=0, numpoints=1, scatterpoints=1) bigax.set_title(time.asctime()) bigax.set_xlim(-0.1,2.6) if ylims is not None: bigax.set_ylim(*ylims) axlist = [bigax] + axlist return axlist
data, wav_params = wavLoad(input_file) fs = wav_params[2] except IOError, e: print "Could not read file: %s" % e sys.exit(-1) # cast to mono if len(data.shape) == 2: data = data.sum(axis=0) # should this be .mean()? window_frames= int(fs * window_seconds) silence_frames = int(fs * silence_seconds) print "Analyzing" move_std = move_std(data, window=window_frames/2) mean_std = nanmean(move_std) print "move_std shape: ", move_std.shape print "len(data): ", len(data) widgets = ["Creating file", Bar(), ETA()] pbar = ProgressBar(widgets=widgets, maxval=len(data)).start() new_data = [] silence_count = 0 for i, d in pbar(enumerate(data)): new_data.append(d) if move_std[i] is not np.nan and move_std[i] < mean_std: silence_count += 1 else:
def m66(current_skyline_app, parent_pid, timeseries, algorithm_parameters): """ A time series data points are anomalous if the 6th median is 6 standard deviations (six-sigma) from the time series 6th median standard deviation and persists for x_windows, where `x_windows = int(window / 2)`. This algorithm finds SIGNIFICANT cahngepoints in a time series, similar to PELT and Bayesian Online Changepoint Detection, however it is more robust to instaneous outliers and more conditionally selective of changepoints. :param current_skyline_app: the Skyline app executing the algorithm. This will be passed to the algorithm by Skyline. This is **required** for error handling and logging. You do not have to worry about handling the argument in the scope of the custom algorithm itself, but the algorithm must accept it as the first agrument. :param parent_pid: the parent pid which is executing the algorithm, this is **required** for error handling and logging. You do not have to worry about handling this argument in the scope of algorithm, but the algorithm must accept it as the second argument. :param timeseries: the time series as a list e.g. ``[[1578916800.0, 29.0], [1578920400.0, 55.0], ... [1580353200.0, 55.0]]`` :param algorithm_parameters: a dictionary of any required parameters for the custom_algorithm and algorithm itself for example: ``algorithm_parameters={ 'nth_median': 6, 'sigma': 6, 'window': 5, 'return_anomalies' = True, }`` :type current_skyline_app: str :type parent_pid: int :type timeseries: list :type algorithm_parameters: dict :return: True, False or Non :rtype: boolean Example CUSTOM_ALGORITHMS configuration: 'm66': { 'namespaces': [ 'skyline.analyzer.run_time', 'skyline.analyzer.total_metrics', 'skyline.analyzer.exceptions' ], 'algorithm_source': '/opt/skyline/github/skyline/skyline/custom_algorithms/m66.py', 'algorithm_parameters': { 'nth_median': 6, 'sigma': 6, 'window': 5, 'resolution': 60, 'minimum_sparsity': 0, 'determine_duration': False, 'return_anomalies': True, 'save_plots_to': False, 'save_plots_to_absolute_dir': False, 'filename_prefix': False }, 'max_execution_time': 1.0 'consensus': 1, 'algorithms_allowed_in_consensus': ['m66'], 'run_3sigma_algorithms': False, 'run_before_3sigma': False, 'run_only_if_consensus': False, 'use_with': ['crucible', 'luminosity'], 'debug_logging': False, }, """ # You MUST define the algorithm_name algorithm_name = 'm66' # Define the default state of None and None, anomalous does not default to # False as that is not correct, False is only correct if the algorithm # determines the data point is not anomalous. The same is true for the # anomalyScore. anomalous = None anomalyScore = None return_anomalies = False anomalies = [] anomalies_dict = {} anomalies_dict['algorithm'] = algorithm_name realtime_analysis = False current_logger = None dev_null = None # If you wanted to log, you can but this should only be done during # testing and development def get_log(current_skyline_app): current_skyline_app_logger = current_skyline_app + 'Log' current_logger = logging.getLogger(current_skyline_app_logger) return current_logger start = timer() # Use the algorithm_parameters to determine the sample_period debug_logging = None try: debug_logging = algorithm_parameters['debug_logging'] except: debug_logging = False if debug_logging: try: current_logger = get_log(current_skyline_app) current_logger.debug( 'debug :: %s :: debug_logging enabled with algorithm_parameters - %s' % (algorithm_name, str(algorithm_parameters))) except Exception as e: # This except pattern MUST be used in ALL custom algortihms to # facilitate the traceback from any errors. The algorithm we want to # run super fast and without spamming the log with lots of errors. # But we do not want the function returning and not reporting # anything to the log, so the pythonic except is used to "sample" any # algorithm errors to a tmp file and report once per run rather than # spewing tons of errors into the log e.g. analyzer.log dev_null = e record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) # Return None and None as the algorithm could not determine True or False del dev_null if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) # Allow the m66 parameters to be passed in the algorithm_parameters window = 6 try: window = algorithm_parameters['window'] except KeyError: window = 6 except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e nth_median = 6 try: nth_median = algorithm_parameters['nth_median'] except KeyError: nth_median = 6 except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e n_sigma = 6 try: n_sigma = algorithm_parameters['sigma'] except KeyError: n_sigma = 6 except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e resolution = 0 try: resolution = algorithm_parameters['resolution'] except KeyError: resolution = 0 except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e determine_duration = False try: determine_duration = algorithm_parameters['determine_duration'] except KeyError: determine_duration = False except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e minimum_sparsity = 0 try: minimum_sparsity = algorithm_parameters['minimum_sparsity'] except KeyError: minimum_sparsity = 0 except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e shift_to_start_of_window = True try: shift_to_start_of_window = algorithm_parameters[ 'shift_to_start_of_window'] except KeyError: shift_to_start_of_window = True except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e save_plots_to = False try: save_plots_to = algorithm_parameters['save_plots_to'] except KeyError: save_plots_to = False except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e save_plots_to_absolute_dir = False try: save_plots_to_absolute_dir = algorithm_parameters[ 'save_plots_to_absolute_dir'] except KeyError: save_plots_to_absolute_dir = False except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e filename_prefix = False try: filename_prefix = algorithm_parameters['filename_prefix'] except KeyError: filename_prefix = False except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e if debug_logging: current_logger.debug('debug :: algorithm_parameters :: %s' % (str(algorithm_parameters))) return_anomalies = False try: return_anomalies = algorithm_parameters['return_anomalies'] except KeyError: return_anomalies = False except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e try: realtime_analysis = algorithm_parameters['realtime_analysis'] except KeyError: realtime_analysis = False except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e save_plots_to = False try: save_plots_to = algorithm_parameters['save_plots_to'] except KeyError: save_plots_to = False except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e save_plots_to_absolute_dir = False try: save_plots_to_absolute_dir = algorithm_parameters[ 'save_plots_to_absolute_dir'] except KeyError: save_plots_to = False except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e filename_prefix = False try: filename_prefix = algorithm_parameters['filename_prefix'] except KeyError: filename_prefix = False except Exception as e: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) dev_null = e try: base_name = algorithm_parameters['base_name'] except Exception as e: # This except pattern MUST be used in ALL custom algortihms to # facilitate the traceback from any errors. The algorithm we want to # run super fast and without spamming the log with lots of errors. # But we do not want the function returning and not reporting # anything to the log, so the pythonic except is used to "sample" any # algorithm errors to a tmp file and report once per run rather than # spewing tons of errors into the log e.g. analyzer.log record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) # Return None and None as the algorithm could not determine True or False dev_null = e del dev_null if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (False, None, anomalies) return (False, None) if debug_logging: current_logger.debug('debug :: %s :: base_name - %s' % (algorithm_name, str(base_name))) anomalies_dict['metric'] = base_name anomalies_dict['anomalies'] = {} use_bottleneck = True if save_plots_to: use_bottleneck = False if use_bottleneck: import bottleneck as bn # ALWAYS WRAP YOUR ALGORITHM IN try and the BELOW except try: start_preprocessing = timer() # INFO: Sorting time series of 10079 data points took 0.002215 seconds timeseries = sorted(timeseries, key=lambda x: x[0]) if debug_logging: current_logger.debug('debug :: %s :: time series of length - %s' % (algorithm_name, str(len(timeseries)))) # Testing the data to ensure it meets minimum requirements, in the case # of Skyline's use of the m66 algorithm this means that: # - the time series must have at least 75% of its full_duration do_not_use_sparse_data = False if current_skyline_app == 'luminosity': do_not_use_sparse_data = True if minimum_sparsity == 0: do_not_use_sparse_data = False total_period = 0 total_datapoints = 0 calculate_variables = False if do_not_use_sparse_data: calculate_variables = True if determine_duration: calculate_variables = True if calculate_variables: try: start_timestamp = int(timeseries[0][0]) end_timestamp = int(timeseries[-1][0]) total_period = end_timestamp - start_timestamp total_datapoints = len(timeseries) except SystemExit as e: if debug_logging: current_logger.debug( 'debug_logging :: %s :: SystemExit called, exiting - %s' % (algorithm_name, e)) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) except: traceback_msg = traceback.format_exc() record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg) if debug_logging: current_logger.error(traceback_msg) current_logger.error( 'error :: debug_logging :: %s :: failed to determine total_period and total_datapoints' % (algorithm_name)) timeseries = [] if not timeseries: if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) if current_skyline_app == 'analyzer': # Default for analyzer at required period to 18 hours period_required = int(FULL_DURATION * 0.75) else: # Determine from timeseries if total_period < FULL_DURATION: period_required = int(FULL_DURATION * 0.75) else: period_required = int(total_period * 0.75) if determine_duration: period_required = int(total_period * 0.75) if do_not_use_sparse_data: # If the time series does not have 75% of its full_duration it does # not have sufficient data to sample try: if total_period < period_required: if debug_logging: current_logger.debug( 'debug :: %s :: time series does not have sufficient data' % (algorithm_name)) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) except SystemExit as e: if debug_logging: current_logger.debug( 'debug_logging :: %s :: SystemExit called, exiting - %s' % (algorithm_name, e)) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) except: traceback_msg = traceback.format_exc() record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg) if debug_logging: current_logger.error(traceback_msg) current_logger.error( 'error :: debug_logging :: %s :: falied to determine if time series has sufficient data' % (algorithm_name)) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) # If the time series does not have 75% of its full_duration # datapoints it does not have sufficient data to sample # Determine resolution from the last 30 data points # INFO took 0.002060 seconds if not resolution: resolution_timestamps = [] metric_resolution = False for metric_datapoint in timeseries[-30:]: timestamp = int(metric_datapoint[0]) resolution_timestamps.append(timestamp) timestamp_resolutions = [] if resolution_timestamps: last_timestamp = None for timestamp in resolution_timestamps: if last_timestamp: resolution = timestamp - last_timestamp timestamp_resolutions.append(resolution) last_timestamp = timestamp else: last_timestamp = timestamp try: del resolution_timestamps except: pass if timestamp_resolutions: try: timestamp_resolutions_count = Counter( timestamp_resolutions) ordered_timestamp_resolutions_count = timestamp_resolutions_count.most_common( ) metric_resolution = int( ordered_timestamp_resolutions_count[0][0]) except SystemExit as e: if debug_logging: current_logger.debug( 'debug_logging :: %s :: SystemExit called, exiting - %s' % (algorithm_name, e)) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) except: traceback_msg = traceback.format_exc() record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg) if debug_logging: current_logger.error(traceback_msg) current_logger.error( 'error :: debug_logging :: %s :: failed to determine if time series has sufficient data' % (algorithm_name)) try: del timestamp_resolutions except: pass else: metric_resolution = resolution minimum_datapoints = None if metric_resolution: minimum_datapoints = int(period_required / metric_resolution) if minimum_datapoints: if total_datapoints < minimum_datapoints: if debug_logging: current_logger.debug( 'debug :: %s :: time series does not have sufficient data, minimum_datapoints required is %s and time series has %s' % (algorithm_name, str(minimum_datapoints), str(total_datapoints))) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) # Is the time series fully populated? # full_duration_datapoints = int(full_duration / metric_resolution) total_period_datapoints = int(total_period / metric_resolution) # minimum_percentage_sparsity = 95 minimum_percentage_sparsity = 90 sparsity = int(total_datapoints / (total_period_datapoints / 100)) if sparsity < minimum_percentage_sparsity: if debug_logging: current_logger.debug( 'debug :: %s :: time series does not have sufficient data, minimum_percentage_sparsity required is %s and time series has %s' % (algorithm_name, str(minimum_percentage_sparsity), str(sparsity))) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) if len(set(item[1] for item in timeseries)) == 1: if debug_logging: current_logger.debug( 'debug :: %s :: time series does not have sufficient variability, all the values are the same' % algorithm_name) anomalous = False anomalyScore = 0.0 if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) end_preprocessing = timer() preprocessing_runtime = end_preprocessing - start_preprocessing if debug_logging: current_logger.debug( 'debug :: %s :: preprocessing took %.6f seconds' % (algorithm_name, preprocessing_runtime)) if not timeseries: if debug_logging: current_logger.debug('debug :: %s :: m66 not run as no data' % (algorithm_name)) anomalies = [] if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) if debug_logging: current_logger.debug('debug :: %s :: timeseries length: %s' % (algorithm_name, str(len(timeseries)))) anomalies_dict['timestamp'] = int(timeseries[-1][0]) anomalies_dict['from_timestamp'] = int(timeseries[0][0]) start_analysis = timer() try: # bottleneck is used because it is much faster # pd dataframe method (1445 data point - 24hrs): took 0.077915 seconds # bottleneck method (1445 data point - 24hrs): took 0.005692 seconds # numpy and pandas rolling # 2021-07-30 12:37:31 :: 2827897 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 136.93 seconds # 2021-07-30 12:44:53 :: 2855884 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 148.82 seconds # 2021-07-30 12:48:41 :: 2870822 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 145.62 seconds # 2021-07-30 12:55:00 :: 2893634 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 139.00 seconds # 2021-07-30 12:59:31 :: 2910443 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 144.80 seconds # 2021-07-30 13:02:31 :: 2922928 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 143.35 seconds # 2021-07-30 14:12:56 :: 3132457 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 129.25 seconds # 2021-07-30 14:22:35 :: 3164370 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 125.72 seconds # 2021-07-30 14:28:24 :: 3179687 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 222.43 seconds # 2021-07-30 14:33:45 :: 3179687 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 244.00 seconds # 2021-07-30 14:36:27 :: 3214047 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 141.10 seconds # numpy and bottleneck # 2021-07-30 16:41:52 :: 3585162 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 73.92 seconds # 2021-07-30 16:46:46 :: 3585162 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 68.84 seconds # 2021-07-30 16:51:48 :: 3585162 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 70.55 seconds # numpy and bottleneck (passing resolution and not calculating in m66) # 2021-07-30 16:57:46 :: 3643253 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 65.59 seconds if use_bottleneck: if len(timeseries) < 10: if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) x_np = np.asarray([x[1] for x in timeseries]) # Fast Min-Max scaling data = (x_np - x_np.min()) / (x_np.max() - x_np.min()) # m66 - calculate to nth_median median_count = 0 while median_count < nth_median: median_count += 1 rolling_median_s = bn.move_median(data, window=window) median = rolling_median_s.tolist() data = median if median_count == nth_median: break # m66 - calculate the moving standard deviation for the # nth_median array rolling_std_s = bn.move_std(data, window=window) std_nth_median_array = np.nan_to_num(rolling_std_s, copy=False, nan=0.0, posinf=None, neginf=None) std_nth_median = std_nth_median_array.tolist() if debug_logging: current_logger.debug( 'debug :: %s :: std_nth_median calculated with bn' % (algorithm_name)) else: df = pd.DataFrame(timeseries, columns=['date', 'value']) df['date'] = pd.to_datetime(df['date'], unit='s') datetime_index = pd.DatetimeIndex(df['date'].values) df = df.set_index(datetime_index) df.drop('date', axis=1, inplace=True) original_df = df.copy() # MinMax scale df = (df - df.min()) / (df.max() - df.min()) # window = 6 data = df['value'].tolist() if len(data) < 10: if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) # m66 - calculate to nth_median median_count = 0 while median_count < nth_median: median_count += 1 s = pd.Series(data) rolling_median_s = s.rolling(window).median() median = rolling_median_s.tolist() data = median if median_count == nth_median: break # m66 - calculate the moving standard deviation for the # nth_median array s = pd.Series(data) rolling_std_s = s.rolling(window).std() nth_median_column = 'std_nth_median_%s' % str(nth_median) df[nth_median_column] = rolling_std_s.tolist() std_nth_median = df[nth_median_column].fillna(0).tolist() # m66 - calculate the standard deviation for the entire nth_median # array metric_stddev = np.std(std_nth_median) std_nth_median_n_sigma = [] anomalies_found = False for value in std_nth_median: # m66 - if the value in the 6th median array is > six-sigma of # the metric_stddev the datapoint is anomalous if value > (metric_stddev * n_sigma): std_nth_median_n_sigma.append(1) anomalies_found = True else: std_nth_median_n_sigma.append(0) std_nth_median_n_sigma_column = 'std_median_%s_%s_sigma' % ( str(nth_median), str(n_sigma)) if not use_bottleneck: df[std_nth_median_n_sigma_column] = std_nth_median_n_sigma anomalies = [] # m66 - only label anomalous if the n_sigma triggers are persisted # for (window / 2) if anomalies_found: current_triggers = [] for index, item in enumerate(timeseries): if std_nth_median_n_sigma[index] == 1: current_triggers.append(index) else: if len(current_triggers) > int(window / 2): for trigger_index in current_triggers: # Shift the anomaly back to the beginning of the # window if shift_to_start_of_window: anomalies.append( timeseries[(trigger_index - (window * int( (nth_median / 2))))]) else: anomalies.append(timeseries[trigger_index]) current_triggers = [] # Process any remaining current_triggers if len(current_triggers) > int(window / 2): for trigger_index in current_triggers: # Shift the anomaly back to the beginning of the # window if shift_to_start_of_window: anomalies.append( timeseries[(trigger_index - (window * int( (nth_median / 2))))]) else: anomalies.append(timeseries[trigger_index]) if not anomalies: anomalous = False if anomalies: anomalous = True anomalies_data = [] anomaly_timestamps = [int(item[0]) for item in anomalies] for item in timeseries: if int(item[0]) in anomaly_timestamps: anomalies_data.append(1) else: anomalies_data.append(0) if not use_bottleneck: df['anomalies'] = anomalies_data anomalies_list = [] for ts, value in timeseries: if int(ts) in anomaly_timestamps: anomalies_list.append([int(ts), value]) anomalies_dict['anomalies'][int(ts)] = value if anomalies and save_plots_to: try: from adtk.visualization import plot metric_dir = base_name.replace('.', '/') timestamp_dir = str(int(timeseries[-1][0])) save_path = '%s/%s/%s/%s' % (save_plots_to, algorithm_name, metric_dir, timestamp_dir) if save_plots_to_absolute_dir: save_path = '%s' % save_plots_to anomalies_dict['file_path'] = save_path save_to_file = '%s/%s.%s.png' % (save_path, algorithm_name, base_name) if filename_prefix: save_to_file = '%s/%s.%s.%s.png' % ( save_path, filename_prefix, algorithm_name, base_name) save_to_path = os_path_dirname(save_to_file) title = '%s\n%s - median %s %s-sigma persisted (window=%s)' % ( base_name, algorithm_name, str(nth_median), str(n_sigma), str(window)) if not os_path_exists(save_to_path): try: mkdir_p(save_to_path) except Exception as e: current_logger.error( 'error :: %s :: failed to create dir - %s - %s' % (algorithm_name, save_to_path, e)) if os_path_exists(save_to_path): try: plot(original_df['value'], anomaly=df['anomalies'], anomaly_color='red', title=title, save_to_file=save_to_file) if debug_logging: current_logger.debug( 'debug :: %s :: plot saved to - %s' % (algorithm_name, save_to_file)) anomalies_dict['image'] = save_to_file except Exception as e: current_logger.error( 'error :: %s :: failed to plot - %s - %s' % (algorithm_name, base_name, e)) anomalies_file = '%s/%s.%s.anomalies_list.txt' % ( save_path, algorithm_name, base_name) with open(anomalies_file, 'w') as fh: fh.write(str(anomalies_list)) # os.chmod(anomalies_file, mode=0o644) data_file = '%s/data.txt' % (save_path) with open(data_file, 'w') as fh: fh.write(str(anomalies_dict)) except SystemExit as e: if debug_logging: current_logger.debug( 'debug_logging :: %s :: SystemExit called during save plot, exiting - %s' % (algorithm_name, e)) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) except Exception as e: traceback_msg = traceback.format_exc() record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg) if debug_logging: current_logger.error(traceback_msg) current_logger.error( 'error :: %s :: failed to plot or save anomalies file - %s - %s' % (algorithm_name, base_name, e)) try: del df except: pass except SystemExit as e: if debug_logging: current_logger.debug( 'debug_logging :: %s :: SystemExit called, during analysis, exiting - %s' % (algorithm_name, e)) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) except: traceback_msg = traceback.format_exc() record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg) if debug_logging: current_logger.error(traceback_msg) current_logger.error( 'error :: debug_logging :: %s :: failed to run on ts' % (algorithm_name)) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) end_analysis = timer() analysis_runtime = end_analysis - start_analysis if debug_logging: current_logger.debug( 'debug :: analysis with %s took %.6f seconds' % (algorithm_name, analysis_runtime)) if anomalous: anomalyScore = 1.0 else: anomalyScore = 0.0 if debug_logging: current_logger.info( '%s :: anomalous - %s, anomalyScore - %s' % (algorithm_name, str(anomalous), str(anomalyScore))) if debug_logging: end = timer() processing_runtime = end - start current_logger.info('%s :: completed in %.6f seconds' % (algorithm_name, processing_runtime)) try: del timeseries except: pass if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) except SystemExit as e: if debug_logging: current_logger.debug( 'debug_logging :: %s :: SystemExit called (before StopIteration), exiting - %s' % (algorithm_name, e)) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore) except StopIteration: # This except pattern MUST be used in ALL custom algortihms to # facilitate the traceback from any errors. The algorithm we want to # run super fast and without spamming the log with lots of errors. # But we do not want the function returning and not reporting # anything to the log, so the pythonic except is used to "sample" any # algorithm errors to a tmp file and report once per run rather than # spewing tons of errors into the log e.g. analyzer.log if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (False, None, anomalies) return (False, None) except: record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc()) # Return None and None as the algorithm could not determine True or False if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (False, None, anomalies) return (False, None) if current_skyline_app == 'webapp': return (anomalous, anomalyScore, anomalies, anomalies_dict) if return_anomalies: return (anomalous, anomalyScore, anomalies) return (anomalous, anomalyScore)
def time_move_std(self, dtype, shape, window): bn.move_std(self.arr, window)
def time_move_std(self, dtype, shape, order, axis, window): bn.move_std(self.arr, window, axis=axis)