def sax_similarity(data, seq_len): from tslearn.piecewise import SymbolicAggregateApproximation print('|--- Calculating the pairwise distance!') ppa_segmet = int(data.shape[0] / seq_len) sax_ins = SymbolicAggregateApproximation(n_segments=ppa_segmet, alphabet_size_avg=10) sax_repre = sax_ins.fit_transform(np.transpose(data)) sax_mx_dist = np.zeros(shape=(data.shape[1], data.shape[1])) for i in range(data.shape[1]): for j in range(data.shape[1]): sax_mx_dist[i, j] = sax_ins.distance_sax(sax_repre[i], sax_repre[j]) return sax_mx_dist
def main(): #FOR NOAA DB influx_url = "http://localhost:8086/query?db=" + dbname + \ "&epoch=ms&q=SELECT+%22water_level%22+FROM+%22h2o_feet%22+WHERE+time+%3E%3D+1440658277944ms+and+time+%3C%3D+1441435694328ms" r = requests.get(influx_url) json_dict = json.loads(r.content) data = json_dict["results"][0]["series"][0]["values"] print(data[0:5]) ## #NOTE:just for NOAA h2o_feet time_interval = data[2][0] - data[0][0] print("time interval:", time_interval) lst2 = [item[1] for item in data] n_segments = len(lst2) print(max(lst2),min(lst2)) original_data_size = len(lst2) print("original data size:", original_data_size) alphabet_size_avg = math.ceil(max(lst2)-min(lst2)) print("alphabet size avg:", alphabet_size_avg) ## a list of sample ratios. ## Want to select the min ratio within the similarity range. ratiolist = [0.025,0.05,0.1,0.15,0.2,0.3,0.4,0.5,0.6] sizelist = [] distlist = [] for ratio in ratiolist: print() print("ratio:",ratio) #generate sample data sample_size = math.floor(original_data_size * ratio) sizelist.append(sample_size) print("sample_size:",sample_size) #NOAA DB: h2o_feet sample_url = "http://localhost:8086/query?db=" + dbname + \ "&epoch=ms&q=SELECT+sample%28%22water_level%22%2C"+str(sample_size) + \ "%29+FROM+%22h2o_feet%22+WHERE+time+%3E%3D+1440658277944ms+and+time+%3C%3D+1441435694328ms" r2 = requests.get(sample_url) json_dict2 = json.loads(r2.content) sampled_data = json_dict2["results"][0]["series"][0]["values"] # [[time, value], ...] sample = [item[1] for item in sampled_data] #[value,...] #fill the sample data with a linear model start_x = data[0][0] end_x = data[-1][0] current_x = start_x current_loc = 0 slope = (sampled_data[current_loc][1]-sampled_data[current_loc+2][1])\ /(sampled_data[current_loc][0] - sampled_data[current_loc+2][0]) ##NOTE! intersection = sampled_data[current_loc][1]-slope*sampled_data[current_loc][0] sample_fit = [] end_sample_x = sampled_data[-1][0] while current_x <= end_sample_x: if current_x >= sampled_data[current_loc+1][0] and current_loc+1 < len(sampled_data)-2: ##NOTE: -2 !! CHANGE TO -1 LATER current_loc+=1 ##NOTE: +2 was just for h2o_feet if (sampled_data[current_loc][0] - sampled_data[current_loc+1][0]) == 0: slope = (sampled_data[current_loc] [1]-sampled_data[current_loc+1][1]) \ /(sampled_data[current_loc][0] - sampled_data[current_loc+2][0]) else: slope = (sampled_data[current_loc] [1]-sampled_data[current_loc+1][1]) \ /(sampled_data[current_loc][0] - sampled_data[current_loc+2][0]) intersection = sampled_data[current_loc][1] - slope*sampled_data[current_loc][0] sample_fit.append([current_x, slope*current_x+intersection]) current_x += time_interval #1000ms #chop the original data to match the linear fit sample data. chopped_data = [] for item in data: if item[0]>= sample_fit[0][0] and item[0] <= sample_fit[-1][0]: chopped_data.append(item) print("size of chopped_data:",len(chopped_data)) chopped_lst2 = [item[1] for item in chopped_data] chopped_len = len(chopped_lst2) #build a sax model for chopped original data sax = SymbolicAggregateApproximation(chopped_len,alphabet_size_avg) scalar = TimeSeriesScalerMeanVariance(mu=0., std=1.) sdb = scalar.fit_transform(chopped_lst2) sax_data = sax.transform(sdb) s3 = sax.fit_transform(sax_data) #build a sax model for linear-fit sampled data sample_fit_extract = [item[1] for item in sample_fit] fit_sample_data = scalar.fit_transform(sample_fit_extract) sax_sample_data = sax.transform(fit_sample_data) s4 = sax.fit_transform(sax_sample_data) #compute the distance between to dataset to calculate the similarity dist = sax.distance_sax(s3[0], s4[0]) print("distance:", dist) norm_dist = 1000*dist/chopped_len distlist.append(norm_dist) print("normalized distance: {:.4f}".format(norm_dist)) plotdist(ratiolist,distlist)
rows, cols = data.shape print(rows, cols) # PAA transform (and inverse transform) of the data n_paa_segments = 10 paa = PiecewiseAggregateApproximation(n_segments=n_paa_segments) paa_dataset_inv = paa.inverse_transform(paa.fit_transform(dataset)) # SAX transform n_sax_symbols = 8 sax = SymbolicAggregateApproximation(n_segments=n_paa_segments, alphabet_size_avg=n_sax_symbols) sax_dataset_inv = sax.inverse_transform(sax.fit_transform(dataset)) sax_data = sax.fit_transform(data) print(sax_data) d = sax.distance_sax(sax_data[0], sax_data[1]) print(d) exit() # 1d-SAX transform n_sax_symbols_avg = 8 n_sax_symbols_slope = 8 one_d_sax = OneD_SymbolicAggregateApproximation( n_segments=n_paa_segments, alphabet_size_avg=n_sax_symbols_avg, alphabet_size_slope=n_sax_symbols_slope) one_d_sax_dataset_inv = one_d_sax.inverse_transform( one_d_sax.fit_transform(dataset)) plt.figure() plt.subplot(2, 2, 1) # First, raw time series
def main(): # fetch original data #for test_quarter db ## influx_url = "http://localhost:8086/query?db=" + dbname + \ ## "&epoch=ms&q=SELECT+%22degrees%22+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1546329600000ms+and+time+%3C%3D+1546329900000ms" #FOR NOAA DB ## influx_url = "http://localhost:8086/query?db=" + dbname + \ ## "&epoch=ms&q=SELECT+%22degrees%22+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1439856000000ms+and+time+%3C%3D+1439992520000ms+and%28%22location%22+%3D+%27santa_monica%27%29" # For test3 influx_url = "http://localhost:8086/query?db=" + dbname + \ "&epoch=ms&q=SELECT+%22degrees%22+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1546355705400ms+and+time+%3C%3D+1548969305400ms" r = requests.get(influx_url) json_dict = json.loads(r.content) data = json_dict["results"][0]["series"][0]["values"] ## print(data[0]) ## print(data[1]) time_interval = data[1][0] - data[0][0] # consistant time interval print("time interval: ", time_interval) lst2 = [item[1] for item in data] n_segments = len(lst2) print("original data size", len(lst2)) alphabet_size_avg = 20 #generate sample data sample_size = 20 ## sample_url = "http://localhost:8086/query?db="+dbname+\ ## "&epoch=ms&q=SELECT+sample%28%22degrees%22%2C" + str(sample_size) +\ ## "%29+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1546329600000ms+and+time+%3C%3D+1546329900000ms" # test3 sample (sin pattern) sample_url = "http://localhost:8086/query?db="+dbname+\ "&epoch=ms&q=SELECT+sample%28%22degrees%22%2C" + str(sample_size) +\ "%29+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1546355705400ms+and+time+%3C%3D+1548969305400ms" ## sample_url = "http://localhost:8086/query?db=" + dbname + \ ## "&epoch=ms&q=SELECT+sample%28%22degrees%22%2C" + str(sample_size) +\ ## "%29+FROM+%22h2o_temperature%22+WHERE+time+%3E%3D+1439856000000ms+and+time+%3C%3D+1442612520000ms+and%28%22location%22+%3D+%27santa_monica%27%29" r2 = requests.get(sample_url) json_dict2 = json.loads(r2.content) sampled_data = json_dict2["results"][0]["series"][0][ "values"] # [[time, value], ...] print("sample length") print(len(sampled_data)) sample = [item[1] for item in sampled_data] #[value,...] #fill the sample data with a linear model start_x = data[0][0] end_x = data[-1][0] current_x = start_x current_loc = 0 slope = (sampled_data[current_loc][1]-sampled_data[current_loc+1][1])\ /(sampled_data[current_loc][0] - sampled_data[current_loc+1][0]) intersection = sampled_data[current_loc][ 1] - slope * sampled_data[current_loc][0] sample_fit = [] end_sample_x = sampled_data[-1][0] while current_x <= end_sample_x: if current_x >= sampled_data[ current_loc + 1][0] and current_loc + 1 < len(sampled_data) - 1: current_loc += 1 slope = (sampled_data[current_loc] [1]-sampled_data[current_loc+1][1]) \ /(sampled_data[current_loc][0] - sampled_data[current_loc+1][0]) intersection = sampled_data[current_loc][ 1] - slope * sampled_data[current_loc][0] sample_fit.append([current_x, slope * current_x + intersection]) current_x += time_interval #1000ms #chop the original data to match the linear fit sample data. chopped_data = [] for item in data: if item[0] >= sample_fit[0][0] and item[0] <= sample_fit[-1][0]: chopped_data.append(item) print("len") print(len(sample_fit), len(chopped_data)) chopped_lst2 = [item[1] for item in chopped_data] chopped_len = len(chopped_lst2) #build a sax model for chopped original data sax = SymbolicAggregateApproximation(chopped_len, alphabet_size_avg) scalar = TimeSeriesScalerMeanVariance(mu=0., std=1.) sdb = scalar.fit_transform(chopped_lst2) sax_data = sax.transform(sdb) s3 = sax.fit_transform(sax_data) #build a sax model for linear-fit sampled data sample_fit_extract = [item[1] for item in sample_fit] fit_sample_data = scalar.fit_transform(sample_fit_extract) sax_sample_data = sax.transform(fit_sample_data) s4 = sax.fit_transform(sax_sample_data) #compute the distance between to dataset to calculate the similarity print("distance") dist = sax.distance_sax(s3[0], s4[0]) print(dist) print("normalized distance") print(dist / chopped_len) #plot the three dataset plot(sample_fit, sampled_data, lst2)