def get_ROP(connection, sku): # R= NORMSINV(service level) x Standard Dev of demand for SKU z = norm.ppf(0.95, loc=0, scale=1) sql = '''select STR_TO_DATE(concat_ws("-",month(transaction.date),year(transaction.date),"01"), "%m-%Y-%d") as monthofsale,sum(quantity) from transaction join transaction_sku on transaction.transaction_id = transaction_sku.transaction_id join product on transaction_sku.sku = product.sku where product.sku='{}' and transaction.reason = 'Sale' group by monthofsale,prod_name order by transaction.date; '''.format(sku) series = read_sql(sql, con=connection, parse_dates=0, index_col=["monthofsale"]) sales = series.values if len(sales) == 0: return 0 sigma_d = np.std(sales) sigma_l = 0.7 #stddev for historical lead time mu_d = np.mean(sales) mu_l = 1.8 ROP = (mu_d * mu_l) + (z * (sqrt((mu_l**2 * sigma_d**2) + (mu_d**2 * sigma_l**2)))) print(sku, sigma_d) print(mu_d) return float(ROP)
def detect_ground(profile): """Automatic detection of ground (end of snowpack). :param snowmicropyn.Profile profile: The profile to detect ground in. :return: Distance where ground was detected. :rtype: float """ force = profile.samples.force distance = profile.samples.distance ground = distance.iloc[-1] if force.max() >= profile.overload: i_ol = force.argmax() i_threshhold = np.where( distance.values >= distance.values[i_ol] - 20)[0][0] f_mean = np.mean(force.iloc[0:i_threshhold]) f_std = np.std(force.iloc[0:i_threshhold]) threshhold = f_mean + 5 * f_std while force.iloc[i_ol] > threshhold: i_ol -= 10 ground = distance.iloc[i_ol] log.info('Detected ground at {:.3f} mm in profile {}'.format( ground, profile)) return ground
def calc_step(spatial_res, forces, cone_area=SMP_CONE_AREA): """Calculate shot noise parameters for a segment of a profile. This is the actual implementation of the algorithm described in the publication and calculates the derived parameters for a single segment of the profile. :param spatial_res: Spatial resolution of profile. :param forces: Iterable containing the force values. :param cone_area: Projected area of cone (tip) of SnowMicroPen in square millimeters. :return: A tuple containing lambda, f0, delta and L. """ n = len(forces) # Mean and variance of force signal k1 = np.mean(forces) k2 = np.var(forces) # Covariance/Autocorrelation (Equation 8 in publication) c_f = np.correlate(forces, forces, mode='full') # Equation 11 in publication delta = -(3. / 2) * c_f[n - 1] / (c_f[n] - c_f[n - 1]) * spatial_res # Equation 12 in publication lambda_ = (4. / 3) * (k1**2) / k2 / delta # Intensity f0 = (3. / 2) * k2 / k1 # According to equation 2 in publication L = (cone_area / lambda_)**(1. / 3) return lambda_, f0, delta, L
def fit(self, X, y, n_iter=None): self.n_iter = self.n_iter if n_iter is None else n_iter for i in range(self.n_iter): deltaW1 = self.delta(X, y) self.W1[0, 0] = self.W1[0, 0] + self.alpha * deltaW1 self.W1[0, 1] = self.W1[0, 1] + self.alpha * np.mean(self.error(X, y)) # self.b1 += self.alpha * deltab1 return self
def validateModel(y_true, y_pred): classify_report = metrics.classification_report(y_true, y_pred) confusion_matrix = metrics.confusion_matrix(y_true, y_pred) overall_accuracy = metrics.accuracy_score(y_true, y_pred) acc_for_each_class = metrics.precision_score(y_true, y_pred, average=None) average_accuracy = np.mean(acc_for_each_class) score = metrics.accuracy_score(y_true, y_pred) print('classify_report : \n', classify_report) print('confusion_matrix : \n', confusion_matrix) print('acc_for_each_class : \n', acc_for_each_class) print('average_accuracy: {0:f}'.format(average_accuracy)) print('overall_accuracy: {0:f}'.format(overall_accuracy)) print('score: {0:f}'.format(score))
def f1(y_hat, y_true, THRESHOLD=0.5): ''' y_hat是未经过sigmoid函数激活的 输出的f1为Macro-F1 ''' epsilon = 1e-7 y_hat = y_hat > THRESHOLD y_hat = np.int8(y_hat) tp = np.sum(y_hat * y_true, axis=0) fp = np.sum(y_hat * (1 - y_true), axis=0) fn = np.sum((1 - y_hat) * y_true, axis=0) p = tp / (tp + fp + epsilon) # epsilon的意义在于防止分母为0,否则当分母为0时python会报错 r = tp / (tp + fn + epsilon) f1 = 2 * p * r / (p + r + epsilon) f1 = np.where(np.isnan(f1), np.zeros_like(f1), f1) return np.mean(f1)
def detect_surface(profile): """Automatic detection of surface (begin of snowpack). :param profile: The profile to detect surface in. :return: Distance where surface was detected. :rtype: float """ # Cut off ca. 1 mm distance = profile.samples.distance.values[250:] force = profile.samples.force.values[250:] force = downsample(force, 20) distance = downsample(distance, 20) force = smooth(force, 242) y_grad = np.gradient(force) y_grad = downsample(y_grad, 3) x_grad = downsample(distance, 3) max_force = np.amax(force) try: for i in np.arange(100, x_grad.size): std = np.std(y_grad[:i - 1]) mean = np.mean(y_grad[:i - 1]) if y_grad[i] >= 5 * std + mean: surface = x_grad[i] break if i == x_grad.size - 1: surface = max_force log.info('Detected surface at {:.3f} mm in profile {}'.format( surface, profile)) return surface except ValueError: log.warning('Failed to detect surface') return max_force
def compute_gap(clustering, data, k_max=5, n_references=5): if len(data.shape) == 1: data = data.reshape(-1, 1) reference = np.random.rand(*data.shape) reference_inertia = [] for k in range(1, k_max + 1): local_inertia = [] for _ in range(n_references): clustering.n_clusters = k assignments = clustering.fit_predict(reference) local_inertia.append(compute_inertia(assignments, reference)) reference_inertia.append(np.mean(local_inertia)) ondata_inertia = [] for k in range(1, k_max + 1): clustering.n_clusters = k assignments = clustering.fit_predict(data) ondata_inertia.append(compute_inertia(assignments, data)) gap = np.log(reference_inertia) - np.log(ondata_inertia) return gap, np.log(reference_inertia), np.log(ondata_inertia)
def evaluate_forecasts(test, forecasts, n_lag, n_seq, shangxian, xiaxian): global time, value mse = [] for i in range(n_seq): actual = [row[i] for row in test] predicted = [forecast[i] for forecast in forecasts] rmse = sqrt(mean_squared_error(actual, predicted)) #rmse=abs(actual-predicted) mse.append(rmse) print('t+%d RMSE: %f' % ((i + 1), rmse)) beta = 0.8 threshold = [] la = 4 for i in range(len(mse) - 1): #平滑处理 mse[i + 1] = mse[i] * beta + (1 - beta) * mse[i + 1] lb = 5 i = 0 z = 0.2 while i * la + lb < len(mse): #lb>la t = np.mean( mse[i * la:(i + 1) * la], dtype=float) + z * np.std(mse[i * la:(i + 1) * la], dtype=float) for j in range(i * la, (i + 1) * la): threshold.append(t) i = i + 1 t = np.mean( mse[i * la:(i + 1) * la], dtype=float) + z * np.std(mse[i * la:(i + 1) * la], dtype=float) for j in range(i * la, len(mse)): threshold.append(t) plt.figure() plt.step(list( range( len(series.values) - n_test + 5, 5 + len(series.values) - n_test + len(threshold))), threshold, label='threshold', color="#8dd3c7", where="pre", lw=2) # plt.plot(list(range(len(series.values) - n_test+5, 5+len(series.values) - n_test + len(threshold))),threshold,label='threshold') plt.plot(list( range( len(series.values) - n_test + 5, 5 + len(series.values) - n_test + len(mse))), mse, label='mse') plt.legend() filename = '4.png' if filename is not None: plt.savefig(filename) else: plt.show() for i in range(len(forecasts[0])): if shangxian[i] < series.values[i + len(series.values) - n_test]: time = i + len(series.values) - n_test + 8 value = abs(shangxian[i] - (series.values[i + len(series.values) - n_test])) - 6 # print("遥测数据出现异常状态的时间发生在:%d"%(i + len(series.values) - n_test)) #print("异常值偏离正常值%.5f"%(abs(shangxian[i]-(series.values[i+len(series.values) - n_test])))) break if xiaxian[i] > series.values[i + len(series.values) - n_test] + 1: time = i + len(series.values) - n_test + 8 value = abs(xiaxian[i] - (series.values[i + len(series.values) - n_test])) - 8 # print("遥测数据出现异常状态的时间发生在:%d"%(i + len(series.values) - n_test)) #print("异常值偏离正常值%.5f"%(abs(xiaxian[i]-(series.values[i+len(series.values) - n_test])))) break return time, value
def analise_distances(path, number, bigOrSmall): path = path + "/" + str(number) + "/" names = [] for i in os.listdir(path): if bigOrSmall: name_to_check = "trajectory-generatedPoints-" else: name_to_check = "trajectory-generate-aSs-" # if os.path.isfile(os.path.join(path, i)) and 'trajectory-generatedPoints-' in i and ".zip" in i: if os.path.isfile(os.path.join( path, i)) and name_to_check in i and ".zip" in i: names.append(i) names = sorted_nicely(names) numb = 0 total_distances_angle = [] total_distances = [] logging.debug("Analysing Trajectories...") for i in tqdm.tqdm(range(len(names))): name = names[i] trajectories_label, json_file = rean_info(path + name) # ----------- distance bearings # real points lat_real = [] lng_real = [] # generated points lat_generated = [] lng_generated = [] label_real = [] label_generated = [] label_trajectory = [] # last point trajectory lat_last = [] lng_last = [] for labels in trajectories_label: for el in json_file[labels]["real"]: if el[0] not in lat_real: lat_real.append(el[0]) lng_real.append(el[1]) label_real.append(json_file[labels]["id"]) for el in json_file[labels]["generated"]: lat_generated.append(el[0]) lng_generated.append(el[1]) label_generated.append(json_file[labels]["id"]) appo_lat = [] appo_lgn = [] for el in json_file[labels]["trajectory"]: appo_lat.append(el[0]) appo_lgn.append(el[1]) lat_last.append(appo_lat[len(appo_lat) - 1]) lng_last.append(appo_lgn[len(appo_lgn) - 1]) label_trajectory.append(json_file[labels]["id"]) distance_per_trajectories = {} # for the trajectories I have for i in range(len(label_real)): # compute real bearing for the current trajectory real_bearing = compute_bearing(lat_last[i], lng_last[i], lat_real[i], lng_real[i]) # find index of the point generated corresponding to this trajectory index = [ j for j, x in enumerate(label_generated) if x == label_real[i] ] index_last_point = [ j for j, x in enumerate(label_trajectory) if x == label_real[i] ] distances = [] for ind in index: bearing = compute_bearing(lat_last[index_last_point[0]], lng_last[index_last_point[0]], lat_generated[ind], lng_generated[ind]) distances.append(fabs(bearing - real_bearing)) array = np.array(distances) distance_per_trajectories.update({ i: (np.max(array), np.min(array), np.mean(array), np.std(array), np.median(array)) }) total_distances_angle.append(distance_per_trajectories) # ----------- distance points # real points lat_real = [] lng_real = [] # generated points lat_generated = [] lng_generated = [] label_real = [] label_generated = [] for labels in trajectories_label: for el in json_file[labels]["real"]: if el[0] not in lat_real: lat_real.append(el[0]) lng_real.append(el[1]) label_real.append(json_file[labels]["id"]) for el in json_file[labels]["generated"]: if el[0] not in lat_generated: lat_generated.append(el[0]) lng_generated.append(el[1]) label_generated.append(json_file[labels]["id"]) distance_per_trajectories = {} # now for every trajectory compute the distance of the generated distance for i in range(len(label_real)): index = [ j for j, x in enumerate(label_generated) if x == label_real[i] ] distances = [] for ind in index: distances.append( float( compute_distance(lat_real[i], lng_real[i], lat_generated[ind], lng_generated[ind]))) array = np.array(distances) distance_per_trajectories.update({ i: (np.max(array), np.min(array), np.mean(array), np.std(array), np.median(array)) }) total_distances.append(distance_per_trajectories) numb += 1 return total_distances, total_distances_angle
x = np.arange(0, len(real_distances)) max_value = [] min_value = [] mean = [] std = [] for el in real_distances: a = [] b = [] c = [] d = [] for k in el.keys(): a.append(el[k][0]) b.append(el[k][1]) c.append(el[k][2]) d.append(el[k][3]) max_value.append(np.mean(np.array(a))) min_value.append(np.mean(np.array(b))) mean.append(np.mean(np.array(c))) std.append(np.mean(np.array(d))) # normalise distances max_median = 20 median_norm = [] std_norm = [] max_value_norm = [] min_value_norm = [] for i in range(len(mean)): median_norm.append((((mean[i] - 0) * (1 - 0)) / (max_median - 0)) + 0) std_norm.append((((std[i] - 0) * (1 - 0)) / (max_median - 0)) + 0)
index_last_point = [ j for j, x in enumerate(label_trajectory) if x == label_real[i] ] distances = [] for ind in index: bearing = computeBearing(lat_last[index_last_point[0]], lng_last[index_last_point[0]], lat_generated[ind], lng_generated[ind]) distances.append(fabs(bearing - real_bearing)) array = np.array(distances) distance_per_trajectories.update({ i: (np.max(array), np.min(array), np.mean(array), np.std(array)) }) total_distances.append(distance_per_trajectories) # # real points # lat_real = [] # lng_real = [] # for el in json_file[trajectories_label[0]]["real"]: # lat_real.append(el[0]) # lng_real.append(el[1]) # # # generated points # lat_generated = [] # lng_generated = [] # for label in trajectories_label: # for el in json_file[label]["generated"]: # lat_generated.append(el[0])
def __init__(self, vector, max_value): self.vector = np.array(vector) self.vector = (self.vector - 0) / (max_value - 0) self.mean = np.mean(self.vector) self.std = np.std(self.vector)
def compute_inertia(a, X): W = [np.mean(pairwise_distances(X[a == c, :])) for c in np.unique(a)] return np.mean(W)
def fit(self, x, y, par_node={}, depth=0): if par_node is None: return None elif len(y) == 0: return None elif self.all_same(y): return {'val': y[0]} elif depth >= self.max_depth: return None else: col, cutoff, entropy = self.find_best_split_of_all(x, y) # find one split given an information gain y_left = y[x[:, col] < cutoff] y_right = y[x[:, col] >= cutoff] par_node = {'col': iris.feature_names[col], 'index_col': col, 'cutoff': cutoff, 'val': np.round(np.mean(y))} par_node['left'] = self.fit(x[x[:, col] < cutoff], y_left, {}, depth + 1) par_node['right'] = self.fit(x[x[:, col] >= cutoff], y_right, {}, depth + 1) self.depth += 1 self.trees = par_node return par_node
def run(self): folders = how_many_fatherFolder(self.path) folders = [s for s in folders if not re.search('txt', s)] folders = [s for s in folders if not re.search('jpg', s)] folders = [s for s in folders if not re.search('png', s)] for experiemnt in folders: logging.debug("Folder under analysis -> " + str(experiemnt)) second_path = self.path + experiemnt + "/" res = how_many_folder(second_path) folders = [s for s in folders if not re.search('txt', s)] folders = [s for s in folders if not re.search('jpg', s)] folders = [s for s in folders if not re.search('png', s)] num_folder = len(res) logging.debug("Folder to analise -> " + str(num_folder)) for el in res: logging.debug("Folder under analysis -> " + str(el)) path_here = second_path + str(el) + "/" names = [] for i in os.listdir(path_here): if os.path.isfile( os.path.join(path_here, i) ) and 'trajectory-generate-aSs-' in i and ".zip" in i: names.append(i) names = sorted_nicely(names) pops = Populations() # find the trajectories ID and Points trajectories = self.read_trajectory_info(path_here + "trajectory.zip") for tra in trajectories: pops.add_population(Population(tra)) # analysing the fitness logging.debug("Analysing the fitness...") max_agent, max_classifier = self.find_max_values_fitness( path_here) agent_generations_info, classifier_generations_info = self.read_fitness( path_here, max_agent, max_classifier) x = np.arange(len(agent_generations_info)) y_agent = [] std_agent = [] for element in agent_generations_info: y_agent.append(element.mean) std_agent.append(element.std) y_classifier = [] std_classifier = [] for element in classifier_generations_info: y_classifier.append(element.mean) std_classifier.append(element.std) # print fitnes self.print_fitnes(x, y_agent, std_agent, y_classifier, std_classifier, path_here) total_distances = [] total_distances_msd = [] std_distances = [] last_generations_values = [] logging.debug("Analysing Trajectories...") for i in tqdm.tqdm(range(len(names))): name = names[i] # obtain info from the file individuals = self.read_info(path_here + name) if i == len(names) - 1: for ind in individuals: for el in ind.array: last_generations_values.append(el) msds = [] for ind in individuals: msds.append(ind.MSD) total_distances.append(np.mean(np.array(msds))) std_distances.append(np.std(np.array(msds))) # store the msd per trajectory distance_per_trajectories = {} for j in range(number_of_trajectories): distances = [] for indiv in individuals: if indiv.trajectoryID == pops.get_population( j).tra.trajectoryID: distances.append(indiv.MSD) array = np.array(distances) MSD = (np.sum(array)) / len(array) distance_per_trajectories.update({j: MSD}) total_distances_msd.append(distance_per_trajectories) # print graph msd per trajectory self.print_graph_msd_per_trajectory(total_distances_msd, path_here) # print graph total msd self.print_graph_msd_total(total_distance, std_distances, path_here) # save the last value array = np.array(last_generations_values) MSD = (np.sum(array)) / len(array) with open(path_here + "/MSD.txt", "w") as text_file: text_file.write(str(MSD))
'channel': channel, 'segment': segment, 'accuracy': acc_test, 'f1-score': f1sc_test }, ignore_index=True) print(" accuracy train " + str(acc_train) + ' vs ' + str(acc_test) + ' test') print("f1-score train " + str(f1sc_train) + ' vs ' + str(f1sc_test) + ' test') df_all.to_csv(csv_file, mode='a', header=False) df_all = df_all.iloc[0:0] acc_avr = np.mean(np.array(accuracies)) acc_std = np.std(np.array(accuracies)) f1_avr = np.mean(np.array(f1scores)) f1_std = np.std(np.array(f1scores)) df_results = df_results.append( { 'bursts': bursts[ind_interval], 'channel': channel, 'segment': segment, 'acc avr': acc_avr, 'acc std_dev': acc_std, 'f1-sc avr': f1_avr, 'f1-sc std_dev': f1_std }, ignore_index=True)