def get_ROP(connection, sku):
# R= NORMSINV(service level) x Standard Dev of demand for SKU
z = norm.ppf(0.95, loc=0, scale=1)
sql = '''select STR_TO_DATE(concat_ws("-",month(transaction.date),year(transaction.date),"01"), "%m-%Y-%d") as monthofsale,sum(quantity)
from transaction join transaction_sku on transaction.transaction_id = transaction_sku.transaction_id
join product on transaction_sku.sku = product.sku
where product.sku='{}'
and transaction.reason = 'Sale'
group by monthofsale,prod_name
order by transaction.date;
'''.format(sku)
series = read_sql(sql,
con=connection,
parse_dates=0,
index_col=["monthofsale"])
sales = series.values
if len(sales) == 0:
return 0
sigma_d = np.std(sales)
sigma_l = 0.7 #stddev for historical lead time
mu_d = np.mean(sales)
mu_l = 1.8
ROP = (mu_d * mu_l) + (z * (sqrt((mu_l**2 * sigma_d**2) +
(mu_d**2 * sigma_l**2))))
print(sku, sigma_d)
print(mu_d)
return float(ROP)
def detect_ground(profile):
"""Automatic detection of ground (end of snowpack).
:param snowmicropyn.Profile profile: The profile to detect ground in.
:return: Distance where ground was detected.
:rtype: float
"""
force = profile.samples.force
distance = profile.samples.distance
ground = distance.iloc[-1]
if force.max() >= profile.overload:
i_ol = force.argmax()
i_threshhold = np.where(
distance.values >= distance.values[i_ol] - 20)[0][0]
f_mean = np.mean(force.iloc[0:i_threshhold])
f_std = np.std(force.iloc[0:i_threshhold])
threshhold = f_mean + 5 * f_std
while force.iloc[i_ol] > threshhold:
i_ol -= 10
ground = distance.iloc[i_ol]
log.info('Detected ground at {:.3f} mm in profile {}'.format(
ground, profile))
return ground
def calc_step(spatial_res, forces, cone_area=SMP_CONE_AREA):
"""Calculate shot noise parameters for a segment of a profile.
This is the actual implementation of the algorithm described in the
publication and calculates the derived parameters for a single segment of
the profile.
:param spatial_res: Spatial resolution of profile.
:param forces: Iterable containing the force values.
:param cone_area: Projected area of cone (tip) of SnowMicroPen in square
millimeters.
:return: A tuple containing lambda, f0, delta and L.
"""
n = len(forces)
# Mean and variance of force signal
k1 = np.mean(forces)
k2 = np.var(forces)
# Covariance/Autocorrelation (Equation 8 in publication)
c_f = np.correlate(forces, forces, mode='full')
# Equation 11 in publication
delta = -(3. / 2) * c_f[n - 1] / (c_f[n] - c_f[n - 1]) * spatial_res
# Equation 12 in publication
lambda_ = (4. / 3) * (k1**2) / k2 / delta # Intensity
f0 = (3. / 2) * k2 / k1
# According to equation 2 in publication
L = (cone_area / lambda_)**(1. / 3)
return lambda_, f0, delta, L
def fit(self, X, y, n_iter=None):
self.n_iter = self.n_iter if n_iter is None else n_iter
for i in range(self.n_iter):
deltaW1 = self.delta(X, y)
self.W1[0, 0] = self.W1[0, 0] + self.alpha * deltaW1
self.W1[0, 1] = self.W1[0, 1] + self.alpha * np.mean(self.error(X, y))
# self.b1 += self.alpha * deltab1
return self
def validateModel(y_true, y_pred):
classify_report = metrics.classification_report(y_true, y_pred)
confusion_matrix = metrics.confusion_matrix(y_true, y_pred)
overall_accuracy = metrics.accuracy_score(y_true, y_pred)
acc_for_each_class = metrics.precision_score(y_true, y_pred, average=None)
average_accuracy = np.mean(acc_for_each_class)
score = metrics.accuracy_score(y_true, y_pred)
print('classify_report : \n', classify_report)
print('confusion_matrix : \n', confusion_matrix)
print('acc_for_each_class : \n', acc_for_each_class)
print('average_accuracy: {0:f}'.format(average_accuracy))
print('overall_accuracy: {0:f}'.format(overall_accuracy))
print('score: {0:f}'.format(score))
def f1(y_hat, y_true, THRESHOLD=0.5):
'''
y_hat是未经过sigmoid函数激活的
输出的f1为Macro-F1
'''
epsilon = 1e-7
y_hat = y_hat > THRESHOLD
y_hat = np.int8(y_hat)
tp = np.sum(y_hat * y_true, axis=0)
fp = np.sum(y_hat * (1 - y_true), axis=0)
fn = np.sum((1 - y_hat) * y_true, axis=0)
p = tp / (tp + fp + epsilon) # epsilon的意义在于防止分母为0，否则当分母为0时python会报错
r = tp / (tp + fn + epsilon)
f1 = 2 * p * r / (p + r + epsilon)
f1 = np.where(np.isnan(f1), np.zeros_like(f1), f1)
return np.mean(f1)
def detect_surface(profile):
"""Automatic detection of surface (begin of snowpack).
:param profile: The profile to detect surface in.
:return: Distance where surface was detected.
:rtype: float
"""
# Cut off ca. 1 mm
distance = profile.samples.distance.values[250:]
force = profile.samples.force.values[250:]
force = downsample(force, 20)
distance = downsample(distance, 20)
force = smooth(force, 242)
y_grad = np.gradient(force)
y_grad = downsample(y_grad, 3)
x_grad = downsample(distance, 3)
max_force = np.amax(force)
try:
for i in np.arange(100, x_grad.size):
std = np.std(y_grad[:i - 1])
mean = np.mean(y_grad[:i - 1])
if y_grad[i] >= 5 * std + mean:
surface = x_grad[i]
break
if i == x_grad.size - 1:
surface = max_force
log.info('Detected surface at {:.3f} mm in profile {}'.format(
surface, profile))
return surface
except ValueError:
log.warning('Failed to detect surface')
return max_force
def compute_gap(clustering, data, k_max=5, n_references=5):
if len(data.shape) == 1:
data = data.reshape(-1, 1)
reference = np.random.rand(*data.shape)
reference_inertia = []
for k in range(1, k_max + 1):
local_inertia = []
for _ in range(n_references):
clustering.n_clusters = k
assignments = clustering.fit_predict(reference)
local_inertia.append(compute_inertia(assignments, reference))
reference_inertia.append(np.mean(local_inertia))
ondata_inertia = []
for k in range(1, k_max + 1):
clustering.n_clusters = k
assignments = clustering.fit_predict(data)
ondata_inertia.append(compute_inertia(assignments, data))
gap = np.log(reference_inertia) - np.log(ondata_inertia)
return gap, np.log(reference_inertia), np.log(ondata_inertia)
def evaluate_forecasts(test, forecasts, n_lag, n_seq, shangxian, xiaxian):
global time, value
mse = []
for i in range(n_seq):
actual = [row[i] for row in test]
predicted = [forecast[i] for forecast in forecasts]
rmse = sqrt(mean_squared_error(actual, predicted))
#rmse=abs(actual-predicted)
mse.append(rmse)
print('t+%d RMSE: %f' % ((i + 1), rmse))
beta = 0.8
threshold = []
la = 4
for i in range(len(mse) - 1): #平滑处理
mse[i + 1] = mse[i] * beta + (1 - beta) * mse[i + 1]
lb = 5
i = 0
z = 0.2
while i * la + lb < len(mse):
#lb>la
t = np.mean(
mse[i * la:(i + 1) * la],
dtype=float) + z * np.std(mse[i * la:(i + 1) * la], dtype=float)
for j in range(i * la, (i + 1) * la):
threshold.append(t)
i = i + 1
t = np.mean(
mse[i * la:(i + 1) * la],
dtype=float) + z * np.std(mse[i * la:(i + 1) * la], dtype=float)
for j in range(i * la, len(mse)):
threshold.append(t)
plt.figure()
plt.step(list(
range(
len(series.values) - n_test + 5,
5 + len(series.values) - n_test + len(threshold))),
threshold,
label='threshold',
color="#8dd3c7",
where="pre",
lw=2)
# plt.plot(list(range(len(series.values) - n_test+5, 5+len(series.values) - n_test + len(threshold))),threshold,label='threshold')
plt.plot(list(
range(
len(series.values) - n_test + 5,
5 + len(series.values) - n_test + len(mse))),
mse,
label='mse')
plt.legend()
filename = '4.png'
if filename is not None:
plt.savefig(filename)
else:
plt.show()
for i in range(len(forecasts[0])):
if shangxian[i] < series.values[i + len(series.values) - n_test]:
time = i + len(series.values) - n_test + 8
value = abs(shangxian[i] -
(series.values[i + len(series.values) - n_test])) - 6
# print("遥测数据出现异常状态的时间发生在：%d"%(i + len(series.values) - n_test))
#print("异常值偏离正常值%.5f"%(abs(shangxian[i]-(series.values[i+len(series.values) - n_test]))))
break
if xiaxian[i] > series.values[i + len(series.values) - n_test] + 1:
time = i + len(series.values) - n_test + 8
value = abs(xiaxian[i] -
(series.values[i + len(series.values) - n_test])) - 8
# print("遥测数据出现异常状态的时间发生在：%d"%(i + len(series.values) - n_test))
#print("异常值偏离正常值%.5f"%(abs(xiaxian[i]-(series.values[i+len(series.values) - n_test]))))
break
return time, value
def analise_distances(path, number, bigOrSmall):
path = path + "/" + str(number) + "/"
names = []
for i in os.listdir(path):
if bigOrSmall:
name_to_check = "trajectory-generatedPoints-"
else:
name_to_check = "trajectory-generate-aSs-"
# if os.path.isfile(os.path.join(path, i)) and 'trajectory-generatedPoints-' in i and ".zip" in i:
if os.path.isfile(os.path.join(
path, i)) and name_to_check in i and ".zip" in i:
names.append(i)
names = sorted_nicely(names)
numb = 0
total_distances_angle = []
total_distances = []
logging.debug("Analysing Trajectories...")
for i in tqdm.tqdm(range(len(names))):
name = names[i]
trajectories_label, json_file = rean_info(path + name)
# ----------- distance bearings
# real points
lat_real = []
lng_real = []
# generated points
lat_generated = []
lng_generated = []
label_real = []
label_generated = []
label_trajectory = []
# last point trajectory
lat_last = []
lng_last = []
for labels in trajectories_label:
for el in json_file[labels]["real"]:
if el[0] not in lat_real:
lat_real.append(el[0])
lng_real.append(el[1])
label_real.append(json_file[labels]["id"])
for el in json_file[labels]["generated"]:
lat_generated.append(el[0])
lng_generated.append(el[1])
label_generated.append(json_file[labels]["id"])
appo_lat = []
appo_lgn = []
for el in json_file[labels]["trajectory"]:
appo_lat.append(el[0])
appo_lgn.append(el[1])
lat_last.append(appo_lat[len(appo_lat) - 1])
lng_last.append(appo_lgn[len(appo_lgn) - 1])
label_trajectory.append(json_file[labels]["id"])
distance_per_trajectories = {}
# for the trajectories I have
for i in range(len(label_real)):
# compute real bearing for the current trajectory
real_bearing = compute_bearing(lat_last[i], lng_last[i],
lat_real[i], lng_real[i])
# find index of the point generated corresponding to this trajectory
index = [
j for j, x in enumerate(label_generated) if x == label_real[i]
]
index_last_point = [
j for j, x in enumerate(label_trajectory) if x == label_real[i]
]
distances = []
for ind in index:
bearing = compute_bearing(lat_last[index_last_point[0]],
lng_last[index_last_point[0]],
lat_generated[ind],
lng_generated[ind])
distances.append(fabs(bearing - real_bearing))
array = np.array(distances)
distance_per_trajectories.update({
i:
(np.max(array), np.min(array), np.mean(array), np.std(array),
np.median(array))
})
total_distances_angle.append(distance_per_trajectories)
# ----------- distance points
# real points
lat_real = []
lng_real = []
# generated points
lat_generated = []
lng_generated = []
label_real = []
label_generated = []
for labels in trajectories_label:
for el in json_file[labels]["real"]:
if el[0] not in lat_real:
lat_real.append(el[0])
lng_real.append(el[1])
label_real.append(json_file[labels]["id"])
for el in json_file[labels]["generated"]:
if el[0] not in lat_generated:
lat_generated.append(el[0])
lng_generated.append(el[1])
label_generated.append(json_file[labels]["id"])
distance_per_trajectories = {}
# now for every trajectory compute the distance of the generated distance
for i in range(len(label_real)):
index = [
j for j, x in enumerate(label_generated) if x == label_real[i]
]
distances = []
for ind in index:
distances.append(
float(
compute_distance(lat_real[i], lng_real[i],
lat_generated[ind],
lng_generated[ind])))
array = np.array(distances)
distance_per_trajectories.update({
i:
(np.max(array), np.min(array), np.mean(array), np.std(array),
np.median(array))
})
total_distances.append(distance_per_trajectories)
numb += 1
return total_distances, total_distances_angle
x = np.arange(0, len(real_distances))
max_value = []
min_value = []
mean = []
std = []
for el in real_distances:
a = []
b = []
c = []
d = []
for k in el.keys():
a.append(el[k][0])
b.append(el[k][1])
c.append(el[k][2])
d.append(el[k][3])
max_value.append(np.mean(np.array(a)))
min_value.append(np.mean(np.array(b)))
mean.append(np.mean(np.array(c)))
std.append(np.mean(np.array(d)))
# normalise distances
max_median = 20
median_norm = []
std_norm = []
max_value_norm = []
min_value_norm = []
for i in range(len(mean)):
median_norm.append((((mean[i] - 0) * (1 - 0)) /
(max_median - 0)) + 0)
std_norm.append((((std[i] - 0) * (1 - 0)) /
(max_median - 0)) + 0)
index_last_point = [
j for j, x in enumerate(label_trajectory) if x == label_real[i]
]
distances = []
for ind in index:
bearing = computeBearing(lat_last[index_last_point[0]],
lng_last[index_last_point[0]],
lat_generated[ind],
lng_generated[ind])
distances.append(fabs(bearing - real_bearing))
array = np.array(distances)
distance_per_trajectories.update({
i:
(np.max(array), np.min(array), np.mean(array), np.std(array))
})
total_distances.append(distance_per_trajectories)
# # real points
# lat_real = []
# lng_real = []
# for el in json_file[trajectories_label[0]]["real"]:
# lat_real.append(el[0])
# lng_real.append(el[1])
#
# # generated points
# lat_generated = []
# lng_generated = []
# for label in trajectories_label:
# for el in json_file[label]["generated"]:
# lat_generated.append(el[0])
def __init__(self, vector, max_value):
self.vector = np.array(vector)
self.vector = (self.vector - 0) / (max_value - 0)
self.mean = np.mean(self.vector)
self.std = np.std(self.vector)
def compute_inertia(a, X):
W = [np.mean(pairwise_distances(X[a == c, :])) for c in np.unique(a)]
return np.mean(W)
def fit(self, x, y, par_node={}, depth=0):
if par_node is None:
return None
elif len(y) == 0:
return None
elif self.all_same(y):
return {'val': y[0]}
elif depth >= self.max_depth:
return None
else:
col, cutoff, entropy = self.find_best_split_of_all(x, y) # find one split given an information gain
y_left = y[x[:, col] < cutoff]
y_right = y[x[:, col] >= cutoff]
par_node = {'col': iris.feature_names[col], 'index_col': col, 'cutoff': cutoff, 'val': np.round(np.mean(y))}
par_node['left'] = self.fit(x[x[:, col] < cutoff], y_left, {}, depth + 1)
par_node['right'] = self.fit(x[x[:, col] >= cutoff], y_right, {}, depth + 1)
self.depth += 1
self.trees = par_node
return par_node
def run(self):
folders = how_many_fatherFolder(self.path)
folders = [s for s in folders if not re.search('txt', s)]
folders = [s for s in folders if not re.search('jpg', s)]
folders = [s for s in folders if not re.search('png', s)]
for experiemnt in folders:
logging.debug("Folder under analysis -> " + str(experiemnt))
second_path = self.path + experiemnt + "/"
res = how_many_folder(second_path)
folders = [s for s in folders if not re.search('txt', s)]
folders = [s for s in folders if not re.search('jpg', s)]
folders = [s for s in folders if not re.search('png', s)]
num_folder = len(res)
logging.debug("Folder to analise -> " + str(num_folder))
for el in res:
logging.debug("Folder under analysis -> " + str(el))
path_here = second_path + str(el) + "/"
names = []
for i in os.listdir(path_here):
if os.path.isfile(
os.path.join(path_here, i)
) and 'trajectory-generate-aSs-' in i and ".zip" in i:
names.append(i)
names = sorted_nicely(names)
pops = Populations()
# find the trajectories ID and Points
trajectories = self.read_trajectory_info(path_here +
"trajectory.zip")
for tra in trajectories:
pops.add_population(Population(tra))
# analysing the fitness
logging.debug("Analysing the fitness...")
max_agent, max_classifier = self.find_max_values_fitness(
path_here)
agent_generations_info, classifier_generations_info = self.read_fitness(
path_here, max_agent, max_classifier)
x = np.arange(len(agent_generations_info))
y_agent = []
std_agent = []
for element in agent_generations_info:
y_agent.append(element.mean)
std_agent.append(element.std)
y_classifier = []
std_classifier = []
for element in classifier_generations_info:
y_classifier.append(element.mean)
std_classifier.append(element.std)
# print fitnes
self.print_fitnes(x, y_agent, std_agent, y_classifier,
std_classifier, path_here)
total_distances = []
total_distances_msd = []
std_distances = []
last_generations_values = []
logging.debug("Analysing Trajectories...")
for i in tqdm.tqdm(range(len(names))):
name = names[i]
# obtain info from the file
individuals = self.read_info(path_here + name)
if i == len(names) - 1:
for ind in individuals:
for el in ind.array:
last_generations_values.append(el)
msds = []
for ind in individuals:
msds.append(ind.MSD)
total_distances.append(np.mean(np.array(msds)))
std_distances.append(np.std(np.array(msds)))
# store the msd per trajectory
distance_per_trajectories = {}
for j in range(number_of_trajectories):
distances = []
for indiv in individuals:
if indiv.trajectoryID == pops.get_population(
j).tra.trajectoryID:
distances.append(indiv.MSD)
array = np.array(distances)
MSD = (np.sum(array)) / len(array)
distance_per_trajectories.update({j: MSD})
total_distances_msd.append(distance_per_trajectories)
# print graph msd per trajectory
self.print_graph_msd_per_trajectory(total_distances_msd,
path_here)
# print graph total msd
self.print_graph_msd_total(total_distance, std_distances,
path_here)
# save the last value
array = np.array(last_generations_values)
MSD = (np.sum(array)) / len(array)
with open(path_here + "/MSD.txt", "w") as text_file:
text_file.write(str(MSD))
'channel': channel,
'segment': segment,
'accuracy': acc_test,
'f1-score': f1sc_test
},
ignore_index=True)
print(" accuracy train " + str(acc_train) + ' vs ' + str(acc_test) +
' test')
print("f1-score train " + str(f1sc_train) + ' vs ' + str(f1sc_test) +
' test')
df_all.to_csv(csv_file, mode='a', header=False)
df_all = df_all.iloc[0:0]
acc_avr = np.mean(np.array(accuracies))
acc_std = np.std(np.array(accuracies))
f1_avr = np.mean(np.array(f1scores))
f1_std = np.std(np.array(f1scores))
df_results = df_results.append(
{
'bursts': bursts[ind_interval],
'channel': channel,
'segment': segment,
'acc avr': acc_avr,
'acc std_dev': acc_std,
'f1-sc avr': f1_avr,
'f1-sc std_dev': f1_std
},
ignore_index=True)