def __store_estimate(self):
for i_method in range(0, self.n_methods):
self.estimate[i_method, :]\
= np.unravel_index(
np.nanargmin(self.criterion_result[i_method, :, :]),
self.criterion_result[i_method, :, :].shape)
self.best_actvt[i_method]\
= self.actvt_result\
[self.estimate[i_method, 0]][self.estimate[i_method, 1]]
self.best_base[i_method]\
= self.base_result\
[self.estimate[i_method, 0]][self.estimate[i_method, 1]]
self.best_completion[i_method]\
= self.completion_result\
[self.estimate[i_method, 0]][self.estimate[i_method, 1]]
if not (self.true_width == None):
for i_method in range(0, self.n_methods):
self.estimate_given_width[i_method]\
= np.nanargmin(self.criterion_result[i_method, self.true_width, :])
self.best_actvt_given_width[i_method]\
= self.actvt_result\
[self.true_width][self.estimate_given_width[i_method]]
self.best_base_given_width[i_method]\
= self.base_result\
[self.true_width][self.estimate_given_width[i_method]]
self.best_completion_given_width[i_method]\
= self.completion_result\
[self.true_width][self.estimate_given_width[i_method]]
def getPerpContourInd(skeleton, skel_ind, contour_side1, contour_side2, contour_width):
#get the closest point in the contour from a line perpedicular to the skeleton.
#get the slop of a line perpendicular to the keleton
dR = skeleton[skel_ind+1] - skeleton[skel_ind-1]
#m = dR[1]/dR[0]; M = -1/m
a = -dR[0]
b = +dR[1]
c = b*skeleton[skel_ind,1]-a*skeleton[skel_ind,0]
max_width_squared = np.max(contour_width)**2
#modified from https://en.wikipedia.org/wiki/Distance_from_a_point_to_a_line
#a = M, b = -1
#make sure we are not selecting a point that get traversed by a coiled worm
dist2cnt1 = np.sum((contour_side1-skeleton[skel_ind])**2,axis=1)
d1 = np.abs(a*contour_side1[:,0] - b*contour_side1[:,1]+ c)
d1[dist2cnt1>max_width_squared] = np.nan
cnt1_ind = np.nanargmin(d1)
dist2cnt2 = np.sum((contour_side2-skeleton[skel_ind])**2,axis=1)
d2 = np.abs(a*contour_side2[:,0] - b*contour_side2[:,1]+ c)
d2[dist2cnt2>max_width_squared] = np.nan
cnt2_ind = np.nanargmin(d2)
return cnt1_ind, cnt2_ind
def get_spans(adult_df, a_variable, method = 'young', fraction = 0.5, reverse_direction = False): ''' Get healthspans as defined by young adult function for a_variable. ''' data_values = adult_df.mloc(measures = [a_variable])[:, 0, :] if reverse_direction: data_values = data_values*-1 if method == 'young': young_adult = np.abs(adult_df.mloc(measures = ['egg_age'])[:, 0, :]) young_adult = np.nanargmin(young_adult, axis = 1) cutoff_value = np.array([data_values[i, young_adult[i]] for i in range(0, young_adult.shape[0])]) cutoff_value = np.mean(cutoff_value)*fraction if method == 'overall_time': all_data = np.ndarray.flatten(data_values) all_data = all_data[~np.isnan(all_data)] cutoff_value = np.percentile(all_data, (1-fraction)*100) # Compute the time of crossing the health-gero threshold. span_side = data_values - cutoff_value health_span_length = np.nanargmin(np.abs(span_side), axis = 1) health_span_length = np.array([adult_df.ages[a_time]*24 for a_time in health_span_length]) # Deal with the special cases of individuals that spend their entire lives in health or gerospan. span_list = [a_span[~np.isnan(a_span)] for a_span in span_side] only_health = np.array([(a_span > 0).all() for a_span in span_list]) only_gero = np.array([(a_span < 0).all() for a_span in span_list]) adultspans = selectData.get_adultspans(adult_df) health_span_length[only_health] = adultspans[only_health] health_span_length[only_gero] = 0 return health_span_length
def divide_spans(complete_df):
'''
Divide individual variables into useful spans and return an array of transition times.
'''
span_variables = list(complete_df.key_measures)
span_cutoffs = []
for a_variable in span_variables:
data_values = complete_df.mloc(measures = [a_variable])[:, 0, :]
# Middle overall value.
if a_variable in ['intensity_95', 'intensity_90', 'intensity_80', 'life_texture']:
cutoff_value = np.ndarray.flatten(data_values)
cutoff_value = cutoff_value[~np.isnan(cutoff_value)]
cutoff_value = np.mean(cutoff_value)
span_side = np.abs(data_values - cutoff_value)
span_side = np.nanargmin(span_side, axis = 1)
span_cutoffs.append(span_side)
# Overall young adult value is 'healthy'.
elif a_variable in ['bulk_movement']:
young_adult = np.abs(complete_df.mloc(measures = ['egg_age'])[:, 0, :])
young_adult = np.nanargmin(young_adult, axis = 1)
cutoff_value = np.array([data_values[i, young_adult[i]] for i in range(0, young_adult.shape[0])])
cutoff_value = np.mean(cutoff_value)/2
span_side = np.abs(data_values - cutoff_value)
span_side = np.nanargmin(span_side, axis = 1)
span_cutoffs.append(span_side)
# Point of own first maximum value.
elif a_variable in ['total_size', 'cumulative_eggs', 'cumulative_area', 'adjusted_size']:
span_side = np.nanargmax(data_values, axis = 1)
span_cutoffs.append(span_side)
# Raise an exception if I can't find the variable.
else:
raise BaseException('Can\'t find ' + a_variable + '.')
return (span_variables, np.array(span_cutoffs))
def mouse_drag(self, event):
'''
'''
if event.inaxes == self.ax and event.button == 1:
# Index of nearest point
i = np.nanargmin(((event.xdata - self.x) / self.nx) ** 2)
j = np.nanargmin(((event.ydata - self.y) / self.ny) ** 2)
if (i == self.last_i) and (j == self.last_j):
return
else:
self.last_i = i
self.last_j = j
# Toggle pixel
if self.aperture[j, i]:
self.aperture[j, i] = 0
else:
self.aperture[j, i] = 1
# Update the contour
self.update()
def test_reductions_2D_int():
x = np.arange(1, 122).reshape((11, 11)).astype('i4')
a = da.from_array(x, chunks=(4, 4))
reduction_2d_test(da.sum, a, np.sum, x)
reduction_2d_test(da.prod, a, np.prod, x)
reduction_2d_test(da.mean, a, np.mean, x)
reduction_2d_test(da.var, a, np.var, x, False) # Difference in dtype algo
reduction_2d_test(da.std, a, np.std, x, False) # Difference in dtype algo
reduction_2d_test(da.min, a, np.min, x, False)
reduction_2d_test(da.max, a, np.max, x, False)
reduction_2d_test(da.any, a, np.any, x, False)
reduction_2d_test(da.all, a, np.all, x, False)
reduction_2d_test(da.nansum, a, np.nansum, x)
with ignoring(AttributeError):
reduction_2d_test(da.nanprod, a, np.nanprod, x)
reduction_2d_test(da.nanmean, a, np.mean, x)
reduction_2d_test(da.nanvar, a, np.nanvar, x, False) # Difference in dtype algo
reduction_2d_test(da.nanstd, a, np.nanstd, x, False) # Difference in dtype algo
reduction_2d_test(da.nanmin, a, np.nanmin, x, False)
reduction_2d_test(da.nanmax, a, np.nanmax, x, False)
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
assert eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
assert eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
def test_reductions_1D(dtype):
x = np.arange(5).astype(dtype)
a = da.from_array(x, chunks=(2,))
reduction_1d_test(da.sum, a, np.sum, x)
reduction_1d_test(da.prod, a, np.prod, x)
reduction_1d_test(da.mean, a, np.mean, x)
reduction_1d_test(da.var, a, np.var, x)
reduction_1d_test(da.std, a, np.std, x)
reduction_1d_test(da.min, a, np.min, x, False)
reduction_1d_test(da.max, a, np.max, x, False)
reduction_1d_test(da.any, a, np.any, x, False)
reduction_1d_test(da.all, a, np.all, x, False)
reduction_1d_test(da.nansum, a, np.nansum, x)
with ignoring(AttributeError):
reduction_1d_test(da.nanprod, a, np.nanprod, x)
reduction_1d_test(da.nanmean, a, np.mean, x)
reduction_1d_test(da.nanvar, a, np.var, x)
reduction_1d_test(da.nanstd, a, np.std, x)
reduction_1d_test(da.nanmin, a, np.nanmin, x, False)
reduction_1d_test(da.nanmax, a, np.nanmax, x, False)
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.argmax(a, axis=0, split_every=2), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0, split_every=2), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0, split_every=2), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0, split_every=2), np.nanargmin(x, axis=0))
def _initialize(self):
""" Enforce bounds """
system = self._system
u = system.vec['u'].array
du = system.vec['du'].array
lower = system.vec['lb'].array
upper = system.vec['ub'].array
self.alpha = 1.0
if not numpy.isnan(lower).all():
lower_const = u + self.alpha*du - lower
ind = numpy.nanargmin(lower_const)
if lower_const[ind] < 0:
self.alpha = (lower[ind] - u[ind]) / du[ind]
if not numpy.isnan(upper).all():
upper_const = upper - u - self.alpha*du
ind = numpy.nanargmin(upper_const)
if upper_const[ind] < 0:
self.alpha = (upper[ind] - u[ind]) / du[ind]
self.info = self.alpha
norm0 = self._norm()
if norm0 == 0.0:
norm0 = 1.0
system.vec['u'].array[:] += self.alpha * system.vec['du'].array[:]
norm = self._norm()
return norm0, norm
def dataindex(x_loc,y_loc,datain_loc):
x_loc = float(x_loc)
y_loc = float(y_loc)
radius = np.sqrt(x_loc**(2.)+y_loc**(2.))
angle = np.arctan(x_loc/y_loc)
line_radius = datain_loc[np.nanargmin((np.abs(datain_loc['r']-(radius))), axis=0)]
radius = line_radius[3]
line_angle = datain_loc[datain_loc['r']==radius][np.nanargmin((np.abs((datain_loc[datain_loc['r']==radius]['theta'])-(angle))), axis=0)]
angle = line_angle[4]
return np.nanargmin((np.abs(datain_loc['r']-(radius)) + np.abs(datain_loc ['theta']-(angle))), axis=0)
def mapmean(tempDF, meta, name = '', option = 0):
import cartopy.crs as ccrs
from cartopy.io.img_tiles import MapQuestOSM
from mpl_toolkits.axes_grid1 import make_axes_locatable
#fig = plt.figure(figsize=(30, 30))
x = meta['location:Longitude'].values
y = meta['location:Latitude'].values
c = tempDF[meta.index].mean()
marker_size = 350
imagery = MapQuestOSM()
fig = plt.figure(figsize=[15,15])
ax = plt.axes(projection=imagery.crs)
ax.set_extent(( meta['location:Longitude'].min()-.005,
meta['location:Longitude'].max()+.005 ,
meta['location:Latitude'].min()-.005,
meta['location:Latitude'].max()+.005))
ax.add_image(imagery, 14)
cmap = matplotlib.cm.OrRd
bounds = np.linspace(round((c.mean()-3)),round((c.mean()+3)),13)
norm = matplotlib.colors.BoundaryNorm(bounds, cmap.N)
plotHandle = ax.scatter(x,y,c = c, s = marker_size, transform=ccrs.Geodetic(),
cmap = cmap,
norm = norm)
if option ==0 :
cbar1 = plt.colorbar(plotHandle, label = 'Temperature in $^\circ $C')
else :
cbar1 = plt.colorbar(plotHandle, label = option)
lon = x[np.nanargmax(c)]
lat = y[np.nanargmax(c)]
at_x, at_y = ax.projection.transform_point(lon, lat,
src_crs=ccrs.Geodetic())
plt.annotate(
'%2.1f'%np.nanmax(c.values), xy=(at_x, at_y), #xytext=(30, 20), textcoords='offset points',
color='black', backgroundcolor='none', size=22,
)
lon = x[np.nanargmin(c)]
lat = y[np.nanargmin(c)]
at_x, at_y = ax.projection.transform_point(lon, lat,
src_crs=ccrs.Geodetic())
plt.annotate(
'%2.1f'%np.nanmin(c.values), xy=(at_x, at_y), #xytext=(30, 20), textcoords='offset points',
color='black', size = 22, backgroundcolor='none')
plt.annotate(
'$\mu = $ %2.1f, $\sigma = $ %2.1f'%(np.nanmean(c.values), np.nanstd(c.values)), (0.01,0.01), xycoords ='axes fraction', #xytext=(30, 20), textcoords='offset points',
color='black', size = 22, backgroundcolor='none')
plt.title('Mean Temperature %s'%name)
filename = './plots/meantempmap%s.eps'%name
plt.savefig(filename, format = 'eps', dpi = 600)
def closest_real_time(complete_df, a_worm, a_time, egg_mode = True):
'''
For a_worm at a_time, find the closest real time point.
'''
time_split = a_time.split('.')
hours_time = int(time_split[0])*24 + int(time_split[1])*3
if not egg_mode:
time_index = np.nanargmin(np.abs(complete_df.raw[a_worm].loc[:, 'age'] - hours_time))
else:
time_index = np.nanargmin(np.abs(complete_df.raw[a_worm].loc[:, 'egg_age'] - hours_time))
real_time = complete_df.raw[a_worm].index[time_index]
return real_time
def test_reductions_2D_nans():
# chunks are a mix of some/all/no NaNs
x = np.full((4, 4), np.nan)
x[:2, :2] = np.array([[1, 2], [3, 4]])
x[2, 2] = 5
x[3, 3] = 6
a = da.from_array(x, chunks=(2, 2))
reduction_2d_test(da.sum, a, np.sum, x, False, False)
reduction_2d_test(da.prod, a, np.prod, x, False, False)
reduction_2d_test(da.mean, a, np.mean, x, False, False)
reduction_2d_test(da.var, a, np.var, x, False, False)
reduction_2d_test(da.std, a, np.std, x, False, False)
reduction_2d_test(da.min, a, np.min, x, False, False)
reduction_2d_test(da.max, a, np.max, x, False, False)
reduction_2d_test(da.any, a, np.any, x, False, False)
reduction_2d_test(da.all, a, np.all, x, False, False)
reduction_2d_test(da.nansum, a, np.nansum, x, False, False)
reduction_2d_test(da.nanprod, a, nanprod, x, False, False)
reduction_2d_test(da.nanmean, a, np.nanmean, x, False, False)
with pytest.warns(None): # division by 0 warning
reduction_2d_test(da.nanvar, a, np.nanvar, x, False, False)
with pytest.warns(None): # division by 0 warning
reduction_2d_test(da.nanstd, a, np.nanstd, x, False, False)
with pytest.warns(None): # all NaN axis warning
reduction_2d_test(da.nanmin, a, np.nanmin, x, False, False)
with pytest.warns(None): # all NaN axis warning
reduction_2d_test(da.nanmax, a, np.nanmax, x, False, False)
assert_eq(da.argmax(a), np.argmax(x))
assert_eq(da.argmin(a), np.argmin(x))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmax(a), np.nanargmax(x))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmin(a), np.nanargmin(x))
assert_eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert_eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert_eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert_eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
def fit(self, t, n=10, retry=True):
"""
Fit for the radius of an event from a given set of
hit times `t`. Seed the initial position by searching
for the most likely radius at `n` radii. If the fit
fails and `retry` is True, try the fit again by seeding
the fit with the best point from `n`*10 trial radii.
"""
def nll(pars):
l, = pars
return -np.log(self.pdf(t,l=l)).sum()
# seed the fit with the best radius in l0
l0 = np.linspace(0,self.r,n)
x0 = [-np.log(self.pdf(t,x)).sum() for x in l0]
x0 = l0[np.nanargmin(x0)]
xopt, fopt, iter, funcalls, warnflag = \
fmin(nll,[x0],disp=False,full_output=True)
if warnflag:
if n < 1e3 and retry:
print 'retrying with n=%i' % (n*10)
return self.fit(t,n*10)
print 'fit failed.'
return xopt
def maximum_posterior(self):
"""Returns the maximum log posterior (minimum negative log posterior)
from the set of chains, along with the position giving the maximum
posterior.
"""
if not self.chains:
raise Exception("There are no chains in the MCMCSet.")
max_posterior = np.inf
max_posterior_position = None
for chain in self.chains:
# Make sure the chain is not empty!
if len(chain.posteriors) > 0:
chain_max_posterior_index = np.nanargmin(chain.posteriors)
chain_max_posterior = \
chain.posteriors[chain_max_posterior_index]
if chain_max_posterior < max_posterior:
max_posterior = chain_max_posterior
max_posterior_position = \
chain.positions[chain_max_posterior_index]
# Check if there are no positions
if max_posterior_position is None:
raise NoPositionsException('The maximum posterior could not be determined '
'because there are no accepted positions.')
return (max_posterior, max_posterior_position)
def apply_roi(self, roi):
if not isinstance(roi, PointROI):
raise NotImplementedError("Only PointROI supported")
if self._layout is None or self.display_data is None:
return
x, y = roi.x, roi.y
if not roi.defined():
return
xs, ys = self._layout[:, ::3]
parent_ys = self._layout[1, 1::3]
delt = np.abs(x - xs)
delt[y > ys] = np.nan
delt[y < parent_ys] = np.nan
if np.isfinite(delt).any():
select = np.nanargmin(delt)
if self.select_substruct:
select = self._substructures(select)
select = np.asarray(select, dtype=np.int)
else:
select = np.array([], dtype=np.int)
state = CategorySubsetState(self.display_data.pixel_component_ids[0],
select)
EditSubsetMode().update(self.collect, state,
focus_data=self.display_data)
def get_closest_inconsistencies(self, vector, distance_measure, max_height=numpy.inf, check_constant=False,
constant_prob=1.0):
num_trees = len(self.trees) if max_height >= len(self.height_levels) else self.height_levels[max_height]
consistent_indices = ~numpy.isnan(vector)
if not numpy.any(consistent_indices):
tree_index = random.randrange(num_trees)
return self.trees[tree_index][:], self.semantic_array[tree_index]
consistent_vector = vector[consistent_indices]
if not numpy.all(numpy.isfinite(consistent_vector)):
tree_index = random.randrange(num_trees)
return self.trees[tree_index][:], self.semantic_array[tree_index]
if is_constant(consistent_vector):
return [gp.Terminal(consistent_vector[0], False, float)], [consistent_vector[0]] * len(vector)
dists = distance_measure(self.semantic_array[:num_trees, consistent_indices], consistent_vector, axis=1)
min_distance_index = numpy.nanargmin(dists)
if check_constant and random.random() < constant_prob:
constant = numpy.median(consistent_vector).item()
constant_distance = distance_measure(consistent_vector, constant)
if constant_distance < dists[min_distance_index]:
return [gp.Terminal(constant, False, float)], [constant] * len(vector)
return self.trees[min_distance_index][:], self.semantic_array[min_distance_index]
def KNNmatch(data1, data2, confidence_list, return_sum):
# Perform KNNmatch algorithm on data
match = np.zeros((len(data1), len(confidence_list)))
pair_scores = np.zeros((len(data1), 2))
pair_index = np.zeros((len(data1),2))
for i in np.arange(len(data1)):
score = np.zeros((len(data2)))
for j in np.arange(len(data2)):
score[j] = sum(np.square(data1[i,:] - data2[j,:]))
imin0 = np.argmin(score)
min0 = score[imin0]
score[imin0] = np.nan
imin1 = np.nanargmin(score)
min1 = np.nanmin(score)
pair_scores[i,:] = min0,min1
pair_index[i,:] = imin0,imin1
for n in np.arange(len(confidence_list)):
if min0 < float(confidence_list[n]) / 100 * min1:
match[i,n] = 1
print np.sum(match, axis = 0), len(data1)
return
return match, pair_scores, pair_index
def maximum_likelihood(self):
"""Returns the maximum log likelihood (minimum negative log likelihood)
from the set of chains, along with the position giving the maximum
likelihood.
"""
if not self.chains:
raise Exception("There are no chains in the MCMCSet.")
max_likelihood = np.inf
max_likelihood_position = None
for chain in self.chains:
# Make sure the chain is not empty!
if len(chain.likelihoods) > 0:
chain_max_likelihood_index = np.nanargmin(chain.likelihoods)
chain_max_likelihood = \
chain.likelihoods[chain_max_likelihood_index]
if chain_max_likelihood < max_likelihood:
max_likelihood = chain_max_likelihood
max_likelihood_position = \
chain.positions[chain_max_likelihood_index]
# Check if there are no positions
if max_likelihood_position is None:
raise NoPositionsException('The maximum likelihood could not be '
'determined because there are no accepted positions.')
return (max_likelihood, max_likelihood_position)
def decide_migration_migrationlikelihood_woi(self): migrate_me_maybe = (self.window_overload_index > self.relocation_thresholds)[0] if np.sum(migrate_me_maybe) > 0: indexes = np.array(np.where(migrate_me_maybe)).tolist()[0] # potential migration sources set_of_vms = list() for i in indexes: partial = (self.location[:, i] == 1).transpose() newly_found = np.array(np.where(partial)).tolist() set_of_vms += newly_found[0] set_of_vms = sorted(set_of_vms) pms = [x.get_pm() for x in self.vms] pm_volumes = np.array([x.get_volume() for x in self.pms]) vm_volumes = np.array([x.get_volume_actual() for x in self.vms]) vm_migrations = np.array([x.get_migrations() for x in self.vms]) available_volume_per_pm = pm_volumes - self.physical_volume_vector available_capacity = [available_volume_per_pm[x.get_pm()] for x in self.vms] plan_coefficients = np.array([x.plan.get_coefficient() for x in self.vms]) minimize_me = -1.0/plan_coefficients * (vm_volumes + available_capacity) + plan_coefficients * vm_migrations vm_migrate = np.nanargmin(minimize_me) pm_source = self.vms[vm_migrate].get_pm() # avoiding to select the source machine as destination by using nan available_volume_per_pm[pm_source] = np.nan pm_destination = np.nanargmax(available_volume_per_pm) self.migrate(vm_migrate, pm_source, pm_destination) self.integrated_overload_index[0,pm_source] = 0
def find_closest(self):
msg = self.current_laser_msg
rngs = msg.ranges
idx = np.nanargmin(rngs)
self.say("idx: " + str(idx))
rad = self.index_to_rad(idx)
return rngs[idx], rad
def johnson(d, debug=False):
"""
Méthode approchée avec garantie de performance (Johnson)
d = Jeu de donnée
"""
assert np.size(d, 1) > 1, "2 machines au moins"
if debug and np.size(d, 1) != 2: print("Attention: Seul les deux premières machines sont pris en compte.")
n, m = np.size(d, 0), 3
A, B, C = range(0,m)
X = list(range(0,n))
G = list()
D = list()
while X:
# Chaque tâche ayant été traité voit ses durées mettre à NaN.
# np.nanargmin récupère ensuite la position dans la matrice
# où la durée minimum est située.
pos = np.nanargmin([[d[i,j] if i in X else np.nan for j in [A,B]] for i in range(n)])
# Je déduit ensuite la ligne et la colonne correspondant à cette position
# i.e. la machine et la tâche.
i, j = pos//(m-1), pos%(n-1)
# Si la durée de la tâche appartient à la machine A
# je la met dans ma liste de gauche G, sinon je l'insère
# dans ma liste de droite en première position.
G.append(i) if j == A else D.insert(0, i)
# Ma tâche en cours est traitée
X.remove(i)
return G + D, evaluation(d, G+D, [])
def otsu(img):
# Find within-class variance for each candidate threshold value. Choose the
# one that minimises it.
n_pix = np.prod(img.shape)
threshold_variances = np.inf * np.ones((254,1))
for i in range(254):
t = i + 1
bin_masks = [img < t, img >= t]
# bin_masks[0] gives the less-than mask.
# bin_masks[1] gives the geq mask.
vals = map(lambda mask : img[mask], bin_masks)
N = map(np.sum, bin_masks)
mu = map(np.mean, vals)
sigma = map(np.std, vals)
variance = map(lambda x : x ** 2, sigma)
threshold_variances[i] = \
(N[0] * variance[0] + N[1] * variance[1]) / n_pix
min_thresh = np.nanargmin(threshold_variances[1:255])
thresholded = np.copy(img)
thresholded[img < min_thresh] = 0
thresholded[img >= min_thresh] = 1
return thresholded
def make_timing_params(self, begin, end, snap_vred=True):
'''Compute tight parameterized time ranges to include given timings.
Calculates appropriate time ranges to cover given begin and end timings
over all GF points in the store. A dict with the following keys is
returned:
* ``'tmin'``: time [s], minimum of begin timing over all GF points
* ``'tmax'``: time [s], maximum of end timing over all GF points
* ``'vred'``, ``'tmin_vred'``: slope [m/s] and offset [s] of reduction
velocity [m/s] appropriate to catch begin timing over all GF points
* ``'tlenmax_vred'``: maximum time length needed to cover all end
timings, when using linear slope given with (`vred`, `tmin_vred`) as
start
'''
data = []
for args in self.config.iter_nodes(level=-1):
tmin = self.t(begin, args)
tmax = self.t(end, args)
x = self.config.get_distance(args)
data.append((x, tmin, tmax))
xs, tmins, tmaxs = num.array(data, dtype=num.float).T
i = num.nanargmin(tmins)
tminmin = tmins[i]
x_tminmin = xs[i]
dx = (xs - x_tminmin)
dx = num.where(dx != 0.0, dx, num.nan)
s = (tmins - tminmin) / dx
sred = num.min(num.abs(s[num.isfinite(s)]))
deltax = self.config.distance_delta
if snap_vred:
tdif = sred*deltax
tdif2 = self.config.deltat * math.floor(tdif / self.config.deltat)
sred = tdif2/self.config.distance_delta
tmin_vred = tminmin - sred*x_tminmin
if snap_vred:
xe = x_tminmin - int(x_tminmin / deltax) * deltax
tmin_vred = float(
self.config.deltat *
math.floor(tmin_vred / self.config.deltat) - xe * sred)
tlenmax_vred = num.nanmax(tmax - (tmin_vred + sred*x))
if sred != 0.0:
vred = 1.0/sred
else:
vred = 0.0
return dict(
tmin=tminmin,
tmax=num.nanmax(tmaxs),
tmin_vred=tmin_vred,
tlenmax_vred=tlenmax_vred,
vred=vred)
def _energyBinning(ws, algorithmLogging):
"""Create common (but nonequidistant) binning for a DeltaE workspace."""
xs = ws.extractX()
minXIndex = numpy.nanargmin(xs[:, 0])
# TODO Fix logging.
dx = BinWidthAtX(InputWorkspace=ws,
X=0.0,
EnableLogging=algorithmLogging)
lastX = numpy.max(xs[:, -1])
binCount = ws.blocksize()
borders = list()
templateXs = xs[minXIndex, :]
currentX = numpy.nan
for i in range(binCount):
currentX = templateXs[i]
borders.append(currentX)
if currentX > 0:
break
i = 1
equalBinStart = borders[-1]
while currentX < lastX:
currentX = equalBinStart + i * dx
borders.append(currentX)
i += 1
borders[-1] = lastX
return numpy.array(borders)
def train(self, alpha):
"""
Finds the agglomerative clustering on the data alpha
:param alpha: angles in radians
:returns: data, cluster ids
"""
assert len(alpha.shape) == 1, 'Clustering works only for 1d data'
n = len(alpha)
cid = np.arange(n, dtype=int)
nu = n
while nu > self.numclust:
mu = np.asarray([descr.mean(alpha[cid == j]) if j in cid else np.Inf for j in range(n)])
D = np.abs(descr.pairwise_cdiff(mu))
idx = np.triu_indices(n,1)
min = np.nanargmin(D[idx])
cid[cid == cid[idx[0][min]]] = cid[idx[1][min]]
nu -= 1
cid2 = np.empty_like(cid)
for i,j in enumerate(np.unique(cid)):
cid2[cid == j] = i
ucid = np.unique(cid2)
self.centroids = np.asarray([descr.mean(alpha[cid2 == i]) for i in ucid])
self.cluster_ids = ucid
self.r = np.asarray([descr.resultant_vector_length(alpha[cid2 == i]) for i in ucid])
return alpha, cid2
def gridsearch(method, gs_name="", njobs=-2, batchsize=3, **params):
if gs_name != "":
gs_name = "_" + gs_name
fn = "data/{}/{}{}.pck".format(name, method.__name__, gs_name)
if os.path.exists(fn):
print "Already exists: {}{}".format(method.__name__, gs_name)
load_result_file(fn)
return
param_names = params.keys()
param_list = list(itertools.product(*[params[k] for k in param_names]))
param_lengths = [len(params[k]) for k in param_names]
k = []
i = 0
while i < len(param_list):
j = min(i + batchsize, len(param_list)+1)
par = [dict(zip(param_names, p)) for p in param_list[i:j]]
k.append(delayed(run)(method, par))
i = j
print "Starting {} {}{}".format(name, method.__name__, gs_name)
res1 = Parallel(n_jobs=njobs, verbose=11)(k)
res = np.vstack(res1)
for i,c in enumerate(criteria):
j = np.nanargmin(res[:,i].flatten())
print "best parameters for {} with value {}:".format(c, np.nanmin(res[:,i]))
print zip(param_names, param_list[j])
res = np.array(res).reshape(*(param_lengths + [len(criteria)]))
if not os.path.exists("data/{name}".format(name=name)):
os.makedirs("data/{name}".format(name=name))
with open(fn, "w") as f:
pickle.dump(dict(res=res, params=param_list, criteria=criteria,
param_names=param_names, **params), f)
print "Finished {} {}{}".format(name, method.__name__, gs_name)
def align_start(x, y, bin_size = 500, verbose = False, window = 2000, search = 100000):
"""Find initial alignment via downsampled data"""
# resample for coarse alignment
xa = average(x, bin_size);
ya = average(y, bin_size);
#align resampled
dd = distance(xa, ya)
min_id = np.argmin(dd);
#search full data
id_search = bin_size * np.array(min_id + np.array([-2, 2]), dtype = int);
dd2 = distance(x[:window+search], y[:window], search = id_search)
start_id = np.nanargmin(dd2);
if verbose:
plt.figure(2); plt.clf();
plt.subplot(1,3,1);
plt.plot(dd)
plt.scatter([min_id], [dd[min_id]], s = 50, c = 'r');
plt.title('cooarse');
plt.subplot(1,3,2);
plt.plot(dd2)
plt.scatter([start_id], [dd2[start_id]], s = 50, c = 'r');
plt.title('fine');
plt.subplot(1,3,3);
plt.plot(x[start_id:start_id+window]);
plt.plot(y[:window])
return start_id;
def log_performance(self):
'''
Record model performance so far, based on validation loss.
'''
handle = open("checkpoints/%s/summary" % self.args.run_string, "w")
for epoch in range(len(self.val_loss)):
handle.write("Checkpoint %d | val loss: %.5f bleu %.2f\n"
% (epoch+1, self.val_loss[epoch],
self.val_metric[epoch]))
logger.info("---") # break up the presentation for clarity
# BLEU is the quickest indicator of performance for our task
# but loss is our objective function
best_bleu = np.nanargmax(self.val_metric)
best_loss = np.nanargmin(self.val_loss)
logger.info("Best Metric: %d | val loss %.5f score %.2f",
best_bleu+1, self.val_loss[best_bleu],
self.val_metric[best_bleu])
handle.write("Best Metric: %d | val loss %.5f score %.2f\n"
% (best_bleu+1, self.val_loss[best_bleu],
self.val_metric[best_bleu]))
logger.info("Best loss: %d | val loss %.5f score %.2f",
best_loss+1, self.val_loss[best_loss],
self.val_metric[best_loss])
handle.write("Best loss: %d | val loss %.5f score %.2f\n"
% (best_loss+1, self.val_loss[best_loss],
self.val_metric[best_loss]))
logger.info("Early stopping marker: wait/patience: %d/%d\n",
self.wait, self.patience)
handle.write("Early stopping marker: wait/patience: %d/%d\n" %
(self.wait, self.patience))
handle.close()
def sample_responses(self, n=1000, condition=None, remove_negative_finishing_times=True):
if condition is None:
condition = np.arange(self.n_conditions)
condition = np.repeat(condition, n)
n = condition.shape[0]
finishing_times = np.zeros((n, self.n_accumulators))
for i, accumulator in enumerate(self.accumulators):
_, finishing_times[:, i] = accumulator.sample_finishing_times(condition=condition)
if remove_negative_finishing_times:
finishing_times[finishing_times < 0] = np.nan
rts = np.nanmin(finishing_times, 1)
responses = np.nanargmin(finishing_times, 1)
mapped_responses = np.ones_like(responses) * np.nan
mapped_responses[~np.isnan(responses)] = self.accumulator2response[responses[~np.isnan(responses)]]
return condition, mapped_responses, rts
def early_stop_decision(self, epoch, val_metric, val_loss):
'''
Stop training if validation loss has stopped decreasing and
validation BLEU score has not increased for --patience epochs.
WARNING: quits with sys.exit(0).
TODO: this doesn't yet support early stopping based on TER
'''
if val_loss < self.best_val_loss:
self.wait = 0
elif val_metric > self.best_val_metric or self.args.no_early_stopping:
self.wait = 0
else:
self.wait += 1
if self.wait >= self.patience:
# we have exceeded patience
if val_loss > self.best_val_loss:
# and loss is no longer decreasing
logger.info("Epoch %d: early stopping", epoch)
handle = open("checkpoints/%s/summary"
% self.args.run_string, "a")
handle.write("Early stopping because patience exceeded\n")
best_bleu = np.nanargmax(self.val_metric)
best_loss = np.nanargmin(self.val_loss)
logger.info("Best Metric: %d | val loss %.5f score %.2f",
best_bleu+1, self.val_loss[best_bleu],
self.val_metric[best_bleu])
logger.info("Best loss: %d | val loss %.5f score %.2f",
best_loss+1, self.val_loss[best_loss],
self.val_metric[best_loss])
handle.close()
sys.exit(0)
temp = a2
a2 = a1
a1 = temp
a1 -= (a2 - temp)
a2 += (a2 - temp)
else:
temp = a1
a1 -= (a2 - temp)
a2 += (a2 - temp)
print("step1a, a1,a2=", a1, a2)
aRange = np.linspace(a1, a2, zoomAccuracy)
for n in range(zoomAccuracy):
Deviation[n] = Dev(x, y, dy, aRange[n], b)
#Shuffle (tranlate)
while (np.nanargmin(Deviation) == 0):
for n in range(zoomAccuracy):
Deviation[n] = Dev(x, y, dy, aRange[n], b)
temp = a1
a1 -= (a2 - temp)
a2 -= (a2 - temp)
if (a1 == a2): break
try:
print("for a, index of min =", np.nanargmin(Deviation),
", just shuffled")
except ValueError:
pass
aRange = np.linspace(
a1, a2, zoomAccuracy
) #if the Deviation(0) is still min, then the loop will restart
def test_benchmark_offline(classify, test_set_x, batch_size):
strides = (2, 5, 8)
vid_root = "../data/YT_seg/"
gt_counts = pickle.load(open("vidGtData.p", "rb"))
gt = numpy.tile(gt_counts, (len(strides), 1))
gt = gt.T
gt1 = gt[:, 1]
tests_num = gt_counts.shape[0]
countArr = numpy.zeros(shape=(tests_num, len(strides)))
entropy = numpy.zeros(shape=(tests_num, len(strides)))
num_entropy = numpy.zeros(shape=(tests_num, len(strides)))
currStride = -1
for nStride in strides:
currStride += 1
for nTestSet in range(tests_num):
print("stride: %i, vid: %i" % (nStride, nTestSet))
fileName = vid_root + "YT_seg_" + str(nTestSet) + ".avi"
mydatasets = load_next_test_data_simple_roi(fileName, nStride)
ns_test_set_x, valid_ds = mydatasets
if valid_ds == 0: # file not axists
continue
test_set_x.set_value(ns_test_set_x, borrow=True)
n_samples = ns_test_set_x.shape[0]
out_list = [classify(i) for i in range(n_samples)]
frame_counter = 0
rep_counter = 0
curr_entropy = 0
ent_cnt = 0
for batch_num in range(len(out_list)):
output_label_batch, pYgivenX = out_list[batch_num]
# Removed index in following line
output_label = (output_label_batch[0] + 3
) # moving from label to cycle length
pYgivenX[pYgivenX == 0] = numpy.float32(
1e-30) # hack to output valid entropy
# calc entropy
curr_entropy = curr_entropy - (pYgivenX *
numpy.log(pYgivenX)).sum()
ent_cnt = ent_cnt + 1
# integrate counting
if batch_num == 0:
rep_counter = 20 / (output_label)
frame_counter = 20 % (output_label)
else:
frame_counter += 1
if frame_counter >= output_label:
rep_counter += 1
frame_counter = 0
countArr[nTestSet, currStride] = rep_counter
entropy[nTestSet, currStride] = curr_entropy
num_entropy[nTestSet, currStride] = ent_cnt
absdiff_o = abs(countArr - gt)
min_err_cnt_o = absdiff_o.min(axis=1)
min_err_perc_o = min_err_cnt_o / gt[:, 1]
err_perc_o = numpy.average(min_err_perc_o) * 100
print("alpha = 1: precentage error: %.2f%%" % (err_perc_o))
print("---------")
med = numpy.median(countArr, axis=1)
medif = numpy.average(abs(med - gt1) / gt1) * 100
print("median stride: precentage error: %.2f%%" % medif)
xx = entropy / num_entropy
chosen_stride = numpy.nanargmin(xx, axis=1)
m = numpy.arange(chosen_stride.shape[0]) * len(strides)
m = m + chosen_stride
flt = countArr.flatten()
ent_chosen_cnt = flt[m]
entdif = numpy.average(abs(ent_chosen_cnt - gt1) / gt1) * 100
print("enropy stride: precentage error: %.2f%%" % entdif)
print("offline entropy counting done")
def run(self, save_sims=''):
"""
Runs evolutionary algorithm
Returns
-------
top_params : list
List of the top parameters from an individual EA.
best_error : list
List of the best errors from an individual EA run.
"""
top_params = np.zeros(
(self.number_of_runs, self.number_of_generations + 1,
len(self.minimums)))
best_error = np.zeros(
(self.number_of_runs, self.number_of_generations + 1))
def scorefxn_helper(ea_individual):
return self.scorefxn(ea_individual, convert=True),
for i in range(self.number_of_runs):
# TYPE
# Create minimizing fitness class w/ single objective:
creator.create('FitnessMin', base.Fitness, weights=(-1.0, ))
# Create individual class:
creator.create('Individual', list, fitness=creator.FitnessMin)
# TOOLBOX
toolbox = base.Toolbox()
# Register function to use initRepeat to fill individual w/ n calls to rand_num:
toolbox.register('individual',
tools.initRepeat,
creator.Individual,
np.random.random,
n=len(self.minimums))
# Register function to use initRepeat to fill population with individuals:
toolbox.register('population', tools.initRepeat, list,
toolbox.individual)
# GENETIC OPERATORS:
# Register evaluate fxn = evaluation function, individual to evaluate given later
toolbox.register('evaluate', scorefxn_helper)
# Register mate fxn = two points crossover function
toolbox.register('mate', tools.cxTwoPoint)
# Register mutate by swapping two points of the individual:
toolbox.register('mutate',
tools.mutPolynomialBounded,
eta=0.1,
low=0.0,
up=1.0,
indpb=0.2)
# Register select = size of tournament set to 3
toolbox.register('select', tools.selTournament, tournsize=3)
# EVOLUTION!
pop = toolbox.population(n=self.number_of_individuals)
hof = tools.HallOfFame(1)
stats = tools.Statistics(key=lambda ind: [ind.fitness.values, ind])
stats.register('all', np.copy)
# using built in eaSimple algo
pop, logbook = algorithms.eaSimple(pop,
toolbox,
cxpb=self.crossover_rate,
mutpb=self.mutation_rate,
ngen=self.number_of_generations,
stats=stats,
halloffame=hof,
verbose=False)
# Save best scores and individuals in population
best_score = []
best_ind = []
for a in range(len(logbook)):
scores = [
logbook[a]['all'][b][0][0]
for b in range(len(logbook[a]['all']))
]
best_score.append(np.nanmin(scores))
# logbook is of type 'deap.creator.Individual' and must be loaded later
# don't want to have to load it to view data everytime, thus numpy
ind_np = np.asarray(logbook[a]['all'][np.nanargmin(scores)][1])
ind_np_conv = self.convert_individual(ind_np)
best_ind.append(ind_np_conv)
# print('Best individual is:\n %s\nwith fitness: %s' %(arr_best_ind[-1],arr_best_score[-1]))
top_params[i] = best_ind
best_error[i] = best_score
if i / self.number_of_runs * 100 % 10 == 0:
print(str((i / self.number_of_runs) * 100) + '% complete.')
if save_sims:
counter = 0
filename = get_filename(save_sims, counter)
while os.path.isfile(filename) == True:
counter += 1
filename = get_filename(save_sims, counter)
with h5py.File(filename, "w") as f:
tparam = f.create_dataset("top_params", data=top_params)
berror = f.create_dataset("best_error", data=best_error)
return top_params, best_error
def closest(cls, query_results, lat, lon, heading, prev_way=None):
results, tree, real_nodes, node_to_way, location_info = query_results
cur_pos = geodetic2ecef((lat, lon, 0))
nodes = tree.query_ball_point(cur_pos, 500)
# If no nodes within 500m, choose closest one
if not nodes:
nodes = [tree.query(cur_pos)[1]]
ways = []
for n in nodes:
real_node = real_nodes[n]
ways += node_to_way[real_node.id]
ways = set(ways)
closest_way = None
best_score = None
for way in ways:
way = Way(way, query_results)
points = way.points_in_car_frame(lat, lon, heading)
on_way = way.on_way(lat, lon, heading, points)
if not on_way:
continue
# Create mask of points in front and behind
x = points[:, 0]
y = points[:, 1]
angles = np.arctan2(y, x)
front = np.logical_and((-np.pi / 2) < angles, angles < (np.pi / 2))
behind = np.logical_not(front)
dists = np.linalg.norm(points, axis=1)
# Get closest point behind the car
dists_behind = np.copy(dists)
dists_behind[front] = np.NaN
closest_behind = points[np.nanargmin(dists_behind)]
# Get closest point in front of the car
dists_front = np.copy(dists)
dists_front[behind] = np.NaN
closest_front = points[np.nanargmin(dists_front)]
# fit line: y = a*x + b
x1, y1, _ = closest_behind
x2, y2, _ = closest_front
a = (y2 - y1) / max((x2 - x1), 1e-5)
b = y1 - a * x1
# With a factor of 60 a 20m offset causes the same error as a 20 degree heading error
# (A 20 degree heading offset results in an a of about 1/3)
score = abs(a) * 60. + abs(b)
# Prefer same type of road
if prev_way is not None:
if way.way.tags.get('highway', '') == prev_way.way.tags.get(
'highway', ''):
score *= 0.5
if closest_way is None or score < best_score:
closest_way = way
best_score = score
return closest_way
def asyncBackTrack(**kwargs):
refDatas = kwargs['refDatas']
inpDatas = kwargs['inpDatas']
matchingCost1 = kwargs['matchingCost1']
matchingCost3 = kwargs['matchingCost3']
totalMatchingCosts = kwargs['totalMatchingCosts']
baseArgs = kwargs['baseArgs']
#inputFinFrameBackTracked, epsilonFinFrameBackTracked = kwargs['argsFinFrameBackTracked']
inputFinFrameBackTracked = np.nanargmin(totalMatchingCosts)
epsilon1FinFrameBackTracked, epsilon3FinFrameBackTracked = \
np.nanargmin(matchingCost1[matchingCost1.shape[0] - 1, inputFinFrameBackTracked]), \
np.nanargmin(matchingCost3[matchingCost3.shape[0] - 1, inputFinFrameBackTracked])
correspondentPoints1, correspondentPoints2, correspondentPoints3 = [], [], []
r, i, epsilon1, epsilon3 = matchingCost1.shape[0] - 1, inputFinFrameBackTracked, \
epsilon1FinFrameBackTracked - limits, epsilon3FinFrameBackTracked - limits
correspondentPoints2.append([r, i])
correspondentPoints1.append([r, i + epsilon1])
correspondentPoints3.append([r, i + epsilon3])
"""
epsilon\i| i i-1 i-2(tmp=0,1,2)
-------------------------
-2 | e+c e+c e+c
-1 | e+c-1 e+c-1 e+c
0 | e+c-2 e+c-1 e+c
1 | e+c-2 e+c-1 e+c
2 | e+c-2 e+c-1 e+c
(epsilon + limits=0,...,4)
"""
forNewEpsilon = [[0, 0, 0], [-1, -1, 0], [-2, -1, 0], [-2, -1, 0],
[-2, -1, 0]]
def _mcosts(iNext, epsilon, matchingCost):
if iNext == 0:
return matchingCost[r - 1, i,
max(epsilon, 0):epsilon + limits + 1]
elif iNext == 1:
return matchingCost[r - 1, i - 1,
max(epsilon + 1, 0):epsilon + limits +
2]
else:
return matchingCost[r - 1, i - 2,
max(epsilon + 2, 0):epsilon + limits +
3]
while r > 0:
tmp = abs(baseArgs[r, i])
c1_ = np.argmin(_mcosts(tmp, epsilon1, matchingCost1))
c3_ = np.argmin(_mcosts(tmp, epsilon3, matchingCost3))
r = r - 1
i = i - tmp
epsilon1 = epsilon1 + c1_[tmp] + forNewEpsilon[epsilon1 +
limits][tmp]
epsilon3 = epsilon3 + c3_[tmp] + forNewEpsilon[epsilon3 +
limits][tmp]
correspondentPoints2.insert(0, [r, i])
correspondentPoints1.insert(0, [r, i + epsilon1])
correspondentPoints3.insert(0, [r, i + epsilon3])
return correspondentPoints1, correspondentPoints2
def waveReleasePointWindsContour(bam,
traj,
ref_loc,
points,
div=37,
mode='ballistic'):
setup = bam.setup
atmos = bam.atmos
steps = 90
alpha = np.linspace(0, 360 * ((steps - 1) / steps), steps)
alpha = np.radians(alpha)
beta = np.linspace(0, 90 * ((steps - 1) / steps), steps)
beta = np.radians(beta)
theta = setup.trajectory.azimuth.rad
phi = setup.trajectory.zenith.rad
tol = 25 #deg
# tol_fact = np.radians(np.linspace(-tol, tol, 10))
WIND = True
u = traj.getTrajVect()
v_list = []
for i in range(steps):
for j in range(steps):
v = np.array([np.sin(alpha[i])*np.sin(beta[j]),\
np.cos(alpha[i])*np.sin(beta[j]),\
-np.cos(beta[j])])
if np.abs(90 - np.degrees(np.arccos(np.dot(u, v)))) <= tol:
v_list.append([alpha[i], beta[j]])
v_list = np.array(v_list)
results = []
# temp hotfix
if mode == 'ballistic_old':
grid_space = 25
p1 = Position(43.8, 13.1, 0)
p2 = Position(47.8, 17.1, 0)
p1.pos_loc(ref_loc)
p2.pos_loc(ref_loc)
xs = np.linspace(p1.x, p2.x, grid_space)
ys = np.linspace(p1.y, p2.y, grid_space)
n_steps = grid_space**2 * len(points)
for xnum, xx in enumerate(xs):
for ynum, yy in enumerate(ys):
angle_list = []
time_list = []
D = [xx, yy, 0]
for pnum, pp in enumerate(points):
step = pnum + ynum * len(points) + xnum * grid_space * len(
points)
loadingBar("Contour Calculation", step, n_steps)
P = Position(pp[0], pp[1], pp[2])
P.pos_loc(ref_loc)
S = P.xyz
R = Position(0, 0, 0)
R.x = D[0]
R.y = D[1]
R.z = D[2]
R.pos_geo(ref_loc)
lats = [P.lat, R.lat]
lons = [P.lon, R.lon]
elev = [P.elev, R.elev]
z_profile, _ = atmos.getSounding(
lats,
lons,
elev,
ref_time=setup.fireball_datetime,
spline=100)
res = cyscan(np.array(S),
np.array(D),
z_profile,
wind=True,
h_tol=330,
v_tol=3000,
debug=False)
alpha = np.radians(res[1])
beta = np.radians(180 - res[2])
res_vector = np.array([np.sin(alpha)*np.sin(beta),\
np.cos(alpha)*np.sin(beta),\
-np.cos(beta)])
angle_list.append(
np.abs(90 - np.degrees(
np.arccos(
np.dot(
u / np.sqrt(u.dot(u)), res_vector /
np.sqrt(res_vector.dot(res_vector)))))))
time_list.append(res[0])
if np.nanmin(angle_list) <= tol:
best_angle_index = np.nanargmin(angle_list)
best_time = time_list[best_angle_index]
if not np.isnan(best_time):
res = [xx, yy, 0, best_time]
results.append(res)
# u = np.array([bam.setup.trajectory.vector.x,
# bam.setup.trajectory.vector.y,
# bam.setup.trajectory.vector.z])
# angle_off = []
# X = []
# for i in range(len(bam.setup.fragmentation_point)):
# az = stn.times.fragmentation[i][0][1]
# tf = stn.times.fragmentation[i][0][2]
# az = np.radians(az)
# tf = np.radians(180 - tf)
# v = np.array([np.sin(az)*np.sin(tf), np.cos(az)*np.sin(tf), -np.cos(tf)])
# angle_off.append(np.degrees(np.arccos(np.dot(u/np.sqrt(u.dot(u)), v/np.sqrt(v.dot(v))))))
# X.append(bam.setup.fragmentation_point[i].position.elev)
# angle_off = np.array(angle_off)
# try:
# best_indx = np.nanargmin(abs(angle_off - 90))
# except ValueError:
# best_indx = None
# a.append(np.array([np.nan, np.nan, np.nan, np.nan]))
# for pert in perturbations:
# a.append(np.array([np.nan, np.nan, np.nan, np.nan]))
# stn.times.ballistic.append(a)
# continue
# np.array([t_arrival, azimuth, takeoff, E[k, l]])
elif mode == 'ballistic':
n_steps = len(v_list) * len(points)
WIND = True
for pp, p in enumerate(points):
for vv, v in enumerate(v_list):
step = vv + pp * len(v_list)
loadingBar("Contour Calculation", step, n_steps)
# v[i] = np.array([np.sin(alpha[i])*np.sin(beta[i]),\
# np.cos(alpha[i])*np.sin(beta[i]),\
# -np.cos(beta[i])])
P = Position(p[0], p[1], p[2])
S = Position(p[0], p[1], 0)
P.pos_loc(ref_loc)
pt = P.xyz
# s = p + p[2]/np.cos(beta[i])*v[i]
# S = Position(0, 0, 0)
# P = Position(0, 0, 0)
# S.x, S.y, S.z = s[0], s[1], s[2]
# P.x, P.y, P.z = p[0], p[1], p[2]
# S.pos_geo(ref_loc)
# P.pos_geo(ref_loc)
lats = [P.lat, S.lat]
lons = [P.lon, S.lon]
elev = [P.elev, S.elev]
# z_profile, _ = supra.Supracenter.cyweatherInterp.getWeather(p, s, setup.weather_type, \
# ref_loc, copy.copy(sounding), convert=True)
if WIND:
z_profile, _ = atmos.getSounding(
lats,
lons,
elev,
ref_time=setup.fireball_datetime,
spline=200)
res = anglescan(pt,
np.degrees(v[0]),
np.degrees(v[1]) + 90,
z_profile,
wind=True,
debug=False)
else:
# if no wind, just draw a straight line
vect = np.array([np.sin(v[0])*np.sin(v[1]),\
np.cos(v[0])*np.sin(v[1]),\
-np.cos(v[1])])
s = -pt[2] / vect[2]
ground_point = pt + s * vect
ground_time = s / 330
res = [
ground_point[0], ground_point[1], ground_point[2],
ground_time
]
# This is the limit in distance from the trajectory (hardcoded)
if res[-1] <= 1000:
# if np.sqrt(res[0]**2 + res[1]**2) <= 150000:
results.append(res)
else:
# n_steps = len(tol_fact)*len(points)*steps
beta = np.linspace(90 + 0.01, 180, steps)
beta = np.radians(beta)
p = points
for ii, i in enumerate(range(steps)):
for jj, j in enumerate(range(steps)):
# print(np.degrees(beta[i]))
step = jj + ii * steps
loadingBar("Contour Calculation", step, n_steps)
v[i*steps + j] = np.array([np.sin(alpha[i])*np.sin(beta[j]),\
np.cos(alpha[i])*np.sin(beta[j]),\
-np.cos(beta[j])])
s = p + p[2] / np.cos(beta[j]) * v[i * steps + j]
S = Position(0, 0, 0)
P = Position(0, 0, 0)
S.x, S.y, S.z = s[0], s[1], s[2]
P.x, P.y, P.z = p[0], p[1], p[2]
S.pos_geo(ref_loc)
P.pos_geo(ref_loc)
lats = [P.lat, S.lat]
lons = [P.lon, S.lon]
elev = [P.elev, S.elev]
z_profile, _ = atmos.getSounding(
lats,
lons,
elev,
ref_time=setup.fireball_datetime,
spline=100)
res = anglescan(p,
np.degrees(alpha[i]),
np.degrees(beta[j]),
z_profile,
wind=True,
debug=False)
# if np.sqrt(res[0]**2 + res[1]**2) <= 200000:
results.append(res)
return results
def asyncBackTrack(**kwargs):
refDatas = kwargs['refDatas']
inpDatas = kwargs['inpDatas']
matchingCost = kwargs['matchingCost']
#inputFinFrameBackTracked, epsilonFinFrameBackTracked = kwargs['argsFinFrameBackTracked']
inputFinFrameBackTracked, epsilonFinFrameBackTracked = np.unravel_index(
np.nanargmin(matchingCost[matchingCost.shape[0] - 1]),
matchingCost[matchingCost.shape[0] - 1].shape)
correspondentPoints1, correspondentPoints2 = [], []
r, i, epsilon = matchingCost.shape[
0] - 1, inputFinFrameBackTracked, epsilonFinFrameBackTracked - limits
correspondentPoints1.append([r, i])
correspondentPoints2.append([r, i + epsilon])
"""
epsilon\i| i i-1 i-2(tmp=0,1,2)
-------------------------
-2 | e+c e+c e+c
-1 | e+c-1 e+c-1 e+c
0 | e+c-2 e+c-1 e+c
1 | e+c-2 e+c-1 e+c
2 | e+c-2 e+c-1 e+c
(epsilon + limits=0,...,4)
"""
forNewEpsilon = [[0, 0, 0], [-1, -1, 0], [-2, -1, 0], [-2, -1, 0],
[-2, -1, 0]]
while r > 0:
if i > 1:
mcosts = [
matchingCost[r - 1, i,
max(epsilon, 0):epsilon + limits + 1],
matchingCost[r - 1, i - 1,
max(epsilon + 1, 0):epsilon + limits + 2],
matchingCost[r - 1, i - 2,
max(epsilon + 2, 0):epsilon + limits + 3]
]
c_, tmp = [], []
for ind, mcost in enumerate(mcosts):
c_.append(np.argmin(mcost))
tmp.append(mcost[c_[ind]])
tmp = np.argmin(tmp)
r = r - 1
i = i - tmp
epsilon = epsilon + c_[tmp] + forNewEpsilon[epsilon +
limits][tmp]
correspondentPoints1.insert(0, [r, i])
correspondentPoints2.insert(0, [r, i + epsilon])
"""
# i -> i
if tmp == 0:
r = r - 1
epsilon = epsilon + c_[tmp] + forNewEpsilon[epsilon + limits][tmp]
correspondentPoints1.insert(0, [r, i])
correspondentPoints2.insert(0, [r, i + epsilon])
# i -> i - 1
elif tmp == 1:
r = r - 1
i = i - 1
epsilon = epsilon + c_[tmp] + forNewEpsilon[epsilon + limits][tmp]
correspondentPoints1.insert(0, [r, i])
correspondentPoints2.insert(0, [r, i + epsilon])
else: # i -> i - 2
r = r - 1
i = i - 2
epsilon = epsilon + c_[tmp] + forNewEpsilon[epsilon + limits][tmp]
correspondentPoints1.insert(0, [r, i])
correspondentPoints2.insert(0, [r, i + epsilon])
"""
elif i == 1:
mcosts = [
matchingCost[r - 1, 1,
max(epsilon, 0):epsilon + limits + 1],
matchingCost[r - 1, 0,
max(epsilon + 1, 0):epsilon + limits + 2]
]
c_, tmp = [], []
for ind, mcost in enumerate(mcosts):
c_.append(np.argmin(mcost))
tmp.append(mcost[c_[ind]])
tmp = np.argmin(tmp)
r = r - 1
i = i - tmp
epsilon = epsilon + c_[tmp] + forNewEpsilon[epsilon +
limits][tmp]
correspondentPoints1.insert(0, [r, i])
correspondentPoints2.insert(0, [r, i + epsilon])
else: # i == 0
c = np.argmin(matchingCost[r - 1, 0,
max(epsilon, 0):epsilon +
limits + 1])
r = r - 1
#i = i
epsilon = epsilon + c + forNewEpsilon[epsilon + limits][0]
correspondentPoints1.insert(0, [r, i])
correspondentPoints2.insert(0, [r, i + epsilon])
return correspondentPoints1, correspondentPoints2
import pandas as pd
import numpy as np
if __name__ == '__main__':
# First, read football.dat into a pandas dataframe
# using any contiguous block of whitespace as the delimiter between columns.
# Upon observing the data, I see that there is a column of hyphens between the 'for' and 'against'
# columns, and this column has no header.
# To account for this, I set header=0 to discard the top row from the file,
# thus allowing me to use my own set of column headers.
# I specify the headers in the 'names' parameter.
table = pd.read_table('football.dat', delim_whitespace=True, header=0, \
names=['number', 'name', 'p', 'w', 'l', 'd', 'for', 'dash', 'against', 'pts'])
# convert the 'for' and 'against' series into numpy float arrays
pts_agnst, pts_for = np.float_(table['against']), np.float_(table['for'])
# create a numpy float array that contains the absolute values of the differences between 'against' and 'for'
spread = abs(pts_agnst - pts_for)
# Now, get the array index of the smallest spread value that is not == nan
# Ignoring nan is necessary to account for the row of hyphens. Another option would have been to
# skip this row in pd.read_table, but the approach I have used is more general.
# numpy.nanargmin does exactly this, finding the index of a float array for the minimum value in
# that array which is not nan
answer_idx = np.nanargmin(spread)
# Print the value in the name column for the row with the smallest spread
print(table['name'][answer_idx])
def SSIdatStaDiag(data,
fs,
br,
ordmax=None,
lim=(0.01, 0.05, 0.02, 0.1),
method='1'):
'''This function return the Stabilization Diagram (Plot) for the given
data calculated according the data driven SSI algorithm.'''
ndat = data.shape[0] # Number of data points
nch = data.shape[1] # Number of channel
# If the maximum order is not given (default) it is set as the maximum
# allowable model order which is: number of block rows * number of channels
if ordmax == None:
ordmax = br * nch
freq_max = fs / 2 # Nyquist Frequency
# unpack the limits used for the construction of the Stab Diag
_lim_f, _lim_s, _lim_ms, _lim_s1 = lim[0], lim[1], lim[2], lim[3]
Yy = np.array(data.transpose()) # Matrice dati trasposta
j = ndat - 2 * br + 1
# DIMENSIONE MATRICE DI HANKEL
H = np.zeros((nch * 2 * br, j)) # PREALLOCA LA MATRICE DI HANKEL
for _k in range(0, 2 * br):
H[_k *
nch:((_k + 1) *
nch), :] = (1 / j**0.5) * Yy[:, _k:_k +
j] # CALCOLO MATRICE DI HANKEL
# FATTORIZZAZIONE LQ DELLA MATRICE DI HANKEL
Q, L = np.linalg.qr(H.T)
L = L.T
Q = Q.T
_a = nch * br
_b = nch
L21 = L[_a:_a + _b, :_a]
L22 = L[_a:_a + _b, _a:_a + _b]
L31 = L[_a + _b:, :_a]
L32 = L[_a + _b:, _a:_a + _b]
Q1 = Q[:_a, :]
Q2 = Q[_a:_a + _b, :]
P_i = np.vstack((L21, L31)) @ Q1 # Projection Matrix P_i
P_im1 = np.hstack((L31, L32)) @ np.vstack((Q1, Q2)) # Projection P_(i-1)
Y_i = np.hstack((L21, L22)) @ np.vstack((Q1, Q2)) # Output sequence
# SINGULAR VALUE DECOMPOSITION
U1, S1, V1_t = np.linalg.svd(P_i, full_matrices=False)
S1 = np.diag(S1)
S1rad = np.sqrt(S1)
# Ciclo per ordine del sistema crescente
Fr = np.full((ordmax, ordmax + 1),
np.nan) # inizializzo la matrice che conterrà le frequenze
Fr_lab = np.full(
(ordmax, ordmax + 1), np.nan
) # inizializzo la matrice che conterrà le labels per i poli(frequenze) da stampare nel diagramma di stabilizzazione
Sm = np.full((ordmax, ordmax + 1),
np.nan) # inizializzo la matrice che conterrà gli smorzamenti
Ms = [] # inizializzo la matrice (3D) che conterrà le forme modali
for z in range(0, int((ordmax) / 2 + 1)):
Ms.append(np.zeros((nch, z * (2))))
for _ind in range(0, ordmax + 1, 2):
# Inizializzo le matrici che mi servono
S11 = np.zeros((_ind, _ind)) # Inizializzo
U11 = np.zeros((br * nch, _ind)) # Inizializzo
V11 = np.zeros((_ind, br * nch)) # Inizializzo
O_1 = np.zeros((br * nch - nch, _ind)) # Inizializzo
O_2 = np.zeros((br * nch - nch, _ind)) # Inizializzo
# Estrazione di sottomatrici per ordine crescente del sistema
S11[:_ind, 0:_ind] = S1rad[
0:_ind, 0:
_ind] # ESTRAZIONE DI UNA SOTTOMATRICE DEI SINGULAR VALUES DA ORDINE MIN A ORD MAX
U11[:br * nch, 0:_ind] = U1[
0:br * nch, 0:
_ind] # ESTRAZIONE DI UNA SOTTOMATRICE DEI LEFT SINGULAR VECTORS DA ORDINE MIN A ORD MAX
V11[0:_ind, :br * nch] = V1_t[0:_ind, 0:br * nch] #
O = U11 @ S11 # Observability matrix
S = np.linalg.pinv(O) @ P_i # Kalman filter state sequence
O_1[:, :] = O[:O.shape[0] - nch, :]
O_2[:, :] = O[nch:, :]
# STIMA DELLA MATRICE DINAMICA A TEMPO DISCRETO
if method == '2': # Method 2
A = np.linalg.pinv(O_1) @ O_2
C = O[:nch, :]
# Ci sarebbero da calcolare le matrici G e R0
else: # METODO 1
Sp1 = np.linalg.pinv(O_1) @ P_im1 # kalman state sequence S_(i+1)
AC = np.vstack((Sp1, Y_i)) @ np.linalg.pinv(S)
A = AC[:Sp1.shape[0]]
C = AC[Sp1.shape[0]:]
# Ci sarebbero da calcolare le matrici G e R0
[_AuVal,
_AuVett] = np.linalg.eig(A) # CALCOLO AUTOVALORI ED AUTOVETTORI
Lambda = (np.log(_AuVal)) * fs # CALCOLO Lambda
fr = abs(Lambda) / (2 * np.pi) # CALCOLO FRQUENZE
smorz = -((np.real(Lambda)) / (abs(Lambda))) # CALCOLO SMORZAMENTO
# calcolo la matrice C per definizione dei modi
# le deformate modali M in forma complessa sono calcolate attraverso
# l'espressione M=CL dove L è la matrice le cui colonne sono gli autovettori
# di A (_AuVett)
Mcomp = C @ _AuVett
Mreal = np.real(C @ _AuVett)
Fr[:len(fr), _ind] = fr # SALVA LE FREQUENZE
Sm[:len(fr), _ind] = smorz # SALVA gli smorzamenti
Ms[int(_ind / 2)] = Mcomp # SALVA le forme modali
# =============================================================================
# Controllo stabilità dei poli
for idx, (_freq, _smor) in enumerate(zip(fr, smorz)):
if _ind == 0 or _ind == 2: # alla prima iterazione/primo ordine sono tutti nuovi poli
Fr_lab[:len(
fr
), _ind] = 0 # 0 è l'etichetta per i nuovi poli e poli instabili
elif np.isnan(_freq) == True:
_freq = 0
Fr[idx, _ind] = 0
else:
# Trovo l'indice del polo che minimizza la differenza alla iterazione/ordine n-1
ind2 = np.nanargmin(
abs(_freq - Fr[:, (_ind) - 2]) -
min(abs(_freq - Fr[:, (_ind) - 2])))
Fi_n = Mcomp[:,
idx] # forma modale all'iterazione = _iteraz (attuale)
Fi_nmeno1 = Ms[int(
_ind / 2 - 1
)][:,
ind2] # forma modale all'iterazione = _iteraz-1 (precedente)
# aMAC = np.abs(Fi_n@Fi_nmeno1)**2 / ((Fi_n@Fi_n)*(Fi_nmeno1@Fi_nmeno1)) # autoMAC
aMAC = MaC(Fi_n.real, Fi_nmeno1.real)
if min(abs(_freq - Fr[:, (_ind) - 2]) /
_freq) < _lim_f: # CONDIZIONE 1 SULLA FREQUENZA
if (_smor - Sm[ind2, (_ind) - 2]
) / _smor < _lim_s: # CONDIZIONE 2 SULLO SMORZAMENTO
if 1 - aMAC < _lim_ms: # CONDIZIONE 3 SULLE FORME MODALI
Fr_lab[
idx,
_ind] = 4 # Se il polo è stabile sia per smorzamento che per forma modale (viene classificato come stabile/verde)
else:
Fr_lab[
idx,
_ind] = 2 # Stabile per smorzamento ma non per forma modale
elif 1 - aMAC < _lim_ms: # CONDIZIONE 3 SULLE FORME MODALI
Fr_lab[
idx,
_ind] = 3 # Stabile per forma modale ma non per smorzamento
else:
Fr_lab[idx, _ind] = 1 # Stabile solo per frequenza
else:
Fr_lab[idx, _ind] = 0 # Nuovo polo o polo instabile
# =============================================================================
# PLOT DIAGRAMMA DI STABILIZZAZZIONE
# =============================================================================
_x = Fr.flatten(order='f')
_y = np.array([_i // len(Fr) for _i in range(len(_x))])
_l = Fr_lab.flatten(order='f')
_d = Sm.flatten(order='f')
_df = pd.DataFrame(dict(Frequency=_x, Order=_y, Label=_l, Damp=_d))
_df1 = _df.copy(
) # DataFrame che mi serve per plot diagramma stabilizzazione
# =============================================================================
# Qui creo un dataframe "ridotto" (senza nan) dove sono salvate: frequenze,
# smorzamenti e indici della forma modale associata al polo (order, emme)
df2 = _df.copy()
df2 = df2.dropna()
emme = []
for effe, order in zip(df2.Frequency, df2.Order):
emme.append(np.nanargmin(abs(effe - Fr[:, order]))) # trovo l'indice
emme = np.array(emme)
df2['Emme'] = emme
df2 = df2.drop_duplicates()
df2 = df2.drop(columns='Label')
# =============================================================================
_df1.Frequency = _df1.Frequency.where(
_df.Damp < _lim_s1
) # qui rimuovo tutti i poli che hanno smorzamento maggiore di lim_s1
_df1.Frequency = _df1.Frequency.where(
_df.Damp >
0) # qui rimuovo tutti i poli che hanno smorzamento negativo (< 0)
_colors = {
0: 'Red',
1: 'darkorange',
2: 'gold',
3: 'yellow',
4: 'Green'
} # assegno i colori alle etichette dei poli
fig1, ax1 = plt.subplots()
ax1 = sns.scatterplot(_df1['Frequency'],
_df1['Order'],
hue=_df1['Label'],
palette=_colors)
ax1.set_xlim(left=0, right=freq_max)
ax1.set_ylim(bottom=0, top=ordmax)
ax1.xaxis.set_major_locator(MultipleLocator(freq_max / 10))
ax1.xaxis.set_major_formatter(FormatStrFormatter('%g'))
ax1.xaxis.set_minor_locator(MultipleLocator(freq_max / 100))
ax1.set_title('''{0} - shift: {1}'''.format('Stabilization Diagram', br))
ax1.set_xlabel('Frequency [Hz]')
mplcursors.cursor()
plt.show()
Results = {}
Results['Data'] = {'Data': data}
Results['Data']['Samp. Freq.'] = fs
Results['All Poles'] = _df1
Results['Reduced Poles'] = df2
Results['Modes'] = Ms
return fig1, Results
def SSIcovStaDiag(data,
fs,
br,
ordmax=None,
lim=(0.01, 0.05, 0.02, 0.1),
method='1'):
ndat = data.shape[0] #NUMERO DI DATI CAMPIONATI
nch = data.shape[1] #NUMERO DI CANALI ACQUISITI
if ordmax == None:
ordmax = br * nch
freq_max = fs / 2 # Frequenza di Nyquist
_lim_f, _lim_s, _lim_ms, _lim_s1 = lim[0], lim[1], lim[2], lim[3]
Yy = np.array(data.transpose())
# =============================================================================
# METODO LIBRO (Rainieri & Fabbrocino)
# =============================================================================
# Mi calcolo tutti i R[i] (con i da 0 a 2*br)
R_is = np.array([
1 / (ndat - _s) * (Yy[:, :ndat - _s] @ Yy[:, _s:].T)
for _s in range(br * 2 + 1)
])
# Mi costruisco la matrice di Toepliz
Tb = np.vstack([
np.hstack([R_is[_o, :, :] for _o in range(br + _l, _l, -1)])
for _l in range(br)
])
# One-lag shifted matrice di Toeplitz per calcolo di A secondo "NExT-ERA"
Tb2 = np.vstack([
np.hstack([R_is[_o, :, :] for _o in range(br + _l, _l, -1)])
for _l in range(1, br + 1)
])
# =============================================================================
# SINGULAR VALUE DECOMPOSITION
# =============================================================================
U1, S1, V1_t = np.linalg.svd(Tb)
S1 = np.diag(S1)
S1rad = np.sqrt(S1)
# =============================================================================
# Ciclo per ordine del sistema crescente
# =============================================================================
Fr = np.full((ordmax, ordmax + 1),
np.nan) # inizializzo la matrice che conterrà le frequenze
Fr_lab = np.full(
(ordmax, ordmax + 1), np.nan
) # inizializzo la matrice che conterrà le labels per i poli(frequenze) da stampare nel diagramma di stabilizzazione
Sm = np.full((ordmax, ordmax + 1),
np.nan) # inizializzo la matrice che conterrà gli smorzamenti
Ms = [] # inizializzo la matrice (3D) che conterrà le forme modali
for z in range(0, int((ordmax) / 2 + 1)):
Ms.append(np.zeros((nch, z * (2))))
# =============================================================================
for _ind in range(0, ordmax + 1, 2):
# Inizializzo le matrici che mi servono
S11 = np.zeros((_ind, _ind)) # Inizializzo
U11 = np.zeros((br * nch, _ind)) # Inizializzo
V11 = np.zeros((_ind, br * nch)) # Inizializzo
O_1 = np.zeros((br * nch - nch, _ind)) # Inizializzo
O_2 = np.zeros((br * nch - nch, _ind)) # Inizializzo
# =============================================================================
# Estrazione di sottomatrici per ordine crescente del sistema
S11[:_ind, 0:_ind] = S1rad[
0:_ind, 0:
_ind] # ESTRAZIONE DI UNA SOTTOMATRICE DEI SINGULAR VALUES DA ORDINE MIN A ORD MAX
U11[:br * nch, 0:_ind] = U1[
0:br * nch, 0:
_ind] # ESTRAZIONE DI UNA SOTTOMATRICE DEI LEFT SINGULAR VECTORS DA ORDINE MIN A ORD MAX
V11[0:_ind, :br * nch] = V1_t[0:_ind, 0:br * nch] #
# =============================================================================
O = U11 @ S11 # CALCOLO MATRICE DI OSSERVABILITA
_GAM = S11 @ V11 # CALCOLO MATRICE DI CONTROLLABILITA
O_1[:, :] = O[:O.shape[0] - nch, :]
O_2[:, :] = O[nch:, :]
# =============================================================================
# STIMA DELLA MATRICE DINAMICA A TEMPO DISCRETO
if method == '2':
A = np.linalg.inv(S11) @ U11.T @ Tb2 @ V11.T @ np.linalg.inv(
S11) # METODO "NExT-ERA"
else:
A = np.linalg.pinv(O_1) @ O_2 # METODO 2 (BALANCED_REALIZATION)
[_AuVal,
_AuVett] = np.linalg.eig(A) # CALCOLO AUTOVALORI ED AUTOVETTORI
Lambda = (np.log(_AuVal)) * fs # CALCOLO Lambda
fr = abs(Lambda) / (2 * np.pi) # CALCOLO FRQUENZE
smorz = -((np.real(Lambda)) / (abs(Lambda))) # CALCOLO SMORZAMENTO
# calcolo la matrice C per definizione dei modi
# le deformate modali M in forma complessa sono calcolate attraverso
# l'espressione M=CL dove L è la matrice le cui colonne sono gli autovettori
# di A (_AuVett)
C = O[:nch, :]
Mcomp = C @ _AuVett
Mreal = np.real(C @ _AuVett)
Fr[:len(fr), _ind] = fr # SALVA LE FREQUENZE
Sm[:len(fr), _ind] = smorz # SALVA gli smorzamenti
Ms[int(_ind / 2)] = Mcomp # SALVA le forme modali
# =============================================================================
# Controllo stabilità dei poli
for idx, (_freq, _smor) in enumerate(zip(fr, smorz)):
if _ind == 0 or _ind == 2: # alla prima iterazione/primo ordine sono tutti nuovi poli
Fr_lab[:len(
fr
), _ind] = 0 # 0 è l'etichetta per i nuovi poli e poli instabili
elif np.isnan(_freq) == True:
_freq = 0
Fr[idx, _ind] = 0
else:
# Trovo l'indice del polo che minimizza la differenza alla iterazione/ordine n-1
ind2 = np.nanargmin(
abs(_freq - Fr[:, (_ind) - 2]) -
min(abs(_freq - Fr[:, (_ind) - 2])))
Fi_n = Mcomp[:,
idx] # forma modale all'iterazione = _iteraz (attuale)
Fi_nmeno1 = Ms[int(
_ind / 2 - 1
)][:,
ind2] # forma modale all'iterazione = _iteraz-1 (precedente)
# aMAC = np.abs(Fi_n@Fi_nmeno1)**2 / ((Fi_n@Fi_n)*(Fi_nmeno1@Fi_nmeno1)) # autoMAC
aMAC = MaC(Fi_n.real, Fi_nmeno1.real)
if min(abs(_freq - Fr[:, (_ind) - 2]) /
_freq) < _lim_f: # CONDIZIONE 1 SULLA FREQUENZA
if (_smor - Sm[ind2, (_ind) - 2]
) / _smor < _lim_s: # CONDIZIONE 2 SULLO SMORZAMENTO
if 1 - aMAC < _lim_ms: # CONDIZIONE 3 SULLE FORME MODALI
Fr_lab[
idx,
_ind] = 4 # Se il polo è stabile sia per smorzamento che per forma modale (viene classificato come stabile/verde)
else:
Fr_lab[
idx,
_ind] = 2 # Stabile per smorzamento ma non per forma modale
elif 1 - aMAC < _lim_ms: # CONDIZIONE 3 SULLE FORME MODALI
Fr_lab[
idx,
_ind] = 3 # Stabile per forma modale ma non per smorzamento
else:
Fr_lab[idx, _ind] = 1 # Stabile solo per frequenza
else:
Fr_lab[idx, _ind] = 0 # Nuovo polo o polo instabile
# =============================================================================
# PLOT DIAGRAMMA DI STABILIZZAZZIONE
# =============================================================================
_x = Fr.flatten(order='f')
_y = np.array([_i // len(Fr) for _i in range(len(_x))])
_l = Fr_lab.flatten(order='f')
_d = Sm.flatten(order='f')
_df = pd.DataFrame(dict(Frequency=_x, Order=_y, Label=_l, Damp=_d))
_df1 = _df.copy(
) # DataFrame che mi serve per plot diagramma stabilizzazione
# =============================================================================
# Qui creo un dataframe "ridotto" (senza nan) dove sono salvate: frequenze,
# smorzamenti e indici della forma modale associata al polo (order, emme)
df2 = _df.copy()
df2 = df2.dropna()
emme = []
for effe, order in zip(df2.Frequency, df2.Order):
emme.append(np.nanargmin(abs(effe - Fr[:, order]))) # trovo l'indice
emme = np.array(emme)
df2['Emme'] = emme
df2 = df2.drop_duplicates()
df2 = df2.drop(columns='Label')
# =============================================================================
_df1.Frequency = _df1.Frequency.where(
_df.Damp < _lim_s1
) # qui rimuovo tutti i poli che hanno smorzamento maggiore di lim_s1
_df1.Frequency = _df1.Frequency.where(
_df.Damp >
0) # qui rimuovo tutti i poli che hanno smorzamento negativo (< 0)
_colors = {
0: 'Red',
1: 'darkorange',
2: 'gold',
3: 'yellow',
4: 'Green'
} # assegno i colori alle etichette dei poli
fig1, ax1 = plt.subplots()
ax1 = sns.scatterplot(_df1['Frequency'],
_df1['Order'],
hue=_df1['Label'],
palette=_colors)
ax1.set_xlim(left=0, right=freq_max)
ax1.set_ylim(bottom=0, top=ordmax)
ax1.xaxis.set_major_locator(MultipleLocator(freq_max / 10))
ax1.xaxis.set_major_formatter(FormatStrFormatter('%g'))
ax1.xaxis.set_minor_locator(MultipleLocator(freq_max / 100))
ax1.set_title('''{0} - shift: {1}'''.format('Stabilization Diagram', br))
ax1.set_xlabel('Frequency [Hz]')
mplcursors.cursor()
plt.show()
Results = {}
Results['Data'] = {'Data': data}
Results['Data']['Samp. Freq.'] = fs
Results['All Poles'] = _df1
Results['Reduced Poles'] = df2
Results['Modes'] = Ms
return fig1, Results
def find_best_cuts_sensitivity(simtelfile_gammas,
simtelfile_protons,
dl2_file_g,
dl2_file_p,
nfiles_gammas,
nfiles_protons,
n_bins_energy,
n_bins_gammaness,
n_bins_theta2,
noff,
obstime=50 * 3600 * u.s):
"""
Main function to find the best cuts to calculate the sensitivity
based on a given a MC data subset
Parameters
---------
simtelfile_gammas: `string` path to simtelfile of gammas with mc info
simtelfile_protons: `string` path to simtelfile of protons with mc info
dl2_file_g: `string` path to h5 file of reconstructed gammas
dl2_file_p: `string' path to h5 file of reconstructed protons
nfiles_gammas: `int` number of simtel gamma files reconstructed
nfiles_protons: `int` number of simtel proton files reconstructed
n_bins_energy: `int` number of bins in energy
n_bins_gammaness: `int` number of bins in gammaness
n_bins_theta2: `int` number of bins in theta2
noff: `float` ratio between the background and the signal region
obstime: `Quantity` Observation time in seconds
Returns
---------
energy: `array` center of energy bins
sensitivity: `array` sensitivity per energy bin
"""
# Read simulated and reconstructed values
gammaness_g, theta2_g, e_reco_g, e_true_g, mc_par_g, events_g = process_mc(
simtelfile_gammas, dl2_file_g, 'gamma')
gammaness_p, angdist2_p, e_reco_p, e_true_p, mc_par_p, events_p = process_mc(
simtelfile_protons, dl2_file_p, 'proton')
mc_par_g['sim_ev'] = mc_par_g['sim_ev'] * nfiles_gammas
mc_par_p['sim_ev'] = mc_par_p['sim_ev'] * nfiles_protons
# Pass units to TeV and cm2
mc_par_g['emin'] = mc_par_g['emin'].to(u.TeV)
mc_par_g['emax'] = mc_par_g['emax'].to(u.TeV)
mc_par_p['emin'] = mc_par_p['emin'].to(u.TeV)
mc_par_p['emax'] = mc_par_p['emax'].to(u.TeV)
mc_par_g['area_sim'] = mc_par_g['area_sim'].to(u.cm**2)
mc_par_p['area_sim'] = mc_par_p['area_sim'].to(u.cm**2)
# Set binning for sensitivity calculation
# TODO: This information should be read from the files
emin_sensitivity = 0.01 * u.TeV # mc_par_g['emin']
emax_sensitivity = 100 * u.TeV # mc_par_g['emax']
energy = np.logspace(np.log10(emin_sensitivity.to_value()),
np.log10(emax_sensitivity.to_value()),
n_bins_energy + 1) * u.TeV
gammaness_bins, theta2_bins = bin_definition(n_bins_gammaness,
n_bins_theta2)
# Extract spectral parameters
dFdE, crab_par = crab_hegra(energy)
dFdEd0, proton_par = proton_bess(energy)
bins = np.logspace(np.log10(emin_sensitivity.to_value()),
np.log10(emax_sensitivity.to_value()),
n_bins_energy + 1)
y0 = mc_par_g['sim_ev'] / (mc_par_g['emax'].to_value()**(mc_par_g['sp_idx'] + 1) \
- mc_par_g['emin'].to_value()**(mc_par_g['sp_idx'] + 1)) \
* (mc_par_g['sp_idx'] + 1)
y = y0 * (bins[1:]**(crab_par['alpha'] + 1) -
bins[:-1]**(crab_par['alpha'] + 1)) / (crab_par['alpha'] + 1)
n_sim_bin = y
# Rates and weights
rate_g = rate("PowerLaw", mc_par_g['emin'], mc_par_g['emax'], crab_par,
mc_par_g['cone'], mc_par_g['area_sim'])
rate_p = rate("PowerLaw", mc_par_p['emin'], mc_par_p['emax'], proton_par,
mc_par_p['cone'], mc_par_p['area_sim'])
w_g = weight("PowerLaw", mc_par_g['emin'], mc_par_g['emax'],
mc_par_g['sp_idx'], rate_g, mc_par_g['sim_ev'], crab_par)
w_p = weight("PowerLaw", mc_par_p['emin'], mc_par_p['emax'],
mc_par_p['sp_idx'], rate_p, mc_par_p['sim_ev'], proton_par)
if (w_g.unit == u.Unit("sr / s")):
print(
"You are using diffuse gammas to estimate point-like sensitivity")
print("These results will make no sense")
w_g = w_g / u.sr # Fix to make tests pass
rate_weighted_g = ((e_true_g / crab_par['e0']) ** (crab_par['alpha'] - mc_par_g['sp_idx'])) \
* w_g
rate_weighted_p = ((e_true_p / proton_par['e0']) ** (proton_par['alpha'] - mc_par_p['sp_idx'])) \
* w_p
p_contained, ang_area_p = ring_containment(angdist2_p, 0.4 * u.deg,
0.3 * u.deg)
# FIX: ring_radius and ring_halfwidth should have units of deg
# FIX: hardcoded at the moment, but ring_radius should be read from
# the gamma file (point-like) or given as input (diffuse).
# FIX: ring_halfwidth should be given as input
area_ratio_p = np.pi * theta2_bins / ang_area_p
# ratio between the area where we search for protons ang_area_p
# and the area where we search for gammas math.pi * t
# Arrays to contain the number of gammas and hadrons for different cuts
final_gamma = np.ndarray(shape=(n_bins_energy, n_bins_gammaness,
n_bins_theta2))
final_hadrons = np.ndarray(shape=(n_bins_energy, n_bins_gammaness,
n_bins_theta2))
pre_gamma = np.ndarray(shape=(n_bins_energy, n_bins_gammaness,
n_bins_theta2))
pre_hadrons = np.ndarray(shape=(n_bins_energy, n_bins_gammaness,
n_bins_theta2))
ngamma_per_ebin = np.ndarray(n_bins_energy)
nhadron_per_ebin = np.ndarray(n_bins_energy)
total_rate_proton = np.sum(rate_weighted_p)
total_rate_gamma = np.sum(rate_weighted_g)
print("Total rate triggered proton {:.3f} Hz".format(total_rate_proton))
print("Total rate triggered gamma {:.3f} Hz".format(total_rate_gamma))
# Weight events and count number of events per bin:
for i in range(0, n_bins_energy): # binning in energy
total_rate_proton_ebin = np.sum(
rate_weighted_p[(e_reco_p < energy[i + 1])
& (e_reco_p > energy[i])])
print("\n******** Energy bin: {:.3f} - {:.3f} TeV ********".format(
energy[i].value, energy[i + 1].value))
total_rate_proton_ebin = np.sum(
rate_weighted_p[(e_reco_p < energy[i + 1])
& (e_reco_p > energy[i])])
total_rate_gamma_ebin = np.sum(
rate_weighted_g[(e_reco_g < energy[i + 1])
& (e_reco_g > energy[i])])
#print("**************")
print("Total rate triggered proton in this bin {:.5f} Hz".format(
total_rate_proton_ebin.value))
print("Total rate triggered gamma in this bin {:.5f} Hz".format(
total_rate_gamma_ebin.value))
for j in range(0, n_bins_gammaness): # cut in gammaness
for k in range(0, n_bins_theta2): # cut in theta2
rate_g_ebin = np.sum(rate_weighted_g[(e_reco_g < energy[i+1]) & (e_reco_g > energy[i]) \
& (gammaness_g > gammaness_bins[j]) & (theta2_g < theta2_bins[k])])
rate_p_ebin = np.sum(rate_weighted_p[(e_reco_p < energy[i+1]) & (e_reco_p > energy[i]) \
& (gammaness_p > gammaness_bins[j]) & p_contained])
final_gamma[i][j][k] = rate_g_ebin * obstime
final_hadrons[i][j][
k] = rate_p_ebin * obstime * area_ratio_p[k]
pre_gamma[i][j][k] = e_reco_g[(e_reco_g < energy[i+1]) & (e_reco_g > energy[i]) \
& (gammaness_g > gammaness_bins[j]) & (theta2_g < theta2_bins[k])].shape[0]
pre_hadrons[i][j][k] = e_reco_p[(e_reco_p < energy[i+1]) & (e_reco_p > energy[i]) \
& (gammaness_p > gammaness_bins[j]) & p_contained].shape[0]
ngamma_per_ebin[i] = np.sum(
rate_weighted_g[(e_reco_g < energy[i + 1])
& (e_reco_g > energy[i])]) * obstime
nhadron_per_ebin[i] = np.sum(
rate_weighted_p[(e_reco_p < energy[i + 1])
& (e_reco_p > energy[i])]) * obstime
n_excesses_5sigma, sensitivity_3Darray = calculate_sensitivity_lima(
final_gamma, final_hadrons * noff,
1 / noff * np.ones_like(final_gamma), n_bins_energy, n_bins_gammaness,
n_bins_theta2)
# Avoid bins which are empty or have too few events:
min_num_events = 5
min_pre_events = 5
# Minimum number of gamma and proton events in a bin to be taken into account for minimization
for i in range(0, n_bins_energy):
for j in range(0, n_bins_gammaness):
for k in range(0, n_bins_theta2):
conditions = (not np.isfinite(sensitivity_3Darray[i,j,k])) or (sensitivity_3Darray[i,j,k]<=0) \
or (final_hadrons[i,j,k] < min_num_events) \
or (pre_gamma[i,j,k] < min_pre_events) \
or (pre_hadrons[i,j,k] < min_pre_events)
if conditions:
sensitivity_3Darray[i][j][k] = np.inf
# Quantities to show in the results
sensitivity = np.ndarray(shape=n_bins_energy)
n_excesses_min = np.ndarray(shape=n_bins_energy)
eff_g = np.ndarray(shape=n_bins_energy)
eff_p = np.ndarray(shape=n_bins_energy)
gcut = np.ndarray(shape=n_bins_energy)
tcut = np.ndarray(shape=n_bins_energy)
ngammas = np.ndarray(shape=n_bins_energy)
nhadrons = np.ndarray(shape=n_bins_energy)
gammarate = np.ndarray(shape=n_bins_energy)
hadronrate = np.ndarray(shape=n_bins_energy)
eff_area = np.ndarray(shape=n_bins_energy)
nevents_gamma = np.ndarray(shape=n_bins_energy)
nevents_proton = np.ndarray(shape=n_bins_energy)
# Calculate the minimum sensitivity per energy bin
for i in range(0, n_bins_energy):
ind = np.unravel_index(np.nanargmin(sensitivity_3Darray[i], axis=None),
sensitivity_3Darray[i].shape)
gcut[i] = gammaness_bins[ind[0]]
tcut[i] = theta2_bins[ind[1]].to_value()
ngammas[i] = final_gamma[i][ind]
nhadrons[i] = final_hadrons[i][ind]
gammarate[i] = final_gamma[i][ind] / (obstime.to(u.min)).to_value()
hadronrate[i] = final_hadrons[i][ind] / (obstime.to(u.min)).to_value()
n_excesses_min[i] = n_excesses_5sigma[i][ind]
sensitivity[i] = sensitivity_3Darray[i][ind]
eff_g[i] = final_gamma[i][ind] / ngamma_per_ebin[i]
eff_p[i] = final_hadrons[i][ind] / nhadron_per_ebin[i]
e_aftercuts = e_true_g[(e_reco_g < energy[i + 1]) & (e_reco_g > energy[i]) \
& (gammaness_g > gammaness_bins[ind[0]]) & (theta2_g < theta2_bins[ind[1]])]
e_aftercuts_p = e_true_p[(e_reco_p < energy[i + 1]) & (e_reco_p > energy[i]) \
& p_contained]
e_aftercuts_w = np.sum(
np.power(e_aftercuts, crab_par['alpha'] - mc_par_g['sp_idx']))
eff_area[i] = e_aftercuts_w.to_value(
) / n_sim_bin[i] * mc_par_g['area_sim'].to(u.m**2).to_value()
nevents_gamma[i] = e_aftercuts.shape[0]
nevents_proton[i] = e_aftercuts_p.shape[0]
# Compute sensitivity in flux units
egeom = np.sqrt(energy[1:] * energy[:-1])
dFdE, par = crab_hegra(egeom)
sensitivity_flux = sensitivity / 100 * (dFdE * egeom * egeom).to(
u.erg / (u.cm**2 * u.s))
list_of_tuples = list(
zip(energy[:energy.shape[0] - 2].to_value(), energy[1:].to_value(),
gcut, tcut, ngammas,
nhadrons, gammarate, hadronrate, n_excesses_min,
sensitivity_flux.to_value(), eff_area, eff_g, eff_p, nevents_gamma,
nevents_proton))
result = pd.DataFrame(list_of_tuples,
columns=[
'ebin_low', 'ebin_up', 'gammaness_cut',
'theta2_cut', 'n_gammas', 'n_hadrons',
'gamma_rate', 'hadron_rate', 'n_excesses_min',
'sensitivity', 'eff_area', 'eff_gamma',
'eff_hadron', 'nevents_g', 'nevents_p'
])
units = [
energy.unit, energy.unit, "", theta2_bins.unit, "", "", u.min**-1,
u.min**-1, "", sensitivity_flux.unit,
mc_par_g['area_sim'].to(u.cm**2).unit, "", "", "", ""
]
# sensitivity_minimization_plot(n_bins_energy, n_bins_gammaness, n_bins_theta2, energy, sensitivity_3Darray)
# plot_positions_survived_events(events_g,
# events_p,
# gammaness_g, gammaness_p,
# theta2_g, p_contained, sensitivity_3Darray, energy, n_bins_energy, g, t)
return energy, sensitivity, result, units, gcut, tcut
def event_to_list_and_dedup(network_events_final, nstations):
#/ add all events to list (includes redundancies if event belongs to multiple pairs)
networkIDs = sorted(network_events_final.keys())
out = np.nan + np.ones((2 * len(networkIDs), nstations + 1))
for i, nid in enumerate(networkIDs):
tmp_dt = network_events_final[nid]['dt']
for stid in xrange(nstations):
if network_events_final[nid][stid]:
if len(network_events_final[nid][stid]) == 2:
out[2 * i, stid] = network_events_final[nid][stid][0]
out[2 * i + 1, stid] = network_events_final[nid][stid][1]
out[2 * i, nstations] = nid
out[2 * i + 1, nstations] = nid
elif len(
network_events_final[nid][stid]
) == 1: #/ if only one event in "pair" (i.e dt is small - TODO: ensure this case can't happen)
out[2 * i, stid] = network_events_final[nid][stid][0]
out[2 * i + 1, stid] = network_events_final[nid][stid][0]
out[2 * i, nstations] = nid
out[2 * i + 1, nstations] = nid
else: # if multiple
tmp_ts = network_events_final[nid][stid]
sidx = [
0,
np.argmax(
[q - p
for p, q in zip(tmp_ts[:-1], tmp_ts[1:])]) + 1
]
out[2 * i, stid] = np.min(tmp_ts[sidx[0]:sidx[1]])
out[2 * i + 1, stid] = np.min(tmp_ts[sidx[1]:])
out[2 * i, nstations] = nid
out[2 * i + 1, nstations] = nid
## remove duplicates from event-list (find and remove entries that are identical up to missing values (nan)) ##
out2 = out[:, 0:nstations]
netids2 = list(out[:, nstations].astype(int))
for sta in xrange(nstations):
row_sort0 = np.argsort(out2[:, sta])
out2 = out2[row_sort0, :]
netids2 = [netids2[x] for x in row_sort0]
n1, n2 = np.shape(out2)
keep_row = np.zeros(n1, dtype=bool)
network_eventlist = list()
tmp_neventlist = list()
for i in xrange(n1 - 1):
if np.all((out2[i, :] == out2[i + 1, :]) | np.isnan(out2[i, :])
| np.isnan(out2[i + 1, :])) & np.any(
out2[i, :] == out2[i + 1, :]): #/ if match or nan
out2[i + 1, :] = np.nanmin((out2[i, :], out2[i + 1, :]),
axis=0) #/ fill in nans
tmp_neventlist.append(netids2[i])
else:
keep_row[i] = True
tmp_neventlist.append(netids2[i]) #/ network id
network_eventlist.append(tmp_neventlist)
tmp_neventlist = list()
if i == n1 - 2: #/ add final event
keep_row[i + 1] = True
tmp_neventlist.append(netids2[i + 1])
network_eventlist.append(tmp_neventlist)
tmp_neventlist = list()
out2 = out2[keep_row, :]
netids2 = network_eventlist
def list_flatten(S):
if S == []:
return S
if isinstance(S[0], list):
return flatten(S[0]) + flatten(S[1:])
return S[:1] + flatten(S[1:])
netids2 = [list_flatten(x) for x in netids2] #/ to check if any missing
nfinal, n2 = np.shape(out2)
tmp = np.nanargmin(out2, axis=1)
row_sort = np.argsort(out2[np.arange(0, nfinal), tmp])
final_eventlist = out2[row_sort, :]
network_eventlist = [netids2[k] for k in row_sort]
return final_eventlist, network_eventlist, nfinal
def _make_trajectory(session, ref_lap):
"""Create telemetry Distance
"""
telemetry = ref_lap.telemetry
if telemetry.size != 0:
x = telemetry['X'].values
y = telemetry['Y'].values
z = telemetry['Z'].values
s = telemetry['Distance'].values
# Assuming constant speed in the last tenth
dt0_ = (ref_lap['LapTime'] - telemetry['Time'].iloc[-1]).total_seconds()
ds0_ = (telemetry['Speed'].iloc[-1] / 3.6) * dt0_
total_s = s[-1] + ds0_
# To prolong start and finish and have a correct linear interpolation
full_s = np.concatenate([s - total_s, s, s + total_s])
full_x = np.concatenate([x, x, x])
full_y = np.concatenate([y, y, y])
full_z = np.concatenate([z, z, z])
reference_s = np.arange(0, total_s, REFERENCE_LAP_RESOLUTION)
reference_x = np.interp(reference_s, full_s, full_x)
reference_y = np.interp(reference_s, full_s, full_y)
reference_z = np.interp(reference_s, full_s, full_z)
ssize = len(reference_s)
"""Build track map and project driver position to one trajectory
"""
def fix_suzuka(projection_index, _s):
"""Yes, suzuka is bad
"""
# For tracks like suzuka (therefore only suzuka) we have
# a beautiful crossing point. So, FOR F**K SAKE, sometimes
# shortest distance may fall below the bridge or viceversa
# gotta do some monotony sort of check. Not the cleanest
# solution.
def moving_average(a, n=3):
ret = np.cumsum(a, dtype=float)
ret[n:] = ret[n:] - ret[:-n]
ma = ret[n - 1:] / n
return np.concatenate([ma[0:n // 2], ma, ma[-n // 2:-1]])
ma_projection = moving_average(_s[projection_index], n=3)
spikes = np.absolute(_s[projection_index] - ma_projection)
# 1000 and 3000, very suzuka specific. Damn magic numbers
sel_bridge = np.logical_and(spikes > 1000, spikes < 3000)
unexpected = np.where(sel_bridge)[0]
max_length = _s[-1]
for p in unexpected:
# Just assuming linearity for this 2 or 3 samples
last_value = _s[projection_index[p - 1]]
last_step = last_value - _s[projection_index[p - 2]]
if (last_value + last_step) > max_length:
# Over the finish line
corrected_distance = -max_length + last_step + last_value
else:
corrected_distance = last_value + last_step
corrected_index = np.argmin(np.abs(_s - corrected_distance))
projection_index[p] = corrected_index
return projection_index
track = np.empty((ssize, 3))
track[:, 0] = reference_x
track[:, 1] = reference_y
track[:, 2] = reference_z
track_tree = scipy.spatial.cKDTree(track)
drivers_list = np.array(list(session.drivers))
stream_length = len(session.pos_data[drivers_list[0]])
dmap = np.empty((stream_length, len(drivers_list)), dtype=int)
fast_query = {'n_jobs': 2, 'distance_upper_bound': 500}
# fast_query < Increases speed
for index, drv in enumerate(drivers_list):
if drv not in session.pos_data.keys():
logging.warning(f"Driver {drv: >2}: No position data. (_make_trajectory)")
continue
trajectory = session.pos_data[drv][['X', 'Y', 'Z']].values
projection_index = track_tree.query(trajectory, **fast_query)[1]
# When tree cannot solve super far points means there is some
# pit shit shutdown. We can replace these index with 0
projection_index[projection_index == len(reference_s)] = 0
dmap[:, index] = fix_suzuka(projection_index.copy(), reference_s)
"""Create transform matrix to change distance point of reference
"""
t_matrix = np.empty((ssize, ssize))
for index in range(ssize):
rref = reference_s - reference_s[index]
rref[rref <= 0] = total_s + rref[rref <= 0]
t_matrix[index, :] = rref
"""Create mask to remove distance elements when car is on track
"""
time = session.pos_data[drivers_list[0]]['Time']
pit_mask = np.zeros((stream_length, len(drivers_list)), dtype=bool)
for driver_index, driver_number in enumerate(drivers_list):
laps = session.laps.pick_driver(driver_number)
in_pit = True
times = [[], []]
for lap_index in laps.index:
lap = laps.loc[lap_index]
if not pd.isnull(lap['PitInTime']) and not in_pit:
times[1].append(lap['PitInTime'])
in_pit = True
if not pd.isnull(lap['PitOutTime']) and in_pit:
times[0].append(lap['PitOutTime'])
in_pit = False
if not in_pit:
# Car crashed, we put a time and 'Status' will take care
times[1].append(lap['Time'])
times = np.transpose(np.array(times))
for inout in times:
out_of_pit = np.logical_and(time >= inout[0], time < inout[1])
pit_mask[:, driver_index] |= out_of_pit
on_track = (session.pos_data[driver_number]['Status'] == 'OnTrack')
pit_mask[:, driver_index] &= on_track.values
"""Calculate relative distances using transform matrix
"""
driver_ahead = {}
stream_axis = np.arange(stream_length)
for my_di, my_d in enumerate(drivers_list):
rel_distance = np.empty(np.shape(dmap))
for his_di, his_d in enumerate(drivers_list):
my_pos_i = dmap[:, my_di]
his_pos_i = dmap[:, his_di]
rel_distance[:, his_di] = t_matrix[my_pos_i, his_pos_i]
his_in_pit = ~pit_mask.copy()
his_in_pit[:, my_di] = False
my_in_pit = ~pit_mask[:, drivers_list == my_d][:, 0]
rel_distance[his_in_pit] = np.nan
closest_index = np.nanargmin(rel_distance, axis=1)
closest_distance = rel_distance[stream_axis, closest_index]
closest_driver = drivers_list[closest_index].astype(object)
closest_distance[my_in_pit] = np.nan
closest_driver[my_in_pit] = None
data = {'DistanceToDriverAhead': closest_distance,
'DriverAhead': closest_driver}
driver_ahead[my_d] = session.pos_data[my_d].join(pd.DataFrame(data), how='outer')
else:
# no data to base calculations on; create empty results
driver_ahead = dict()
for drv in session.drivers:
data = {'DistanceToDriverAhead': (), 'DriverAhead': ()}
driver_ahead[drv] = session.pos_data[drv].join(pd.DataFrame(data), how='outer')
return driver_ahead
def fit(self, X, y, n_epochs=None,
validation_set=None, max_fail=5, thresold_fail=1.5):
print(self.dropout)
print(self.batch_size)
print(self.optimizer_classes)
self.p_init = self.dropout[0]
self.p_hidden = self.dropout[1]
self.net_class_init = functools.partial(self.net_class, n_features=X.shape[1], n_classes=len(np.unique(y)),
dropout=self.dropout)
if n_epochs == None:
n_epochs = self.n_epochs
if validation_set == None:
validation_set = self.validation_set
## Load the data
input, target = torch.from_numpy(X), torch.from_numpy(y)
dataset = TensorDataset(input, target)
trainloader = self.load_data(dataset)
## Create the different initialisation (i.e create different nets)
net_list = self.multi_init()
## Train the nets
print("BEGINNING TRAINING")
loss_list = list()
loss_list_validation = list()
best_net_on_validation = [None, np.inf]
for epoch in range(n_epochs):
for net in net_list:
net[2] = 0.0
n_iter = 0
for data in trainloader:
input, target = data
target = target.view(-1)
input, target = Variable(input).float(), Variable(target).long()
for i in range(len(net_list)):
net, optimizer, _ = net_list[i]
optimizer.zero_grad()
output = net(input)
loss = self.criterion(output, target)
loss.backward()
optimizer.step()
net_list[i][2] += loss.data[0]
n_iter += 1
index_min = np.nanargmin([net[2] for net in net_list])
print(net_list[index_min][2] / n_iter)
loss_list.append(net_list[index_min][2] / n_iter)
## Check the loss on the validation set and stop if it's increasing for at least max_fail epochs
# or go back above thresold_fail * (minimum loss attained on the validation set)
if validation_set:
# copy the best net and make it in evaluation mode for scoring
best_net = copy.deepcopy(net_list[index_min][0])
weight_scaling_inference_rule(best_net, self.p_init, self.p_hidden)
best_net.eval()
score = self.score(validation_set[0], validation_set[1], best_net)
print(score)
loss_list_validation.append(score)
if score < best_net_on_validation[1]: # WARNING : loss or score ???
best_net_on_validation = [best_net, score]
if (score > thresold_fail * best_net_on_validation[1] or
max_fail < len(loss_list_validation)
and (np.array([loss_list_validation[i] - loss_list_validation[i - 1] for i in
range(- 1, - max_fail - 1, - 1)]) > 0).all()
):
print("EARLY STOPPING")
self.trained_net = best_net_on_validation[0]
return loss_list, loss_list_validation
##
if len(net_list) > 1:
del net_list[np.argmax([net[2] for net in net_list])]
print(epoch)
self.trained_net = best_net_on_validation[0]
# self.trained_net = net_list[-1][0] # for training on train
l = loss_list, loss_list_validation # before
# return self #for sklearn
return l
def test_nanargmin(self):
tgt = np.argmin(self.mat)
for mat in self.integer_arrays():
assert_equal(np.nanargmin(mat), tgt)
def argf(self, *args, **kwargs): return np.nanargmin(*args, **kwargs) class nanmax(A_extremum):
def argf(self, *args, **kwargs):
return np.nanargmin(*args, **kwargs)
def main():
# df_trn = pd.read_csv("df_trn_pool_SD_reddots.csv")
df_trn = pd.read_csv("df_trn_SD_AS2019_PA_CQT.csv")
tag_trn = df_trn['SDkey']
f = lambda x: tuple([str(s) for s in x.split(",")])
tag_trn = tag_trn.apply(f)
df_trn['tag_trn'] = tag_trn
# df_test = pd.read_csv("df_val_SD_AS2019_PA_CQT.csv")
df_test = pd.read_csv("df_eval_SD_AS2019_PA_CQT.csv")
tag_val = df_test['SDkey']
f = lambda x: tuple([str(s) for s in x.split(",")])
tag_val = tag_val.apply(f)
df_test['tag_eval'] = tag_val
# datagen=ImageDataGenerator(rescale=1./255.,validation_split=0.20)
datagen = ImageDataGenerator(rescale=1. / 255.)
test_datagen = ImageDataGenerator(rescale=1. / 255.)
train_generator = datagen.flow_from_dataframe(
dataframe=df_trn,
directory=None,
x_col="Filenames",
y_col="tag_trn",
# subset="training",
color_mode='grayscale',
batch_size=bs,
seed=66,
shuffle=True,
class_mode="categorical",
classes=trn_dev_classes,
target_size=(863, 400))
'''
valid_generator=datagen.flow_from_dataframe(
dataframe=df_trn,
directory=None,
x_col="Filenames",
y_col="tag_trn",
subset="validation",
color_mode='grayscale',
batch_size=bs,
seed=66,
shuffle=True,
class_mode="categorical",
classes=trn_dev_classes,
target_size=(400,80))
'''
test_generator = test_datagen.flow_from_dataframe(dataframe=df_test,
directory=None,
x_col="Filenames",
y_col="tag_eval",
color_mode='grayscale',
batch_size=1,
seed=66,
shuffle=False,
class_mode="categorical",
target_size=(863, 400))
model = load_model('model.hdf5',
custom_objects={'asoftmax_loss': asoftmax_loss})
# model.compile(optimizers.Adam(lr=5e-5, beta_1=0.9, beta_2=0.999),loss="categorical_crossentropy",metrics=["accuracy"])
model.compile(optimizers.Adam(lr=5e-5, beta_1=0.9, beta_2=0.999),
loss=asoftmax_loss,
metrics=["accuracy"])
STEP_SIZE_TRAIN = train_generator.n // train_generator.batch_size
STEP_SIZE_TEST = test_generator.n // test_generator.batch_size
test_generator.reset()
pred = model.predict_generator(test_generator,
steps=STEP_SIZE_TEST,
verbose=1)
np.savetxt("pred.csv", pred, delimiter=',', fmt='%1.4f')
predicted_class_indices = np.argmax(pred, axis=1)
labels = (train_generator.class_indices)
labels = dict((v, u) for u, v in labels.items())
predictions = [labels[u] for u in predicted_class_indices]
filenames = test_generator.filenames
results = pd.DataFrame({"Filename": filenames, "Predictions": predictions})
results.to_csv("results.csv", index=False)
# EER calculation
fpr, tpr, thresholds = roc_curve(test_generator.classes,
pred[:, 0],
pos_label=0)
fnr = 1 - tpr
EER = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
det_data = pd.DataFrame({"fpr": fpr, "fnr": fnr})
det_data.to_csv("DET.csv", index=False)
print EER
def file_loop(f):
print('Doing file: ' + f)
dic = xr.open_dataset(f)
edate = pd.Timestamp(dic.time.values)
out = dictionary()
res = []
outt = dic['tc_lag0'].values
outp = dic['p'].values
out['lon'] = dic['lon'].values
out['lat'] = dic['lat'].values
out['hour'] = dic['time.hour'].item()
out['month'] = dic['time.month'].item()
out['year'] = dic['time.year'].item()
out['date'] = dic['time'].values
if np.nanmin(dic['tc_lag0'].values) > -53:
return
#ipdb.set_trace()
out['clat'] = np.min(out['lat'])+((np.max(out['lat'])-np.min(out['lat']))*0.5)
out['clon'] = np.min(out['lon']) + ((np.max(out['lon']) - np.min(out['lon'])) * 0.5)
if (out['clat']<9) | (out['clon']<-15) | (out['clon']>15):
print('MCS out of box')
return
# if edate.hour < 17:
# return
try:
era_pl = xr.open_dataset(cnst.ERA5_HOURLY_PL+'ERA5_'+str(dic['time.year'].values)+'_'+str(dic['time.month'].values).zfill(2)+'_pl.nc')
except:
print('ERA5 missing')
return
try:
era_srfc = xr.open_dataset(cnst.ERA5_HOURLY_SRFC+'ERA5_'+str(dic['time.year'].values)+'_'+str(dic['time.month'].values).zfill(2)+'_srfc.nc')
except:
print('ERA5 srfc missing')
return
era_pl = uda.flip_lat(era_pl)
era_srfc = uda.flip_lat(era_srfc)
edate = edate.replace(hour=12, minute=0)
era_pl_day = era_pl.sel(time=edate, longitude=slice(-16,17), latitude=slice(4,26))
era_srfc_day = era_srfc.sel(time=edate, longitude=slice(-16,17), latitude=slice(4,26))
tminpos = np.where(dic['tc_lag0'].values == np.nanmin(dic['tc_lag0'].values)) # era position close to min temp
if len(tminpos[0])>1:
ptmax = np.nanmax((dic['p'].values)[tminpos])
if ptmax > 0:
prpos = np.where((dic['p'].values)[tminpos] == ptmax)
tminpos = ((tminpos[0])[prpos], (tminpos[1])[prpos] )
else:
tminpos = ((tminpos[0])[0], (tminpos[1])[0])
elon = dic['lon'].values[tminpos]
elat = dic['lat'].values[tminpos]
era_day = era_pl_day.sel(latitude=elat, longitude=elon , method='nearest') # take point of minimum T
era_day_srfc = era_srfc_day.sel(latitude=elat, longitude=elon , method='nearest') # take point of minimum T
del era_srfc_day
e925 = era_day.sel(level=925).mean()
e850 = era_pl_day['t'].sel(level=850)
elow = era_day.sel(level=slice(925,850)).mean('level').mean()
e650 = era_day.sel(level=650).mean()
emid = era_day.sel(level=slice(600,700)).mean('level').mean()
srfc = era_day_srfc.mean()
t_thresh = -50 # -40C ~ 167 W m-2
mask = np.isfinite(outp) & (outt<=t_thresh) & np.isfinite(outt)
mask_area = (outt<=t_thresh) & np.isfinite(outt)
mask70 = (outt<=-70) & np.isfinite(outt)
if np.sum(mask) < 3:
return
print(np.nanmax(outt[mask])) # can be bigger than cutout threshold because of interpolation to 5km grid after cutout
out['area'] = np.sum(mask_area)
out['area70'] = np.sum(mask70)
out['tmin'] = np.min(outt[mask])
out['tmean'] = np.mean(outt[mask])
maxpos = np.unravel_index(np.nanargmax(outp), outp.shape)
out['pmax'] = np.nanmean(ua.cut_kernel(outp,maxpos[1], maxpos[0],1)) #np.max(outp[mask])
out['pmean'] = np.mean(outp[mask])
dbox = e850.copy(deep=True)
minlon = era_pl_day.sel(latitude=8, longitude=np.min(out['lon']), method='nearest')
maxlon = era_pl_day.sel(latitude=8, longitude=np.max(out['lon']), method='nearest')
del era_pl_day
tgrad = dbox.sel(longitude=slice(minlon.longitude.values, maxlon.longitude.values)).mean('longitude')
tmin = np.nanargmin(tgrad.values)
tmax = np.nanargmax(tgrad.values)
tgrad = tgrad.isel(latitude=slice(tmin, tmax))
lingress = uda.linear_trend_lingress(tgrad)
out['tgrad'] = lingress['slope'].values
tgrad2 = dbox.sel(longitude=slice(np.min(out['lon']), np.max(out['lon'])), latitude=slice(10, 20)).mean(
['longitude', 'latitude']) - \
dbox.sel(longitude=slice(np.min(out['lon']), np.max(out['lon'])), latitude=slice(5, 7)).mean(['longitude', 'latitude'])
out['tbox'] = tgrad2.values
try:
out['q925'] =float(e925['q'])
except TypeError:
return
out['q650'] = float(e650['q'])
out['v925'] = float(e925['v'])
out['v650'] = float(e925['v'])
out['u925'] = float(e925['u'])
out['u650'] = float(e650['u'])
out['w925'] = float(e925['w'])
out['w650'] = float(e650['w'])
out['rh925'] = float(e925['r'])
out['rh650'] = float(e650['r'])
out['t925'] = float(e925['t'])
out['t650'] = float(e650['t'])
out['pv925'] = float(e925['pv'])
out['pv650'] = float(e650['pv'])
out['div925'] = float(e925['d'])
out['div650'] = float(e650['d'])
out['q_low'] = float(elow['q'])
out['q_mid'] = float(emid['q'])
out['tcwv'] = float(srfc['tcwv'])
out['shear'] = float(e650['u']-e925['u'])
theta_down = u_met.theta_e(925,e925['t']-273.15, e925['q'])
theta_up = u_met.theta_e(650,e650['t']-273.15, e650['q'])
out['dtheta'] = (theta_down-theta_up).values
out['thetaup'] = theta_up.values
out['thetadown'] = theta_down.values
out['pgt30'] = np.sum(outp[mask]>=30)
out['isvalid'] = np.sum(mask)
out['pgt01'] = np.sum(outp[mask]>=0.1)
#
out['p'] = outp[mask]
out['t'] = outt[mask]
#ipdb.set_trace()
dic.close()
return out
def summary_gene_feature(qtl_results_file='qtl_results_',snp_metadata_file='snp_metadata_', feature_metadata_file='feature_metadata_',output_file='qtl_results_genome',\
feature_report='ensembl_gene_id',chr_list=[9],path_data=None,folder_qtl=None, \
p_value_field='p_value',p_value_raw_field='p_value',local_adjustment_method='Bonferroni', exclude_snps=['']):
_doc = " aggregates qtl results to feature_report level"
iichr = 0
for ichr, chr in enumerate(chr_list):
print('chromosome: ' + str(chr))
try:
frez = h5py.File(
path_data + '/' + folder_qtl + '/' + qtl_results_file +
str(chr) + '.h5', 'r')
frezkeys = np.array(
[k.replace('_i_', '') for k in list(frez.keys())])
ffea = pandas.read_table(path_data + '/' + folder_qtl + '/' +
feature_metadata_file + str(chr) + '.txt',
sep='\t')
fsnp = pandas.read_table(path_data + '/' + folder_qtl + '/' +
snp_metadata_file + str(chr) + '.txt',
sep='\t').set_index(
'snp_id', drop=False).transpose()
print(ffea.columns.values)
except:
print('chromosome' + str(chr) + ' missing')
continue
if ~np.in1d(feature_report, ffea.columns.values):
ffea[feature_report] = ffea['feature_id']
indexnan = np.where(
(ffea[feature_report]) != (ffea[feature_report]))[0]
for i in indexnan:
ffea[feature_report][i] = 'gene_' + str(
ffea['chromosome'][i]) + '_' + str(ffea['start'][i])
ffea_feature = ffea.set_index('feature_id', drop=False).transpose()
ffea_report_feature = ffea.set_index(feature_report,
drop=False).transpose()
if iichr == 0:
fOut = h5py.File(
path_data + '/' + folder_qtl + '_' + feature_report + '_' +
output_file + '.h5', 'w')
iichr = 1
else:
fOut = h5py.File(
path_data + '/' + folder_qtl + '_' + feature_report + '_' +
output_file + '.h5', 'r+')
# for each report_feature create h5 groups
count = 0
for ifea, report_feature in enumerate(
np.unique(list(ffea_report_feature))):
# print (report_feature)
#select features for which qtl was computed
features = np.intersect1d(
np.array(ffea_report_feature[report_feature].transpose()
['feature_id']), frezkeys)
if len(features) >= 1:
fg = fOut.create_group(report_feature)
pv = np.array([frez[f][p_value_field] for f in features])
for i, ppv in enumerate(pv):
ppv[ppv != ppv] = 1
pv2 = np.array([frez[f][p_value_raw_field] for f in features])
for i, ppv in enumerate(pv2):
ppv[ppv != ppv] = 1
beta = np.array([frez[f]['beta'] for f in features])
for i, b in enumerate(beta):
b[b != b] = 1
beta_se = np.array([frez[f]['beta_se'] for f in features])
for i, b in enumerate(beta_se):
b[b != b] = 1
snp_id = np.array([frez[f]['snp_id'] for f in features])
position = np.array([
fsnp[snp_id[indf].astype('U')].transpose()['position']
for indf, f in enumerate(features)
])
for i, p in enumerate(position):
p[p != p] = 1
fgm = fg.create_group('metadata')
for key in ffea_feature[features[0]].keys():
if isinstance(ffea_feature[features[0]][key], int):
fgm.create_dataset(key,
data=np.array([
ffea_feature[f][key]
for f in features
]))
else:
fgm.create_dataset(key,
data=np.array([
ffea_feature[f][key]
for f in features
]).astype('S'))
fgd = fg.create_group('data')
fgd.create_dataset('features', data=features.astype('S'))
fgdp = fgd.create_group(p_value_field)
for indf, f in enumerate(features):
fgdp.create_dataset(f, data=pv[indf])
fgdp2 = fgd.create_group('p_value_raw')
for indf, f in enumerate(features):
fgdp2.create_dataset(f, data=pv2[indf])
fgdb = fgd.create_group('beta')
for indf, f in enumerate(features):
fgdb.create_dataset(f, data=beta[indf])
fgdbse = fgd.create_group('beta_se')
for indf, f in enumerate(features):
fgdbse.create_dataset(f, data=beta_se[indf])
fgdpo = fgd.create_group('position')
for indf, f in enumerate(features):
fgdpo.create_dataset(f, data=position[indf].astype(int))
fgds = fgd.create_group('snp_id')
for indf, f in enumerate(features):
fgds.create_dataset(f, data=snp_id[indf])
fgs = fg.create_group('summary_data')
fgs.create_dataset('min_p_value',
data=np.nanmin(np.hstack(pv))[None])
fgs.create_dataset('min_p_value_beta',
data=np.hstack(beta)[np.nanargmin(
np.hstack(pv))][None])
fgs.create_dataset('min_p_value_beta_se',
data=np.hstack(beta_se)[np.nanargmin(
np.hstack(pv))][None])
p_bonf = np.nanmin(
local_adjustment((pv), method=local_adjustment_method))
if p_bonf > 1: p_bonf = np.array(1)
fgs.create_dataset('min_p_value_local_adjusted',
data=p_bonf[None])
minfeatureindex = np.argmin(
[np.nanmin(ppv) * len(ppv) for ppv in pv])
fgs.create_dataset(
'min_p_value_feature_id',
data=np.array(
fgm['feature_id'][minfeatureindex].astype('S'))[None])
min_snp_id = frez[features[minfeatureindex]]['snp_id'][
np.nanargmin(pv[minfeatureindex])].astype('U')
fgs.create_dataset('min_p_value_snp_id',
data=np.array(min_snp_id).astype('S')[None])
fgs.create_dataset('min_p_value_position',
data=np.array(
fsnp[min_snp_id]['position'])[None])
count += 1
frez.close()
fOut.close()
def merge_all(distances):
'''
calculate the closest cluster for each cluster in distances
merge each cluster with its closest one
merge clusters that share clusters until no sharers left
'''
# get the closest cluster for each cluster
# generates a Series with pointer lists
closest_clusters = pd.Series(index=range(len(distances)), dtype='object')
for i in distances.index:
target_index = np.nanargmin(distances[i]).item()
# only one value now, but we will add values later
closest_clusters[i] = list()
closest_clusters[i].append(target_index)
cluster_groups = closest_clusters
# generate initial groups by adding the index to the target
for i, group in cluster_groups.iteritems():
# first value is the initial closest cluster
target = group[0]
cluster_groups[target].append(i)
# merge until there are only loners and groups with a pointer loop
# a loner is a cluster that has a target minimum distance cluster that does not point back to the loner
# a pointer loop is when two cluster point towards each other, even over multiple cluster between
# if a loner points to a loner the dependencies are pushed up until they are in a pointer loop group
finished = False
while not finished:
finished = True
# merge dependencies
for i, group in cluster_groups.iteritems():
# loner check
if len(group) > 1:
# first value is the initial closest cluster
target = group[0]
# rest of the values are pointers added by dependent groups
pointers = group[1:]
# check whether this is a dependent group without a pointer loop
if (target not in pointers):
# still dependent groups left, we need to iterate at least one more time
finished = False
# add own index to target
cluster_groups[target].append(i)
# sanity check whether looping is required
if (type(pointers) is list):
# multiple entries we can loop
for x in pointers:
if (x not in cluster_groups[target]):
cluster_groups[target].append(x)
else:
print(pointers)
cluster_groups[target].append(pointers[0])
# dependent group is spent, create loner
cluster_groups[i] = list()
cluster_groups[i].append(target)
# clear loners
for i, group in cluster_groups.iteritems():
# loner check
if (len(group) <= 1):
target = group[0]
# sanity check
if target in cluster_groups.index:
if len(cluster_groups[target]) > 1:
# loner points to pointer loop group
# push own index to target
cluster_groups[target].append(i)
# drop loner
cluster_groups = cluster_groups.drop(i)
# dress up the group list
# sort it and remove duplicates
merged_groups = list()
for i, group in cluster_groups.iteritems():
# add own index
temp = group
temp.append(i)
# generate sorted list without duplicates
temp = sorted(list(set(temp)))
merged_groups.append(temp)
merged_groups = sorted(merged_groups)
# merge connected groups and remove duplicates
for i, group_a in enumerate(merged_groups):
for k, group_b in enumerate(merged_groups):
# sanity check to not operate on the same list
if k is not i:
for x in group_a:
if x in set(group_b):
merged_groups[i] = list(
set(group_a).union(set(group_b)))
# both will point to the same list
merged_groups[k] = merged_groups[i]
# sort all cluster groups and don't add duplicates
clean = list()
for group in merged_groups:
sgroup = sorted(group)
# duplicate check
if sgroup not in clean:
clean.append(sgroup)
clean = sorted(clean)
# print cluster count information
print("total clusters: " +
str(len(list(set(list(itertools.chain.from_iterable(clean)))))))
print("total cluster groups: " + str(len(clean)))
return clean
def check_history_consistency(self, start: OptimizerResult):
# TODO other implementations
assert isinstance(start.history, (CsvHistory, MemoryHistory))
def xfull(x_trace):
return self.problem.get_full_vector(x_trace,
self.problem.x_fixed_vals)
if isinstance(start.history, CsvHistory):
it_final = np.nanargmin(start.history.get_fval_trace())
if isinstance(it_final, np.ndarray):
it_final = it_final[0]
it_start = int(
np.where(
np.logical_not(np.isnan(
start.history.get_fval_trace())))[0][0])
assert np.allclose(xfull(start.history.get_x_trace(it_start)),
start.x0)
assert np.allclose(xfull(start.history.get_x_trace(it_final)),
start.x)
assert np.isclose(start.history.get_fval_trace(it_start),
start.fval0)
funs = {
FVAL:
self.obj.get_fval,
GRAD:
self.obj.get_grad,
HESS:
self.obj.get_hess,
RES:
self.obj.get_res,
SRES:
self.obj.get_sres,
CHI2:
lambda x: res_to_chi2(self.obj.get_res(x)),
SCHI2:
lambda x: sres_to_schi2(*self.obj(
x, (
0,
1,
), pypesto.objective.constants.MODE_RES))
}
for var, fun in funs.items():
for it in range(5):
x_full = xfull(start.history.get_x_trace(it))
val = getattr(start.history, f'get_{var}_trace')(it)
if not getattr(self.history_options, f'trace_record_{var}',
True):
assert np.isnan(val)
continue
if np.all(np.isnan(val)):
continue
if var in [FVAL, CHI2]:
# note that we can expect slight deviations here since
# this fval/chi2 may be computed without sensitivities
# while the result here may be computed with with
# sensitivies activated. If this fails to often,
# increase atol/rtol
assert np.isclose(val, fun(x_full), rtol=1e-3,
atol=1e-4), var
elif var in [RES]:
# note that we can expect slight deviations here since
# this res is computed without sensitivities while the
# result here may be computed with with sensitivies
# activated. If this fails to often, increase atol/rtol
assert np.allclose(val, fun(x_full), rtol=1e-3,
atol=1e-4), var
elif var in [SRES]:
assert np.allclose(
val,
fun(x_full)[:, self.problem.x_free_indices],
), var
elif var in [GRAD, SCHI2]:
assert np.allclose(
val,
self.problem.get_reduced_vector(fun(x_full)),
), var
elif var in [HESS]:
assert np.allclose(
val,
self.problem.get_reduced_matrix(fun(x_full)),
), var
else:
raise RuntimeError('missing test implementation')
def main(opt):
ctx = mx.gpu() if opt.use_gpu else mx.cpu()
testclasspaths = []
testclasslabels = []
print('loading test files')
filename = '_testlist.txt'
with open(opt.dataset + "_" + opt.expname + filename, 'r') as f:
for line in f:
testclasspaths.append(line.split(' ')[0])
if int(line.split(' ')[1]) == -1:
testclasslabels.append(0)
else:
testclasslabels.append(1)
neworder = range(len(testclasslabels))
neworder = shuffle(neworder)
c = list(zip(testclasslabels, testclasspaths))
print('shuffling')
random.shuffle(c)
im_mean = mean_image.load_mean()
im_mean = im_mean.broadcast_to(
(opt.batch_size, np.shape(im_mean)[0], np.shape(im_mean)[1],
np.shape(im_mean)[2])
) #im_mean = nd.transpose(im_mean, (2, 0, 1))#testclasslabels, testclasspaths = zip(*c)
#testclasslabels = testclasslabels[1:5000]
#testclasspaths = testclasspaths[1:5000]
ltnt = 4096
print('loading pictures')
test_data = load_image.load_test_images(testclasspaths, testclasslabels,
opt.batch_size, opt.img_wd,
opt.img_ht, ctx, opt.noisevar)
print('picture loading done')
netEn, netDe, netD, netD2, netDS = set_network(opt.depth, ctx, 0, 0,
opt.ndf, opt.ngf,
opt.append)
netEn.load_params('checkpoints/' + opt.expname + '_' + str(opt.epochs) +
'_En.params',
ctx=ctx)
netDe.load_params('checkpoints/' + opt.expname + '_' + str(opt.epochs) +
'_De.params',
ctx=ctx)
netD.load_params('checkpoints/' + opt.expname + '_' + str(opt.epochs) +
'_D.params',
ctx=ctx)
netD2.load_params('checkpoints/' + opt.expname + '_' + str(opt.epochs) +
'_D2.params',
ctx=ctx)
netDS.load_params('checkpoints/' + opt.expname + '_' + str(opt.epochs) +
'_SD.params',
ctx=ctx)
print('Model loading done')
lbllist = []
scorelist1 = []
scorelist2 = []
scorelist3 = []
scorelist4 = []
test_data.reset()
count = 0
for batch in (test_data):
count += 1
print(str(count)) #, end="\r")
real_in = batch.data[0].as_in_context(ctx) - im_mean.as_in_context(ctx)
real_out = batch.data[1].as_in_context(ctx) - im_mean.as_in_context(
ctx)
lbls = batch.label[0].as_in_context(ctx)
code = netEn((real_out))
code = code + nd.random.normal(
loc=0, scale=0.002, shape=code.shape, ctx=ctx)
outnn = (netDe(code))
out_concat = nd.concat(real_out, outnn, dim=1) if opt.append else outnn
output4 = nd.mean((netD(out_concat)), (1, 3, 2)).asnumpy()
code = netEn(real_in)
#code=codet+nd.random.normal(loc=0, scale=0.0000001, shape=code.shape,ctx=ctx)
#code2=codet+nd.random.normal(loc=0, scale=0.000001, shape=code.shape,ctx=ctx)
#eq_code = heq(code.asnumpy(),2)
#code = nd.array(eq_code, ctx=ctx)
out = netDe(code)
#out2 = netDe(code2)
out_concat = nd.concat(real_in, out, dim=1) if opt.append else out
output = netD(out_concat) #Denoised image
output3 = nd.mean((out - real_out)**2,
(1, 3, 2)).asnumpy() #denoised-real
output = nd.mean(output, (1, 3, 2)).asnumpy()
out_concat = nd.concat(real_out, real_out,
dim=1) if opt.append else real_out
output2 = netDS(netDe(code)) #Image with no noise
output2 = nd.mean(output2, (1, 3, 2)).asnumpy()
lbllist = lbllist + list(lbls.asnumpy())
scorelist1 = scorelist1 + list(output)
scorelist2 = scorelist2 + list(output2)
scorelist3 = scorelist3 + list(output3)
scorelist4 = scorelist4 + list(output4)
fake_img1 = nd.concat(real_in[0], real_out[0], out[0], outnn[0], dim=1)
fake_img2 = nd.concat(real_in[1], real_out[1], out[1], outnn[1], dim=1)
fake_img3 = nd.concat(real_in[2], real_out[2], out[2], outnn[2], dim=1)
fake_img4 = nd.concat(real_in[3], real_out[3], out[3], outnn[3], dim=1)
fake_img = nd.concat(fake_img1, fake_img2, fake_img3, fake_img4, dim=2)
#print(np.shape(fake_img))
visual.visualize(fake_img)
plt.savefig('outputs/T_' + opt.expname + '_' + str(count) + '.png')
if not opt.isvalidation:
fpr, tpr, _ = roc_curve(lbllist, scorelist1, 1)
roc_auc1 = auc(fpr, tpr)
fpr, tpr, _ = roc_curve(lbllist, scorelist2, 1)
roc_auc2 = auc(fpr, tpr)
fpr, tpr, _ = roc_curve(lbllist, scorelist3, 1)
roc_auc3 = auc(fpr, tpr)
EER = fpr[np.nanargmin(np.absolute((1 - tpr - fpr)))]
fpr, tpr, _ = roc_curve(lbllist, scorelist4, 1)
roc_auc4 = auc(fpr, tpr)
plt.gcf().clear()
plt.clf()
sns.set(color_codes=True)
posscores = [
scorelist3[i] for i, v in enumerate(lbllist) if int(v) == 1
]
negscores = [
scorelist3[i] for i, v in enumerate(lbllist) if int(v) == 0
]
#sns.distplot(posscores, hist=False, label="Known Classes" ,rug=True)
sns.kdeplot(posscores, label="Known Classes")
sns.kdeplot(negscores, label="Unnown Classes")
#plt.hold()
#sns.distplot(negscores, hist=False, label = "Unknown Classes", rug=True);
plt.legend()
plt.savefig('outputs/matdist_' + opt.expname + '_.png')
plt.gcf().clear()
inputT = nd.zeros((ltnt, ltnt, 1, 1), ctx=ctx)
for i in range(0, ltnt):
inputT[i, i, :, :] = -1
out = netDe(inputT)
count = 0
for i in range(int(math.ceil(math.sqrt(ltnt)))):
for j in range(int(math.ceil(math.sqrt(ltnt)))):
if count < ltnt:
plt.subplot(math.ceil(math.sqrt(ltnt)),
math.ceil(math.sqrt(ltnt)), count + 1)
plt.imshow(
((out[count].asnumpy().transpose(1, 2, 0) + 1.0) *
127.5).astype(np.uint8))
plt.axis('off')
count += 1
plt.savefig('outputs/atoms_' + opt.expname + '_.png')
plt.gcf().clear()
plt.clf()
print(EER)
return ([roc_auc1, roc_auc2, roc_auc3, roc_auc4])
else:
return ([0, 0, 0, 0])
fakecode = nd.random_normal(loc=0,
scale=1,
shape=(16, 4096, 1, 1),
ctx=ctx)
out = netDe(fakecode)
fake_img1 = nd.concat(out[0], out[1], out[2], out[3], dim=1)
fake_img2 = nd.concat(out[7], out[6], out[5], out[4], dim=1)
fake_img3 = nd.concat(out[8], out[9], out[10], out[11], dim=1)
fake_img4 = nd.concat(out[15], out[14], out[13], out[12], dim=1)
fake_img = nd.concat(fake_img1, fake_img2, fake_img3, fake_img4, dim=2)
#print(np.shape(fake_img))
visual.visualize(fake_img)
plt.savefig('outputs/fakes_' + opt.expname + '_.png')
def add_step(data, jpos, means, variances, steps, lamb=0.1, gamma0=0.5):
''' Try adding a negative or positive step to the steps already found. Only the part of the trace
before the penultimate step will be considered.
'''
mb = means[0]
vb = variances[0]
mf = means[1] - means[0]
vf = variances[1] - variances[0]
if vf < 0:
vf = variances[1]
sz = len(data)
# initial sic
sicmin = posterior_single(data, jpos, means, variances, steps)
stp, ct = np.unique(np.abs(steps), return_counts=True)
# add one to steps with zero occupancy
ct[0] += 1
step_out = steps
jpos_out = jpos
# build arrays for squared diff, variance and length
i_in = np.cumsum(np.hstack((0, steps)))
steploc = np.hstack((0, jpos, sz))
diffar = np.tile(
np.array(list(map(sum, (np.split(data, jpos) - i_in * mf - mb)**2))),
[sz - jpos[1], 1])
diffar = np.hstack((diffar, np.zeros([sz - jpos[1], 1])))
varphi = np.tile(np.hstack((i_in * vf + vb, 0)), [sz - jpos[1], 1])
nphi = np.tile(np.hstack((np.diff(steploc), 0)), [sz - jpos[1], 1])
sics = []
steppos = []
for stp in (-1, 1):
for I in range(2, len(steploc) - 1):
# length
stepl = int(steploc[I + 1] - steploc[I])
if I < len(steploc) - 2:
# shift values after current range by one and recalculate with i + / - 1
try:
dd = np.array(
list(
map(sum, (np.split(data[steploc[I + 1]:],
jpos[I + 1:] - steploc[I + 1]) -
(i_in[I + 1:] + stp) * mf - mb)**2)))
except ValueError:
dd = np.array(
list(
map(sum, (np.split(data[steploc[I + 1]:],
jpos[I + 1:] - steploc[I + 1]) -
np.expand_dims(
(i_in[I + 1:] + stp) * mf - mb,
axis=1))**2)))
diffar[steploc[I] - jpos[1]:steploc[I + 1] - jpos[1],
I + 2:] = np.tile(dd, [stepl, 1])
varphi[steploc[I] - jpos[1]:steploc[I + 1] - jpos[1],
I + 2:] = np.tile((i_in[I + 1:] + stp) * vf + vb,
[stepl, 1])
nphi[steploc[I] - jpos[1]:steploc[I + 1] - jpos[1],
I + 2:] = np.tile(np.diff(steploc[I + 1:]), [stepl, 1])
# diffs under the diagonal (before added step)
dar = np.tile(data[steploc[I]:steploc[I + 1]], [stepl, 1])
dar[np.triu(np.ones([stepl, stepl]), 1).astype(
bool)] = i_in[I] * mf + mb # diff values will be zeros
diffar[steploc[I] - jpos[1]:steploc[I + 1] - jpos[1], I] = np.sum(
(dar - i_in[I] * mf - mb)**2, 1)
# diffs above diagonal (after added step)
dar = np.tile(data[steploc[I]:steploc[I + 1]], [stepl, 1])
dar[np.tril(np.ones([stepl, stepl]),
0).astype(bool)] = (i_in[I] + stp) * mf + mb
diffar[steploc[I] - jpos[1]:steploc[I + 1] - jpos[1],
I + 1] = np.sum((dar - (i_in[I] + stp) * mf - mb)**2, 1)
# varphi for the current range
varphi[steploc[I] - jpos[1]:steploc[I + 1] - jpos[1],
I] = i_in[I] * vf + vb
varphi[steploc[I] - jpos[1]:steploc[I + 1] - jpos[1],
I + 1] = (i_in[I] + stp) * vf + vb
# length for current range
nphi[steploc[I] - jpos[1]:steploc[I + 1] - jpos[1],
I] = np.arange(stepl) + 1
nphi[steploc[I] - jpos[1]:steploc[I + 1] - jpos[1],
I + 1] = stepl - np.arange(stepl) - 1
# SIC calculation
m = np.sum(np.abs(steps)) + 1
K = len(jpos) + 1
# This sometimes throws warnings invalid value in log
sic = np.sum(nphi * np.log(varphi) + diffar / varphi, 1)
sic += 2 * (-K * np.log(lamb) - np.log(
factorial(m - K) * factorial(K) / factorial(m - 1)) +
np.sum(np.log(factorial(ct))))
sic += 2 * gamma0 * (m - K + 1) / K + np.log(m - K + 2) + np.log(
m - K + 1) - np.log(m - K + 2 - (m - K + 1) * np.exp(-gamma0 / K))
# exclude positions around existing steps and edges
jignore = np.ravel(np.tile(jpos, [5, 1]).T +
np.arange(-2, 3)) - jpos[1]
jignore = jignore[(jignore > 0) * (jignore < len(sic) - 2)]
ind = np.arange(len(sic) - 2)
ind = np.delete(ind, jignore)
sic = sic[ind]
sics.append(np.min(sic))
steppos.append(ind[np.argmin(sic)] + jpos[1] + 1)
# nanmin in case of negative fluorophores, maybe can be done more elegantly
# sometimes warns about all nan axis
if np.nanmin(sics) < sicmin:
# found new step, add it
sicmin = np.nanmin(sics)
newstep = steppos[np.nanargmin(sics)]
stepsign = np.nanargmin(sics) * 2 - 1
insertpos = np.max(np.where(jpos < newstep)[0]) + 1
step_out = np.hstack((steps[:insertpos], stepsign, steps[insertpos:]))
jpos_out = sorted(np.hstack((jpos, newstep)))
# calculate variances and means
variances = [np.var(ar) for ar in np.split(data, jpos_out)]
means = [np.mean(ar) for ar in np.split(data, jpos_out)]
return sicmin, jpos_out, step_out, means, variances
def test_lfw(self, set='test', model='ssdm_resnet', visualize=True):
if set is 'train':
pairfile = 'pairsDevTrain.txt'
else:
pairfile = 'pairsDevTest.txt'
lfw_path = DeepFaceConfs.get()['dataset']['lfw']
path = os.path.join(lfw_path, pairfile)
with open(path, 'r') as f:
lines = f.readlines()[1:]
pairs = []
for line in lines:
elms = line.split()
if len(elms) == 3:
pairs.append((elms[0], int(elms[1]), elms[0], int(elms[2])))
elif len(elms) == 4:
pairs.append((elms[0], int(elms[1]), elms[2], int(elms[3])))
else:
logger.warning('line should have 3 or 4 elements, line=%s' %
line)
detec = FaceDetectorDlib.NAME
if model == 'baseline':
recog = FaceRecognizerVGG.NAME
just_name = 'vgg'
elif model == 'baseline_resnet':
recog = FaceRecognizerResnet.NAME
just_name = 'resnet'
elif model == 'ssdm_resnet':
recog = FaceRecognizerResnet.NAME
just_name = 'resnet'
detec = 'detector_ssd_mobilenet_v2'
else:
raise Exception('invalid model name=%s' % model)
logger.info('pair length=%d' % len(pairs))
test_result = [] # score, label(1=same)
for name1, idx1, name2, idx2 in tqdm(pairs):
img1_path = os.path.join(lfw_path, name1,
'%s_%04d.jpg' % (name1, idx1))
img2_path = os.path.join(lfw_path, name2,
'%s_%04d.jpg' % (name2, idx2))
img1 = cv2.imread(img1_path, cv2.IMREAD_COLOR)
img2 = cv2.imread(img2_path, cv2.IMREAD_COLOR)
if img1 is None:
logger.warning('image not read, path=%s' % img1_path)
if img2 is None:
logger.warning('image not read, path=%s' % img2_path)
result1 = self.run(image=img1,
detector=detec,
recognizer=recog,
visualize=False)
result2 = self.run(image=img2,
detector=detec,
recognizer=recog,
visualize=False)
if len(result1) == 0:
logger.warning('face not detected, name=%s(%d)! %s(%d)' %
(name1, idx1, name2, idx2))
test_result.append((0.0, name1 == name2))
continue
if len(result2) == 0:
logger.warning('face not detected, name=%s(%d) %s(%d)!' %
(name1, idx1, name2, idx2))
test_result.append((0.0, name1 == name2))
continue
feat1 = result1[0].face_feature
feat2 = result2[0].face_feature
similarity = feat_distance_cosine(feat1, feat2)
test_result.append((similarity, name1 == name2))
# calculate accuracy TODO
accuracy = sum([
label ==
(score > DeepFaceConfs.get()['recognizer'][just_name]['score_th'])
for score, label in test_result
]) / float(len(test_result))
logger.info('accuracy=%.8f' % accuracy)
# ROC Curve, AUC
tps = []
fps = []
accuracy0 = []
accuracy1 = []
acc_th = []
for th in range(0, 100, 5):
th = th / 100.0
tp = 0
tn = 0
fp = 0
fn = 0
for score, label in test_result:
if score >= th and label == 1:
tp += 1
elif score >= th and label == 0:
fp += 1
elif score < th and label == 0:
tn += 1
elif score < th and label == 1:
fn += 1
tpr = tp / (tp + fn + 1e-12)
fpr = fp / (fp + tn + 1e-12)
tps.append(tpr)
fps.append(fpr)
accuracy0.append(tn / (tn + fp + 1e-12))
accuracy1.append(tp / (tp + fn + 1e-12))
acc_th.append(th)
fpr, tpr, thresh = roc_curve([x[1] for x in test_result],
[x[0] for x in test_result])
fnr = 1 - tpr
eer = fnr[np.nanargmin(np.absolute((fnr - fpr)))]
logger.info('1-eer=%.4f' % (1.0 - eer))
with open('./etc/test_lfw.pkl', 'rb') as f:
results = pickle.load(f)
if visualize in [True, 'True', 'true', 1, '1']:
fig = plt.figure()
a = fig.add_subplot(1, 2, 1)
plt.title('Experiment on LFW')
plt.plot(fpr, tpr,
label='%s(%.4f)' % (model, 1 - eer)) # TODO : label
for model_name in results:
if model_name == model:
continue
fpr_prev = results[model_name]['fpr']
tpr_prev = results[model_name]['tpr']
eer_prev = results[model_name]['eer']
plt.plot(fpr_prev,
tpr_prev,
label='%s(%.4f)' % (model_name, 1 - eer_prev))
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
a.legend()
a.set_title('Receiver operating characteristic')
a = fig.add_subplot(1, 2, 2)
plt.plot(accuracy0, acc_th, label='Accuracy_diff')
plt.plot(accuracy1, acc_th, label='Accuracy_same')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
a.legend()
a.set_title('%s : TP, TN' % model)
fig.savefig('./etc/roc.png', dpi=300)
plt.show()
plt.draw()
with open('./etc/test_lfw.pkl', 'wb') as f:
results[model] = {
'fpr': fpr,
'tpr': tpr,
'acc_th': acc_th,
'accuracy0': accuracy0,
'accuracy1': accuracy1,
'eer': eer
}
pickle.dump(results, f, pickle.HIGHEST_PROTOCOL)
return 1.0 - eer
n = inputData['n']
R = inputData['R']
epsilon = inputData['epsilon']
x0 = inputData['x']
y0 = inputData['y']
phi = data['phi']
phiCount = phi.size
Tphi = data['Tphi']
frequencies = data['frequencies']
countf = frequencies.size
Rk = data['Rk']
S = 2 * pi / phiCount * Rk * np.sum(Tphi, axis=1)
print(np.nanargmax(S)) # = 840
print(np.nanargmin(S[100:]) + 100)
print(np.nanargmax(S[900:]) + 900)
mpl.rcParams['mathtext.fontset'] = 'stix'
mpl.rcParams['font.family'] = 'STIXGeneral'
mpl.rcParams['legend.fontsize'] = 'medium'
mpl.rcParams['axes.labelsize'] = 'large'
plt.figure(figsize=(7, 3))
plt.plot(frequencies, S)
plt.ylim(top=10, bottom=5e-4)
plt.yscale("log")
plt.xlabel(r"$f$")
plt.ylabel(r"$S$")
plt.title(r"1 hexagon, $R=0.45$, $\epsilon = (1.1 \pm 0.1\mathrm{i})^2$")
def decode(self, bits_in, debug=0):
len_trellis = (len(bits_in) / 6) #+1
state_prob = [1] + [0] * 63
prev_state = numpy.zeros((64, len_trellis + 1))
acc_err = numpy.empty((64, len_trellis + 1))
acc_err[:] = numpy.nan
acc_err[0][0] = 0
for m in range(0, len_trellis):
#print(self.state_prop)
#print( bits_in[0+m*6:6+m*6] )
for n in range(0, 64):
#print( state_prob[n])
if state_prob[n] != 0:
b_0 = self.calc_error(n, bits_in[0 + m * 6:6 + m * 6], 0)
b_1 = self.calc_error(n, bits_in[0 + m * 6:6 + m * 6], 1)
next_state = self.trans_0[n]
#if debug:
#if acc_err[n][m] == 0:
#print( str(int(m)) + ' State ' + str(n) + ' 0:' + str( int(b_0) ) + ' 1:' + str(int(b_1) ) )
#self.calc_error_dbug(n,bits_in[0+m*6:6+m*6], 0)
#self.calc_error_dbug(n,bits_in[0+m*6:6+m*6], 1)
if (acc_err[next_state][m + 1] > acc_err[n][m] +
b_0) or numpy.isnan(acc_err[next_state][m + 1]):
if numpy.isnan(acc_err[n][m]):
acc_err[next_state][m + 1] = b_0
else:
acc_err[next_state][m + 1] = acc_err[n][m] + b_0
prev_state[next_state][m + 1] = n
next_state = self.trans_1[n]
if (acc_err[next_state][m + 1] > acc_err[n][m] +
b_1) or numpy.isnan(acc_err[next_state][m + 1]):
if numpy.isnan(acc_err[n][m]):
acc_err[next_state][m + 1] = b_1
else:
acc_err[next_state][m + 1] = acc_err[n][m] + b_1
prev_state[next_state][m + 1] = n
state_prob = numpy.dot(state_prob, self.trans_prob)
# Pfad mit kleinstem Fehler suchen
#print( [x[len_trellis] for x in acc_err] )
best_res = numpy.nanargmin([x[len_trellis] for x in acc_err])
if debug:
print('Bester Pfad : ' + str(best_res) + ' Fehler: ' +
str(acc_err[best_res][len_trellis]))
#Traceback
bits_out = numpy.zeros(len_trellis)
state = best_res
#print( 'Trace: ' + str(state))
for n in range(0, len_trellis):
#print(self.trans_0[int(prev_state[state][int(len_trellis-n)])])
if self.trans_0[int(prev_state[int(state)][int(len_trellis -
n)])] == state:
bits_out[len_trellis - 1 - n] = int(0)
else:
bits_out[len_trellis - 1 - n] = int(1)
state = prev_state[int(state)][int(len_trellis - n)]
#print( 'Trace: ' + str(state))
return bits_out
def ref_impl(a):
return np.nanargmin(a)
def asyncCalc(refDatas, inpDatas):
if len(refDatas) != 3 or len(inpDatas) != 3:
raise ValueError(
'The length of both refDatas and inpDatas must be three, but got ref:{0} and inp:{1}'
.format(len(refDatas), len(inpDatas)))
R, I = refDatas[0].shape[0], inpDatas[0].shape[0]
localCost1 = _asyncLocalCost(refDatas[1], refDatas[0], inpDatas[1],
inpDatas[0], limits, 2)
localCost3 = _asyncLocalCost(refDatas[1], refDatas[2], inpDatas[1],
inpDatas[2], limits, 2)
matchingCost1 = np.zeros((R, I, 2 * limits + 1))
matchingCost3 = np.zeros((R, I, 2 * limits + 1))
matchingCost1[0, :, :] = localCost1[0, :, :]
matchingCost3[0, :, :] = localCost3[0, :, :]
baseArgs = np.zeros((R, I, 2 * limits + 1))
baseArgs[0, :, :] = np.nan
for r in range(1, R):
for index, epsilon in enumerate(range(-limits, limits + 1)):
# i = 0
matchingCost1[r, 0,
index] = localCost1[r, 0, index] + np.min(
matchingCost1[r - 1, 0,
max(epsilon, 0):index + 1])
matchingCost3[r, 0,
index] = localCost3[r, 0, index] + np.min(
matchingCost3[r - 1, 0,
max(epsilon, 0):index + 1])
#baseArgs[r, 0, index] = 0
#baseArgs[r - 1, 0, index] = 0
# i = 1
cumulativeCost1 = np.array([
np.min(matchingCost1[r - 1, 1,
max(epsilon, 0):index + 1]),
np.min(matchingCost1[r - 1, 0,
max(epsilon + 1, 0):index + 2])
])
cumulativeCost3 = np.array([
np.min(matchingCost3[r - 1, 1,
max(epsilon, 0):index + 1]),
np.min(matchingCost3[r - 1, 0,
max(epsilon + 1, 0):index + 2])
])
arg = np.nanargmin(cumulativeCost1 + cumulativeCost3,
axis=0)
matchingCost1[
r, 1,
index] = localCost1[r, 1, index] + cumulativeCost1[arg]
matchingCost3[
r, 1,
index] = localCost3[r, 1, index] + cumulativeCost3[arg]
baseArgs[r, 1, index] = -arg
#baseArgs[r - 1, 1, index] = -arg
# i = 2...
cumulativeCost1 = np.array([
np.min(matchingCost1[r - 1, 2:,
max(epsilon, 0):index + 1],
axis=1),
np.min(matchingCost1[r - 1, 1:-1,
max(epsilon + 1, 0):index + 2],
axis=1),
np.min(matchingCost1[r - 1, :-2,
max(epsilon + 2, 0):index + 3],
axis=1)
])
cumulativeCost3 = np.array([
np.min(matchingCost3[r - 1, 2:,
max(epsilon, 0):index + 1],
axis=1),
np.min(matchingCost3[r - 1, 1:-1,
max(epsilon + 1, 0):index + 2],
axis=1),
np.min(matchingCost3[r - 1, :-2,
max(epsilon + 2, 0):index + 3],
axis=1)
])
arg = np.nanargmin(cumulativeCost1 + cumulativeCost3,
axis=0)
matchingCost1[r, 2:,
index] = localCost1[r, 2:, index] + np.diag(
cumulativeCost1[arg])
matchingCost3[r, 2:,
index] = localCost3[r, 2:, index] + np.diag(
cumulativeCost3[arg])
baseArgs[r, 2:, index] = -arg
#baseArgs[r - 1, 2:, index] = -arg
# calculate total matching cost
totalMatchingCosts = np.ones(I) * np.inf
for index, epsilon in enumerate(range(-limits, limits + 1)):
# i = 0
totalMatchingCosts[0] = np.min(matchingCost1[R - 1, 0, max(epsilon, 0):index + 1]) + \
np.min(matchingCost3[R - 1, 0, max(epsilon, 0):index + 1])
#baseArgs[R, 0, index] = 0
#baseArgs[R - 1, 0, index] = 0
# i = 1
cumulativeCost1 = np.array([
np.min(matchingCost1[R - 1, 1,
max(epsilon, 0):index + 1]),
np.min(matchingCost1[R - 1, 0,
max(epsilon + 1, 0):index + 2])
])
cumulativeCost3 = np.array([
np.min(matchingCost3[R - 1, 1,
max(epsilon, 0):index + 1]),
np.min(matchingCost3[R - 1, 0,
max(epsilon + 1, 0):index + 2])
])
arg = np.nanargmin(cumulativeCost1 + cumulativeCost3, axis=0)
totalMatchingCosts[
1] = cumulativeCost1[arg] + cumulativeCost3[arg]
#baseArgs[R, 1, index] = -arg
#baseArgs[R - 1, 1, index] = -arg
# i >= 2
cumulativeCost1 = np.array([
np.min(matchingCost1[R - 1, 2:,
max(epsilon, 0):index + 1],
axis=1),
np.min(matchingCost1[R - 1, 1:-1,
max(epsilon + 1, 0):index + 2],
axis=1),
np.min(matchingCost1[R - 1, :-2,
max(epsilon + 2, 0):index + 3],
axis=1)
])
cumulativeCost3 = np.array([
np.min(matchingCost3[R - 1, 2:,
max(epsilon, 0):index + 1],
axis=1),
np.min(matchingCost3[R - 1, 1:-1,
max(epsilon + 1, 0):index + 2],
axis=1),
np.min(matchingCost3[R - 1, :-2,
max(epsilon + 2, 0):index + 3],
axis=1)
])
arg = np.nanargmin(cumulativeCost1 + cumulativeCost3, axis=0)
totalMatchingCosts[2:] = np.diag(
cumulativeCost1[arg]) + np.diag(cumulativeCost3[arg])
#baseArgs[R, 2:, index] = -arg
#baseArgs[R - 1, 2:, index] = -arg
if (np.sum(np.isinf(matchingCost1[R - 1, :, :])) == matchingCost1.shape[1] * matchingCost1.shape[2]) or \
(np.sum(np.isinf(matchingCost3[R - 1, :, :])) == matchingCost3.shape[1] * matchingCost3.shape[2]): # all matching cost are infinity
raise OverflowError('all matching cost are infinity')
return {
'matchingCost1': matchingCost1,
'matchingCost3': matchingCost3,
'totalMatchingCosts': totalMatchingCosts,
'baseArgs': baseArgs
}
classifier_dat = all_data_red_long[:, :, time_inds]
plt.figure()
for component in range(classifier_dat.shape[0]):
plt.subplot(1, classifier_dat.shape[0], component + 1)
data.imshow(classifier_dat[component, :, :])
# Downsample data to increase number of trials
## Don't split downsampled trials from same batch across training and testing data
train_fraction = 0.5
total_features = classifier_dat.shape[0] * classifier_dat.shape[2]
train_trial_num = classifier_dat.shape[1] * train_fraction
downsample_ratio_vec = np.linspace(0.1, 10, 100)
down_sample_ratio = np.ceil(downsample_ratio_vec[np.nanargmin(
((total_features / downsample_ratio_vec) -
(train_trial_num * downsample_ratio_vec))**2)])
down_sample_ratio = down_sample_ratio.astype('int')
new_time_points = (classifier_dat.shape[-1] // down_sample_ratio)
downsample_inds = np.asarray([x for x in range(down_sample_ratio)])
batched_classifier_dat = np.zeros(
(classifier_dat.shape[0], classifier_dat.shape[1] * down_sample_ratio,
new_time_points))
for time in range(new_time_points):
for trial in range(classifier_dat.shape[1]):
batched_classifier_dat[:,trial*down_sample_ratio : (trial*down_sample_ratio)+down_sample_ratio,time] = \
classifier_dat[:,trial,(time*down_sample_ratio) : (time*down_sample_ratio)+down_sample_ratio]
