def getgeodesicpts(m):
"""
computes the lat/lon values of the points on the surface of the sphere
corresponding to a twenty-sided (icosahedral) geodesic.
@param m: the number of points on the edge of a single geodesic triangle.
There are 10*(m-1)**2+2 total geodesic points, including the poles.
@return: C{B{lats, lons}} - rank 1 numpy float32 arrays containing
the latitudes and longitudes of the geodesic points (in degrees). These
points are nearly evenly distributed on the surface of the sphere.
"""
x,y,z = _spherepack.ihgeod(m)
# convert cartesian coords to lat/lon.
rad2dg = 180./math.pi
r1 = x*x+y*y
r = numpy.sqrt(r1+z*z)
r1 = numpy.sqrt(r1)
xtmp = numpy.where(numpy.logical_or(x,y),x,numpy.ones(x.shape,numpy.float32))
ztmp = numpy.where(numpy.logical_or(r1,z),z,numpy.ones(z.shape,numpy.float32))
lons = rad2dg*numpy.arctan2(y,xtmp)+180.
lats = rad2dg*numpy.arctan2(r1,ztmp)-90.
lat = numpy.zeros(10*(m-1)**2+2,numpy.float32)
lon = numpy.zeros(10*(m-1)**2+2,numpy.float32)
# first two points are poles.
lat[0] = 90; lat[1] = -90.
lon[0] = 0.; lon[1] = 0.
lat[2:] = lats[0:2*(m-1),0:m-1,:].flatten()
lon[2:] = lons[0:2*(m-1),0:m-1,:].flatten()
return lat,lon
def __init__(self, strFilename, reconst=False, mask=None, pointfile=None, dSpacing=None, wavelength=None):
"""
@param: input image filename
@param reconst: shall mased part or negative values be reconstructed (wipe out problems with pilatus gaps)
"""
self.strFilename = strFilename
self.data = fabio.open(strFilename).data.astype("float32")
if mask is not None:
mask = mask.astype(bool)
view = self.data.ravel()
flat_mask = mask.ravel()
min_valid = view[numpy.where(flat_mask == False)].min()
view[numpy.where(flat_mask)] = min_valid
self.shape = self.data.shape
self.points = ControlPoints(pointfile, dSpacing=dSpacing, wavelength=wavelength)
# self.lstPoints = []
self.fig = None
self.fig2 = None
self.fig2sp = None
self.ax = None
self.ct = None
self.msp = None
if reconstruct and (reconst is not False):
if mask is None:
mask = self.data < 0
self.massif = Massif(reconstruct(self.data, mask))
else:
self.massif = Massif(self.data)
self._sem = threading.Semaphore()
self._semGui = threading.Semaphore()
self.defaultNbPoints = 100
def vis_result(image, seg, gt, title1='Segmentation', title2='Ground truth', savefile=None):
indices = np.where(seg >= 0.5)
indices_gt = np.where(gt >= 0.5)
im_norm = image / image.max()
rgb_image = color.gray2rgb(im_norm)
multiplier = [0., 1., 1.]
multiplier_gt = [1., 1., 0.]
im_seg = rgb_image.copy()
im_gt = rgb_image.copy()
im_seg[indices[0], indices[1], :] *= multiplier
im_gt[indices_gt[0], indices_gt[1], :] *= multiplier_gt
fig = plt.figure()
a = fig.add_subplot(1, 2, 1)
plt.imshow(im_seg)
a.set_title(title1)
a = fig.add_subplot(1, 2, 2)
plt.imshow(im_gt)
a.set_title(title2)
if savefile is None:
plt.show()
else:
plt.savefig(savefile)
plt.close()
def test_float_modulus_exact(self):
# test that float results are exact for small integers. This also
# holds for the same integers scaled by powers of two.
nlst = list(range(-127, 0))
plst = list(range(1, 128))
dividend = nlst + [0] + plst
divisor = nlst + plst
arg = list(itertools.product(dividend, divisor))
tgt = list(divmod(*t) for t in arg)
a, b = np.array(arg, dtype=int).T
# convert exact integer results from Python to float so that
# signed zero can be used, it is checked.
tgtdiv, tgtrem = np.array(tgt, dtype=float).T
tgtdiv = np.where((tgtdiv == 0.0) & ((b < 0) ^ (a < 0)), -0.0, tgtdiv)
tgtrem = np.where((tgtrem == 0.0) & (b < 0), -0.0, tgtrem)
for dt in np.typecodes['Float']:
msg = 'dtype: %s' % (dt,)
fa = a.astype(dt)
fb = b.astype(dt)
# use list comprehension so a_ and b_ are scalars
div = [self.floordiv(a_, b_) for a_, b_ in zip(fa, fb)]
rem = [self.mod(a_, b_) for a_, b_ in zip(fa, fb)]
assert_equal(div, tgtdiv, err_msg=msg)
assert_equal(rem, tgtrem, err_msg=msg)
def plotResult(self, nn):
cmask = np.where(self.y==1);
plot(self.X[cmask,0], self.X[cmask,1], 'or', markersize=4)
cmask = np.where(self.y==2);
plot(self.X[cmask,0], self.X[cmask,1], 'ob', markersize=4)
cmask = np.where(self.y==3);
plot(self.X[cmask,0], self.X[cmask,1], 'og', markersize=4)
minX = min(self.X[:,0])
minY = min(self.X[:,1])
maxX = max(self.X[:,0])
maxY = max(self.X[:,1])
grid_range = [minX, maxX, minY, maxY];
delta = 0.05; levels = 100
a = arange(grid_range[0],grid_range[1],delta)
b = arange(grid_range[2],grid_range[3],delta)
A, B = meshgrid(a, b)
values = np.zeros(A.shape)
for i in range(len(a)):
for j in range(len(b)):
values[j,i] = nn.getNetworkOutput( [ a[i], b[j] ] )
contour(A, B, values, levels=[1], colors=['k'], linestyles='dashed')
contourf(A, B, values, levels=linspace(values.min(),values.max(),levels), cmap=cm.RdBu)
def do_cluster(raw_data, mode):
cluster_labels=[]
filter_zeros=np.where(raw_data[:, 1] != 0)[0]
#% OPTIMIZE DATA QUALITY
clear_data = raw_data[filter_zeros,:] # ignore the zeros on X
filter_zeros=np.where(clear_data[:, 2] != 0)[0]
clear_data = clear_data[filter_zeros,:] # ignore the zeros on Y
rospy.loginfo('filtered data')
#filter2=np.where(clear_data[:, 1] > 0.5)[0]
#clear_data = clear_data[filter2, :] # ignore mishits
#% TRAINING
if mode == 0:
cluster_labels =np.zeros((len(clear_data),1),int)
eps = 0.5
min_points = 3
rospy.loginfo('call DBscan ')
[core_samples,cluster_labels, n_clusters, human]=scikit_dbscan.dbscan(clear_data, eps, min_points,mode)
#% TESTING
if mode == 1:
rospy.loginfo('call DBscan ')
[core_samples,cluster_labels,n_clusters, human]=scikit_dbscan.dbscan(clear_data,eps, min_points)
return core_samples,cluster_labels,human
def soft_threshold(lamda,b):
th = float(lamda)/2.0
print ("(lamda,Threshold)",lamda,th)
print("The type of b is ..., its len is ",type(b),b.shape,len(b[0]))
if(lamda == 0):
return b
m,n = b.shape
x = np.zeros((m,n))
k = np.where(b > th)
# print("(b > th)",k)
#print("Number of elements -->(b > th) ",type(k))
x[k] = b[k] - th
k = np.where(np.absolute(b) <= th)
# print("abs(b) <= th",k)
# print("Number of elements -->abs(b) <= th ",len(k))
x[k] = 0
k = np.where(b < -th )
# print("(b < -th )",k)
# print("Number of elements -->(b < -th ) <= th",len(k))
x[k] = b[k] + th
x = x[:]
return x
def factorize(self):
""" factorize the filter into a minimal phase and maximal phase
Returns
-------
(Fmin,Fmax) : tuple of filter
Fmin : minimal phase filter
Fmax : maximal phase filter
"""
if self.fir:
# selectionne les zeros interne et externe
internal = np.where(abs(self.z) < 1)[0]
external = np.where(abs(self.z) >= 1)[0]
zi = self.z[internal]
ze = self.z[external]
Fmin = DF()
Fmin.z = zi
Fmax = DF()
Fmax.z = ze
return (Fmin, Fmax)
def compute_cost( X, y, theta, lam ):
'''Compute cost for logistic regression.'''
# Number of training examples
m = y.shape[0]
# Compute the prediction based on theta and X
predictions = X.dot( theta )
# Preprocessing values before sending to sigmoid function.
# If the argument to sigmoid function >= 0, we know that the
# sigmoid value is 1. Similarly for the negative values.
predictions[ where( predictions >= 20 ) ] = 20
predictions[ where( predictions <= -500 ) ] = -500
hypothesis = sigmoid( predictions )
hypothesis[ where( hypothesis == 1.0 ) ] = 0.99999
# Part of the cost function without regularization
J1 = ( -1.0 / m ) * sum( ( y * np.log( hypothesis ) ) +
( ( 1.0 - y ) * np.log( 1.0 - hypothesis ) ) )
# Computing the regularization term
J2 = lam / ( 2.0 * m ) * sum( theta[ 1:, ] * theta[ 1:, ] )
error = hypothesis - y
return J1 + J2
def select_id(self, id, roi=True):
"""Convert a ROI id into an index to be used to index features safely.
Parameters
----------
id : any hashable type, must be in self.get_id()
The id of the region one wants to access.
roi : bool
If True (default), return the ROI index in the ROI list.
If False, return the indices of the voxels of the ROI with the given
id. That way, internal access to self.label can be made.
Returns
-------
index : int or np.array of shape (roi.size, )
Either the position of the ROI in the ROI list (if roi == True),
or the positions of the voxels of the ROI with id `id`
with respect to the self.label array.
"""
if id not in self.get_id():
raise ValueError("Unexisting `id` provided")
if roi:
index = int(np.where(self.get_id() == id)[0])
else:
index = np.where(self.label == np.where(self.get_id() == id)[0])[0]
return index
def infos() :
'''Some information about the neuronal populations for each structures'''
print "Striatal populations: %d" % len(STR)
print "Pallidal populations: %d" % len(GP)
print
C = (W_STR_STR > 0).sum(axis=1) + (W_STR_STR < 0).sum(axis=1)
L = W_STR_STR[np.where(W_STR_STR != 0)]
print "Collateral striatal connections"
print "Mean number: %g (+/- %g)" % (C.mean(), C.std())
print "Mean length: %g (+/- %g)" % (L.mean(), L.std())
print
C = (W_GP_GP > 0).sum(axis=1) + (W_GP_GP < 0).sum(axis=1)
L = W_GP_GP[np.where(W_GP_GP != 0)]
print "Collateral pallidal connections"
print "Mean number: %g (+/- %g)" % (C.mean(), C.std())
print "Mean length: %g (+/- %g)" % (L.mean(), L.std())
print
C = (W_STR_GP > 0).sum(axis=1) + (W_STR_GP < 0).sum(axis=1)
L = W_STR_GP[np.where(W_STR_GP != 0)]
print "Striato-pallidal connections"
print "Mean number: %g (+/- %g)" % (C.mean(), C.std())
print "Mean length: %g (+/- %g)" % (L.mean(), L.std())
print
print "Mean # collateral striato-pallidal connections: %g (+/- %g)" % (C.mean(), C.std())
C = (W_GP_STR > 0).sum(axis=1) + (W_GP_STR < 0).sum(axis=1)
L = W_GP_STR[np.where(W_GP_STR != 0)]
print "Pallido-striatal connections"
print "Mean number: %g (+/- %g)" % (C.mean(), C.std())
print "Mean length: %g (+/- %g)" % (L.mean(), L.std())
print
def prime_to_pixel(self, xprime, yprime, color=0):
color0 = self._get_ricut()
g0, g1, g2, g3 = self._get_drow()
h0, h1, h2, h3 = self._get_dcol()
px, py, qx, qy = self._get_cscc()
# #$(%*&^(%$%*& bad documentation.
(px,py) = (py,px)
(qx,qy) = (qy,qx)
qx = qx * np.ones_like(xprime)
qy = qy * np.ones_like(yprime)
xprime -= np.where(color < color0, px * color, qx)
yprime -= np.where(color < color0, py * color, qy)
# Now invert:
# yprime = y + g0 + g1 * x + g2 * x**2 + g3 * x**3
# xprime = x + h0 + h1 * x + h2 * x**2 + h3 * x**3
x = xprime - h0
# dumb-ass Newton's method
dx = 1.
# FIXME -- should just update the ones that aren't zero
# FIXME -- should put in some failsafe...
while np.max(np.abs(np.atleast_1d(dx))) > 1e-10:
xp = x + h0 + h1 * x + h2 * x**2 + h3 * x**3
dxpdx = 1 + h1 + h2 * 2*x + h3 * 3*x**2
dx = (xprime - xp) / dxpdx
x += dx
y = yprime - (g0 + g1 * x + g2 * x**2 + g3 * x**3)
return (x, y)
def processTrafficData(self):
for index, row in self.traffic_data.iterrows():
adjacent_list = []
if index > 1 and index < 18467:
if self.traffic_data.ix[index - 1]['inter1'] == row[0]:
key1 = self.traffic_data.ix[index - 1]['inter1'] + ', ' + self.traffic_data.ix[index - 1]['inter2']
adjacent_list.append(key1)
if self.traffic_data.ix[index + 1]['inter1'] == row[0]:
key2 = self.traffic_data.ix[index + 1]['inter1'] + ', ' + self.traffic_data.ix[index + 1]['inter2']
adjacent_list.append(key2)
keylist = np.where(self.traffic_data['inter1'] == row[1])[0]
keylist1 = np.where(self.traffic_data['inter2'] == row[0])[0]
ind_list = np.intersect1d(keylist, keylist1)
if len(ind_list) >= 1:
ind = ind_list[0]
if ind > 1 and ind < 18467:
if self.traffic_data.ix[ind - 1]['inter1'] == row[1]:
key3 = self.traffic_data.ix[ind - 1]['inter1'] + ', ' + self.traffic_data.ix[ind - 1]['inter2']
adjacent_list.append(key3)
if self.traffic_data.ix[ind + 1]['inter1'] == row[1]:
key4 = self.traffic_data.ix[ind + 1]['inter1'] + ', ' + self.traffic_data.ix[ind + 1]['inter2']
adjacent_list.append(key4)
node = row['node']
node.setAdjacents(adjacent_list)
def stats(t, snp=None):
'''Return a record array with imputation statistics.'''
T = t.sample_index_to_impute
imputed = t.imputed_data[:, T, :]
tot_to_impute = 2 * imputed.shape[1]
snp = snp if snp is not None else np.arange(t.num_snps)
stats = np.zeros((len(snp),),
dtype=[
('dist_cm', 'f4'), # Genetic distance from beginning of chromosome
('count', '(2,)i4'), # Allele count
('frequency', '(2,)f4'), # Allele frequency
('call_rate', 'f4'), # Imputation Call rate
('call_rate_training', 'f4') # Imputation Call rate
])
call_rate_training = 1.0 * np.sum(np.sum(t.imputed_data[:, t.sample_index, :] != 0, axis=2), axis=1)# / (2 * len(t.sample_index))
for row, snp_index in enumerate(snp):
# TODO: replace by a bulk group-by/hist?
# g = t.__t.training_data[snp_index, :, :]
i = imputed[snp_index, :]
(c1, c2) = (len(np.where(i == 1)[0]), len(np.where(i == 2)[0]))
c = c1 + c2 + SMALL_FLOAT
f1, f2 = (1.0 * c1) / c, (1.0 * c2) / c
call_rate = 1.0 * len(i.nonzero()[0]) / tot_to_impute
# print 'c1 %4d c2 %4d f1 %.2f f2 %.2f call rate %5.2f' % (c1, c2, f1, f2, call_rate)
stats[row] = (t.snp['dist_cm'][snp_index], [c1, c2], [f1, f2], call_rate, call_rate_training[snp_index])
return stats
def pixel_to_prime(self, x, y, color=0):
# Secret decoder ring:
# http://www.sdss.org/dr7/products/general/astrometry.html
# (color)0 is called riCut;
# g0, g1, g2, and g3 are called
# dRow0, dRow1, dRow2, and dRow3, respectively;
# h0, h1, h2, and h3 are called
# dCol0, dCol1, dCol2, and dCol3, respectively;
# px and py are called csRow and csCol, respectively;
# and qx and qy are called ccRow and ccCol, respectively.
color0 = self._get_ricut()
g0, g1, g2, g3 = self._get_drow()
h0, h1, h2, h3 = self._get_dcol()
px, py, qx, qy = self._get_cscc()
# #$(%*&^(%$%*& bad documentation.
(px,py) = (py,px)
(qx,qy) = (qy,qx)
yprime = y + g0 + g1 * x + g2 * x**2 + g3 * x**3
xprime = x + h0 + h1 * x + h2 * x**2 + h3 * x**3
# The code below implements this, vectorized:
# if color < color0:
# xprime += px * color
# yprime += py * color
# else:
# xprime += qx
# yprime += qy
qx = qx * np.ones_like(x)
qy = qy * np.ones_like(y)
xprime += np.where(color < color0, px * color, qx)
yprime += np.where(color < color0, py * color, qy)
return (xprime, yprime)
def prep_data(dataset):
df = dataset.copy()
latlon = list(zip(df.lat, df.lon))
dist = np.array([distance(latlon[i + 1], latlon[i]) for i in range(len((latlon[:-1])))])
df["dist"] = np.concatenate(([0], np.cumsum(dist)))
slope = np.abs(100 * np.diff(df.alt) / (1000 * dist))
slope[np.where( slope < 4) ] = 0 # "green"
slope[np.where((slope >= 4) & (slope < 6))] = 1 # "yellow"
slope[np.where((slope >= 6) & (slope < 10))] = 2 # "pink"
slope[np.where((slope >= 10) & (slope < 15))] = 3 # "orange"
slope[np.where( slope >= 15 )] = 4 # "red"
slope = im.median_filter(slope, 6)
colors = np.empty_like(slope, dtype=object)
colors[np.where(slope == 0)] = "green"
colors[np.where(slope == 1)] = "yellow"
colors[np.where(slope == 2)] = "pink"
colors[np.where(slope == 3)] = "orange"
colors[np.where(slope == 4)] = "red"
df["colors"] = list(colors) + [None] # NOTE: add [None] just make pandas happy
return df
def myTradingSystem(DATE, CLOSE, settings):
''' This system uses mean reversion techniques to allocate capital into the desired equities '''
# This strategy evaluates two averages over time of the close over a long/short
# scale and builds the ratio. For each day, "smaQuot" is an array of "nMarkets"
# size.
nMarkets = numpy.shape(CLOSE)[1]
periodLong = 200
periodShort = 40
smaLong = numpy.sum(CLOSE[-periodLong:, :], axis=0)/periodLong
smaRecent = numpy.sum(CLOSE[-periodShort:, :], axis=0)/periodShort
smaQuot = smaRecent / smaLong
# For each day, scan the ratio of moving averages over the markets and find the
# market with the maximum ratio and the market with the minimum ratio:
longEquity = numpy.where(smaQuot == numpy.nanmin(smaQuot))
shortEquity = numpy.where(smaQuot == numpy.nanmax(smaQuot))
# Take a contrarian view, going long the market with the minimum ratio and
# going short the market with the maximum ratio. The array "pos" will contain
# all zero entries except for those cases where we go long (1) and short (-1):
pos = numpy.zeros((1, nMarkets))
pos[0, longEquity[0][0]] = 1
pos[0, shortEquity[0][0]] = -1
# For the position sizing, we supply a vector of weights defining our
# exposure to the markets in settings['markets']. This vector should be
# normalized.
pos = pos/numpy.nansum(abs(pos))
return pos, settings
def getFracMelt(self, time=None, scale=1.0):
"""Get fractional area where basal melting occurs.
time: if None, return data for all time slices
if list/etc of size two, interpret as array selection
if single value, get only this time slice"""
(tarray, t) = CFchecklist(time, self.file.variables["time"])
values = []
fact = self.deltax * self.deltay * scale
if tarray:
for i in range(t[0], t[1] + 1):
ih = self.getIceArea(time=i, scale=scale)
if ih > 0:
mlt = numpy.where(self.file.variables["bmlt"][i, :, :] > 0.0, 1, 0).flat
values.append(sum(mlt) * fact / ih)
else:
values.append(0.0)
return values
ih = self.getIceArea(time=t, scale=scale)
if ih > 0:
mlt = numpy.where(self.file.variables["bmlt"][t, :, :] > 0.0, 1, 0).flat
return sum(mlt) * fact / ih
else:
return 0.0
def get_absline(self, inp):
""" Returns an AbsLine from the AbsSystem
Parameters
----------
inp : str or Quantity
str -- Name of the transition, e.g. 'CII 1334'
Quantity -- Rest wavelength of the transition, e.g. 1334.53*u.AA
to 0.01 precision
Returns
-------
absline -- AbsLine object or list of Abslines
More than one will be returned if this line exists in
multiple components. The returned quantity will then
be a list instead of a single object
"""
# Generate the lines
abslines = self.list_of_abslines()
if isinstance(inp,basestring):
names = np.array([absline.name for absline in abslines])
mt = np.where(names == inp)[0]
elif isinstance(inp,Quantity):
wrest = Quantity([absline.wrest for absline in abslines])
mt = np.where(np.abs(wrest-inp) < 0.01*u.AA)[0]
else:
raise IOError("Bad input to absline")
# Finish
if len(mt) == 0:
warnings.warn("No absline with input={}".format(inp))
return None
elif len(mt) == 1:
return abslines[mt]
else:
return [abslines[ii] for ii in mt]
def multi_where(vec1, vec2):
'''Given two vectors, multi_where returns a tuple of indices where those
two vectors overlap.
****THIS FUNCTION HAS NOT BEEN TESTED ON N-DIMENSIONAL ARRAYS*******
Inputs:
2 numpy vectors
Output:
(xy, yx) where xy is a numpy vector containing the indices of the
elements in vector 1 that are also in vector 2. yx is a vector
containing the indices of the elements in vector 2 that are also
in vector 1.
Example:
>> x = np.array([1,2,3,4,5])
>> y = np.array([3,4,5,6,7])
>> (xy,yx) = multi_where(x,y)
>> xy
array([2,3,4])
>> yx
array([0,1,2])
'''
OneInTwo = np.array([])
TwoInOne = np.array([])
for i in range(vec1.shape[0]):
if np.where(vec2 == vec1[i])[0].shape[0]:
OneInTwo = np.append(OneInTwo,i)
TwoInOne = np.append(TwoInOne, np.where(vec2 == vec1[i])[0][0])
return (np.int8(OneInTwo), np.int8(TwoInOne))
def hist_mask(data, threshold=.95, keep='lower'):
"""
Returns boolean mask of values below a frequency percentage threshold (0-1).
Args:
-data (1D array)
-threshold (float): 0-1
-keep (str): lower, greater, middle. If middle, threshold is ignored,
and a single cluster is searched out.
"""
bins = len(data)/100 if keep.lower() == 'middle' else len(data) / 2
freq, val = np.histogram(data, bins=bins)
freq = freq / np.sum(freq).astype(float) # Normalize frequency data
if keep.lower() in ('lower', 'upper'):
cutoff_value = val[np.where(np.diff(np.cumsum(freq) < threshold))[0] + 1]
cutoff_value = val[1] if len(cutoff_value)==0 else cutoff_value
if keep.lower() == 'lower':
return data < cutoff_value
else:
return data > cutoff_value
else:
histmask = np.ones(data.shape[0], dtype=bool) # Initializing mask with all True values
# Slowly increment the parameter until a strong single central cluster is found
for param in np.arange(0.0005, .02, .0003):
cutoff_values = val[np.where(np.diff(freq < param))[0]]
if len(cutoff_values) == 2:
histmask &= data > cutoff_values[0]
histmask &= data < cutoff_values[1]
return histmask
else:
return data > -100000. # Return an all-true mask
print("Warning: Histogram filter not finding a good parameter to form a central cluster. Please try again.")
def implicit_black_box(propensities, V, X, w, h, deter_vector, stoc_positions, positions, valid, deriv):
# Adjustment for systems reaching steady state
temp = derivative_G(propensities, V, X, w, deter_vector, stoc_positions, positions, valid)
# pdb.set_trace()
valid_adjust_pos = np.where(np.sum(np.abs(temp), axis=0) < 1e-10, True, False)
valid_adjust = valid[:, :]
valid_adjust[valid_adjust_pos, :] = False
# print(" Reached Steady State %d"%(np.sum(valid_adjust_pos)))
from scipy.integrate import ode
# pdb.set_trace()
deter_ode = ode(f).set_integrator("lsoda", method="adams", with_jacobian=False)
deter_ode.set_initial_value(X[deter_vector, :].flatten(), 0).set_f_params(
[propensities, V, X, deter_vector, stoc_positions, positions, valid_adjust, w]
)
# pdb.set_trace()
while deter_ode.successful() and deter_ode.t < h:
deter_ode.integrate(h)
# print("Black Box: \n"+ str(deter_ode.y))
# print("iterator : \n:"+str(next_X[deter_vector,:]))
X[deter_vector, :] = deter_ode.y.reshape((np.sum(deter_vector), X.shape[1]))
# Another adjust to compensate for non negative
X = np.where(X < 0.0, 0.0, X)
return X
def word2ind(word, vocab, utok_ind=None):
ind = np.where(vocab == unicode(word))
if len(ind[0]) == 0:
if not utok_ind:
utok_ind = np.where(vocab == UTOK)
ind = utok_ind
return ind[0][0]
def _lininterp(self,x,X,Y):
if hasattr(x,'__len__'):
xtype = 'array'
xx=np.asarray(x).astype(np.float)
else:
xtype = 'scalar'
xx=np.asarray([x]).astype(np.float)
idx = X.searchsorted(xx)
yy = xx*0
yy[idx>len(X)-1] = Y[-1] # over
yy[idx<=0] = Y[0] # under
wok = np.where((idx>0) & (idx<len(X))) # the good ones
iok=idx[wok]
yywok = Y[iok-1] + ( (Y[iok]-Y[iok-1])/(X[iok]-X[iok-1])
* (xx[wok]-X[iok-1]) )
w = np.where( ((X[iok]-X[iok-1]) == 0) ) # where are the nan ?
yywok[w] = Y[iok[w]-1] # replace by previous value
wl = np.where(xx[wok] == X[0])
yywok[wl] = Y[0]
wh = np.where(xx[wok] == X[-1])
yywok[wh] = Y[-1]
yy[wok] = yywok
if xtype == 'scalar':
yy = yy[0]
return yy
def corrtag_image(in_data,xtype='XCORR',ytype='YCORR',pha=(2,30),bins=(1024,16384),times=None,ranges=((0,1023),(0,16384)),binning=(1,1),NUV=False):
try: histogram2d
except NameError: from numpy import histogram2d,where,zeros
try: getdata
except NameError: from pyfits import getdata
try: events=getdata(in_data,1)
except: events=in_data
xlength = (ranges[1][1]+1)
ylength = (ranges[0][1]+1)
xbinning = binning[1]
ybinning = binning[0]
if NUV:
bins = (1024,1024)
pha = (-1,1)
ranges = ( (0,1023), (0,1023) )
if times != None:
index = where( (events['TIME']>=times[0]) & (events['TIME'] <= times[1]) )
events= events[index]
index = where((events['PHA']>=pha[0])&(events['PHA']<=pha[1]))
if len(index[0]):
image,y_r,x_r = histogram2d(events[ytype][index],events[xtype][index],bins=bins,range=ranges)
else:
image = zeros( (bins[0]//binning[0],bins[1]//binning[1]) )
return image
def mutual_information(self, x_index, y_index, log_base, debug=False):
"""
Calculate and return Mutual information between two random variables
"""
# Check if index are into the bounds
assert (0 <= x_index <= self.n_rows)
assert (0 <= y_index <= self.n_rows)
# Variable to return MI
summation = 0.0
# Get uniques values of random variables
values_x = set(self.data[x_index])
values_y = set(self.data[y_index])
# Print debug info
if debug:
print 'MI between'
print self.data[x_index]
print self.data[y_index]
# For each random
for value_x in values_x:
for value_y in values_y:
px = shape(where(self.data[x_index] == value_x))[1] / self.n_cols
py = shape(where(self.data[y_index] == value_y))[1] / self.n_cols
pxy = len(where(in1d(where(self.data[x_index] == value_x)[0],
where(self.data[y_index] == value_y)[0]) == True)[0]) / self.n_cols
if pxy > 0.0:
summation += pxy * math.log((pxy / (px * py)), log_base)
if debug:
print '(%d,%d) px:%f py:%f pxy:%f' % (value_x, value_y, px, py, pxy)
return summation
def __init__(self,turn,elem,single,name,s,x,xp,y,yp,pc,de,tau,**args):
apc=float(pc[0])*1e9
ade=float(de[0])
self.m0=self.pmass
en=np.sqrt(apc**2+self.pmass**2)
self.e0=en-ade
self.p0c=np.sqrt(self.e0**2-self.m0**2)
# structure
self.elem=np.array(elem,dtype=int)
self.turn=np.array(turn,dtype=int)
d0=np.where(np.diff(self.elem)!=0)[0][0]+1
d1=(np.where(np.diff(self.turn)!=0)[0][0]+1)/d0
d2=len(self.turn)/d1/d0
self.single=np.array(single,dtype=int)
self.name=np.array(name,dtype=str)
self.s =np.array(s ,dtype=float)
self.x =np.array(x ,dtype=float)
self.y =np.array(y ,dtype=float)
self.tau=-np.array(tau,dtype=float)*self.clight
opd=np.array(pc,dtype=float)*(1e9/self.p0c)
self.delta=opd-1
self.pt=np.array(de,dtype=float)/self.p0c
self.px=np.array(xp,dtype=float)*opd
self.py=np.array(yp,dtype=float)*opd
for nn,vv in self.__dict__.items():
if hasattr(vv,'__len__') and len(vv)==d0*d1*d2:
setattr(self,nn,vv.reshape(d2,d1,d0))
def test_background_model(tmpdir):
data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/')
bgmaker = OffDataBackgroundMaker(data_store, outdir=str(tmpdir))
bgmaker.select_observations(selection='all')
table = Table.read('run.lis', format='ascii.csv')
assert table['OBS_ID'][1] == 23526
bgmaker.group_observations()
table = ObservationTable.read(str(tmpdir / 'obs.fits'))
assert list(table['GROUP_ID']) == [0, 0, 0, 1]
table = ObservationTable.read(str(tmpdir / 'group-def.fits'))
assert list(table['ZEN_PNT_MAX']) == [49, 90]
# TODO: Fix 3D code
# bgmaker.make_model("3D")
# bgmaker.save_models("3D")
# model = CubeBackgroundModel.read(str(tmpdir / 'background_3D_group_001_table.fits.gz'))
# assert model.counts_cube.data.sum() == 1527
bgmaker.make_model("2D")
bgmaker.save_models("2D")
model = EnergyOffsetBackgroundModel.read(str(tmpdir / 'background_2D_group_001_table.fits.gz'))
assert model.counts.data.value.sum() == 1398
index_table_new = bgmaker.make_total_index_table(data_store, "2D", None, None)
table_bkg = index_table_new[np.where(index_table_new["HDU_NAME"] == "bkg_2d")]
name_bkg_run023523 = table_bkg[np.where(table_bkg["OBS_ID"] == 23523)]["FILE_NAME"]
assert str(tmpdir) + "/" + name_bkg_run023523[0] == str(tmpdir) + '/background_2D_group_001_table.fits.gz'
name_bkg_run023526 = table_bkg[np.where(table_bkg["OBS_ID"] == 23526)]["FILE_NAME"]
assert str(tmpdir) + "/" + name_bkg_run023526[0] == str(tmpdir) + '/background_2D_group_000_table.fits.gz'
def entropy(self, x_index, y_index, log_base, debug=False):
"""
Calculate the entropy between two random variable
"""
assert (0 <= x_index <= self.n_rows)
assert (0 <= y_index <= self.n_rows)
# Variable to return MI
summation = 0.0
# Get uniques values of random variables
values_x = set(self.data[x_index])
values_y = set(self.data[y_index])
# Print debug info
if debug:
print 'Entropy between'
print self.data[x_index]
print self.data[y_index]
# For each random
for value_x in values_x:
for value_y in values_y:
pxy = len(where(in1d(where(self.data[x_index] == value_x)[0],
where(self.data[y_index] == value_y)[0]) == True)[0]) / self.n_cols
if pxy > 0.0:
summation += pxy * math.log(pxy, log_base)
if debug:
print '(%d,%d) pxy:%f' % (value_x, value_y, pxy)
if summation == 0.0:
return summation
else:
return - summation
def CalcErange(inWS,ns,erange,binWidth):
#length of array in Fortran
array_len = 4096
binWidth = int(binWidth)
bnorm = 1.0/binWidth
#get data from input workspace
_,X,Y,E = GetXYE(inWS,ns,array_len)
Xdata = mtd[inWS].readX(0)
#get all x values within the energy range
rangeMask = (Xdata >= erange[0]) & (Xdata <= erange[1])
Xin = Xdata[rangeMask]
#get indicies of the bounds of our energy range
minIndex = np.where(Xdata==Xin[0])[0][0]+1
maxIndex = np.where(Xdata==Xin[-1])[0][0]
#reshape array into sublists of bins
Xin = Xin.reshape(len(Xin)/binWidth, binWidth)
#sum and normalise values in bins
Xout = [sum(bin_val) * bnorm for bin_val in Xin]
#count number of bins
nbins = len(Xout)
nout = [nbins, minIndex, maxIndex]
#pad array for use in Fortran code
Xout = PadArray(Xout,array_len)
return nout,bnorm,Xout,X,Y,E
def load_hdf5(self, restartFolder, timestep, tXY):
"""
This function reads and allocates **badlands** parameters & variables from a previous
simulation. These parameters are obtained from a specific time HDF5 file.
Important:
This function is only called when a **restart simulation** is ran... |:bomb:|
Args:
restartFolder: restart folder name as defined in tge XML input file.
timestep: time step to load defined based on output interval number.
tXY: 2D numpy array TIN grid local coordinates.
Returns
-------
elev
numpy 1D array containing the updated elevation from the restart model.
cum
numpy 1D array containing the updated erosion/deposition values from the restart model.
"""
if os.path.exists(restartFolder):
folder = restartFolder + "/h5/"
fileCPU = "tin.time%s.hdf5" % timestep
restartncpus = len(glob.glob1(folder, fileCPU))
if restartncpus == 0:
raise ValueError(
"The requested time step for the restart simulation cannot be found in the restart folder."
)
else:
raise ValueError(
"The restart folder is missing or the given path is incorrect."
)
df = h5py.File("%s/h5/tin.time%s.hdf5" % (restartFolder, timestep),
"r")
coords = numpy.array((df["/coords"]))
cumdiff = numpy.array((df["/cumdiff"]))
cumhill = numpy.array((df["/cumhill"]))
cumfail = numpy.array((df["/cumfail"]))
x, y, z = numpy.hsplit(coords, 3)
c = cumdiff
h = cumhill
f = cumfail
XY = numpy.column_stack((x, y))
tree = cKDTree(XY)
distances, indices = tree.query(tXY, k=3)
if len(z[indices].shape) == 3:
z_vals = z[indices][:, :, 0]
c_vals = c[indices][:, :, 0]
h_vals = h[indices][:, :, 0]
f_vals = f[indices][:, :, 0]
else:
z_vals = z[indices]
c_vals = c[indices]
h_vals = h[indices]
f_vals = f[indices]
with numpy.errstate(divide="ignore"):
elev = numpy.average(z_vals, weights=(1.0 / distances), axis=1)
cum = numpy.average(c_vals, weights=(1.0 / distances), axis=1)
hcum = numpy.average(h_vals, weights=(1.0 / distances), axis=1)
fcum = numpy.average(f_vals, weights=(1.0 / distances), axis=1)
onIDs = numpy.where(distances[:, 0] == 0)[0]
if len(onIDs) > 0:
if len(z[indices].shape) == 3:
elev[onIDs] = z[indices[onIDs, 0], 0]
cum[onIDs] = c[indices[onIDs, 0], 0]
hcum[onIDs] = h[indices[onIDs, 0], 0]
fcum[onIDs] = f[indices[onIDs, 0], 0]
else:
elev[onIDs] = z[indices[onIDs, 0]]
cum[onIDs] = c[indices[onIDs, 0]]
hcum[onIDs] = h[indices[onIDs, 0]]
fcum[onIDs] = f[indices[onIDs, 0]]
return elev, cum, hcum, fcum
def sample_homography(shape,
perspective=True,
scaling=True,
rotation=True,
translation=True,
n_scales=5,
n_angles=25,
scaling_amplitude=0.1,
perspective_amplitude_x=0.1,
perspective_amplitude_y=0.1,
patch_ratio=0.5,
max_angle=np.pi / 2,
allow_artifacts=False,
translation_overflow=0.):
"""Sample a random valid homography.
Computes the homography transformation between a random patch in the original image
and a warped projection with the same image size.
As in `tf.contrib.image.transform`, it maps the output point (warped patch) to a
transformed input point (original patch).
The original patch, which is initialized with a simple half-size centered crop, is
iteratively projected, scaled, rotated and translated.
Arguments:
shape: A rank-2 `Tensor` specifying the height and width of the original image.
perspective: A boolean that enables the perspective and affine transformations.
scaling: A boolean that enables the random scaling of the patch.
rotation: A boolean that enables the random rotation of the patch.
translation: A boolean that enables the random translation of the patch.
n_scales: The number of tentative scales that are sampled when scaling.
n_angles: The number of tentatives angles that are sampled when rotating.
scaling_amplitude: Controls the amount of scale.
perspective_amplitude_x: Controls the perspective effect in x direction.
perspective_amplitude_y: Controls the perspective effect in y direction.
patch_ratio: Controls the size of the patches used to create the homography.
max_angle: Maximum angle used in rotations.
allow_artifacts: A boolean that enables artifacts when applying the homography.
translation_overflow: Amount of border artifacts caused by translation.
Returns:
A `Tensor` of shape `[1, 8]` corresponding to the flattened homography transform.
"""
# Corners of the output image
pts1 = np.stack([[0., 0.], [0., 1.], [1., 1.], [1., 0.]], axis=0)
# Corners of the input patch
margin = (1 - patch_ratio) / 2
pts2 = margin + np.array([[0, 0], [0, patch_ratio],
[patch_ratio, patch_ratio], [patch_ratio, 0]],
dtype=np.float32)
# Random perspective and affine perturbations
if perspective:
if not allow_artifacts:
perspective_amplitude_x = min(perspective_amplitude_x, margin)
perspective_amplitude_y = min(perspective_amplitude_y, margin)
perspective_displacement = truncated_normal(
[1], 0., perspective_amplitude_y / 2)
h_displacement_left = truncated_normal([1], 0.,
perspective_amplitude_x / 2)
h_displacement_right = truncated_normal([1], 0.,
perspective_amplitude_x / 2)
pts2 += np.stack([
np.concatenate([h_displacement_left, perspective_displacement], 0),
np.concatenate([h_displacement_left, -perspective_displacement],
0),
np.concatenate([h_displacement_right, perspective_displacement],
0),
np.concatenate([h_displacement_right, -perspective_displacement],
0)
])
# Random scaling
# sample several scales, check collision with borders, randomly pick a valid one
if scaling:
scales = np.concatenate(
[[1.],
truncated_normal([n_scales], 1, scaling_amplitude / 2)], 0)
center = np.mean(pts2, axis=0, keepdims=True)
scaled = np.expand_dims(pts2 - center, axis=0) * np.expand_dims(
np.expand_dims(scales, 1), 1) + center
if allow_artifacts:
valid = np.arange(n_scales) # all scales are valid except scale=1
else:
a = np.all((scaled >= 0.) & (scaled < 1.), (1, 2))
valid = np.where(np.all((scaled >= 0.) & (scaled < 1.),
(1, 2)))[:][0]
idx = valid[np.random.random_integers(0, high=np.shape(valid)[0])]
pts2 = scaled[idx]
# Random translation
if translation:
t_min, t_max = np.min(pts2, axis=0), np.min(1 - pts2, axis=0)
if allow_artifacts:
t_min += translation_overflow
t_max += translation_overflow
pts2 += np.expand_dims(np.stack([
np.random.uniform(-t_min[0], t_max[0]),
np.random.uniform(-t_min[1], t_max[1])
]),
axis=0)
# Random rotation
# sample several rotations, check collision with borders, randomly pick a valid one
if rotation:
angles = np.linspace(-max_angle, max_angle, n_angles)
angles = np.concatenate([[0.], angles],
axis=0) # in case no rotation is valid
center = np.mean(pts2, axis=0, keepdims=True)
rot_mat = np.reshape(
np.stack([
np.cos(angles), -np.sin(angles),
np.sin(angles),
np.cos(angles)
],
axis=1), [-1, 2, 2])
rotated = np.matmul(
np.tile(np.expand_dims(pts2 - center, axis=0),
[n_angles + 1, 1, 1]), rot_mat) + center
if allow_artifacts:
valid = np.arange(n_angles) # all angles are valid, except angle=0
else:
valid = np.where(
np.all((rotated >= 0.) & (rotated < 1.), axis=(1, 2)))[:][0]
idx = valid[np.random.random_integers(0, high=np.shape(valid)[0])]
pts2 = rotated[idx]
# Rescale to actual size
shape = shape[::-1] # different convention [y, x]
pts1 *= np.expand_dims(shape, axis=0).astype(np.float32)
pts2 *= np.expand_dims(shape, axis=0).astype(np.float32)
def ax(p, q):
return [p[0], p[1], 1, 0, 0, 0, -p[0] * q[0], -p[1] * q[0]]
def ay(p, q):
return [0, 0, 0, p[0], p[1], 1, -p[0] * q[1], -p[1] * q[1]]
a_mat = np.stack([f(pts1[i], pts2[i]) for i in range(4) for f in (ax, ay)],
axis=0)
p_mat = np.transpose(
np.stack([[pts2[i][j] for i in range(4) for j in range(2)]], axis=0))
homography = np.transpose(np.linalg.solve(a_mat, p_mat))
return homography
def get_current_frame(self, draw_options={}):
with self.current_frame_lock:
frame_copy = np.copy(self._current_frame)
frame_time = self.current_frame_time
tracked_objects = {
k: v.to_dict()
for k, v in self.tracked_objects.items()
}
motion_boxes = self.motion_boxes.copy()
regions = self.regions.copy()
frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_YUV2BGR_I420)
# draw on the frame
if draw_options.get('bounding_boxes'):
# draw the bounding boxes on the frame
for obj in tracked_objects.values():
thickness = 2
color = COLOR_MAP[obj['label']]
if obj['frame_time'] != frame_time:
thickness = 1
color = (255, 0, 0)
# draw the bounding boxes on the frame
box = obj['box']
draw_box_with_label(
frame_copy,
box[0],
box[1],
box[2],
box[3],
obj['label'],
f"{int(obj['score']*100)}% {int(obj['area'])}",
thickness=thickness,
color=color)
if draw_options.get('regions'):
for region in regions:
cv2.rectangle(frame_copy, (region[0], region[1]),
(region[2], region[3]), (0, 255, 0), 2)
if draw_options.get('zones'):
for name, zone in self.camera_config.zones.items():
thickness = 8 if any([
name in obj['current_zones']
for obj in tracked_objects.values()
]) else 2
cv2.drawContours(frame_copy, [zone.contour], -1, zone.color,
thickness)
if draw_options.get('mask'):
mask_overlay = np.where(self.camera_config.motion.mask == [0])
frame_copy[mask_overlay] = [0, 0, 0]
if draw_options.get('motion_boxes'):
for m_box in motion_boxes:
cv2.rectangle(frame_copy, (m_box[0], m_box[1]),
(m_box[2], m_box[3]), (0, 0, 255), 2)
if draw_options.get('timestamp'):
time_to_show = datetime.datetime.fromtimestamp(
frame_time).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(frame_copy,
time_to_show, (10, 30),
cv2.FONT_HERSHEY_SIMPLEX,
fontScale=.8,
color=(255, 255, 255),
thickness=2)
return frame_copy
def _diff(self):
# Much of the code for comparing columns is similar to the code for
# comparing headers--consider refactoring
colsa = self.a.columns
colsb = self.b.columns
if len(colsa) != len(colsb):
self.diff_column_count = (len(colsa), len(colsb))
# Even if the number of columns are unequal, we still do comparison of
# any common columns
colsa = dict((c.name.lower(), c) for c in colsa)
colsb = dict((c.name.lower(), c) for c in colsb)
if '*' in self.ignore_fields:
# If all columns are to be ignored, ignore any further differences
# between the columns
return
# Keep the user's original ignore_fields list for reporting purposes,
# but internally use a case-insensitive version
ignore_fields = set([f.lower() for f in self.ignore_fields])
# It might be nice if there were a cleaner way to do this, but for now
# it'll do
for fieldname in ignore_fields:
fieldname = fieldname.lower()
if fieldname in colsa:
del colsa[fieldname]
if fieldname in colsb:
del colsb[fieldname]
colsa_set = set(colsa.values())
colsb_set = set(colsb.values())
self.common_columns = sorted(colsa_set.intersection(colsb_set),
key=lambda c: c.name)
self.common_column_names = set([col.name.lower()
for col in self.common_columns])
left_only_columns = dict((col.name.lower(), col)
for col in colsa_set.difference(colsb_set))
right_only_columns = dict((col.name.lower(), col)
for col in colsb_set.difference(colsa_set))
if left_only_columns or right_only_columns:
self.diff_columns = (left_only_columns, right_only_columns)
self.diff_column_names = ([], [])
if left_only_columns:
for col in self.a.columns:
if col.name.lower() in left_only_columns:
self.diff_column_names[0].append(col.name)
if right_only_columns:
for col in self.b.columns:
if col.name.lower() in right_only_columns:
self.diff_column_names[1].append(col.name)
# If the tables have a different number of rows, we don't compare the
# columns right now.
# TODO: It might be nice to optionally compare the first n rows where n
# is the minimum of the row counts between the two tables.
if len(self.a) != len(self.b):
self.diff_rows = (len(self.a), len(self.b))
return
# If the tables contain no rows there's no data to compare, so we're
# done at this point. (See ticket #178)
if len(self.a) == len(self.b) == 0:
return
# Like in the old fitsdiff, compare tables on a column by column basis
# The difficulty here is that, while FITS column names are meant to be
# case-insensitive, PyFITS still allows, for the sake of flexibility,
# two columns with the same name but different case. When columns are
# accessed in FITS tables, a case-sensitive is tried first, and failing
# that a case-insensitive match is made.
# It's conceivable that the same column could appear in both tables
# being compared, but with different case.
# Though it *may* lead to inconsistencies in these rare cases, this
# just assumes that there are no duplicated column names in either
# table, and that the column names can be treated case-insensitively.
for col in self.common_columns:
name_lower = col.name.lower()
if name_lower in ignore_fields:
continue
cola = colsa[name_lower]
colb = colsb[name_lower]
for attr, _ in _COL_ATTRS:
vala = getattr(cola, attr, None)
valb = getattr(colb, attr, None)
if diff_values(vala, valb):
self.diff_column_attributes.append(
((col.name.upper(), attr), (vala, valb)))
arra = self.a[col.name]
arrb = self.b[col.name]
if (np.issubdtype(arra.dtype, float) and
np.issubdtype(arrb.dtype, float)):
diffs = where_not_allclose(arra, arrb, atol=0.0,
rtol=self.tolerance)
elif 'P' in col.format:
diffs = ([idx for idx in xrange(len(arra))
if not np.allclose(arra[idx], arrb[idx], atol=0.0,
rtol=self.tolerance)],)
else:
diffs = np.where(arra != arrb)
self.diff_total += len(set(diffs[0]))
if self.numdiffs >= 0:
if len(self.diff_values) >= self.numdiffs:
# Don't save any more diff values
continue
# Add no more diff'd values than this
max_diffs = self.numdiffs - len(self.diff_values)
else:
max_diffs = len(diffs[0])
last_seen_idx = None
for idx in islice(diffs[0], 0, max_diffs):
if idx == last_seen_idx:
# Skip duplicate indices, which my occur when the column
# data contains multi-dimensional values; we're only
# interested in storing row-by-row differences
continue
last_seen_idx = idx
self.diff_values.append(((col.name, idx),
(arra[idx], arrb[idx])))
total_values = len(self.a) * len(self.a.dtype.fields)
self.diff_ratio = float(self.diff_total) / float(total_values)
def synthetic_data_generator(size,dgp,parameters,noise,plots = True):
"""
synthetic_data_generator(size,dgp,plots = True)
# ------- PARAMETERS -------- #
size: int
size of the sample
dgp: str
data generating process ; "sigmoid", "quadratic", "noise"
parameters: dict
true parameters, dict keys are "alpha","beta1","beta2"
plots : Bool
True if you wants the plots to be savedù
noise : float
Noise coefficient
# ------- OUTPUT ------------ #
data: np.array
data generated
"""
# get parameters
alpha = parameters["alpha"] ; beta1 = parameters["beta1"] ; beta2 = parameters["beta2"]
# --------------------------- generate data on independent variables --------------------------- #
# generate the independent variables
x1 = np.random.normal(loc = 200, scale = 100, size = size)
x2 = np.linspace(-300,300,size)
# generate the y from the data based on the case
if dgp == "sigmoid":
# case 1 use a simple logistic
true_values = sigmoid(alpha + beta1 * x1 + beta2 * x2)
values = sigmoid(alpha + beta1 * x1 + beta2 * x2 + np.random.normal(loc = 0, scale = noise * np.abs((x1 - x1.mean()))))
# set threshold
threshold = values.mean()
y = (values >= threshold).astype(int)
elif dgp == "quadratic":
# case 2 circle
true_values = alpha + beta1 * x1**2 + beta2 * x2**2
values = true_values + np.random.normal(loc = 0, scale = noise * (x1 - x1.mean())**2)
# set threshold
threshold = values.mean()
y = (values >= threshold).astype(int)
elif dgp == "noise":
# purely random y
y = ss.bernoulli.rvs(p = 0.5, size = size)
else:
raise Exception
# --------------------------- plot the data ---------------------------------------- #
if plots == True:
_,ax = plt.subplots()
ax.scatter(x1[np.where(y == 0)],x2[np.where(y == 0)],color="blue")
ax.scatter(x1[np.where(y == 1)],x2[np.where(y == 1)],color="yellow")
if (dgp == "sigmoid"):
ax.plot(x1[np.argsort(x1)],(inv_sigmoid(threshold) - alpha - beta1 * x1[np.argsort(x1)])/beta2)
boundary = []
boundary.append(np.array([x1[np.argsort(x1)],(inv_sigmoid(threshold) - alpha - beta1 * x1[np.argsort(x1)])/beta2]).T)
elif (dgp == "quadratic"):
ax.plot(x1[np.argsort(x1)],np.sqrt((threshold - alpha - beta1 * x1[np.argsort(x1)]**2)/beta2))
ax.plot(x1[np.argsort(x1)],-np.sqrt((threshold - alpha - beta1 * x1[np.argsort(x1)]**2)/beta2))
boundary = []
boundary.append(np.array([x1[np.argsort(x1)],-np.sqrt((threshold - alpha - beta1 * x1[np.argsort(x1)]**2)/beta2)]).T)
boundary.append(np.array([x1[np.argsort(x1)],np.sqrt((threshold - alpha - beta1 * x1[np.argsort(x1)]**2)/beta2)]).T)
else:
boundary = []
ax.set_xlabel("x1")
ax.set_ylabel("x2")
graph_name = "SyntheticData_scatter_" + dgp
plt.savefig("/Users/davideferri/Documents/Repos/sklearn_applications/BivariateClassification_MLexample/graphs/" + graph_name)
# -------------------------- return the data ------------------------------------ #
# get the data in a single array
data = np.array([y,x1,x2]).T
return data,boundary
sim = 'SIMBA' snapnum = 33 realizations = 66 # images parameters grid = 250 splits = 5 # parameters of the density field routine periodic = True verbose = False x_min, y_min = 0.0, 0.0 ##################################################################################### # find the numbers that each cpu will work with numbers = np.where(np.arange(realizations)%nprocs==myrank)[0] # define the matrix hosting all the maps maps_T_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_T_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Z_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Z_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_P_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_P_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Mg_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Mg_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Mc_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Mc_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Ms_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Ms_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Vg_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32)
def load_hdf5_flex(self, restartFolder, timestep, tXY):
"""
This function reads and allocates **badlands** parameters & variables from a previous
simulation with flexural isostasy turned on.
These parameters are obtained from a specific time HDF5 file.
Args:
restartFolder: restart folder name as defined in tge XML input file.
timestep: time step to load defined based on output interval number.
tXY: 2D numpy array TIN grid local coordinates.
Returns
-------
elev
numpy 1D array containing the updated elevation from the restart model.
cum
numpy 1D array containing the updated erosion/deposition values from the restart model.
fcum
numpy 1D array containing the cumulative flexural isostasy values from the restart model.
Note:
Flexural isostasy is obtained from the gFlex_ package available on Github!
Wickert, A. D. (2016), Open-source modular solutions for flexural isostasy: gFlex v1.0,
Geosci. Model Dev., 9(3), 997–1017, `doi:10.5194/gmd-9-997-2016`_.
.. _`doi:10.5194/gmd-9-997-2016`: https://doi.org/10.5194/gmd-9-997-2016
.. _gflex: https://github.com/awickert/gFlex
"""
if os.path.exists(restartFolder):
folder = restartFolder + "/h5/"
fileCPU = "tin.time%s.hdf5" % timestep
restartncpus = len(glob.glob1(folder, fileCPU))
if restartncpus == 0:
raise ValueError(
"The requested time step for the restart simulation cannot be found in the restart folder."
)
else:
raise ValueError(
"The restart folder is missing or the given path is incorrect."
)
df = h5py.File("%s/h5/tin.time%s.hdf5" % (restartFolder, timestep),
"r")
coords = numpy.array((df["/coords"]))
cumdiff = numpy.array((df["/cumdiff"]))
cumhill = numpy.array((df["/cumhill"]))
cumfail = numpy.array((df["/cumfail"]))
cumflex = numpy.array((df["/cumflex"]))
x, y, z = numpy.hsplit(coords, 3)
c = cumdiff
h = cumhill
s = cumfail
f = cumflex
XY = numpy.column_stack((x, y))
tree = cKDTree(XY)
distances, indices = tree.query(tXY, k=3)
if len(z[indices].shape) == 3:
z_vals = z[indices][:, :, 0]
c_vals = c[indices][:, :, 0]
h_vals = h[indices][:, :, 0]
s_vals = s[indices][:, :, 0]
f_vals = f[indices][:, :, 0]
else:
z_vals = z[indices]
c_vals = c[indices]
s_vals = s[indices]
h_vals = h[indices]
f_vals = f[indices]
distances[distances < 0.0001] = 0.0001
with numpy.errstate(divide="ignore"):
elev = numpy.average(z_vals, weights=(1.0 / distances), axis=1)
cum = numpy.average(c_vals, weights=(1.0 / distances), axis=1)
hcum = numpy.average(h_vals, weights=(1.0 / distances), axis=1)
scum = numpy.average(s_vals, weights=(1.0 / distances), axis=1)
fcum = numpy.average(f_vals, weights=(1.0 / distances), axis=1)
onIDs = numpy.where(distances[:, 0] <= 0.0001)[0]
if len(onIDs) > 0:
if len(z[indices].shape) == 3:
elev[onIDs] = z[indices[onIDs, 0], 0]
cum[onIDs] = c[indices[onIDs, 0], 0]
hcum[onIDs] = h[indices[onIDs, 0], 0]
scum[onIDs] = s[indices[onIDs, 0], 0]
fcum[onIDs] = f[indices[onIDs, 0], 0]
else:
elev[onIDs] = z[indices[onIDs, 0]]
cum[onIDs] = c[indices[onIDs, 0]]
hcum[onIDs] = h[indices[onIDs, 0]]
scum[onIDs] = s[indices[onIDs, 0]]
fcum[onIDs] = f[indices[onIDs, 0]]
return elev, cum, hcum, scum, fcum
def beam_search(self, data_loader, k=12, max_len=100,
avoid_rep=False, avoid_unk=False):
"""Performs beam search over split and returns the hyps."""
results = []
inf = 1e3
bos = self.trg_vocab['<bos>']
eos = self.trg_vocab['<eos>']
for batch in pbar(data_loader, unit='batch'):
n = batch.size
# Mask to apply to pdxs.view(-1) to fix indices
nk_mask = torch.arange(n * k).long().cuda()
pdxs_mask = (nk_mask / k) * k
# Tile indices to use in the loop to expand first dim
tile = nk_mask / k
# We can fill this to represent the beams in tensor format
beam = torch.zeros((max_len, n, k)).long().cuda()
# Encode source sentences into ctxs (S*N*C)
ctxs, ctx_mask = self.enc(to_var(batch[self.sl], volatile=True))
# Get initial decoder state (N*H)
h_t = self.dec.f_init(ctxs, ctx_mask)
# Initial y_t for <bos> embs: N x emb_dim
y_t = self.dec.emb(to_var(
torch.ones(n).long() * bos, volatile=True))
log_p, h_t = self.dec.f_next(ctxs, ctx_mask, y_t, h_t)
nll, beam[0] = log_p.data.topk(k, sorted=False, largest=False)
for t in range(1, max_len):
cur_tokens = beam[t - 1].view(-1)
fini_idxs = (cur_tokens == eos).nonzero()
n_fini = fini_idxs.numel()
if n_fini == n * k:
break
# Fetch embs for the next iteration (N*K, E)
y_t = self.dec.emb(to_var(cur_tokens, volatile=True))
# Get log_probs and new LSTM states (log_p, N*K, V)
ctxs, ctx_mask = ctxs[:, tile], ctx_mask[:, tile]
log_p, h_t = self.dec.f_next(ctxs, ctx_mask, y_t, h_t[tile])
log_p = log_p.data
# Suppress probabilities of previous tokens
if avoid_rep:
log_p.view(-1).index_fill_(
0, cur_tokens + (nk_mask * self.n_trg_vocab), inf)
# Avoid <unk> tokens
if avoid_unk:
log_p[:, self.trg_vocab['<unk>']] = inf
# Favor finished hyps to generate <eos> again
# Their nll scores will not increase further and they will
# always be kept in the beam.
if n_fini > 0:
fidxs = fini_idxs[:, 0]
log_p.index_fill_(0, fidxs, inf)
log_p.view(-1).index_fill_(
0, fidxs * self.n_trg_vocab + eos, 0)
# Expand to 3D, cross-sum scores and reduce back to 2D
nll = (nll.unsqueeze(2) + log_p.view(n, k, -1)).view(n, -1)
# Reduce (N, K*V) to k-best
nll, idxs = nll.topk(k, sorted=False, largest=False)
# previous indices into the beam and current token indices
pdxs = idxs / self.n_trg_vocab
# Insert current tokens
beam[t] = idxs % self.n_trg_vocab
# Permute all hypothesis history according to new order
beam[:t] = beam[:t].gather(2, pdxs.repeat(t, 1, 1))
# Compute correct previous indices
# Mask is needed since we're in flattened regime
tile = pdxs.view(-1) + pdxs_mask
# Put an explicit <eos> to make idxs_to_sent happy
beam[max_len - 1] = eos
# Get beam and scores to CPU
beam = beam.cpu().numpy()
nll = nll.cpu().numpy()
# Normalize scores by length
lens, idxs = np.where(beam.reshape(max_len, -1) == eos)
idxs = idxs.tolist()
# Get sequence lengths
lens = [lens[idxs.index(i)] for i in range(n * k)]
# Avoid divide-by-zero in extreme case where hyp is only <eos>
lens = [len_ if len_ != 0 else 1 for len_ in lens]
nll /= np.array(lens).reshape(n, k)
# Get best hyp for each sample in the batch
hyps = beam[:, range(n), nll.argmin(axis=1)].T
results.extend([self.trg_vocab.idxs_to_sent(hy) for hy in hyps])
return results
def objective(limit, target): w = np.where(like>limit) count = histogram[w] return count.sum() - target
img_disp_target[pos_mask] = targets
return img_disp_target
if __name__ == '__main__':
# 缺点： 中心采样策略无法反映hw变化，而且既然叫做半径，为啥mask区域不是圆形，而是正方形
center_sampling = True # 是否使用中心采样策略
feature_shape = (100, 100, 3)
strides = 4
radius = 3.5 # 默认1.5
center_sample_radius = radius * strides # 扩展半径radius，值越大，扩展面积越大
gt_boox = [20, 30, 80, 71] # 特征图size xyxy
gt_bbox = torch.as_tensor(gt_boox, dtype=torch.float32).view(-1, 4)
pos_mask, bbox_targets = get_target_mask(gt_bbox, feature_shape, center_sample_radius, center_sampling)
# 可视化
pos_mask1 = pos_mask[..., 0].numpy()
gray_img = np.where(pos_mask1 > 0, 255, 0).astype(np.uint8)
# 绘制原始bbox
img = mmcv.gray2bgr(gray_img)
cv2.rectangle(img, (gt_boox[0], gt_boox[1]), (gt_boox[2], gt_boox[3]), color=(255, 0, 0))
cv2.namedWindow('img', 0)
mmcv.imshow(img, 'img')
# 显示centerness
centerness_targets = centerness_target(pos_mask, bbox_targets)
centerness_targets = centerness_targets.view(feature_shape[0], feature_shape[1])
plt.imshow(centerness_targets)
plt.show()
def get_hist_data(self, contract, durationStr='10 D', barSizeSetting='1 day', show='TRADES'):
"""
:param symbol: e.g. 'SPY'
:param type: e.g. 'STK', 'FUT', 'OPT'
:param durationStr: [S Seconds, D Day, W Week, M Month, Y Year ]
:param barSizeSetting: [1 secs, 5 secs, 10 secs, 15 secs, 30 secs, 1 min, 2 mins, 3 mins, 5 mins, 10 mins ,15 mins,
20 mins, 30 mins, 1 hour, 2 hours, 3 hours, 4 hours, 8 hours, 1 day, 1 week, 1 month]
:param show: [OPTION_IMPLIED_VOLATILITY, TRADES]
:return:
:return:
"""
# k = 0
# while k < 10:
# contract_details_list = self.tws.resolve_ib_contract(symbol, type)
# try:
# contract_details = contract_details_list[0]
# break
# except TypeError:
# sleep(2)
# k = k + 1
# if k == 10:
# return -1
#
# contract = contract_details.contract
id = str(contract.symbol) + str(contract.secType) + str(contract.exchange) + str(contract.primaryExchange) + show + barSizeSetting
if id in self.data.keys():
date_updated = self.data[id]['UpdateDate']
hist_data_old = self.data[id]['Data']
if date_updated.date() == datetime.datetime.today().date():
hist_data = hist_data_old
else:
if hist_data_old.size > 0:
# date_last_str = hist_data_old[-1][BAR_DICT['Date']]
date_last_date = hist_data_old[-1][BAR_DICT['Date']]
date_now = datetime.datetime.now()
date_delta = date_now - date_last_date
date_delta_days = date_delta.days
if date_delta_days > 0:
hist_data_new = self.tws.get_IB_historical_data(contract,
durationStr='{} D'.format(date_delta_days),
barSizeSetting=barSizeSetting,
show=show)
hist_data_new = Database.convert_hist_data(hist_data_new)
z = np.where(hist_data_old[:, 0] == date_last_date)[0][0] - 1
if z is not None and hist_data_new.size > 0:
hist_data_old = hist_data_old[:z]
hist_data = np.concatenate((hist_data_old, hist_data_new), axis=0)
else:
hist_data = hist_data_old
else:
hist_data = hist_data_old
else:
hist_data = self.tws.get_IB_historical_data(contract, durationStr=durationStr,
barSizeSetting=barSizeSetting, show=show)
hist_data = Database.convert_hist_data(hist_data)
sleep(2)
self.data[id]['Data'] = hist_data
self.data[id]['UpdateDate'] = datetime.datetime.now()
self.save(self.data, 'HistData')
else:
hist_data = self.tws.get_IB_historical_data(contract, durationStr=durationStr,
barSizeSetting=barSizeSetting, show=show)
hist_data = Database.convert_hist_data(hist_data)
sleep(5)
if hist_data.size != 0:
self.data[id] = {'Data': None, 'UpdateDate': None}
self.data[id]['Data'] = hist_data
self.data[id]['UpdateDate'] = datetime.datetime.now()
self.save(self.data, 'HistData')
else:
return -1
# self.data[id]['Data'] = hist_data
# self.data[id]['UpdateDate'] = datetime.datetime.now()
return hist_data
def rg_update(s , r , ac , af , s_max , rg):
rg = np.where(s>s_max , rg_cal(r , ac , af) , rg)
s_max = np.maximum(s , s_max)
return rg , s_max
def _trim_cell(relative_axes, cell, symprec):
"""Trim overlapping atoms
Parameters
----------
relative_axes: ndarray
Transformation matrix to transform supercell to a smaller cell such as:
trimmed_lattice = np.dot(relative_axes.T, cell.get_cell())
shape=(3,3)
cell: PhonopyAtoms
A supercell
symprec: float
Tolerance to find overlapping atoms in the trimmed cell
Returns
-------
tuple
trimmed_cell, extracted_atoms, mapping_table
"""
positions = cell.get_scaled_positions()
numbers = cell.get_atomic_numbers()
masses = cell.get_masses()
magmoms = cell.get_magnetic_moments()
lattice = cell.get_cell()
trimmed_lattice = np.dot(relative_axes.T, lattice)
trimmed_positions = []
trimmed_numbers = []
if masses is None:
trimmed_masses = None
else:
trimmed_masses = []
if magmoms is None:
trimmed_magmoms = None
else:
trimmed_magmoms = []
extracted_atoms = []
positions_in_new_lattice = np.dot(positions,
np.linalg.inv(relative_axes).T)
positions_in_new_lattice -= np.floor(positions_in_new_lattice)
trimmed_positions = np.zeros_like(positions_in_new_lattice)
num_atom = 0
mapping_table = np.arange(len(positions), dtype='intc')
for i, pos in enumerate(positions_in_new_lattice):
is_overlap = False
if num_atom > 0:
diff = trimmed_positions[:num_atom] - pos
diff -= np.rint(diff)
# Older numpy doesn't support axis argument.
# distances = np.linalg.norm(np.dot(diff, trimmed_lattice), axis=1)
# overlap_indices = np.where(distances < symprec)[0]
distances = np.sqrt(
np.sum(np.dot(diff, trimmed_lattice)**2, axis=1))
overlap_indices = np.where(distances < symprec)[0]
if len(overlap_indices) > 0:
assert len(overlap_indices) == 1
is_overlap = True
mapping_table[i] = extracted_atoms[overlap_indices[0]]
if not is_overlap:
trimmed_positions[num_atom] = pos
num_atom += 1
trimmed_numbers.append(numbers[i])
if masses is not None:
trimmed_masses.append(masses[i])
if magmoms is not None:
trimmed_magmoms.append(magmoms[i])
extracted_atoms.append(i)
# scale is not always to become integer.
scale = 1.0 / np.linalg.det(relative_axes)
if len(numbers) == np.rint(scale * len(trimmed_numbers)):
trimmed_cell = PhonopyAtoms(
numbers=trimmed_numbers,
masses=trimmed_masses,
magmoms=trimmed_magmoms,
scaled_positions=trimmed_positions[:num_atom],
cell=trimmed_lattice,
pbc=True)
return trimmed_cell, extracted_atoms, mapping_table
else:
return False
def full_date_pkr(x):
return int(x.game_id[3:11])
def full_date_pkr2(x):
return int(x.game_id[3:12])
read_df['full_date'] = read_df.apply(full_date_pkr, axis=1)
read_df['full_date2'] = read_df.apply(full_date_pkr2, axis=1)
df = read_df.copy()
df['h'] = np.where(df.h_fl > 0, 1, 0)
df = df.groupby(by=['full_date', 'full_date2', 'bat_id']).agg({'h': 'sum'})
df = df.reset_index()
df = df.merge(players, left_on='bat_id', right_on='retroID')
def debut_pkr(x):
return int(x.debut.replace("-", ""))
df['debut2'] = df.apply(debut_pkr, axis=1)
df = df.query("debut2 >= 19900101")
player_list = list(df.bat_id.unique())
def add_reg_noise(data_name, X, y, lat, lon, pr_noise, nlabel, y_perc = np.nan, cutoff = 1., region_name='ENSO'):
lat = lat.flatten()
lon = lon.flatten()
NSAMPLES = np.shape(X)[0]
#-----------------------------------------------
# define the tranquil and noisy regions
if data_name.find('mixedLabels')==0:
inoise = np.arange(0,len(y))
tranquil = is_tranquil(NSAMPLES,inoise)
elif data_name.find('tranquilFOO') == 0:
reg_lats, reg_lons = get_region(region_name = region_name)
ilat = np.where(np.logical_and(lat>=reg_lats[0],lat<=reg_lats[1]))[0]
ilon = np.where(np.logical_and(lon>=reg_lons[0],lon<=reg_lons[1]))[0]
reg_avg = np.nansum(np.nansum(X[:,:,ilon],axis=-1)[:,ilat],axis=-1)/(len(ilat)*len(ilon))
print('region shape = ' + str(len(ilat)) + ' x ' + str(len(ilon)))
inoise = np.where(reg_avg<=cutoff)[0]
tranquil = is_tranquil(NSAMPLES,inoise)
elif data_name.find('noNoise') == 0:
inoise = []
tranquil = np.ones((NSAMPLES,))
else:
raise ValueError('no such data noise type.')
#-----------------------------------------------
#change the labels on the noisy data
y_new = np.copy(y)
corrupt = np.zeros(np.shape(tranquil))
for i in inoise:
if np.random.random() < pr_noise:
i_rand_sample = np.random.randint(0,NSAMPLES)
value = y[i_rand_sample]
corrupt_value = 1
else:
value = y[i]
corrupt_value = 0
y_new[i] = value
corrupt[i] = corrupt_value
itranquil = np.where(tranquil==1)[0]
i = np.where(corrupt==1)[0]
k = np.where(np.logical_and(tranquil==1, corrupt==1))[0]
print('\n----Mislabeled----')
print('# tranquil = ' + str(len(itranquil)) + ' out of ' + str(NSAMPLES) + ' samples')
print('percent tranquil = ' + str(np.round(100.*len(itranquil)/len(tranquil))) + '%')
try:
print('tranquil mislabeled = ' + str(np.round(100.*len(k)/len(itranquil))) + '%')
print('non-tranquil mislabeled = ' + str(np.round(100.*len(i)/len(inoise))) + '%')
except:
pass
print('total mislabeled = ' + str(np.round(100.*len(i)/len(y_new))) + '%')
#-----------------------------------------------
return X, y_new, tranquil, corrupt
def add_bbox_regression_targets(roidb):
"""Add information needed to train bounding-box regressors."""
assert len(roidb) > 0
assert 'max_classes' in roidb[0], 'Did you call prepare_roidb first?'
num_images = len(roidb)
# Infer number of classes from the number of columns in gt_overlaps
num_classes = roidb[0]['gt_overlaps'].shape[1]
for im_i in xrange(num_images):
rois = roidb[im_i]['boxes']
max_overlaps = roidb[im_i]['max_overlaps']
max_classes = roidb[im_i]['max_classes']
roidb[im_i]['bbox_targets'] = \
_compute_targets(rois, max_overlaps, max_classes)
if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED:
# Use fixed / precomputed "means" and "stds" instead of empirical values
means = np.tile(
np.array(cfg.TRAIN.BBOX_NORMALIZE_MEANS), (num_classes, 1))
stds = np.tile(
np.array(cfg.TRAIN.BBOX_NORMALIZE_STDS), (num_classes, 1))
else:
# Compute values needed for means and stds
# var(x) = E(x^2) - E(x)^2
class_counts = np.zeros((num_classes, 1)) + cfg.EPS
sums = np.zeros((num_classes, 4))
squared_sums = np.zeros((num_classes, 4))
for im_i in xrange(num_images):
targets = roidb[im_i]['bbox_targets']
for cls in xrange(1, num_classes):
cls_inds = np.where(targets[:, 0] == cls)[0]
if cls_inds.size > 0:
class_counts[cls] += cls_inds.size
sums[cls, :] += targets[cls_inds, 1:].sum(axis=0)
squared_sums[cls, :] += \
(targets[cls_inds, 1:] ** 2).sum(axis=0)
means = sums / class_counts
stds = np.sqrt(squared_sums / class_counts - means ** 2)
print 'bbox target means:'
print means
print means[1:, :].mean(axis=0) # ignore bg class
print 'bbox target stdevs:'
print stds
print stds[1:, :].mean(axis=0) # ignore bg class
# Normalize targets
if cfg.TRAIN.BBOX_NORMALIZE_TARGETS:
print "Normalizing targets"
for im_i in xrange(num_images):
targets = roidb[im_i]['bbox_targets']
for cls in xrange(1, num_classes):
cls_inds = np.where(targets[:, 0] == cls)[0]
roidb[im_i]['bbox_targets'][cls_inds, 1:] -= means[cls, :]
roidb[im_i]['bbox_targets'][cls_inds, 1:] /= stds[cls, :]
else:
print "NOT normalizing targets"
# These values will be needed for making predictions
# (the predicts will need to be unnormalized and uncentered)
return means.ravel(), stds.ravel()
def iterar(self):
#Verifica os novos infectados por infectantes do tipo 1 e 2
#print(self.lista_infectados_tipo_1+self.lista_infectados_tipo_2)
lista_novos_infectados_tipo1, lista_novos_infectados_tipo2 = self.verificar_infeccao(self.lista_infectados_tipo_1+self.lista_infectados_tipo_2)
for indice in lista_novos_infectados_tipo1:
self.criar_individuo(self.INFECTADO_TIPO_1, indice)
for indice in lista_novos_infectados_tipo2:
self.criar_individuo(self.INFECTADO_TIPO_2, indice)
#Verifica morte dos infectados tipo 2
lista_mortos = self.checagem_morte_lista(self.lista_infectados_tipo_2)
#retira os indices dos individuos mortos da lista de infectados
self.lista_infectados_tipo_2 = [indice for indice in self.lista_infectados_tipo_2 if indice not in lista_mortos]
#Instancia individuos mortos na matriz
for indice in lista_mortos:
self.criar_individuo(self.MORTO, indice)
#atualiza o número de mortos na matriz
self.num_mortos = self.num_mortos + len(lista_mortos)
#Verifica cura dos infectados tipo 1
lista_curados_t1 = self.checagem_cura_lista(self.lista_infectados_tipo_1)
#Verifica cura dos infectados tipo 2
lista_curados_t2 = self.checagem_cura_lista(self.lista_infectados_tipo_2 )
#Instancia individuos mortos na matriz
for indice in lista_curados_t1+lista_curados_t2:
self.criar_individuo(self.CURADO, indice)
#atualiza o número de curados na matriz
self.num_curados = self.num_curados + len(lista_curados_t1 + lista_curados_t2)
#Atualiza a lista de infectados após a cura dos individuos
self.lista_infectados_tipo_1 = [indice for indice in self.lista_infectados_tipo_1 if indice not in lista_curados_t1]
self.lista_infectados_tipo_2 = [indice for indice in self.lista_infectados_tipo_2 if indice not in lista_curados_t2]
#movimentação
nova_lista_t1 = []
for indice in self.lista_infectados_tipo_1:
nova_lista_t1.append(self.mover_infectante(indice))
self.lista_infectados_tipo_1 = nova_lista_t1
#print(self.lista_infectados_tipo_1)
nova_lista_t2 = []
for indice in self.lista_infectados_tipo_2:
nova_lista_t2.append(self.mover_infectante(indice))
self.lista_infectados_tipo_2 = nova_lista_t2
#print(self.lista_infectados_tipo_2)
# matriz_infectantes = matriz_infectantes[matriz_infectantes < 3]
indices_infectados = list(zip(*np.where((self.matriz_status == 1) + (self.matriz_status == 2))))
# indices_infectados = list(zip(*self.matriz_status.nonzero()))
#indices_infectados = [indice for indice in indices_infectados if indice not in self.lista_infectados_tipo_1 + self.lista_infectados_tipo_2]
# self.num_curados = 0
#self.num_mortos = 0
self.lista_infectados_tipo_1 = []
self.lista_infectados_tipo_2 = []
#novos_t1 = []
#novos_t2 = []
for indice in indices_infectados:
#if indice not in self.lista_infectados_tipo_1 and indice not in self.lista_infectados_tipo_2:
# print(indice)
# print(self.matriz_status.shape)
status = self.matriz_status[indice[0], indice[1]]
if status == self.INFECTADO_TIPO_1:
self.lista_infectados_tipo_1.append(indice)
#novos_t1.append(indice)
if status == self.INFECTADO_TIPO_2:
self.lista_infectados_tipo_2.append(indice)
#novos_t2.append(indice)
#self.lista_infectados_tipo_1 = self.lista_infectados_tipo_1 + novos_t1
#self.lista_infectados_tipo_2 = self.lista_infectados_tipo_2 + novos_t2
dict = {'num_sadios': self.populacao_inicial - len(self.lista_infectados_tipo_1) -len(self.lista_infectados_tipo_2) -self.num_curados-self.num_mortos,
'num_infect_t1': len(self.lista_infectados_tipo_1),
'num_infect_t2': len(self.lista_infectados_tipo_2),
'num_curados': self.num_curados,
'num_mortos': self.num_mortos}
self.dataframe = self.dataframe.append(dict, ignore_index=True)
self.salvar_posicionamento()
#adiciona 1 ao número de atualizações realizadas na matriz
self.num_atualizacoes +=1
import cv2
import numpy as np
mask_path = '/mnt/data7/slice_test_seg/ct_lesion-type_20200311_177'
lung_path = '/home/xzw/lung_seg'
img_path = '/mnt/data6/lung_data'
output_path_slices = '/mnt/data7/slice_test_seg/mask_jpgs_new'
output_path_raw = '/mnt/data7/slice_test_seg/raw_jpgs_new'
os.makedirs(output_path_raw, exist_ok=True)
os.makedirs(output_path_slices, exist_ok=True)
all_files = glob.glob(mask_path + '/*mask-label.nii')
for item in all_files:
mask = sitk.ReadImage(item)
mask = sitk.GetArrayFromImage(mask)
zz, yy, xx = np.where(mask > 0)
name = item.split('/')[-1].split('-mask')[0]
if int(name.split('_')[0]) < 100:
full_name = os.path.join(
img_path, 'lung_1st',
name.split('_')[0] + '_' + name.split('_')[1] + '.nii')
full_name_lung = os.path.join(lung_path,
'illPatient1_' + name + '_label.nii')
else:
full_name = os.path.join(
img_path, 'lung_2rd',
str(int(name.split('_')[0]) - 100) + '_' + name.split('_')[1] +
'.nii')
full_name_lung = os.path.join(
lung_path, 'illPatient2_' + str(int(name.split('_')[0]) - 100) +
def calc_statistic_outliers(self, factor: float = 3.0) -> Tuple[pd.DataFrame, pd.DataFrame]:
"""
Optional:
factor (float) - prefactor for stddev to detect statistic outliers
Returns:
data (DataFrame) - pandas DataFrame with columns 'object_id', 'time', 'cluster_id', 'outlier'
outlier_result (DataFrame) - pandas DataFrame with columns 'object_id', 'start_time', 'end_time',
'cluster_end_time', 'rating', 'distance' and 'outlier'
"""
timestamps = self._data[self._time_column_name].unique()
timestamps.sort()
self._outlier_rating = pd.DataFrame(
columns=[self._object_column_name, 'start_time', 'end_time', 'rating', 'distance', 'outlier'])
for t1 in range(len(timestamps) - 1):
for t2 in range(t1 + 1, len(timestamps)):
subsequence_ratings = self.calc_subsequence_ratings(timestamps[t1], timestamps[t2])
subsequence_ratings = subsequence_ratings.assign(distance=0)
subsequence_ratings = subsequence_ratings.assign(outlier=0)
std_dev = np.std(subsequence_ratings[subsequence_ratings['rating'] >= 0]['rating'].values)
mean = np.mean(subsequence_ratings[subsequence_ratings['rating'] >= 0]['rating'].values)
subsequence_ratings = subsequence_ratings.assign(distance=lambda x: abs(x['rating'] - mean))
subsequence_ratings[subsequence_ratings['rating'] >= 0] = subsequence_ratings[subsequence_ratings['rating'] >= 0].assign(outlier=lambda x: (x['distance'] <= factor * std_dev) - 1)
self._outlier_rating = self._outlier_rating.append(subsequence_ratings)
noise_object_ids = self._data[self._data[self._cluster_column_name] < 0][self._object_column_name].unique()
for object_id in noise_object_ids:
noisy_timestamps = self._data[(self._data[self._object_column_name] == object_id) &
(self._data[self._cluster_column_name] < 0)][
self._time_column_name].unique().tolist()
noisy_timestamps.sort()
start_index = np.where(timestamps == noisy_timestamps[0])[0][0]
counter = 1
for i in range(1, len(noisy_timestamps)):
if noisy_timestamps[i] == timestamps[start_index + counter]:
counter += 1
else:
if counter > 1:
self._outlier_rating = self._outlier_rating.append({self._object_column_name: object_id,
'start_time': timestamps[start_index],
'end_time': timestamps[
start_index + counter - 1],
'rating': -1,
'distance': -1,
'outlier': -2}, ignore_index=True)
start_index = np.where(timestamps == noisy_timestamps[i])[0][0]
counter = 1
if counter > 1:
self._outlier_rating = self._outlier_rating.append({self._object_column_name: object_id,
'start_time': timestamps[start_index],
'end_time': timestamps[start_index + counter - 1],
'rating': -1,
'distance': -1,
'outlier': -2}, ignore_index=True)
self._outlier_result = self._outlier_rating[self._outlier_rating['outlier'] < 0]
timestamps = timestamps.tolist()
self._data = self._data.assign(outlier=0)
for index, row in self._outlier_result.iterrows():
for time_point in timestamps[timestamps.index(int(row['start_time'])): timestamps.index(
int(row['end_time'])) + 1]:
self._data.loc[(self._data[self._time_column_name] == time_point) & (
self._data[self._object_column_name] == row[self._object_column_name]), 'outlier'] = row['outlier']
return self._data, self._outlier_result
def detect_pnet20(self, im):
"""Get face candidates through pnet
Parameters:
----------
im: numpy array
input image array
Returns:
-------
boxes: numpy array
detected boxes before calibration
boxes_c: numpy array
boxes after calibration
"""
h, w, c = im.shape
net_size = 20
current_scale = float(
net_size) / self.min_face_size # find initial scale
im_resized = self.resize_image(im, current_scale)
_, _, current_height, current_width = im_resized.shape
if self.slide_window:
# sliding window
temp_rectangles = list()
rectangles = list(
) # list of rectangles [x11, y11, x12, y12, confidence] (corresponding to original image)
all_cropped_ims = list()
while min(current_height, current_width) > net_size:
current_y_list = range(0, current_height - net_size + 1, self.stride) if (current_height - net_size) % self.stride == 0 \
else range(0, current_height - net_size + 1, self.stride) + [current_height - net_size]
current_x_list = range(0, current_width - net_size + 1, self.stride) if (current_width - net_size) % self.stride == 0 \
else range(0, current_width - net_size + 1, self.stride) + [current_width - net_size]
for current_y in current_y_list:
for current_x in current_x_list:
cropped_im = im_resized[:, :,
current_y:current_y + net_size,
current_x:current_x + net_size]
current_rectangle = [
int(w * float(current_x) / current_width),
int(h * float(current_y) / current_height),
int(w * float(current_x) / current_width) +
int(w * float(net_size) / current_width),
int(h * float(current_y) / current_height) +
int(w * float(net_size) / current_width), 0.0
]
temp_rectangles.append(current_rectangle)
all_cropped_ims.append(cropped_im)
current_scale *= self.scale_factor
im_resized = self.resize_image(im, current_scale)
_, _, current_height, current_width = im_resized.shape
'''
# helper for setting PNet batch size
num_boxes = len(all_cropped_ims)
batch_size = self.pnet_detector.batch_size
ratio = float(num_boxes) / batch_size
if ratio > 3 or ratio < 0.3:
print "You may need to reset PNet batch size if this info appears frequently, \
face candidates:%d, current batch_size:%d"%(num_boxes, batch_size)
'''
all_cropped_ims = np.vstack(all_cropped_ims)
cls_scores, reg = self.pnet_detector.predict(all_cropped_ims)
cls_scores = cls_scores[:, 1].flatten()
keep_inds = np.where(cls_scores > self.thresh[0])[0]
if len(keep_inds) > 0:
boxes = np.vstack(temp_rectangles[ind] for ind in keep_inds)
boxes[:, 4] = cls_scores[keep_inds]
reg = reg[keep_inds].reshape(-1, 4)
else:
return None, None
keep = py_nms(boxes, 0.7, 'Union')
boxes = boxes[keep]
boxes_c = self.calibrate_box(boxes, reg[keep])
else:
# fcn
all_boxes = list()
while min(current_height, current_width) > net_size:
cls_map, reg = self.pnet_detector.predict(im_resized)
cls_map = cls_map.asnumpy()
reg = reg.asnumpy()
boxes = self.generate_bbox(cls_map[0, 1, :, :], reg,
current_scale, self.thresh[0])
current_scale *= self.scale_factor
im_resized = self.resize_image(im, current_scale)
_, _, current_height, current_width = im_resized.shape
if boxes.size == 0:
continue
keep = py_nms(boxes[:, :5], 0.5, 'Union')
boxes = boxes[keep]
all_boxes.append(boxes)
if len(all_boxes) == 0:
return None, None
all_boxes = np.vstack(all_boxes)
# merge the detection from first stage
keep = py_nms(all_boxes[:, 0:5], 0.7, 'Union')
all_boxes = all_boxes[keep]
boxes = all_boxes[:, :5]
bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1
bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1
# refine the boxes
boxes_c = np.vstack([
all_boxes[:, 0] + all_boxes[:, 5] * bbw,
all_boxes[:, 1] + all_boxes[:, 6] * bbh,
all_boxes[:, 2] + all_boxes[:, 7] * bbw,
all_boxes[:, 3] + all_boxes[:, 8] * bbh, all_boxes[:, 4]
])
boxes_c = boxes_c.T
return boxes, boxes_c
def predict(self, A):
return np.where(A > 0.5, 1., 0.)
def operations(img1,img2,threshold):
image_1 = cv2.imread(img1)
image_2 = cv2.imread(img2)
###transforming input images to grayscale images and then binarization
##image 1
blue_component1 = image_1[:,:,0]
green_component1 = image_1[:,:,1]
red_component1 = image_1[:,:,2]
gray_image_1 = (0.11*blue_component1 + 0.59*green_component1 + 0.3*red_component1)
binary_image_1 = np.where(gray_image_1 > threshold, 255,0)
print(binary_image_1)
##image 2
blue_component2 = image_2[:,:,0]
green_component2 = image_2[:,:,1]
red_component2 = image_2[:,:,2]
gray_image_2 = (0.11*blue_component2 + 0.59*green_component2 + 0.3*red_component2)
binary_image_2 = np.where(gray_image_2 > threshold,255,0)
##addition
#image_3 = binary_image_1 + binary_image_2
image_3_add = cv2.add(binary_image_1,binary_image_2)
##substraction
#image_3 = binary_image_1 - binary_image_2
image_3_sub = cv2.subtract(binary_image_1,binary_image_2)
### logical xor
xor = np.logical_xor(binary_image_1, binary_image_2)
image_3_xor = np.where(xor > 0,255,0)
### logical andd
#andd = np.logical_and(binary_image_1,binary_image_2)
image_3_andd = np.where((binary_image_1*binary_image_2>0), 255,0)
### logical or
orr = np.logical_or(binary_image_1,binary_image_2)
image_3_orr = np.where(orr > 0,255,0)
cv2.imwrite("image_add.jpg",image_3_add)
image_add = cv2.imread("image_add.jpg")
cv2.imshow('Addition of Images',image_add)
cv2.imwrite("image_sub.jpg",image_3_sub)
image_sub = cv2.imread("image_sub.jpg")
cv2.imshow('Subtraction of Images',image_sub)
cv2.imwrite("image_xor.jpg",image_3_xor)
image_xor = cv2.imread("image_xor.jpg")
cv2.imshow('Logical XOR of Images',image_xor)
cv2.imwrite("image_and.jpg",image_3_andd)
image_andd = cv2.imread("image_and.jpg")
cv2.imshow('Logical AND of Images',image_andd)
cv2.imwrite("image_or.jpg",image_3_orr)
image_orr = cv2.imread("image_or.jpg")
cv2.imshow('Logical OR of Images',image_orr)
cv2.waitKey(0)
cv2.destroyAllWindows()
sys.exit()
plt.contourf(xx, yy, Z, cmap=cmap, alpha=0.25)
plt.contour(xx, yy, Z, colors='k', linewidths=0.7)
plt.scatter(X[0], X[1], c=Y[0], cmap=cmap, edgecolors='k')
plt.savefig('decision_boundary_%s_%s.png' % (layer, node))
if __name__ == '__main__':
X1 = np.random.randn(2, 40)
X1 /= 1.5 * np.linalg.norm(X1, axis=0)
Y1 = np.array([1.] * 40).reshape((1, 40))
X2 = np.random.randn(2, 40)
X2 /= np.linalg.norm(X2, axis=0)
Y2 = np.array([0.] * 40).reshape((1, 40))
X = np.concatenate([X1, X2], axis=1)
Y = np.concatenate([Y1, Y2], axis=1)
layers = initiate_layers([(3, tanh()), (1, sigmoid())], X)
num_iterations = 10000
losses = []
for i in range(num_iterations):
activations = forward_propogation(X, layers)
layers = backward_propogation(Y, 1.0, activations, layers)
P = np.where(activations[-1] > 0.5, 1., 0)
losses.append(np.squeeze(loss(Y, activations[-1])))
print(metrics.accuracy_score(Y[0], P[0]))
for i, (W, B, g) in enumerate(layers):
for j in range(W.shape[0]):
plot_decision_boundary(layers, X, Y, layer=i, node=j)
plt.figure(figsize=(5, 5))
plt.plot(losses)
plt.savefig('loss.png')
#year forest
g = GeoTiff(lossyear)
ex = grid.extent_in_crs(crs=wgs84) # l, r, b, t
g.set_subset(corners=((ex[0], ex[2]), (ex[1], ex[3])),
crs=wgs84, margin=10)
ls = g.get_vardata()
ls = np.array(ls, dtype=float)
# ls[ls >100] = np.nan
# ls = grid.map_gridded_data(ls, g.grid)
ls = np.round(ls).astype(int)
year_on_ds = ds18_present.salem.lookup_transform(ls, grid=g.grid, lut=lut, method=lambda x:np.bincount(x[np.where(x>0)]).argmax())
year_on_ds = year_on_ds.assign_coords(lon=grid.ll_coordinates[0][0,:], lat=grid.ll_coordinates[1][:,0])
# deforestation before 2009: set to 0
#pdb.set_trace()
#lst_on_ds[np.where(year_on_ds<=9)]=0
lst_on_ds.values[np.where(year_on_ds.values<=9)]=0
# mask_river = grid.region_of_interest(shape=river)
# mask_lakes = grid.region_of_interest(shape=lakes, roi=mask_river, all_touched=True)
# mask_river[mask_lakes]= 1
# larr = ds18_present.copy()
# larr.values = mask_river
ds2f = ds18_present.values.flatten()
dsf = ds18_hist.values.flatten()
def SE_BDD(H_mat, Z_mat, Threshold, Verbose = "True"):
import numpy as np
from numpy.random import seed
from numpy.random import randn
from numpy import mean
from numpy import std
#print("\n ************** State Estimation ************* \n")
# takes H matrix Z measurements, Threshold and Verbose as input
std_cutOff_th = 1.5
Max_Threshold = 10
Threshold_step = 2
doneFlag = 0
fullRank = True
numberOfMeasurements = H_mat.shape[0]
numberOfStates = H_mat.shape[1]
#print("Considering only the taken measurements")
############# starting State Estimation ###############
Z_mat = Z_mat[Z_mat[:,0].argsort(kind='mergesort')]
Z_mat_Copy = Z_mat.copy()
# Space holder for Z_est
Z_est = np.zeros(numberOfMeasurements)
list_of_removed_sensors = []
while(doneFlag == 0):
#list_of_removed_sensors = []
#considering only the taken measurements
Z_msr = Z_mat[Z_mat[:,1]==1][:,2]
# considering only the corresponding columns in H
H_msr = H_mat[Z_mat[:,1]==1]
#Chekcing rank of H
Rank = np.linalg.matrix_rank(H_msr)
if Verbose == "True":
print("Current Rank", Rank)
print("H_msr Shape: ", H_msr.shape)
###### H is full rank --> Start Estimating
if Rank == numberOfStates:
#Estimating the states using WMSE estimator
States = (np.linalg.inv(H_msr.T@H_msr)@H_msr.T)@Z_msr
# Estimating the measurement from the estimated states
Z_est = H_mat@States
if Verbose == "True":
pass
#print("\n Initital Z_m: "+str(Z_mat))
#print("\n Sates: \n"+str(States))
#print("\n Z_est: \n"+str(Z_est))
##################### Checking for Bad Data ##################################
# Calculating the Noise
M_Noise, P_Noise, Noisy_Index = Calc_Percent_of_Noise (Z_est, Z_mat, Threshold, fullRank, Verbose =="False")
##### If there are some noisy data #############
if( len(Noisy_Index) > 0):
#print("***************** Bad Data Detected *********************")
#changing the noisy measurement parameter (taken) from 1 to 0
# Analyzig the percentage of noise
Noise_mean, Noise_std = mean(P_Noise), std(P_Noise)
# identify outliers
Noise_cut_off = Noise_mean + Noise_std * std_cutOff_th
indxtbremoved = np.where(P_Noise > Noise_cut_off)[0]
target_noisy_indx = indxtbremoved if indxtbremoved.size > 0 else Noisy_Index #np.argsort(P_Noise)[::-1]
if Verbose == "True":
print(f"Noise_mean: {Noise_mean}, Noise_std: {Noise_std}")
print("Removing sensor: ", target_noisy_indx)
#############################################################
# removing noisy sensors from the state estimation process
Z_mat[target_noisy_indx,1] = -1
list_of_removed_sensors = list_of_removed_sensors + target_noisy_indx.tolist()
else:
if Verbose == "True": print("!!!! Done !!!\nNo more noisy data")
doneFlag = 1
else:
if Threshold < Max_Threshold:
Threshold += Threshold_step
if Verbose == "True": print(f"Relaxing the nosie threshold to {Threshold}")
list_of_removed_sensors = []
Z_mat = Z_mat_Copy.copy()
else:
doneFlag = -1
if Verbose == "True": print(f"\n\n\nSystem Unobservable !, Rank = {Rank}")
##### Returning ##############
fullRank = False
Z_est = np.zeros(numberOfMeasurements)
States = np.zeros(numberOfStates)
############### Returning ########################
#print("list_of_removed_sensors:", list_of_removed_sensors)
return States, Z_est, Z_mat, fullRank
def predict(self, A):
return np.where(A > 0, 1., -1.)
def pattern_gen(n, capacity):
import numpy as np
r = len(capacity)
if r == 0:
print(" number of sections cant be zero")
return
list_ = []
avg_frac = n/r
avg_int = n//r
diff = (avg_frac - avg_int)
if diff >= 0.5:
diff = 1 - diff
step = avg_int + 1
ceil_sign = -1
else:
step = avg_int
ceil_sign = +1
liability = 0
repeat = 0
while repeat < r:
repeat = repeat + 1
liability = liability + diff * ceil_sign
if abs(liability) >= 0.99:
added = round(step + liability)
list_.append(added)
n = n - added
liability = liability - ceil_sign
else:
list_.append(round(step))
n = n - round(step)
##################################
######### checking capacity ##3##
#print(list_)
cap = np.array(capacity)
dis = np.array(list_)
diff = cap - dis
#print("diff:", diff)
lagg = np.sum(diff[np.where(diff<0)])
#print("lagg:", lagg)
index = 0
while(lagg < 0):
positiveIndex = np.where(diff > 0)[0]
if index >= positiveIndex.size:
index = 0
if positiveIndex.size == 0:
print("Distribution not possible!!!!!")
break
# print(positiveIndex)
# print(index)
# print(diff[positiveIndex[index]])
diff[positiveIndex[index]] -= 1
lagg += 1
index += 1
#print(diff, lagg)
diff[np.where(diff < 0)] = 0
dis = cap - diff
#print(dis)
return dis.astype(int).tolist()
def SE_BDD_(H_mat, Z_mat, W_list, Threshold_min, Threshold_max, Verbose = "True"):
import numpy as np
from numpy.random import seed
from numpy.random import randn
from numpy import mean
from numpy import std
if Verbose == "True":
print("\n ************** State Estimation ************* \n")
doneFlag = 0
fullRank = True
Threshold = Threshold_min
Threshold_step = 2
numberOfMeasurements = H_mat.shape[0]
numberOfStates = H_mat.shape[1]
############# starting State Estimation ###############
#Z_mat = Z_mat[Z_mat[:, 0].argsort(kind='mergesort')]
Z_mat_Copy = Z_mat.copy()
W_list = np.array(W_list)
# Space holder for Z_est
#Z_est = np.zeros(numberOfMeasurements)
#list_of_removed_sensors = []
################# Starting the loop ###################################
while(doneFlag == 0):
#list_of_removed_sensors = []
#considering only the taken measurements
consideredIndx = np.where(Z_mat[:,1] == 1)[0]
#print(Z_mat)
Z_msr = Z_mat[consideredIndx][:,2]
# considering only the corresponding columns in H
H_msr = H_mat[consideredIndx]
# Measurement Covarriance Residual Matrix
R_msr = np.diag(W_list[consideredIndx])
#print(R_msr)
R_inv = np.linalg.inv(R_msr)
#print(R_inv)
#Chekcing rank of H
Rank = np.linalg.matrix_rank(H_msr) if H_msr.shape[0] > 0 else 0
if Verbose == "True":
print("Current Rank", Rank)
print("H_msr Shape: ", H_msr.shape)
###### H is full rank --> Start Estimating
if Rank == numberOfStates:
#Estimating the states using WMSE estimator
inv__Ht_Rinv_H__Ht = np.linalg.inv(H_msr.T@R_inv@H_msr)@H_msr.T
States = inv__Ht_Rinv_H__Ht@R_inv@Z_msr
# Ω = R − H. (H_T.R−1.H)−1.HT
# Omega is a residual covarience matrix
Omega_mat = R_msr - (H_msr@inv__Ht_Rinv_H__Ht)
#print("Check :\n", R_msr - ([email protected](H_msr.T@R_inv@H_msr)@H_msr.T))
#print(f"R_msr: {R_msr} \n Shape {R_msr.shape}")
#print(f"Omega_mat: {Omega_mat} \n Shape {Omega_mat.shape}")
# Estimating the measurement from the estimated states
Z_est = H_mat@States
if Verbose == "True":
print("\n Initital Z_m: "+str(Z_mat))
print("\n Sates: \n"+str(States))
print("\n Z_est: \n"+str(Z_est))
print("Calling Noise.. CheckNoise...")
##################### Checking for Bad Data ##################################
# Calculating the Noise
M_Noise, P_Noise, doneFlag = CheckNoise(
Z_est, Z_mat, Omega_mat, R_msr, Threshold, fullRank, Verbose =="False")
# H is not a full rank matrix ............. abort estimation
else:
if Threshold < Threshold_max:
Threshold += Threshold_step
Z_mat = Z_mat_Copy.copy()
if Verbose == "True": print(f"Relaxing the threshold to {Threshold}")
else:
doneFlag = -1 #system unobservable
if Verbose == "True": print(f"\n\n\nSystem Unobservable !, Rank = {Rank}")
##### Returning ##############
fullRank = False
Z_est = np.zeros(numberOfMeasurements)
States = np.zeros(numberOfStates)
M_Noise = np.zeros(numberOfMeasurements)
##############################################################################
Noisy_Indx = np.where(Z_mat[:,1] == -1)[0]
return States, Z_est, Z_mat, M_Noise, Noisy_Indx, fullRank, Threshold
# Iterative read file ; each time 100000 records are read and processed
for i in range(1, 10):
data = np.genfromtxt(
"C:\\Users\\Tejal\\Documents\\Tejal\\College\\INF552\\Project\\rating.csv",
delimiter=",",
max_rows=i * 100000,
usecols=(0, 1, 2),
skip_header=((i - 1) * 100000))
if i == 1:
hdr = list(data[0])
data = np.delete(data, [0], axis=0)
# Get unique users
s = list(set(data[:, 0]))
#Find number of occurences of the user
for user in s:
c = np.where(data[:, 0] == user)
# Select only those users who have rated atleast 100 movies
if total_users == 3000 or len(fin_data) > 200000:
flag = 1
break
if len(c[0]) > 100:
total_users += 1
for j in c:
fin_data = np.append(fin_data, data[j], axis=0)
if flag == 1:
break
# Keep ratings of only those movies that have atleast 10 ratings
# Get unique movies
movie = list(set(fin_data[:, 1]))
def CheckNoiseCor (Z_est, Z_mat, Omega_mat, R_msr, Threshold, fullRank, Corr, Verbose):
import math
import numpy as np
if fullRank != True:
# if Verbose == "True":
# print("System Unobservable!"
return None, None, Z_mat[:,0]
Z_msr = Z_mat[Z_mat[:, 1] == 1][:,2].copy()
#################################################### Here -------------------->
'''boolean index did not match indexed array along dimension 0; dimension is 53
but corresponding boolean dimension is 55'''
if Verbose == "True":
print("Starting BDD")
print("Z_est: ", Z_est.shape)
#print("Z_mat[:, 1] == 1", Z_mat[:, 1] == 1)
#print(Z_mat)
Z_est_msr = Z_est[Z_mat[:, 1] == 1]
# Calculating the measurement error
M_Noise = (Z_msr - Z_est_msr)
M_Noise_norm = M_Noise.copy()
# Calculating the normalized residuals
for index, _ in enumerate(M_Noise):
if index == 0: continue
try:
M_Noise_norm [index] = np.absolute(M_Noise [index])/math.sqrt(Omega_mat[index, index])
except:
M_Noise_norm [index] = 0
if Verbose == "True":
print("index: ", index, np.absolute(M_Noise [index]))
print(f" Value Error, Expected postive, Got {Omega_mat[index, index]}")
# Noise_mat_actual = np.zeros(Z_mat.shape[0])
# Noise_mat_actual[Z_mat[:,1] == 1] = M_Noise
# Noise_mat_norm = np.zeros(Z_mat.shape[0])
# Noise_mat_norm[Z_mat[:,1] == 1] = M_Noise_norm
# print(M_Noise.shape)
Noise_mat_actual = M_Noise.copy()
Noise_mat_norm = M_Noise_norm.copy()
active_idx = np.where(Z_mat[:,1] == 1)[0]
# Checking for Noisy data
if np.max(Noise_mat_norm) > Threshold:
tIndx = np.argmax(Noise_mat_norm)
if Verbose == "True":
print(f"targetedIndx in cut: {tIndx}--> Value : {Noise_mat_norm[tIndx]}")
print("Updating Z_mat...")
print("Before: ", Z_mat[tIndx])
#print("R_msr: ", R_msr.shape, Omega_mat.shape)
if Corr == True:
Z_mat[active_idx[tIndx], 2] = Z_mat[active_idx[tIndx], 2] - R_msr[tIndx,tIndx]/Omega_mat[tIndx,tIndx]*M_Noise[tIndx]
else:
Z_mat[active_idx[tIndx], 1] = -1
doneFlag = 0
if Verbose == "True":
print("After: ", Z_mat[tIndx])
else:
if Verbose == "True": print("No Bad Data Detected....")
doneFlag = 1
Noise_mat_actual = np.zeros(Z_mat.shape[0])
Noise_mat_actual[Z_mat[:,1] == 1] = M_Noise.copy()
Noise_mat_norm = np.zeros(Z_mat.shape[0])
Noise_mat_norm[Z_mat[:,1] == 1] = M_Noise_norm.copy()
##############################################
return Noise_mat_actual, Noise_mat_norm, doneFlag