def pcprint(self):
if self.doBoth==1:
subplot(1,2,2)
fs=12
else:
fs=15
self.c1=self.data1.copy() #for copying histogram value
data2=np.isfinite(self.data1)
data3=self.data1[data2]
nn=str(self.lineEdit_4.text())
b=nn.split(',')
if len(b)==1:
m=int(b[0])
n=int(b[0])
else:
m=int(b[0])
n=int(b[1])
dmax=nanmax(data3)
dmin=nanmin(data3)
total=dmax-dmin
low=dmin+(total*m)/100
high=dmax-(total*n)/100
datal = np.where(data3 < low ,low,data3)
datah = np.where(datal > high ,high,datal)
if self.checkBox.isChecked(): # for fix Legend
self.c1[0][0]=self.maxdata
self.c1[0][1]=self.mindata
np.place(self.c1,data2,datah)
plt.imshow(self.c1,cmap=get_cmap(self.colormap),extent=[self.originX,self.lastX,self.lastY,self.originY])
plt.text(79.2,31,self.image,fontsize=fs)
plt.title('Percentile Clipped')
plt.colorbar()
def remove_wrongly_sized_connected_components(a, min_size,
max_size=None,
in_place=False, bin_out=False):
"""
Copied from lazyflow.operators.opFilterLabels.py
Originally adapted from http://github.com/jni/ray/blob/develop/ray/morpho.py
(MIT License)
"""
original_dtype = a.dtype
if not in_place:
a = a.copy()
if min_size == 0 and (max_size is None or max_size > np.prod(a.shape)): # shortcut for efficiency
if (bin_out):
np.place(a,a,1)
return a
try:
component_sizes = np.bincount( a.ravel() )
except TypeError:
# On 32-bit systems, must explicitly convert from uint32 to int
# (This fix is just for VM testing.)
component_sizes = np.bincount( np.asarray(a.ravel(), dtype=int) )
bad_sizes = component_sizes < min_size
if max_size is not None:
np.logical_or( bad_sizes, component_sizes > max_size, out=bad_sizes )
bad_locations = bad_sizes[a]
a[bad_locations] = 0
if (bin_out):
# Replace non-zero values with 1
np.place(a,a,1)
return np.array(a, dtype=original_dtype)
def matrixDiscreteMaker(t):
'''
this function converts the frequencies matrix t into an integer valued-matrix
rules used:
less than 1% or nan --> nan
between 1% and 5% --> 1
between 5% and 10% --> 2
between 10% and 20% --> 3
more than 20% --> 4
'''
# nan, convert to 0 to avoid numerical warnings
numpy.place(t,numpy.isnan(t),0)
# between 1% and 5% --> 1
t[numpy.where(numpy.logical_and(t>=0.01, t<0.05))]=1
# between 5% and 10% --> 2
t[numpy.where(numpy.logical_and(t>=0.05, t<0.1))]=2
# between 10% and 20% --> 3
t[numpy.where(numpy.logical_and(t>=0.1, t<0.2))]=3
# more than 20% --> 4
t[numpy.where(numpy.logical_and(t>=0.2, t<1-1e-10))]=4
# less than 1% or nan --> nan, which maps as white. this line should be last in order to avoid numerical warnings
numpy.place(t,t<0.01,float('nan'))
return t
def sdprint(self):
if self.doBoth==1:
subplot(1,2,2)
fs=12
else:
fs=15
self.c1=self.data1.copy() #for copying histogram value
data2=np.isfinite(self.data1)
data3=self.data1[data2]
mean=np.mean(data3)
sd=np.std(data3)
nn=str(self.lineEdit_4.text())
b=nn.split(',')
if len(b)==1:
m=int(b[0])
n=int(b[0])
else:
m=int(b[0])
n=int(b[1])
low=mean-m*sd
high=mean+n*sd
## print sd,mean,low,high
datal = np.where(data3 < low ,low,data3)
datah = np.where(datal > high ,high,datal)
np.place(self.c1,data2,datah)
if self.checkBox.isChecked(): # for Fix-Legend
self.c1[0][0]=self.maxdata
self.c1[0][1]=self.mindata
plt.imshow(self.c1,cmap=get_cmap(self.colormap),extent=[self.originX,self.lastX,self.lastY,self.originY])
plt.text(79.2,31,self.image,fontsize=fs)
plt.title('Standard Deviation')
plt.colorbar()
def IDCTnDequantize(prop): #To dequantize img3 = prop.image iHeight, iWidth = img3.shape[:2] img2 = np.zeros((iHeight,iWidth,3), np.uint8) #print img2.dtype for startY in range(0, iHeight, 8): for startX in range(0, iWidth, 8): for c in range(0, 3): block = img3[startY:startY+8, startX:startX+8, c:c+1].reshape(8,8) blockf = np.float32(block) # float conversion dst = cv2.idct(blockf) # inverse dct np.place(dst, dst>255.0, 255.0) # saturation np.place(dst, dst<0.0, 0.0) # grounding block = np.uint8(np.around(dst)) # store the results for y in range(8): for x in range(8): img2[startY+y, startX+x, c] = block[y, x] # convert to BGR img2 = cv2.cvtColor(img2, cv2.COLOR_YCR_CB2BGR) prop.image = img2
def micro_step(self):
""" Defining microphysical step """
print "qc min, max przed mikro", self.state["rc"].min(), self.state["rc"].max()
print "nc min, max przed mikro", self.state["nc"].min(), self.state["nc"].max()
# dot_ variables have to be zero before rhs_cellwise
for k in ("dot_th_d", "dot_rv", "dot_rc", "dot_nc", "dot_rr", "dot_nr"):
self.state_dot[k] *= 0.
#pdb.set_trace()
libcl.blk_2m.rhs_cellwise(self.opts,
self.state_dot["dot_th_d"], self.state_dot["dot_rv"],
self.state_dot["dot_rc"], self.state_dot["dot_nc"],
self.state_dot["dot_rr"], self.state_dot["dot_nr"],
self.state["rho_d"], self.state["th_d"],
self.state["rv"], self.state["rc"], self.state["nc"],
self.state["rr"], self.state["nr"], self.dt)
print "qc min, max po mikro", self.state["rc"].min(), self.state["rc"].max()
print "nc min, max po mikro", self.state["nc"].min(), self.state["nc"].max()
for k in ("th_d", "rv", "rc", "nc", "rr", "nr"):
self.state[k] += self.state_dot["dot_"+k] * self.dt
# rc, nc can be sometimes smaller than zero -- TODO!!
np.place(self.state["rc"], self.state["rc"]<0, 0)
np.place(self.state["nc"], self.state["nc"]<0, 0)
print "qc min, max po place", self.state["rc"].min(), self.state["rc"].max()
print "nc min, max po place", self.state["nc"].min(), self.state["nc"].max()
def generate(input):
""" Testing function, all parameters are in a single tuple """
seed,gen,p0,p1,n,method,MCsize= input
npran.seed(seed)
if gen.lower() == "ising":
X = ising_X(p1+p0,n)
ynor = norm_y(X,p1)
elif gen.lower() == "genetic":
genes = np.genfromtxt('data/SNPdata.txt', delimiter=',')
np.place(genes,genes!=0,1)
X = given_X(p1+p0,n,genes)
ybin = bern_y(X,p1)
ynor = norm_y(X,p1)
# Logit
bin_logit = ko.knockoff_logit(ybin,X,.2,
knockoff='binary',
method=method,
MCsize=MCsize,
intercept=True
)
bin_logit.fit()
ori_logit = ko.knockoff_logit(ybin,X,.2,
knockoff='original',
intercept=False
)
ori_logit.fit()
trueS = (np.arange(p0+p1)<p1).astype(int)
bin_FDR = np.dot(bin_logit.S,1-trueS)/max(np.sum(bin_logit.S),1)
bin_power = np.dot(bin_logit.S,trueS) /max(p1,1)
ori_FDR = np.dot(ori_logit.S,1-trueS)/max(np.sum(ori_logit.S),1)
ori_power = np.dot(ori_logit.S,trueS) /max(p1,1)
corr = np.corrcoef(ori_logit.S,bin_logit.S)[0,1]
ko_corr = [cor for cor in bin_logit.emp_ko_corr if not np.isnan(cor)]
with open('data/logit_test_'+str(p0+p1)+'_w_n.txt','a') as f:
f.write("%d\t%s\t%d\t%d\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\n" % (seed, gen, p1, n, bin_logit.M_distortion, np.mean(ko_corr), bin_FDR, bin_power, np.mean(ori_logit.emp_ko_corr), ori_FDR, ori_power, corr))
# LASSO
bin_lasso = ko.knockoff_lasso(ynor,X,.2,
knockoff='binary',
method=method,
MCsize=MCsize,
intercept=True
)
bin_lasso.fit(bin_logit.X_lrg)
ori_lasso = ko.knockoff_lasso(ynor,X,.2,
knockoff='original',
intercept=False
)
ori_lasso.fit()
trueS = (np.arange(p0+p1)<p1).astype(int)
bin_FDR = np.dot(bin_lasso.S,1-trueS)/max(np.sum(bin_lasso.S),1)
bin_power = np.dot(bin_lasso.S,trueS) /max(p1,1)
ori_FDR = np.dot(ori_lasso.S,1-trueS)/max(np.sum(ori_lasso.S),1)
ori_power = np.dot(ori_lasso.S,trueS) /max(p1,1)
corr = np.corrcoef(ori_lasso.S,bin_lasso.S)[0,1]
with open('data/lasso_test_'+str(p0+p1)+'_w_n.txt','a') as f:
f.write("%d\t%s\t%d\t%d\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\n" % (seed, gen, p1, n, bin_logit.M_distortion, np.mean(ko_corr), bin_FDR, bin_power, np.mean(ori_lasso.emp_ko_corr), ori_FDR, ori_power, corr))
def logsumexp_array(x, axis=None):
"""
Compute log(sum(exp(x))) along a particular axis for Numpy arrays.
"""
# This implementation hasn't been tested on arrays with dimension > 2!
if axis is None:
max_ent = x.max()
bias = max_ent
else:
max_ent = x.max(axis)
bias = max_ent if axis == 0 else max_ent[:, numpy.newaxis]
# In the no-axis case, if -Inf is the max value, it means *all* the entries
# were -Inf, so we already know what's going to happen and we can skip doing
# real work.
if axis is None and max_ent == LOG_ZERO:
return LOG_ZERO
# Otherwise, there's a bit of trickiness here - subtracting -Inf results in
# Nan, so we never want to use that as a bias.
mask = (max_ent == LOG_ZERO)
if mask.any():
# If some rows or columns have only -Inf values, use a bias of 0 in just
# those rows or cols.
numpy.place(max_ent, mask, 0.0)
numpy.place(bias, bias == LOG_ZERO, 0.0)
return max_ent + quiet_log(numpy.sum(numpy.exp(x - bias), axis=axis))
def _lazywhere(cond, arrays, f, fillvalue=None, f2=None):
"""
np.where(cond, x, fillvalue) always evaluates x even where cond is False.
This one only evaluates f(arr1[cond], arr2[cond], ...).
For example,
>>> a, b = np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8])
>>> def f(a, b):
return a*b
>>> _lazywhere(a > 2, (a, b), f, np.nan)
array([ nan, nan, 21., 32.])
Notice it assumes that all `arrays` are of the same shape, or can be
broadcasted together.
"""
if fillvalue is None:
if f2 is None:
raise ValueError("One of (fillvalue, f2) must be given.")
else:
fillvalue = np.nan
else:
if f2 is not None:
raise ValueError("Only one of (fillvalue, f2) can be given.")
arrays = np.broadcast_arrays(*arrays)
temp = tuple(np.extract(cond, arr) for arr in arrays)
tcode = np.mintypecode([a.dtype.char for a in arrays])
out = _valarray(np.shape(arrays[0]), value=fillvalue, typecode=tcode)
np.place(out, cond, f(*temp))
if f2 is not None:
temp = tuple(np.extract(~cond, arr) for arr in arrays)
np.place(out, ~cond, f2(*temp))
return out
def __init__ (self, raw, lambdas_in):
self.lambdas = lambdas_in
self.previous_prediction = None
self.num_obs_fall = 0
self.num_obs_rise = 0
self.num_predictions = 0
self.num_correct_short_predictions = 0
self.num_correct_long_predictions = 0
self.num_uncertain_predictions = 0
self.alpha_hat = {}
lambdas = self.lambdas
for key in lambdas:
split = lambdas[key]['split']
A = np.array (lambdas[key]['A'])
B = np.array (lambdas[key]['B'])
Pi = np.array (lambdas[key]['Pi'])
N = Pi.shape[0]
M = B.shape[1]
T = len (raw)
O = np.zeros (T, dtype=np.int8)
np.place (O, raw >= split, 1)
alpha_bar = np.multiply (Pi, B[:, O[0]])
alpha_hat = alpha_bar / alpha_bar.sum ()
for t in range (0, T - 1):
# multiply is elementwise, dot is matrix multiplication
alpha_bar = np.multiply (np.dot (alpha_hat, A), B[:, O[t+1]])
alpha_hat = alpha_bar / alpha_bar.sum ()
self.alpha_hat[key] = alpha_hat
def __init__(self, V_nM, S_MM, No, pwfmask, lcaoindices=None):
Nw = V_nM.shape[1]
self.V_oM = V_nM[:No]
dtype = V_nM.dtype
# Do PWF optimization for pwfbasis submatrix only!
Npwf = len(pwfmask.nonzero()[0])
pwfmask2 = np.outer(pwfmask, pwfmask)
s_MM = S_MM[pwfmask2].reshape(Npwf, Npwf)
v_oM = self.V_oM[:, pwfmask]
f_MM = s_MM - np.dot(v_oM.T.conj(), v_oM)
nw = len(s_MM)
assert No <= nw
u_ow, u_lw, u_Ml = get_rot(f_MM, v_oM, nw - No)
u_Mw = np.dot(u_Ml, u_lw)
u_ow = u_ow - np.dot(v_oM, u_Mw)
# Determine U for full lcao basis
self.U_ow = np.zeros((No, Nw), dtype)
for U_w, u_w in zip(self.U_ow, u_ow):
np.place(U_w, pwfmask, u_w)
self.U_Mw = np.identity(Nw, dtype)
np.place(self.U_Mw, pwfmask2, u_Mw.flat)
if lcaoindices is not None:
for i in lcaoindices:
self.U_ow[:, i] = 0.0
self.U_Mw[:, i] = 0.0
self.U_Mw[i, i] = 1.0
self.S_ww = self.rotate_matrix(np.ones(1), S_MM)
self.norms_n = None
def make_mask_map(data, infile, outfile, index=None):
from neurosynth.base.mask import Masker
from neurosynth.base import imageutils
# Load image with masker
masker = Masker(infile)
img = imageutils.load_imgs(infile, masker)
data = list(data)
if index is None:
index = np.arange(0, len(data))
rev_index = None
else:
all_reg = np.arange(0, img.max())
rev_index = all_reg[np.invert(np.in1d(all_reg, index))]
min_val = img.min()
for num, value in enumerate(data):
n = index[num]
np.place(img, img == n + min_val, [value])
if rev_index is not None:
for value in rev_index:
np.place(img, img == value + min_val, 0)
img = img.astype('float32')
imageutils.save_img(img, outfile, masker)
def conditional_logsumexp(where, axis):
masked = -np.ones(a.shape) * np.inf
np.copyto(masked, a, where = where)
masked_sum = logsumexp(masked, axis = axis)
#np.copyto(masked_sum, -np.ones(masked_sum.shape) * np.inf, where = np.isnan(masked_sum))
np.place(masked_sum, np.isnan(masked_sum), -np.inf)
return masked_sum
def factorial2(n,exact=0):
"""n!! = special.gamma(n/2+1)*2**((m+1)/2)/sqrt(pi) n odd
= 2**(n) * n! n even
If exact==0, then floating point precision is used, otherwise
exact long integer is computed.
Notes:
- Array argument accepted only for exact=0 case.
- If n<0, the return value is 0.
"""
if exact:
if n < -1:
return 0L
if n <= 0:
return 1L
val = 1L
for k in xrange(n,0,-2):
val *= k
return val
else:
from scipy import special
n = asarray(n)
vals = zeros(n.shape,'d')
cond1 = (n % 2) & (n >= -1)
cond2 = (1-(n % 2)) & (n >= -1)
oddn = extract(cond1,n)
evenn = extract(cond2,n)
nd2o = oddn / 2.0
nd2e = evenn / 2.0
place(vals,cond1,special.gamma(nd2o+1)/sqrt(pi)*pow(2.0,nd2o+0.5))
place(vals,cond2,special.gamma(nd2e+1) * pow(2.0,nd2e))
return vals
def remove_wrongly_sized_connected_components(self, a, min_size, max_size, in_place):
"""
Adapted from http://github.com/jni/ray/blob/develop/ray/morpho.py
(MIT License)
"""
bin_out = self.BinaryOut.value
original_dtype = a.dtype
if not in_place:
a = a.copy()
if min_size == 0 and (max_size is None or max_size > numpy.prod(a.shape)): # shortcut for efficiency
return a
try:
component_sizes = numpy.bincount( a.ravel() )
except TypeError:
# On 32-bit systems, must explicitly convert from uint32 to int
# (This fix is just for VM testing.)
component_sizes = numpy.bincount( numpy.asarray(a.ravel(), dtype=int) )
bad_sizes = component_sizes < min_size
if max_size is not None:
numpy.logical_or( bad_sizes, component_sizes > max_size, out=bad_sizes )
bad_locations = bad_sizes[a]
a[bad_locations] = 0
if (bin_out):
# Replace non-zero values with 1
numpy.place(a,a,1)
return numpy.array(a, dtype=original_dtype)
def investment(positions, num_trials):
'''this program accepts two inputs: positions and num_trials:
positions: list of number of shares to buy in parallel
num_trials: how many times to randomly repeat the test
the program returns the outcomes of num_trials times'''
final_ret = np.zeros((num_trials,1))
'''for every position, using the position value:
1. calculate cumulative return that day with 51% chance of doubling and 49% of losing everythin
2. calculate daily return based on cumulative return
if there are 10 positions, each would be a random independent trial and the cumulative return is the sum of the outcome of the 10 positions'''
for position in positions:
position_value = 1000 / position
win = position_value * 2
odds = np.random.rand(num_trials,position)
np.place(odds, odds > 0.51, 0)
np.place(odds, odds != 0, win)
cumu_ret = np.sum(odds, axis = 1)
daily_ret = cumu_ret / 1000 -1
daily_ret = np.expand_dims(daily_ret, axis =1)
final_ret = np.concatenate((final_ret, daily_ret),axis = 1)
final_ret = np.delete(final_ret,0,1)
return final_ret
def __call__(self, x):
x_ = np.asarray(x)
if x_.size == 1:
x_.shape = (1,)
np.place(x_, x_ > 0, [1])
np.place(x_, x_ < 0, [-1])
return x_
def solve_tangency(R, C, rf, fit_func=lambda mean, var, rf: mean-rf/(var**.5)):
"""Calculates the tangency portfolio given a set of expected returns (R), covariances (C),
risk-free rate (rf), and a fitness function (fit_func) which defaults to the Sharpe ratio.
Returns the weights of the tangency portfolio.
"""
n = len(R)
#Begin with equal weights
W = np.ones([n])/n
# Replace expected returns that are less than rf with rf + a super small value
# since if it's less than rf, the sharpe ratio is negative, which ruins minimization
np.place(R, R<rf, rf+0.00001)
# Set boundaries on weights - no shorting or leverage allowed. Can probably incorporate
# this functionality easily, though.
bounds = [(0., 1.) for i in xrange(n)]
# Set constraints as defined in SciPy's documentation for minimize. 'fun' forces the weights to sum to 1
constraints = ({'type':'eq',
'fun': lambda W: sum(W)-1.})
# Minimize fitness by changing W with (R, C, rf, fit_func) as given using the SLQSP method
# based on the above defined constraints and bounds
tangency = spo.minimize(fun=fitness,
x0=W,
args=(R, C, rf, fit_func),
method='SLSQP',
constraints=constraints,
bounds=bounds)
if not tangency.success:
raise BaseException(tangency.message)
return tangency.x
def get_normalized_data():
print("Reading in and transforming data...")
if not os.path.exists('../large_files/train.csv'):
print('Looking for ../large_files/train.csv')
print('You have not downloaded the data and/or not placed the files in the correct location.')
print('Please get the data from: https://www.kaggle.com/c/digit-recognizer')
print('Place train.csv in the folder large_files adjacent to the class folder')
exit()
df = pd.read_csv('../large_files/train.csv')
data = df.values.astype(np.float32)
np.random.shuffle(data)
X = data[:, 1:]
Y = data[:, 0]
Xtrain = X[:-1000]
Ytrain = Y[:-1000]
Xtest = X[-1000:]
Ytest = Y[-1000:]
# normalize the data
mu = Xtrain.mean(axis=0)
std = Xtrain.std(axis=0)
np.place(std, std == 0, 1)
Xtrain = (Xtrain - mu) / std
Xtest = (Xtest - mu) / std
return Xtrain, Xtest, Ytrain, Ytest
def _getdatafromsql(connection, tmp_table, query):
"""
Private function creating a ndarray from the current table.
Parameters
----------
connection: sqlite3.Connection
Current SQL connection.
tmp_table: string
Name of the temporary table created for the purpose of keeping ids when WHERE is used
query: string
SQL query.
"""
# Transforms the typestr into dtypes
# Define and execute the query
connection.execute("CREATE TEMPORARY TABLE %s AS %s"%(tmp_table, query))
# Get the list of names and types from the pragma
pragmastr = "PRAGMA TABLE_INFO(%s)"%tmp_table
(names, typestr) = zip(*(_[1:3] for _ in connection.execute(pragmastr).fetchall()))
ndtype = []
for (i, (n, t)) in enumerate(zip(names, typestr)):
# Transform the name into a regular string (not unicode)
n = str(n)
if t =='INTEGER':
ndtype.append((n, int))
elif t =='TEXT':
ndtype.append((n, '|S30'))
elif t == 'BLOB':
ndtype.append((n, object))
else:
ndtype.append((n, float))
# Construct the ndarray
connection.row_factory = sqlite3.Row
data = connection.execute("SELECT * FROM %s"%tmp_table).fetchall()
try:
return np.array(data, dtype=ndtype)
except TypeError:
output = ma.empty(len(data), dtype=ndtype)
# Find the index of the first row (0 or 1)?
rowidref = connection.execute("SELECT rowid FROM %s LIMIT 1"%tmp_table).fetchone()[0]
# Loop through the different fields identifying the null fields to mask
maskstr_template = "SELECT rowid FROM %s WHERE %%s IS NULL"%tmp_table
datastr_template = "SELECT %%s FROM %s WHERE %%s IS NOT NULL"%tmp_table
for (i, field) in enumerate(names):
current_output = output[field]
current_mask = current_output._mask
maskstr = maskstr_template % field
maskidx = [_[0] - rowidref for _ in connection.execute(maskstr).fetchall()]
current_mask[maskidx] = True
datastr = datastr_template % (field, field)
np.place(current_output._data, ~current_mask,
[_[0] for _ in connection.execute(datastr).fetchall()])
connection.execute("DROP TABLE %s"%tmp_table)
return output
def testDropNegatives(self):
# Note: the test is done by replacing segment_ids with 8 to -1
# for index and replace values generated by numpy with 0.
dtypes = [
dtypes_lib.float32, dtypes_lib.float64, dtypes_lib.int64,
dtypes_lib.int32, dtypes_lib.complex64, dtypes_lib.complex128
]
indices_flat = np.array([0, 4, 0, 8, 3, 8, 4, 7, 7, 3])
num_segments = 12
for indices in indices_flat, indices_flat.reshape(5, 2):
shape = indices.shape + (2,)
for dtype in dtypes:
with self.test_session(use_gpu=True):
tf_x, np_x = self._input(shape, dtype=dtype)
np_ans = self._segmentReduce(
indices, np_x, np.add, op2=None, num_segments=num_segments)
# Replace np_ans[8] with 0 for the value
np_ans[8:] = 0
# Replace 8 with -1 in indices
np.place(indices, indices == 8, [-1])
s = math_ops.unsorted_segment_sum(
data=tf_x, segment_ids=indices, num_segments=num_segments)
tf_ans = s.eval()
self.assertAllClose(np_ans, tf_ans)
self.assertShapeEqual(np_ans, s)
def substitute_values(self, vect):
"""
Internal method to substitute integers into the vector, and construct
metadata to convert back to the original vector.
np.nan is always given -1, all other objects are given integers in
order of apperence.
Parameters
----------
vect : np.array
the vector in which to substitute values in
"""
try:
unique = np.unique(vect)
except:
unique = set(vect)
unique = [
x for x in unique if not isinstance(x, float) or not isnan(x)
]
arr = np.copy(vect)
for new_id, value in enumerate(unique):
np.place(arr, arr==value, new_id)
self.metadata[new_id] = value
arr = arr.astype(np.float)
np.place(arr, np.isnan(arr), -1)
self.arr = arr
if -1 in arr:
self.metadata[-1] = self._missing_id
def add_noise_batch(self, shingle_imgs, shingle_dim, rng):
# assumes RGBA stain > shingle in both dim
# assumes shingle is 'L'
assert shingle_imgs is not None
assert shingle_dim is not None
assert rng is not None
randseq = rng.randint(len(self.images), size=len(shingle_imgs))
return_images = np.zeros(shingle_imgs.shape)
# Choose a random image and a random section inside of the image
for i, randnoise in enumerate(randseq):
stain_img = self.images[randnoise]
stain_w, stain_h = stain_img.size
max_rand_x = stain_w - shingle_dim[1]
max_rand_y = stain_h - shingle_dim[0]
startx = rng.randint(max_rand_x)
starty = rng.randint(max_rand_y)
# Crop the image
rand_cropped_stain = stain_img.crop((startx, starty, startx + shingle_dim[1], starty + shingle_dim[0]))
# Invert both images and add them together
shingle_inv = (255 - shingle_imgs[i]) # .astype(np.uint16)
stain_inv = (255 - np.asarray(rand_cropped_stain)) # .astype(np.uint16)
final_img = shingle_inv + stain_inv
np.place(final_img, final_img > 255, 255)
assert final_img is not None
# Invert image back to 255
return_images[i] = 255 - final_img
return return_images
def equalise_histogram(image):
histo = generate_histogram(image)
new_image = np.zeros(image.shape)
height, width = image.shape
total_pixels = height * width
freq_sum = np.sum(histo)
for i, freq in enumerate(reversed(histo)):
intensity = 255 - i
new_intensity = round(255 * freq_sum / total_pixels)
temp = image.copy()
np.place(temp, temp != intensity, 0)
np.place(temp, temp == intensity, new_intensity)
new_image += temp
freq_sum -= freq
new_image = np.array(new_image, dtype=np.uint8)
new_histo = generate_histogram(new_image)
plot_histogram(new_histo)
return new_image
def __call__(self, wave):
""" Return dimensionless throughput of bandpass at given wavelength in nanometers.
Note that outside of the wavelength range defined by the `blue_limit` and `red_limit`
attributes, the throughput is assumed to be zero.
@param wave Wavelength in nanometers.
@returns the dimensionless throughput.
"""
# figure out what we received, and return the same thing
# option 1: a NumPy array
if isinstance(wave, np.ndarray):
wgood = (wave >= self.blue_limit) & (wave <= self.red_limit)
ret = np.zeros(wave.shape, dtype=np.float)
np.place(ret, wgood, self.func(wave[wgood]))
return ret
# option 2: a tuple
elif isinstance(wave, tuple):
return tuple([self.func(w) if (w >= self.blue_limit and w <= self.red_limit) else 0.0 for w in wave])
# option 3: a list
elif isinstance(wave, list):
return [self.func(w) if (w >= self.blue_limit and w <= self.red_limit) else 0.0 for w in wave]
# option 4: a single value
else:
return self.func(wave) if (wave >= self.blue_limit and wave <= self.red_limit) else 0.0
def get_iris(catagory):
data=np.loadtxt('iris.txt')
np.place(data[:,0],data[:,0]!=catagory,-1)
np.place(data[:,0],data[:,0]==catagory,1)
return [np.concatenate((data[0:25],data[75:125])), np.concatenate((data[25:75],data[125:150]))]
def make_mask_map_4d(data, infile, outfile):
""" Make mask map with 4d dimeions
data: values for levels in infile. Shape = [4th dimension, regions]
infile: input file to replace levels with values
outfile: output file name
"""
from neurosynth.base.mask import Masker
from neurosynth.base import imageutils
from nibabel import nifti1
data = np.array(data)
# Load image with masker
masker = Masker(infile)
img = imageutils.load_imgs(infile, masker)
header = masker.get_header()
shape = header.get_data_shape()[0:3] + (data.shape[0],)
header.set_data_shape(shape)
result = []
for t_dim, t_val in enumerate(data):
result.append(img.copy())
for num, value in enumerate(t_val):
np.place(result[t_dim], img == num + 1, [value])
result = np.hstack(result)
header.set_data_dtype(result.dtype) # Avoids loss of precision
img = nifti1.Nifti1Image(masker.unmask(result).squeeze(), None, header)
img.to_filename(outfile)
def score_MCC(ground_truth, scores):
'''
assuming the model output is the probability of being default,
then this probability can be used for ranking. Then using the fraction of
default in validation data to assign the proper threshold to the prediction
'''
if isinstance(scores, pd.Series):
scores = scores.values
if isinstance(ground_truth, pd.Series):
ground_truth = ground_truth.values
tmp_ground_truth = np.copy(ground_truth)
fault_frac = tmp_ground_truth.mean()
#print 'score shape:', scores.shape,
print 'mean of groud truth:', fault_frac
thres_value = np.percentile(scores, 100.*(1-fault_frac), axis=0)
print 'threshold for preds:', thres_value
binary_scores = scores > thres_value
binary_scores = binary_scores.astype(int)
## convert to sk-learn format
np.place(binary_scores, binary_scores==0, -1)
np.place(tmp_ground_truth, tmp_ground_truth==0, -1)
return matthews_corrcoef(tmp_ground_truth, binary_scores)
def predict(self, z):
zClass = []
within_dist = []
inputScaler = preprocessing.StandardScaler().fit(self.training_set)
z = inputScaler.transform(z)
for zz in z:
zClass.append([self.pfn(zz)])
within = False
normalized_zz = self.normalize(zz)
for i in xrange(len(self.bests)):
# pdb.set_trace()
# within = within or (self.get_distance(self.normalized_best[i], normalized_zz) < self.best_distance[i])
# pdb.set_trace()
within = (
within
or (
abs(self.normalized_best[i] - normalized_zz)
< abs(self.normalized_best[i] - self.best_distance[i])
).all()
)
within_dist.append([within])
# try:
# within_dist.append([self.get_distance(self.normalized_best, zz)] < self.best_distance)
# except:
# within_dist.append([False])
# import pdb
result = array(zClass)
within_dist = array(within_dist)
place(result, ~logical_or(result > self.limit, within_dist), [0.0])
# place(result, within_dist,[1.0])
# place(result, ~within_dist,[0.0])
# result = 1/(1+exp(-result*60+30)) ## sigmoid function
# result = result * result * result * result * result * result
return result
def add_noise_np(self, shingle_img, shingle_dim, rng):
# assumes RGBA stain > shingle in both dim
# assumes shingle is 'L'
assert shingle_img is not None
assert shingle_dim is not None
assert rng is not None
# Choose a random image and a random section inside of the image
stain_img = self.images[rng.randint(len(self.images))]
stain_w, stain_h = stain_img.size
max_rand_x = stain_w - shingle_dim[1]
max_rand_y = stain_h - shingle_dim[0]
startx = rng.randint(max_rand_x)
starty = rng.randint(max_rand_y)
# Crop the image
rand_cropped_stain = stain_img.crop((startx, starty, startx + shingle_dim[1], starty + shingle_dim[0]))
# Invert both images and add them together
shingle_inv = (255 - shingle_img) # .astype(np.uint16)
stain_inv = (255 - np.asarray(rand_cropped_stain)) # .astype(np.uint16)
final_img = shingle_inv + stain_inv
np.place(final_img, final_img > 255, 255)
assert final_img is not None
# Invert image back to 255
final_img = 255 - final_img
return final_img
def Estimate_sf_And_dist(locs,
data,
cellGraph,
gmmDict,
test_genes,
tissue_mat,
plot=False,
unary_scale_factor=100,
label_cost=10,
algorithm='expansion'):
'''
Relationship smooth factor with distance for estimating smooth factor.
the function just like recal_dist_to_tissue() with tissue_mat,
but getting best sf by using min(hamming_dist).
Return Best_smooth_factor, related size of gmm labels' components and three distance.
'''
result_df_new = pd.DataFrame()
new_hamming = []
new_jaccard = []
new_hausdorff = []
con_list = []
for geneID in test_genes: #zeor_boundGenes: #de_counts:
#if geneID not in result_df.index:
count = data.loc[:, geneID].values
gmm = gmmDict[geneID]
Labels = gmm.predict(count.reshape(-1, 1))
zero_label = gmm.predict(np.min(count[count > 0]).reshape(-1, 1))
if sum(count == 0):
np.place(Labels, count == 0, zero_label)
con = count_component(locs, cellGraph, Labels)
con_list.append(len(con))
hamming_dist_list = []
jaccard_dist_list = []
hausdorff_dist_list = []
test_df_list = pd.DataFrame()
#figsize(10,8)
for sf in np.arange(0, 110, 10): # 'Cldn9' need start from sf=0
temp_factor = sf
newLabels = cut_graph_general(cellGraph, count, gmm,
unary_scale_factor, temp_factor,
label_cost, algorithm)
#print(geneID)
reverLabels = compute_p_reverLabels(locs, newLabels, gmm, count,
cellGraph)
p, node, com = compute_p_CSR(locs, reverLabels, gmm, count,
cellGraph)
# figsize(8,6)
#print(sf,p)
labels_array = np.array(reverLabels).reshape(1, -1)
data_array = np.array((geneID, p, min(p), temp_factor, node),
dtype=object).reshape(1, -1)
t_array = np.hstack((data_array, labels_array))
c_labels = ['p_value', 'fdr', 'smooth_factor', 'nodes']
for i in np.arange(labels_array.shape[1]) + 1:
temp_label = 'label_cell_' + str(i)
c_labels.append(temp_label)
test_df = pd.DataFrame(t_array[:, 1:],
index=t_array[:, 0],
columns=c_labels)
dist_df, false_genes = calc_distance_df(locs, data, cellGraph,
gmmDict, test_df,
tissue_mat)
if plot == True:
plot_voronoi_boundary(geneID, locs, count, reverLabels, min(p))
print(sf, p, dist_df)
hamming_dist_list.append(dist_df.iloc[:, 0].values)
jaccard_dist_list.append(dist_df.iloc[:, 1].values)
hausdorff_dist_list.append(dist_df.iloc[:, 2].values)
test_df_list = pd.concat([test_df_list, test_df])
if len(jaccard_dist_list) > 0:
min_inde = np.argmin(
hamming_dist_list
) ## multiple min(p) select min(hamming_dist) amony
new_hamming.append(hamming_dist_list[min_inde][0])
new_jaccard.append(jaccard_dist_list[min_inde][0])
new_hausdorff.append(hausdorff_dist_list[min_inde][0])
best_test_df = test_df_list.iloc[min_inde:min_inde + 1]
result_df_new = pd.concat([result_df_new, best_test_df])
best_sf = result_df_new.smooth_factor.values
dist_df_new = pd.DataFrame(
[best_sf, con_list, new_hamming, new_jaccard,
new_hausdorff]).T ### no norm dist
dist_df_new.columns = [
'smooth_factor', 'con_size', 'Hamming', 'Jaccard', 'Hausdorff'
]
dist_df_new.index = result_df_new.index
return result_df_new, dist_df_new
def compute_p_reverLabels(locs, newLabels, gmm, exp, cellGraph):
'''
compute p values for each subgraph, and reverse labels of bad subgraph.
return new reversal labels.
'''
com_factor = 1
p_values = list()
node_lists = list()
reverLabel = newLabels.copy()
a = exp[exp > 0]
gmm_pred = gmm.predict(a.reshape(-1, 1))
zero_label = gmm.predict(np.min(a).reshape(-1, 1))[0]
label_pred = gmm.predict(exp.reshape(-1, 1))
if len(np.where(exp == 0)[0]) > 0:
np.place(label_pred, exp == 0, zero_label)
gmm_pred = label_pred
unique, counts = np.unique(gmm_pred, return_counts=True)
con_components = count_component(locs, cellGraph, newLabels)
for j in np.arange(
len(con_components)): ## all subgraphs are normal,score all
if len(
con_components[j]
) <= 9: # len(exp)/2+10: ## too big pattern is not reverse labels
node_list = con_components[j]
com_size = len(node_list)
temp_exp = exp[np.array(list(node_list))]
gmm_pred_com = gmm.predict(temp_exp.reshape(-1, 1))
if sum(temp_exp == 0) > 0:
# a=temp_exp[temp_exp>0]
# zero_label = gmm.predict(np.min(a).reshape(-1,1))[0]
np.place(gmm_pred_com, temp_exp == 0, zero_label)
# check 0s
unique_com, counts_com = np.unique(gmm_pred_com,
return_counts=True)
major_label = unique_com[np.where(
counts_com == counts_com.max())[0][0]]
label_count = counts[np.where(unique == major_label)[0]]
count_in_com = counts_com.max()
cover = exp.shape[0] / com_size
pmf = poisson.pmf(count_in_com,
com_size * (label_count / exp.shape[0]))[0]
psf = poisson.sf(count_in_com,
com_size * (label_count / exp.shape[0]))[0]
prob_sf = (pmf + psf) * cover
prob = (1 - poisson.cdf(count_in_com,
com_size *
(label_count / exp.shape[0]))[0]) * cover
# if prob>=1:
# prob=1
# p_values.append(prob)
#print(prob_sf,prob,major_label,com_size,cover)
# node_lists.append(np.array(list(node_list)))
# for i in node_list:
# plt.text(locs[i,0],locs[i,1],str(i))
# plot_voronoi_boundary(geneID,locs,exp,newLabels,prob)
## 1. find connected pattern of noise pattern
if prob_sf > 0.1:
test_node = node_list
node_otherPattern = []
for i in test_node:
if i in cellGraph[:, 0:2]:
node = cellGraph[np.where(i == cellGraph[:, 0:2])[0],
0:2]
node_array = node.reshape(1, -1)[0]
other_node = [
int(val) for val in list(
set(node_array) - set((test_node)))
]
node_otherPattern.extend(other_node)
# print(i, ' ',node_array,other_node)
node_otherPattern = np.unique(node_otherPattern)
# print(test_node,node_otherPattern)
## 2. judege nodes of connencted other pattern in one same pattern
diff_con = []
for con in range(len(con_components)):
if con != j:
diff = set(node_otherPattern) - set(
con_components[con])
if len(diff) == 0:
diff_con.append(con)
if len(diff_con) == 1:
#reverLabel=newLabels.copy()
rever = newLabels[list(con_components[diff_con[0]])][0]
reverLabel[list(test_node)] = rever
# plot_voronoi_boundary(geneID,locs,exp,reverLabel,prob)
return reverLabel
def Min_Max(nt, nc, NumC, Bay, Height):
it = 0
Movement = 0
relocation = 0
while it < NumC:
p_l_c = np.where(Bay == Bay.min(
keepdims=True)) #p_l_c = The position of lowest container
'''
print(Bay)
print(p_l_c)
'''
p_r = int(p_l_c[0]) #the row position of p_l_c
#print('p_r =', p_r)
p_c = int(p_l_c[1]) #the column position of p_l_c
if p_r == nt - int(
Height[p_c]
): #if target container is on the top of a stack, directly retrieving it
Bay[p_r][p_c] = NumC + 1
Height[p_c] = Height[p_c] - 1
'''
print('Height =', Height)
'''
Movement += 1
np.place(Bay, Bay == NumC + 1, 0)
'''
print('Round =',Movement,'\n',Bay)
'''
np.place(Bay, Bay == 0, NumC + 1)
'''
print('\n')
'''
elif p_r > nt - Height[p_c]:
r = nt - Height[p_c]
'''
print('r = ', r)
'''
while p_r > r: #while loop concept
i = 0
Height_m = [
] #create an empty list for column which is up to maximum height
c_s_i = Bay.min(axis=0) - Bay[r][
p_c] #candidate stack including target container
while nc > i:
if Height[i] == nt:
Height_m.append(
i) #add column with maximum height to list
i = i + 1
Height_m.append(p_c) # add the target column to list
c_s = np.delete(
c_s_i, Height_m, None
) #candidate stack after deleting target container and stack up to height limit
if np.max(c_s) > 0: #find arg number
arg_c = min(i for i in c_s if i > 0)
else:
arg_c = max(c_s)
l_arg_c = []
for i in range(
0, nc
): # I ingore if there are many arg numbers, it will do the same things. It will influence the value of Height.
if c_s_i[i] == arg_c: # find the location of arg_c
l_arg_c.append(i) # add them to list
r_arg_c = random.choice(l_arg_c) #random choose one of them
Bay[nt - Height[r_arg_c] -
1][r_arg_c] = Bay[r][p_c] #relocation
Bay[r][p_c] = NumC + 1
Height[p_c] = Height[p_c] - 1
Height[r_arg_c] = Height[r_arg_c] + 1
'''
print('\n')
print('Height =', Height)
'''
relocation += 1
Movement += 1
np.place(Bay, Bay == NumC + 1, 0)
'''
print('Round =',Movement,'\n',Bay)
'''
np.place(Bay, Bay == 0, NumC + 1)
'''
print('relocation =', relocation)
print('\n')
'''
r = r + 100
'''
Bay[p_r][p_c] = NumC + 1
Height[p_c] = Height[p_c] - 1
#print('Height =', Height)
Movement += 1
'''
#print('Height =', Height)
np.place(Bay, Bay == NumC + 1, 0)
'''
print('Round =',Movement,'\n',Bay)
'''
np.place(Bay, Bay == 0, NumC + 1)
break
'''
print('\n')
'''
it = it + 100
np.place(Bay, Bay == NumC + 1, 0)
'''
print(Bay)
'''
'''
np.place(Bay, Bay == 0, NumC + 1)
'''
'''
print("Total movements =", Movement)
print("Total relocation =", relocation)
'''
#print('Height =', Height)
#print('Bay =', Bay)
return Height
def bay_test(Initial_Bay, NumC):
Bay_test = Initial_Bay
np.place(Initial_Bay, Initial_Bay == 0,
NumC + 1) # strange syntax, transfer 0 to NumCon + 1
return Bay_test
x_values = np.linspace(x_min, x_max, x_max - x_min)
# get the comparison values:
yvalues_1 = Cls_1(x_values)
yvalues_2 = Cls_2(x_values)
# protect against zeroes:
yvalues_1_temp = np.abs(yvalues_1)
yvalues_2_temp = np.abs(yvalues_2)
try:
min2val_1 = np.amin(yvalues_1_temp[np.nonzero(yvalues_1_temp)])
min2val_2 = np.amin(yvalues_2_temp[np.nonzero(yvalues_2_temp)])
except:
min2val_1 = cutoff
min2val_2 = cutoff
np.place(yvalues_1, yvalues_1 == 0.0, min2val_1)
np.place(yvalues_2, yvalues_2 == 0.0, min2val_2)
# computation of the percentual comparison:
yvalues_comp_1 = (yvalues_1 -
yvalues_2) / np.abs(yvalues_1) * 100.0
yvalues_comp_2 = (yvalues_1 -
yvalues_2) / np.abs(yvalues_2) * 100.0
# protection against values too small:
np.place(yvalues_comp_1, abs(yvalues_comp_1) < cutoff, [cutoff])
np.place(yvalues_comp_2, abs(yvalues_comp_2) < cutoff, [cutoff])
# get the Cls values:
if do_abs:
yvalues_comp_1 = np.abs(yvalues_comp_1)
yvalues_comp_2 = np.abs(yvalues_comp_2)
# get the plot bounds for comp 1:
def square(t, duty=0.5):
"""
Return a periodic square-wave waveform.
The square wave has a period ``2*pi``, has value +1 from 0 to
``2*pi*duty`` and -1 from ``2*pi*duty`` to ``2*pi``. `duty` must be in
the interval [0,1].
Note that this is not band-limited. It produces an infinite number
of harmonics, which are aliased back and forth across the frequency
spectrum.
Parameters
----------
t : array_like
The input time array.
duty : array_like, optional
Duty cycle. Default is 0.5 (50% duty cycle).
If an array, causes wave shape to change over time, and must be the
same length as t.
Returns
-------
y : ndarray
Output array containing the square waveform.
Examples
--------
A 5 Hz waveform sampled at 500 Hz for 1 second:
>>> from scipy import signal
>>> import matplotlib.pyplot as plt
>>> t = np.linspace(0, 1, 500, endpoint=False)
>>> plt.plot(t, signal.square(2 * np.pi * 5 * t))
>>> plt.ylim(-2, 2)
A pulse-width modulated sine wave:
>>> plt.figure()
>>> sig = np.sin(2 * np.pi * t)
>>> pwm = signal.square(2 * np.pi * 30 * t, duty=(sig + 1)/2)
>>> plt.subplot(2, 1, 1)
>>> plt.plot(t, sig)
>>> plt.subplot(2, 1, 2)
>>> plt.plot(t, pwm)
>>> plt.ylim(-1.5, 1.5)
"""
t, w = asarray(t), asarray(duty)
w = asarray(w + (t - t))
t = asarray(t + (w - w))
if t.dtype.char in ['fFdD']:
ytype = t.dtype.char
else:
ytype = 'd'
y = zeros(t.shape, ytype)
# width must be between 0 and 1 inclusive
mask1 = (w > 1) | (w < 0)
place(y, mask1, nan)
# on the interval 0 to duty*2*pi function is 1
tmod = mod(t, 2 * pi)
mask2 = (1 - mask1) & (tmod < w * 2 * pi)
place(y, mask2, 1)
# on the interval duty*2*pi to 2*pi function is
# (pi*(w+1)-tmod) / (pi*(1-w))
mask3 = (1 - mask1) & (1 - mask2)
place(y, mask3, -1)
return y
def LA_N(Bay_test, N, NumC):
lowest_con = 1
movement = 0
relocation = 0
NumC = 7
while lowest_con <= NumCon:
row_lowest_con, column_lowest_con = Find_con_position(
Bay_test, lowest_con)
r = 1
N = min(2, NumC)
#print('N =', N)
#print('NumC =', NumC)
while Bay_test[row_lowest_con -
1][column_lowest_con] <= NumCon and row_lowest_con != 0:
Stack_N = Find_Stack_N(Bay_test, N, lowest_con)
Stack_Height, Stack_not_N = not_in_Stack_N(Stack_N, Height)
#print(int(min(Stack_Height)))
while int(min(Stack_Height)) == NTier or len(Stack_N) == NTier:
N -= 1
Stack_N = Find_Stack_N(Bay_test, N, lowest_con)
Stack_Height, Stack_not_N = not_in_Stack_N(Stack_N, Height)
#print('N =', N)
#print('lowest container=', lowest_con)
#print('Stack_N =', Stack_N)
#print('Stack_Height =', Stack_Height)
#print('Stack_not_N =', Stack_not_N)
Top_Stack_N = Find_Top_Stack_N(Bay_test, Stack_N)
#print('Top_Stack_N =', Top_Stack_N)
Low_Stack = Find_Lowest_con_in_column(Bay_test, Stack_not_N)
#print('Low_Stack =', Low_Stack)
#print('r =', r)
#print('len(Stack_N) =', len(Stack_N))
while r <= len(Stack_N):
n = max(Top_Stack_N)
#print('n = ',n)
#print(lowest_con)
row_n, column_n = Find_con_position(Bay_test, n)
#print('row_n =', row_n, 'column_n =', column_n)
if column_n == column_lowest_con:
if min(Low_Stack) == NumCon + 1:
for j in range(0, NCol):
if Bay_test[NTier - 1][j] == NumCon + 1:
row_low, column_low = NTier, j
#print('row_low = ',row_low, 'column_low =', column_low)
break
elif max(Low_Stack) - n > 0:
while min(Low_Stack) - n < 0:
Low_Stack.remove(min(Low_Stack))
if min(Low_Stack) == NumCon + 1:
for j in range(0, NCol):
if Bay_test[NTier - 1][j] == NumCon + 1:
row_low, column_low = NTier, j
#print('row_low = ',row_low, 'column_low =', column_low)
else:
row, column_low = Find_con_position(
Bay_test, min(Low_Stack))
row_low = NTier - Height[column_low]
else:
row, column_low = Find_con_position(
Bay_test, max(Low_Stack))
row_low = NTier - Height[column_low]
#print('row_low =', row_low, 'column_low =', column_low)
break
elif Bay_test[row_n][column_n] <= min(
list(Bay_test[i][column_n] for i in range(0, NTier))):
#print('min =', min(list(Bay_test[i][column_n] for i in range(0, NTier))))
for i in Stack_N:
rounds = 0
for j in range(0, NTier - 1):
if Bay_test[j][i] < Bay_test[j + 1][i]:
rounds += 1
elif Bay_test[j][i] == NumCon + 1:
rounds += 1
if Bay_test[i][NTier] != NumCon + 1:
rounds += 1
if rounds == NTier:
Stack_N.remove(i)
if r == len(Stack_N):
row, column_low = Find_con_position(
Bay_test, min(Low_Stack))
row_low = NTier - Height[column_low]
break
else:
r += 1
elif max(Low_Stack) > n:
if min(Low_Stack) == NumCon + 1:
for j in range(0, NCol):
if Bay_test[NTier - 1][j] == NumCon + 1:
row_low, column_low = NTier, j
#print('row_low = ',row_low, 'column_low =', column_low)
break
else:
while min(Low_Stack) - n < 0:
Low_Stack.remove(min(Low_Stack))
row, column_low = Find_con_position(
Bay_test, min(Low_Stack))
row_low = NTier - Height[column_low]
#print(min(Low_Stack))
break
elif r == len(Stack_N):
row, column_low = Find_con_position(
Bay_test, max(Low_Stack))
row_low = NTier - Height[column_low]
break
Top_Stack_N.remove(max(Top_Stack_N))
#print('Top_Stack_N =', Top_Stack_N)
r += 1
Bay_test[row_low - 1][column_low] = Bay_test[row_n][column_n]
Bay_test[row_n][column_n] = NumCon + 1
Height[column_low] += 1
Height[column_n] -= 1
np.place(Bay_test, Bay_test == NumCon + 1, 0)
#print(Bay_test)
#np.place(Bay_test, Bay_test == 0, NumCon + 1)
movement += 1
relocation += 1
break
if movement == 0:
Bay_test[row_lowest_con][column_lowest_con] = NumCon + 1
Height[column_lowest_con] -= 1
np.place(Bay_test, Bay_test == NumCon + 1, 0)
break
#np.place(Bay_test, Bay_test == NumCon +1, 0)
#print(Bay_test)
np.place(Bay_test, Bay_test == 0, NumCon + 1)
movement += 1
lowest_con += 1
NumC -= 1
return Bay_test
def richardson_lucy(image, psf, iterations=50, clip=True):
"""Richardson-Lucy deconvolution.
Parameters
----------
image : ndarray
Input degraded image (can be N dimensional).
psf : ndarray
The point spread function.
iterations : int
Number of iterations. This parameter plays the role of
regularisation.
clip : boolean, optional
True by default. If true, pixel value of the result above 1 or
under -1 are thresholded for skimage pipeline compatibility.
Returns
-------
im_deconv : ndarray
The deconvolved image.
Examples
--------
>>> from skimage import color, data, restoration
>>> camera = color.rgb2gray(data.camera())
>>> from scipy.signal import convolve2d
>>> psf = np.ones((5, 5)) / 25
>>> camera = convolve2d(camera, psf, 'same')
>>> camera += 0.1 * camera.std() * np.random.standard_normal(camera.shape)
>>> deconvolved = restoration.richardson_lucy(camera, psf, 5)
References
----------
.. [1] http://en.wikipedia.org/wiki/Richardson%E2%80%93Lucy_deconvolution
"""
# compute the times for direct convolution and the fft method. The fft is of
# complexity O(N log(N)) for each dimension and the direct method does
# straight arithmetic (and is O(n*k) to add n elements k times)
direct_time = np.prod(image.shape + psf.shape)
fft_time = np.sum([n * np.log(n) for n in image.shape + psf.shape])
# see whether the fourier transform convolution method or the direct
# convolution method is faster (discussed in scikit-image PR #1792)
time_ratio = 40.032 * fft_time / direct_time
if time_ratio <= 1 or len(image.shape) > 2:
convolve_method = fftconvolve
else:
convolve_method = convolve
image = image.astype(np.float)
psf = psf.astype(np.float)
im_deconv = 0.5 * np.ones(image.shape)
psf_mirror = psf[::-1, ::-1]
for _ in range(iterations):
eps = np.finfo(image.dtype).eps
x = convolve_method(im_deconv, psf, 'same')
np.place(x, x == 0, eps)
relative_blur = image / x + eps
im_deconv *= convolve_method(relative_blur, psf_mirror, 'same')
if clip:
im_deconv[im_deconv > 1] = 1
im_deconv[im_deconv < -1] = -1
return im_deconv
# ===============================================================
# Buy or sell indicator array (1 = buy, 0 = sell)
# Note: the "_bh" suffix indicates the corresponding arrays for a buy-and-hold AAPL strategy
buy_or_sell = Y_test_pred.values
buy_or_sell_bh = np.zeros_like(Y_test_pred)
buy_or_sell_bh[0] = 1
# Convert prices array from type object to type float
prices = open_prices[split_ind:].astype(float).values
# The change in shares
# positive = shares bought on day i (number of shares increases)
# negative = shares sold on day i (number of shares decreases)
shares_change = np.zeros_like(buy_or_sell).astype(int)
np.place(shares_change, buy_or_sell == 1, shares)
np.place(shares_change, buy_or_sell == 0, -shares)
shares_change_bh = np.zeros_like(buy_or_sell_bh).astype(int)
shares_change_bh[0] = shares
# The change in cash
# positive = gross proceeds from selling shares
# negative = purchase amount for buying shares
cash_change = np.zeros_like(buy_or_sell).astype(float)
# Populate cash_change array
for i in range(len(buy_or_sell)):
# Buy
if buy_or_sell[i] == 1:
amount = shares * prices[i] + commission
cash_change[i] = -amount
def maybe_upcast_putmask(result, mask, other):
"""
A safe version of putmask that potentially upcasts the result.
The result is replaced with the first N elements of other,
where N is the number of True values in mask.
If the length of other is shorter than N, other will be repeated.
Parameters
----------
result : ndarray
The destination array. This will be mutated in-place if no upcasting is
necessary.
mask : boolean ndarray
other : ndarray or scalar
The source array or value
Returns
-------
result : ndarray
changed : boolean
Set to true if the result array was upcasted
Examples
--------
>>> result, _ = maybe_upcast_putmask(np.arange(1,6),
np.array([False, True, False, True, True]), np.arange(21,23))
>>> result
array([1, 21, 3, 22, 21])
"""
if not isinstance(result, np.ndarray):
raise ValueError("The result input must be a ndarray.")
if mask.any():
# Two conversions for date-like dtypes that can't be done automatically
# in np.place:
# NaN -> NaT
# integer or integer array -> date-like array
if is_datetimelike(result.dtype):
if is_scalar(other):
if isna(other):
other = result.dtype.type("nat")
elif is_integer(other):
other = np.array(other, dtype=result.dtype)
elif is_integer_dtype(other):
other = np.array(other, dtype=result.dtype)
def changeit():
# try to directly set by expanding our array to full
# length of the boolean
try:
om = other[mask]
except (IndexError, TypeError):
# IndexError occurs in test_upcast when we have a boolean
# mask of the wrong shape
# TypeError occurs in test_upcast when `other` is a bool
pass
else:
om_at = om.astype(result.dtype)
if (om == om_at).all():
new_result = result.values.copy()
new_result[mask] = om_at
result[:] = new_result
return result, False
# we are forced to change the dtype of the result as the input
# isn't compatible
r, _ = maybe_upcast(result, fill_value=other, copy=True)
np.place(r, mask, other)
return r, True
# we want to decide whether place will work
# if we have nans in the False portion of our mask then we need to
# upcast (possibly), otherwise we DON't want to upcast (e.g. if we
# have values, say integers, in the success portion then it's ok to not
# upcast)
new_dtype, _ = maybe_promote(result.dtype, other)
if new_dtype != result.dtype:
# we have a scalar or len 0 ndarray
# and its nan and we are changing some values
if is_scalar(other) or (isinstance(other, np.ndarray) and other.ndim < 1):
if isna(other):
return changeit()
# we have an ndarray and the masking has nans in it
else:
if isna(other).any():
return changeit()
try:
np.place(result, mask, other)
except TypeError:
# e.g. int-dtype result and float-dtype other
return changeit()
return result, False
def maybe_upcast_putmask(result, mask, other):
"""
A safe version of putmask that potentially upcasts the result
Parameters
----------
result : ndarray
The destination array. This will be mutated in-place if no upcasting is
necessary.
mask : boolean ndarray
other : ndarray or scalar
The source array or value
Returns
-------
result : ndarray
changed : boolean
Set to true if the result array was upcasted
"""
if mask.any():
# Two conversions for date-like dtypes that can't be done automatically
# in np.place:
# NaN -> NaT
# integer or integer array -> date-like array
if is_datetimelike(result.dtype):
if is_scalar(other):
if isna(other):
other = result.dtype.type('nat')
elif is_integer(other):
other = np.array(other, dtype=result.dtype)
elif is_integer_dtype(other):
other = np.array(other, dtype=result.dtype)
def changeit():
# try to directly set by expanding our array to full
# length of the boolean
try:
om = other[mask]
om_at = om.astype(result.dtype)
if (om == om_at).all():
new_result = result.values.copy()
new_result[mask] = om_at
result[:] = new_result
return result, False
except:
pass
# we are forced to change the dtype of the result as the input
# isn't compatible
r, _ = maybe_upcast(result, fill_value=other, copy=True)
np.place(r, mask, other)
return r, True
# we want to decide whether place will work
# if we have nans in the False portion of our mask then we need to
# upcast (possibly), otherwise we DON't want to upcast (e.g. if we
# have values, say integers, in the success portion then it's ok to not
# upcast)
new_dtype, _ = maybe_promote(result.dtype, other)
if new_dtype != result.dtype:
# we have a scalar or len 0 ndarray
# and its nan and we are changing some values
if (is_scalar(other) or
(isinstance(other, np.ndarray) and other.ndim < 1)):
if isna(other):
return changeit()
# we have an ndarray and the masking has nans in it
else:
if isna(other[mask]).any():
return changeit()
try:
np.place(result, mask, other)
except:
return changeit()
return result, False
def main():
sim_config = simulation_configuration()
sim_config.set_scenario_local_ref(
h_t = 13.82e6, # m
h_r = 20e3, # meters
elevation = 60.0*np.pi/180,
v_t = np.array([-2684.911, 1183.799, -671.829]), # m/s
v_r = np.array([20, 20, 20]) # m/s
)
#sim_config.jacobian_type = 'spherical'
sim_config.receiver_antenna_gain = lambda p1,p2: 12.589
sim_config.rcs = lambda p1,p2: target_rcs.radar_cross_section(p1, 0, p2)
sim_config.u_10 = 8.0
#sim_config.delay_chip = 1/gps_ca_chips_per_second # seconds
delay_chip = sim_config.delay_chip
number_of_delay_pixels = 128 - 50
number_of_doppler_pixels = 20 + 50
sim_config.doppler_increment_start = -50
sim_config.doppler_increment_end = 50
sim_config.doppler_resolution = (sim_config.doppler_increment_end - sim_config.doppler_increment_start)/number_of_doppler_pixels/3
sim_config.delay_increment_start = -0.5*delay_chip
sim_config.delay_increment_end = 2*delay_chip
#sim_config.delay_resolution = 0.01*delay_chip
sim_config.delay_resolution = (sim_config.delay_increment_end - sim_config.delay_increment_start)/number_of_delay_pixels/3
sim_config.coherent_integration_time = 2e-2 # sec
delay_increment_start = sim_config.delay_increment_start
delay_increment_end = sim_config.delay_increment_end
delay_resolution = sim_config.delay_resolution
doppler_increment_start = sim_config.doppler_increment_start
doppler_increment_end = sim_config.doppler_increment_end
doppler_resolution = sim_config.doppler_resolution
doppler_specular_point = eq_doppler_absolute_shift(np.array([0,0,0]), sim_config)
rescaled_doppler_resolution = (doppler_increment_end - doppler_increment_start)/number_of_doppler_pixels
rescaled_delay_resolution_chips = (delay_increment_end - delay_increment_start)/delay_chip/number_of_delay_pixels
print("doppler res: {0}".format(rescaled_doppler_resolution))
print("delay res: {0}".format(rescaled_delay_resolution_chips))
# Surface mesh
x_0 = 0
x_1 = 6e3 # meters
n_x = 800
y_0 = -1e3
y_1 = 6e3 # meters
n_y = 800
x_grid, y_grid = np.meshgrid(
np.linspace(x_0, x_1, n_x),
np.linspace(y_0, y_1, n_y)
)
r = np.array([x_grid, y_grid, 0])
# Isolines and RCS
z_grid_delay_chip = eq_delay_incremet(r, sim_config)/delay_chip
doppler_specular_point = eq_doppler_absolute_shift(np.array([0,0,0]), sim_config)
z_grid_doppler_increment = eq_doppler_absolute_shift(r, sim_config) - doppler_specular_point
z_rcs = sim_config.rcs(r, sim_config)
# Plot
fig_rcs, ax_rcs = plt.subplots(1,figsize=(10, 4))
#contour_delay_chip = ax_rcs.contour(
# x_grid, y_grid, z_grid_delay_chip,
# np.arange(0, 2, 0.1),
# cmap='winter', alpha = 0.3
# )
#contour_doppler = ax_rcs.contour(
# x_grid, y_grid, z_grid_doppler_increment,
# np.arange(-50, 50, 1),
# cmap='jet', alpha = 0.3
# )
contour_rcs = ax_rcs.contourf(x_grid, y_grid, z_rcs, 55, cmap='jet', alpha = 0.8)
ax_rcs.set_title('RCS')
plt.xlabel('[km]')
plt.ylabel('[km]')
#fig_rcs.colorbar(contour_delay_chip, label='C/A chips')
#fig_rcs.colorbar(contour_doppler, label='Hz')
fig_rcs.colorbar(contour_rcs, label='Gain')
target_delay_increment = 0.54
target_doppler_increment = 17.35
target_delay = 0.35
target_doppler = 12.5
target_iso_delay = ax_rcs.contour(x_grid, y_grid, z_grid_delay_chip,
#[target_delay_increment-0.1],
[target_delay - rescaled_delay_resolution_chips],
colors='red',
linewidths = 2.5,
linestyles='dashed',
)
target_iso_delay = ax_rcs.contour(x_grid, y_grid, z_grid_delay_chip,
#[target_delay_increment+0.1],
[target_delay + rescaled_delay_resolution_chips],
colors='red',
linewidths = 2.5,
linestyles='dashed',
)
target_iso_delay = ax_rcs.contour(x_grid, y_grid, z_grid_doppler_increment,
#[target_doppler_increment-0.5],
[target_doppler - rescaled_doppler_resolution],
colors='red',
linewidths = 2.5,
linestyles='dashed',
)
target_iso_delay = ax_rcs.contour(x_grid, y_grid, z_grid_doppler_increment,
#[target_doppler_increment+0.5],
[target_doppler + rescaled_doppler_resolution],
colors='red',
linewidths = 2.5,
linestyles='dashed',
)
ticks_y = ticker.FuncFormatter(lambda y, pos: '{0:g}'.format(y/1000))
ticks_x = ticker.FuncFormatter(lambda x, pos: '{0:g}'.format(x/1000))
ax_rcs.xaxis.set_major_formatter(ticks_x)
ax_rcs.yaxis.set_major_formatter(ticks_y)
# DDM
ddm_sim = np.copy(simulate_ddm(sim_config))
sim_config.rcs = sea_rcs.radar_cross_section
sim_config.rcs = lambda p1,p2: target_rcs.radar_cross_section(p1, 10, p2)
sim_config.u_10 = 8.02 # m/s
ddm_sim_1 = np.copy(simulate_ddm(sim_config))
ddm_diff = np.abs(ddm_sim - ddm_sim_1)
fig_diff, ax_diff = plt.subplots(1,figsize=(10, 4))
plt.title('DDM diff simulation')
plt.xlabel('C/A chips')
plt.ylabel('Hz')
contour_diff = ax_diff.imshow(ddm_diff, cmap='jet',
extent=(
delay_increment_start/delay_chip, delay_increment_end/delay_chip,
doppler_increment_end, doppler_increment_start),
aspect="auto"
)
fig_diff.colorbar(contour_diff, label='Correlated Power [W]')
fig_ddm, ax_ddm = plt.subplots(1,figsize=(10, 4))
plt.title('DDM original simulation')
plt.xlabel('C/A chips')
plt.ylabel('Hz')
contour_sim = ax_ddm.imshow(ddm_sim, cmap='jet',
extent=(
delay_increment_start/delay_chip, delay_increment_end/delay_chip,
doppler_increment_end, doppler_increment_start),
aspect="auto"
)
fig_ddm.colorbar(contour_sim, label='Correlated Power [W]')
fig_ddm_1, ax_ddm_1 = plt.subplots(1,figsize=(10, 4))
plt.title('DDM original simulation 1')
plt.xlabel('C/A chips')
plt.ylabel('Hz')
contour_sim_1 = ax_ddm_1.imshow(ddm_sim_1, cmap='jet',
extent=(
delay_increment_start/delay_chip, delay_increment_end/delay_chip,
doppler_increment_end, doppler_increment_start),
aspect="auto"
)
fig_ddm_1.colorbar(contour_sim_1, label='Correlated Power [W]')
# Image downscaling to desired resolution:
# TODO: This is just an average of the pixels around the area
# This is not valid, summation i srequired:
# Di Simone > From a physical viewpoint,
# such an approach should call for summation instead of averaging
# https://stackoverflow.com/questions/48121916/numpy-resize-rescale-image
fig_ddm_rescaled_diff, ax_ddm_rescaled_diff = plt.subplots(1,figsize=(10, 4))
plt.title('Simulation')
plt.xlabel('C/A chips')
plt.ylabel('Hz')
#ddm_rescaled = rescale(ddm_diff, number_of_doppler_pixels, number_of_delay_pixels)
ddm_sim_res = rescale(ddm_sim, number_of_doppler_pixels, number_of_delay_pixels)
ddm_sim_1_res = rescale(ddm_sim_1, number_of_doppler_pixels, number_of_delay_pixels)
ddm_diff_res = np.abs(ddm_sim_res - ddm_sim_1_res)
contour_diff = ax_ddm_rescaled_diff.imshow(ddm_diff_res, cmap='jet',
extent=(
delay_increment_start/delay_chip, delay_increment_end/delay_chip,
doppler_increment_end, doppler_increment_start),
aspect='auto'
)
fig_ddm_rescaled_diff.colorbar(contour_diff, label='Correlated Power [W]')
fig_ddm_rescaled, ax_ddm_rescaled = plt.subplots(1,figsize=(10, 4))
plt.title('Simulation Rescaled')
plt.xlabel('C/A chips')
plt.ylabel('Hz')
contour_rescaled = ax_ddm_rescaled.imshow(ddm_sim_res, cmap='jet',
extent=(
delay_increment_start/delay_chip, delay_increment_end/delay_chip,
doppler_increment_end, doppler_increment_start),
aspect='auto'
)
fig_ddm_rescaled.colorbar(contour_rescaled, label='Correlated Power [W]')
# DDM Noise
T_noise_receiver = 225
k_b = 1.38e-23 # J/K
y_noise = 1/sim_config.coherent_integration_time*k_b*T_noise_receiver
print("expected SNR: {}".format(y_noise))
fig_snr, ax_snr = plt.subplots(1,figsize=(10, 4))
plt.title('SNR')
plt.xlabel('C/A chips')
plt.ylabel('Hz')
ddm_snr = 10*np.log10(np.abs(ddm_diff_res)/y_noise)
np.place(ddm_snr, ddm_snr < -30, np.nan)
contour_snr = ax_snr.imshow(ddm_snr, cmap='jet',
extent=(
delay_increment_start/delay_chip, delay_increment_end/delay_chip,
doppler_increment_end, doppler_increment_start),
aspect='auto'
)
fig_snr.colorbar(contour_snr, label='Correlated Power [W]')
plt.show()
train_com = np.concatenate((cl1_train, cl2_train), axis=0)
train_lab = np.concatenate((cl1_label, cl2_label), axis=0)
[train_sff, train_labs] = shuffle(train_com, train_lab)
fig = plt.figure()
for i in range(16):
plt.subplot(4, 4, i + 1)
plt.tight_layout()
plt.imshow(train_sff[i], cmap='gray', interpolation='none')
plt.title("Digit: {}".format(train_labs[i]))
plt.xticks([])
plt.yticks([])
fig
np.place(train_labs, train_labs == cl1, [0])
np.place(train_labs, train_labs == cl2, [1])
############################################################################################################
train_labs_cat = keras.utils.to_categorical(train_labs, 2)
#
train_sff = train_sff.astype('float32')
train_sff /= 255
ftrain_sff = train_sff.reshape(train_labs_cat.shape[0], img_rows * img_cols)
################################################################################################################
model = Sequential()
model.add(Dense(1024, input_dim=784, activation='sigmoid'))
model.add(Dense(512, activation='sigmoid'))
model.add(Dense(256, activation='sigmoid'))
def sawtooth(t, width=1):
"""
Return a periodic sawtooth or triangle waveform.
The sawtooth waveform has a period ``2*pi``, rises from -1 to 1 on the
interval 0 to ``width*2*pi``, then drops from 1 to -1 on the interval
``width*2*pi`` to ``2*pi``. `width` must be in the interval [0, 1].
Note that this is not band-limited. It produces an infinite number
of harmonics, which are aliased back and forth across the frequency
spectrum.
Parameters
----------
t : array_like
Time.
width : array_like, optional
Width of the rising ramp as a proportion of the total cycle.
Default is 1, producing a rising ramp, while 0 produces a falling
ramp. `width` = 0.5 produces a triangle wave.
If an array, causes wave shape to change over time, and must be the
same length as t.
Returns
-------
y : ndarray
Output array containing the sawtooth waveform.
Examples
--------
A 5 Hz waveform sampled at 500 Hz for 1 second:
>>> from scipy import signal
>>> import matplotlib.pyplot as plt
>>> t = np.linspace(0, 1, 500)
>>> plt.plot(t, signal.sawtooth(2 * np.pi * 5 * t))
"""
t, w = asarray(t), asarray(width)
w = asarray(w + (t - t))
t = asarray(t + (w - w))
if t.dtype.char in ['fFdD']:
ytype = t.dtype.char
else:
ytype = 'd'
y = zeros(t.shape, ytype)
# width must be between 0 and 1 inclusive
mask1 = (w > 1) | (w < 0)
place(y, mask1, nan)
# take t modulo 2*pi
tmod = mod(t, 2 * pi)
# on the interval 0 to width*2*pi function is
# tmod / (pi*w) - 1
mask2 = (1 - mask1) & (tmod < w * 2 * pi)
tsub = extract(mask2, tmod)
wsub = extract(mask2, w)
place(y, mask2, tsub / (pi * wsub) - 1)
# on the interval width*2*pi to 2*pi function is
# (pi*(w+1)-tmod) / (pi*(1-w))
mask3 = (1 - mask1) & (1 - mask2)
tsub = extract(mask3, tmod)
wsub = extract(mask3, w)
place(y, mask3, (pi * (wsub + 1) - tsub) / (pi * (1 - wsub)))
return y
def active_to_full(self, active_joints, intial_position):
full_joints = np.array(intial_position, copy=True, dtype=np.float)
np.place(full_joints, self.active_joints_mask, active_joints)
return full_joints
def num_roads(osmdata, nom, lineTbl, polyTbl, folder, cellsize, srs,
rstTemplate):
"""
Select Roads and convert To Raster
"""
import datetime
import os
import numpy as np
from osgeo import gdal
from threading import Thread
from gasp.fm.rst import rst_to_array
from gasp.anls.exct import sel_by_attr
from gasp.sql.anls.prox import splite_buffer
from gasp.to.rst import shp_to_raster, array_to_raster
from gasp.sql.mng.tbl import row_num
time_a = datetime.datetime.now().replace(microsecond=0)
NR = row_num(osmdata, lineTbl, where="roads IS NOT NULL", api='sqlite')
time_b = datetime.datetime.now().replace(microsecond=0)
if not NR: return None, {0: ('count_rows_roads', time_b - time_a)}
timeGasto = {0: ('count_rows_roads', time_b - time_a)}
# Get Roads Buffer
LULC_CLS = '1221' if nom != "GLOBE_LAND_30" else '801'
bfShps = []
def exportAndBuffer():
time_cc = datetime.datetime.now().replace(microsecond=0)
roadFile = splite_buffer(osmdata,
lineTbl,
"bf_roads",
"geometry",
os.path.join(folder, 'bf_roads.gml'),
whrClause="roads IS NOT NULL",
outTblIsFile=True,
dissolve=None)
time_c = datetime.datetime.now().replace(microsecond=0)
distRst = shp_to_raster(roadFile,
None,
cellsize,
-1,
os.path.join(folder, 'rst_roads.tif'),
epsg=srs,
rst_template=rstTemplate,
api="gdal")
time_d = datetime.datetime.now().replace(microsecond=0)
bfShps.append(distRst)
timeGasto[1] = ('buffer_roads', time_c - time_cc)
timeGasto[2] = ('to_rst_roads', time_d - time_c)
BUILDINGS = []
def exportBuild():
time_ee = datetime.datetime.now().replace(microsecond=0)
NB = row_num(osmdata,
polyTbl,
where="building IS NOT NULL",
api='sqlite')
time_e = datetime.datetime.now().replace(microsecond=0)
timeGasto[3] = ('check_builds', time_e - time_ee)
if not NB:
return
bShp = sel_by_attr(
osmdata,
"SELECT geometry FROM {} WHERE building IS NOT NULL".format(
polyTbl),
os.path.join(folder, 'road_builds.shp'),
api_gis='ogr')
time_f = datetime.datetime.now().replace(microsecond=0)
bRst = shp_to_raster(bShp,
None,
cellsize,
-1,
os.path.join(folder, 'road_builds.tif'),
epsg=srs,
rst_template=rstTemplate,
api='gdal')
time_g = datetime.datetime.now().replace(microsecond=0)
BUILDINGS.append(bRst)
timeGasto[4] = ('export_builds', time_f - time_e)
timeGasto[5] = ('builds_to_rst', time_g - time_f)
thrds = [
Thread(name="build-th", target=exportBuild),
Thread(name='roads-th', target=exportAndBuffer)
]
for t in thrds:
t.start()
for t in thrds:
t.join()
if not len(BUILDINGS):
return {LULC_CLS: bfShps[0]}
time_x = datetime.datetime.now().replace(microsecond=0)
BUILD_ARRAY = rst_to_array(BUILDINGS[0], with_nodata=True)
rst_array = rst_to_array(bfShps[0], with_nodata=True)
np.place(rst_array, BUILD_ARRAY == 1, 0)
newRaster = array_to_raster(rst_array,
os.path.join(folder, 'fin_roads.tif'),
rstTemplate,
srs,
gdal.GDT_Byte,
noData=-1,
gisApi='gdal')
time_z = datetime.datetime.now().replace(microsecond=0)
timeGasto[6] = ('sanitize_roads', time_z - time_x)
return {int(LULC_CLS): newRaster}, timeGasto
arr : ndarray
Array to put data into.
mask : array_like
Boolean mask array. Must have the same size as `arr`. !!!
vals : 1-D sequence
Values to put into `arr`.
Only the first N elements are used, where N is the number of True values in `mask`.
If `vals` is smaller than N, it will be repeated, and if elements of `arr` are to be masked,
this sequence must be non-empty.
See also
copyto, put, take, extract
"""
arr = np.arange(6).reshape(2, 3)
arr
np.place(arr, arr>2, [10, 20]) #!!! inplace
arr
#array([[ 0, 1, 2],
# [10, 20, 10]])
# also
arr = np.arange(6).reshape(2, 3)
arr
np.putmask(arr, (arr>2), [10, 20]) #!!! the same
arr
# while
np.copyto(arr, [10, 20], where=arr>2) # ValueError: could not broadcast input array from shape (2) into shape (2,3)
np.place(arr, arr>1, [10, 20, 30])
arr
def update_globe_land_cover(original_globe_raster, osm_urban_atlas_raster,
osm_globe_raster, epsg, updated_globe_raster,
detailed_globe_raster):
"""
Update the original Glob Land 30 with the result of the conversion of
OSM DATA to the Globe Land Cover nomenclature;
Also updates he previous updated Glob Land 30 with the result of the
conversion of osm data to the Urban Atlas Nomenclature
"""
import os
import numpy as np
from gasp.gt.fmrst import rst_to_array
from gasp.gt.prop.rst import get_cellsize, get_nodata
from gasp.gt.torst import obj_to_rst
# ############################# #
# Convert images to numpy array #
# ############################# #
np_globe_original = rst_to_array(original_globe_raster)
np_globe_osm = rst_to_array(osm_globe_raster)
np_ua_osm = rst_to_array(osm_urban_atlas_raster)
# ################################## #
# Check the dimension of both images #
# ################################## #
if np_globe_original.shape != np_globe_osm.shape:
return (
'The Globe Land 30 raster (original) do not have the same number'
' of columns/lines comparing with the Globe Land 30 derived '
'from OSM data')
elif np_globe_original.shape != np_ua_osm.shape:
return (
'The Globe Land 30 raster (original) do not have the same '
'number of columns/lines comparing with the Urban Atlas raster '
'derived from OSM data')
elif np_globe_osm.shape != np_ua_osm.shape:
return (
'The Globe Land 30 derived from OSM data do not have the same '
'number of columns/lines comparing with the Urban Atlas raster '
'derived from OSM data')
# ############## #
# Check Cellsize #
# ############## #
cell_of_rsts = get_cellsize(
[original_globe_raster, osm_globe_raster, osm_urban_atlas_raster],
xy=True,
gisApi='gdal')
cell_globe_original = cell_of_rsts[original_globe_raster]
cell_globe_osm = cell_of_rsts[osm_globe_raster]
cell_ua_osm = cell_of_rsts[osm_urban_atlas_raster]
if cell_globe_original != cell_globe_osm:
return (
'The cellsize of the Globe Land 30 raster (original) is not the '
'same comparing with the Globe Land 30 derived from OSM data')
elif cell_globe_original != cell_ua_osm:
return (
'The cellsize of the Globe Land 30 raster (original) is not the '
'same comparing with the Urban Atlas raster derived from OSM data')
elif cell_ua_osm != cell_globe_osm:
return (
'The cellsize of the Globe Land 30 derived from OSM data is not '
'the same comparing with the Urban Atlas raster derived from '
'OSM data')
# ############################# #
# Get the Value of Nodata Cells #
# ############################# #
nodata_glob_original = get_nodata(original_globe_raster, gisApi='gdal')
nodata_glob_osm = get_nodata(osm_globe_raster, gisApi='gdal')
nodata_ua_osm = get_nodata(osm_urban_atlas_raster, gisApi='gdal')
# ######################################## #
# Create a new map - Globe Land 30 Updated #
# ######################################## #
"""
Create a new array with zeros...
1) The zeros will be replaced by the values in the Globe Land derived from
OSM.
2) The zeros will be replaced by the values in the Original Globe Land at
the cells with NULL data in the Globe Land derived from OSM.
The meta array will identify values origins in the updated raster:
1 - Orinal Raster
2 - OSM Derived Raster
"""
update_array = np.zeros(
(np_globe_original.shape[0], np_globe_original.shape[1]))
update_meta_array = np.zeros(
(np_globe_original.shape[0], np_globe_original.shape[1]))
# 1)
np.copyto(update_array, np_globe_osm, 'no',
np_globe_osm != nodata_glob_osm)
# 1) meta
np.place(update_meta_array, update_array != 0, 2)
# 2) meta
np.place(update_meta_array, update_array == 0, 1)
# 2)
np.copyto(update_array, np_globe_original, 'no', update_array == 0)
# 2) meta
np.place(update_meta_array, update_array == nodata_glob_original,
int(nodata_glob_original))
# noData to int
np.place(update_array, update_array == nodata_glob_original,
int(nodata_glob_original))
updated_meta = os.path.join(
os.path.dirname(updated_globe_raster), '{n}_meta{e}'.format(
n=os.path.splitext(os.path.basename(updated_globe_raster))[0],
e=os.path.splitext(os.path.basename(updated_globe_raster))[1]))
# Create Updated Globe Cover 30
obj_to_rst(update_array,
updated_globe_raster,
original_globe_raster,
noData=int(nodata_glob_original))
# Create Updated Globe Cover 30 meta
obj_to_rst(update_meta_array,
updated_meta,
original_globe_raster,
noData=int(nodata_glob_original))
# ################################################# #
# Create a new map - Globe Land 30 Detailed with UA #
# ################################################# #
np_update = rst_to_array(updated_globe_raster)
detailed_array = np.zeros((np_update.shape[0], np_update.shape[1]))
detailed_meta_array = np.zeros((np_update.shape[0], np_update.shape[1]))
"""
Replace 80 Globe Land for 11, 12, 13, 14 of Urban Atlas
The meta array will identify values origins in the detailed raster:
1 - Updated Raster
2 - UA Derived Raster from OSM
"""
# Globe - Mantain some classes
np.place(detailed_array, np_update == 30, 8)
np.place(detailed_array, np_update == 30, 1)
np.place(detailed_array, np_update == 40, 9)
np.place(detailed_array, np_update == 40, 1)
np.place(detailed_array, np_update == 50, 10)
np.place(detailed_array, np_update == 50, 1)
np.place(detailed_array, np_update == 10, 5)
np.place(detailed_array, np_update == 10, 1)
# Water bodies
np.place(detailed_array, np_ua_osm == 50 or np_update == 60, 7)
np.place(detailed_meta_array, np_ua_osm == 50 or np_update == 60, 1)
# Urban - Where Urban Atlas IS NOT NULL
np.place(detailed_array, np_ua_osm == 11, 1)
np.place(detailed_meta_array, np_ua_osm == 11, 2)
np.place(detailed_array, np_ua_osm == 12, 2)
np.place(detailed_meta_array, np_ua_osm == 12, 2)
np.place(detailed_array, np_ua_osm == 13, 3)
np.place(detailed_meta_array, np_ua_osm == 13, 2)
np.place(detailed_array, np_ua_osm == 14, 4)
np.place(detailed_meta_array, np_ua_osm == 14, 2)
# Urban Atlas - Class 30 to 6
np.place(detailed_array, np_ua_osm == 30, 6)
np.place(detailed_meta_array, np_ua_osm == 30, 2)
# Create Detailed Globe Cover 30
obj_to_rst(detailed_array,
detailed_globe_raster,
original_globe_raster,
noData=0)
# Create Detailed Globe Cover 30 meta
detailed_meta = os.path.join(
os.path.dirname(detailed_globe_raster), '{n}_meta{e}'.format(
n=os.path.splitext(os.path.basename(detailed_meta))[0],
e=os.path.splitext(os.path.basename(detailed_meta))[1]))
obj_to_rst(detailed_meta_array,
detailed_meta,
original_globe_raster,
noData=0)
def ntirogiannis2014(im):
lib.debug_prefix.append('ng2014')
debug_imwrite('input.png', im)
im_h, _ = im.shape
N, BG_prime = ng2014_normalize(im)
O = otsu(N)
debug_imwrite('O.png', O)
letters = algorithm.all_letters(O)
height_map = HeightMap(letters)
ratio_sum = 0
for h in range(1, height_map.max_height() + 1):
if len(height_map[h]) == 0: continue
ratio_sum += height_map.ratio_pixels(h) / height_map.ratio_components(
h)
if ratio_sum > 1:
break
min_height = h
if lib.debug: print('Accept components only >= height', h)
OP = O.copy()
for h in range(1, min_height):
for letter in height_map[h]:
sliced = letter.slice(OP)
np.place(sliced, letter.raster(), 255)
debug_imwrite('OP.png', OP)
strokes = fast_stroke_width(OP)
debug_imwrite('strokes.png', normalize_u8(strokes.clip(0, 10)))
SW = int(round(strokes.sum() / np.count_nonzero(strokes)))
if lib.debug: print('SW =', SW)
S = skeleton(OP)
debug_imwrite('S.png', S)
S_inv = ~S
# S_inv_32 = S_inv.astype(np.int32)
# FG_count = np.count_nonzero(S_inv)
FG_pos = im[S_inv.astype(bool)]
FG_avg = FG_pos.mean()
FG_std = FG_pos.std()
# FG = (S_inv & im).astype(np.int32)
# FG_avg = FG.sum() / float(FG_count)
# FG_std = np.sqrt(((S_inv_32 & (FG - FG_avg)) ** 2).sum() / float(FG_count))
if lib.debug: print('FG:', FG_avg, FG_std)
BG_avg = BG_prime.mean()
BG_std = BG_prime.std()
if lib.debug: print('BG:', BG_avg, BG_std)
if FG_avg + FG_std != 0:
C = -50 * np.log10((FG_avg + FG_std) / (BG_avg - BG_std))
k = -0.2 - 0.1 * C / 10
else: # This is the extreme case when the FG is 100% black, check the article explaination page before equation 5
C = -50 * np.log10((2.5) / (BG_avg - BG_std))
k = -0.2 - 0.1 * C / 10
if lib.debug: print('niblack:', C, k)
local = niblack(N, window_size=(2 * SW) | 1, k=k)
debug_imwrite('local.png', local)
local_CCs = algorithm.all_letters(local)
# NB: paper uses OP here, which results in neglecting all small components.
O_inv = ~O
O_inv_32 = O_inv.astype(np.int8,
copy=False).astype(np.int32).astype(np.uint32,
copy=False)
label_map_O_inv = O_inv_32 & local_CCs[0].label_map
CO_inv = np.zeros(im.shape, dtype=np.uint8)
for cc in local_CCs:
if np.count_nonzero(cc.slice(label_map_O_inv) == cc.label) / float(
cc.area()) >= C / 100:
CO_sliced = cc.slice(CO_inv)
np.place(CO_sliced, cc.raster(), 255)
CO = ~CO_inv
debug_imwrite('CO.png', CO)
CO_inv_dilated = cv2.dilate(CO_inv, rect33)
FB = ~(CO_inv | ((~O) & CO_inv_dilated))
debug_imwrite('FB.png', FB)
lib.debug_prefix.pop()
return FB
def netCDF_filter_nan(data_file, phase_str, interim_path):
'''
phase_str: train, val or test
'''
data_dic = {'input_obs': None, 'input_ruitu': None, 'ground_truth': None}
print('processing...:', data_file)
ori_data = nc.Dataset(data_file) # 读取nc文件
ori_dimensions, ori_variables = ori_data.dimensions, ori_data.variables # 获取文件中的维度和变量
date_index, fortime_index, station_index = 1, 2, 3 # 根据三个维度，读取该变量在特定日期、特定站点的特定预报时刻的数值
var_obs = [] # var name list
var_all = []
var_ruitu = []
for v in ori_variables:
var_all.append(v)
if v.find("_obs") != -1:
var_obs.append(v)
elif v.find('_M') != -1:
var_ruitu.append(v)
sta_id = ori_variables['station'][:].data
print('sta_id:', sta_id)
hour_index = ori_variables['foretimes'][:].data
print('hour_index:', hour_index)
day_index = ori_variables['date'][:].data
print('day_index:', day_index)
print(str(list(day_index)[-1]).split('.')[0])
# build a map for staion and its index
station_dic = {}
for i, s in enumerate(sta_id):
station_dic[s] = i
print(station_dic)
NUMS = ori_dimensions['date'].size # 1188 for train
input_ruitu_nan_list = []
input_obs_nan_list = []
#output_obs_nan_list=[]
for i in range(NUMS - 1):
input_ruitu_nan = (
ori_variables['t2m_M'][i, :, :].data == -9999.).all()
input_obs_nan = (
ori_variables['t2m_obs'][i, :, :].data == -9999.).all()
if input_ruitu_nan:
input_ruitu_nan_list.append(i)
if input_obs_nan:
input_obs_nan_list.append(i)
input_ruitu_nan_list_minus1 = [i - 1 for i in input_ruitu_nan_list]
print('input_ruitu_nan_list_minus1:', input_ruitu_nan_list_minus1)
print('input_obs_nan_list', input_obs_nan_list)
deleted_obs_days = input_ruitu_nan_list_minus1 + input_obs_nan_list
#bad_days
print('deleted_obs_days:', (deleted_obs_days))
print('deleted_obs_days_nums:', len(deleted_obs_days))
input_obs_dic = dict.fromkeys(var_obs, None)
input_ruitu_dic = dict.fromkeys(var_ruitu, None)
ground_truth_dic = dict.fromkeys(var_obs, None)
good_obs_days = [i for i in range(NUMS - 1) if i not in deleted_obs_days]
print('The number of not seriously NaN days:', len(good_obs_days))
good_groundtruth_days = [i + 1 for i in good_obs_days]
for v in var_obs:
input_obs_dic[v] = ori_variables[v][good_obs_days, :, :].data
ground_truth_dic[v] = ori_variables[v][
good_groundtruth_days, :, :].data
for v in var_ruitu:
input_ruitu_dic[v] = ori_variables[v][good_groundtruth_days, :, :].data
for v in var_obs:
np.place(input_obs_dic[v], input_obs_dic[v] == -9999., np.nan)
# Fill missing value with mean value
mean_ = np.nanmean(input_obs_dic[v]) # Calculate mean except np.nan
where_are_NaNs = np.isnan(input_obs_dic[v])
input_obs_dic[v][where_are_NaNs] = mean_
np.place(ground_truth_dic[v], ground_truth_dic[v] == -9999., np.nan)
mean_ = np.nanmean(ground_truth_dic[v]) # Calculate mean except np.nan
where_are_NaNs = np.isnan(ground_truth_dic[v])
ground_truth_dic[v][where_are_NaNs] = mean_
data_dic['input_obs'] = input_obs_dic
data_dic['ground_truth'] = ground_truth_dic
for v in var_ruitu:
np.place(input_ruitu_dic[v], input_ruitu_dic[v] == -9999., np.nan)
mean_ = np.nanmean(input_ruitu_dic[v]) # Calculate mean except np.nan
where_are_NaNs = np.isnan(input_ruitu_dic[v])
input_ruitu_dic[v][where_are_NaNs] = mean_
data_dic['input_ruitu'] = input_ruitu_dic
save_pkl(data_dic, interim_path, '{}_non_NaN.dict'.format(phase_str))
return '{}_non_NaN.dict'.format(phase_str)
def generate(self, area, sigma=None, verbose=False):
""" Generates simulated maps.
Parameters
----------
area: float
Area of generated maps, in deg^2
sigma: ndarray or None
Map instrument noise, in Jy. If None, no instrument
noise is added.
verbose: bool
Print informational messages as it runs.
Returns
-------
A tuple containing the input maps. If truthtable is
set on initialization, also includes the truth table of
positions and fluxes, where the positions are relative
to the first map.
Notes
-----
Remember that the returned maps follow astropy.io.fits conventions,
so the indexing into the maps is [y, x]
"""
if area <= 0.0:
raise ValueError("Invalid (non-positive) area")
if sigma is None:
int_sigma = np.zeros(self._nbands, dtype=np.float32)
elif type(sigma) == list:
if len(sigma) != self._nbands:
if len(sigma) == 1:
int_sigma = sigma[0] * np.ones_like(self._wave)
else:
raise ValueError("Number of sigmas doesn't match number"
" of wavelengths")
else:
int_sigma = np.asarray(sigma, dtype=np.float32)
elif type(sigma) == np.ndarray:
if len(sigma) != self._nbands:
if len(sigma) == 1:
int_sigma = sigma[0] * np.ones_like(self._wave)
else:
raise ValueError("Number of sigmas doesn't match number"
" of wavelengths")
else:
int_sigma = sigma.astype(np.float32, copy=False)
else:
int_sigma = float(sigma) * np.ones_like(self._wave)
if int_sigma.min() < 0:
raise ValueError("Invalid (negative) instrument sigma")
# Make the non-convolved images
# The first step is to initialize the output maps.
# Since we do the catalog in chunks (in typical applications
# the catalog takes more memory than the maps), we must hold
# all the maps in memory at once
nextent = np.empty(self._nbands, dtype=np.int32)
truearea = np.empty(self._nbands, dtype=np.float32)
maps = []
for i in range(self._nbands):
pixarea = (self._pixsize[i] / 3600.0)**2
nextent[i] = math.ceil(math.sqrt(area / pixarea))
truearea[i] = nextent[i]**2 * pixarea
maps.append(np.zeros((nextent[i], nextent[i]), dtype=np.float32))
# Figure out how many sources to make
truearea = truearea.mean()
nsources_base = self._npersr * (math.pi / 180.0)**2 * truearea
nsources = np.random.poisson(lam=nsources_base)
if verbose:
print("True area: {:0.2f} [deg^2]".format(truearea))
print("Number of sources to generate: {:d}".format(nsources))
# We do this in chunks
if self._gensize == 0:
# One big chunk
nchunks = 1
chunks = np.array([nsources], dtype=np.int64)
else:
# Recall this is python 3 -- floating point division
nchunks = math.ceil(nsources / self._gensize)
chunks = self._gensize * np.ones(nchunks, dtype=np.int64)
chunks[-1] = nsources - (nchunks - 1) * self._gensize
assert chunks.sum() == nsources
# Source generation loop
nexgen = float(nextent[0])
if verbose:
print("Generating sources")
for i, nsrc in enumerate(chunks):
if verbose and nchunks > 1:
print(" Doing chunk {0:d} of {1:d}".format(i + 1, nchunks))
# Generate positions in base image, uniformly distributed
# Note these are floating point
xpos = nexgen * np.random.rand(nsrc)
ypos = nexgen * np.random.rand(nsrc)
# Get fluxes (in Jy)
cat = self._gencat.generate(nsrc, wave=self._wave)
fluxes = cat[-1].copy()
# Set up truth table if needed
if self._returntruth:
truthtable = {
'x': xpos,
'y': ypos,
'z': cat[0],
'log10M': cat[1],
'sb': cat[2],
'log10sSFR': cat[3],
'log10Lir': cat[4],
'fluxdens': fluxes
}
# Try to be good about freeing up memory
del cat
# Add to first map without rescaling.
# Note this has to happen in a for loop because multiple
# sources can go in the -same- pixel
cmap = maps[0]
xf = np.floor(xpos)
yf = np.floor(ypos)
nx, ny = cmap.shape
np.place(xf, xf > nx - 1, nx - 1)
np.place(yf, yf > ny - 1, ny - 1)
for cx, cy, cf in zip(xf, yf, fluxes[:, 0]):
cmap[cy, cx] += cf # Note transpose
# Other bands, with pixel scale adjustment
for mapidx in range(1, self._nbands):
posrescale = self._pixsize[0] / self._pixsize[mapidx]
xf = np.floor(posrescale * xpos)
yf = np.floor(posrescale * ypos)
cmap = maps[mapidx]
nx, ny = cmap.shape
np.place(xf, xf > nx - 1, nx - 1)
np.place(yf, yf > ny - 1, ny - 1)
for cx, cy, cf in zip(xf, yf, fluxes[:, mapidx]):
cmap[cy, cx] += cf # Note transpose
if not self._returntruth:
del fluxes, xpos, ypos, xf, yf
else:
del xf, yf
# Now image details -- convolution, instrument noise
for mapidx in range(self._nbands):
if verbose:
msg = "Preparing map for wavelength {0:5.1f} um "\
"extent: {1:d} x {2:d}"
print(
msg.format(self._wave[mapidx], maps[mapidx].shape[0],
maps[mapidx].shape[1]))
print(" Convolving")
beam = self.get_gauss_beam(mapidx)
maps[mapidx] = convolve(maps[mapidx], beam, boundary='wrap')
if int_sigma[mapidx] > 0:
if verbose:
msg = " Adding instrument noise: {:0.4f} [Jy]"
print(msg.format(int_sigma[mapidx]))
maps[mapidx] += np.random.normal(scale=int_sigma[mapidx],
size=maps[mapidx].shape)
if self._returntruth:
maps.append(truthtable)
return maps
def dereverberate(wave,
fs,
params=None,
estimate_execution_time=True,
show_progress_bar=True):
"""
Estimates the impulse response in a room the recording took place
:param wave: 1-D ndarray of wave samples
:param fs: int - sampling frequency
:param params: dict containing the algorithm parameters - keys:
:param estimate_execution_time: should we print estimated execution time for each next frame
:param show_progress_bar: should we print progress bar of estimation
:returns: (h_stft_pow) 2-D ndarray power STFT of h_rir,
(wave_dry) 1-D ndarray of the dry signal,
(wave_wet) 1-D ndarray of the wet signal
"""
# estimating execution time
loop_times = np.zeros(10)
# =================== Parameters ===================
if params is None:
params = DEF_PARAMS
# ==================== Windowing ===================
win_len_ms = params["win_len"]
win_ovlap_p = params["win_ovlap"]
# ================ Times to samples ================
win_len = int(win_len_ms / 1000 * fs)
win_ovlap = int(win_len * win_ovlap_p)
window = np.hanning(win_len)
# =================== Signal stft ==================
nfft = first_larger_square(win_len)
sig_stft = STFT(wave, window, win_ovlap, nfft)
sig_stft = sig_stft[:, 0:nfft // 2 + 1]
frame_count, frequency_count = sig_stft.shape
# ==================== Constants ===================
# length of the impulse response
blocks = params["blocks"]
# minimum gain of dry signal per frequency
min_gain_dry = params["min_gain_dry"]
# maximum impulse response estimate
# max_h, blocks = read_impulse_response("deverb_test_samples/stalbans_a_mono.wav", fs, nfft, win_len_ms, win_ovlap_p)
max_h = get_max_h_matrix('const', frequency_count, blocks)
# bias used to keep magnitudes from getting stuck on a wrong minimum
bias = params["bias"]
# alpha and gamma - smoothing factors for impulse response magnitude and gain
alpha = params["alpha"]
gamma = params["gamma"]
# ==================== Algorithm ===================
# dry_stft and wet_stft are the estimated dry and reverberant signals in frequency-time domain
dry_stft = np.zeros((frame_count, frequency_count), dtype=np.csingle)
wet_stft = np.zeros((frame_count, frequency_count), dtype=np.csingle)
# h_stft_pow is the estimated impulse response in frequency-time domain
h_stft_pow = max_h / 2
# matrices with the information of currently estimated raw and dry signal (power spectra)
raw_frames = np.ones((blocks, frequency_count))
dry_frames = np.zeros((blocks, frequency_count))
# c is a matrix to keep the raw estimated powers of the impulse response
c = np.zeros((blocks, frequency_count))
# gain_dry and gain_wet are the frequency gains of the dry and wet signals
gain_dry = np.ones(frequency_count)
gain_wet = np.zeros(frequency_count)
for i in range(frame_count):
if estimate_execution_time:
remaining = round(np.mean(loop_times) * (frame_count - i))
loop_times[1:] = loop_times[0:-1]
loop_times[0] = time.time()
print(
"Processing frame {} of {}, estimated time left: {} ms".format(
i + 1, frame_count, remaining))
frame = sig_stft[i, :]
frame_power = np.power(np.abs(frame), 2)
# estimate signals based on i-th frame
for b in range(blocks):
estimate = frame_power / raw_frames[b, :]
np.place(estimate, estimate >= h_stft_pow[b, :],
h_stft_pow[b, :] * bias + np.finfo(np.float32).eps)
np.fmin(estimate, max_h[b, :], out=c[b, :])
h_stft_pow[b, :] = alpha * h_stft_pow[b, :] + (1 - alpha) * c[b, :]
# calculating gains
new_gain_dry = 1 - np.sum(dry_frames * h_stft_pow,
axis=0) / frame_power
np.place(new_gain_dry, new_gain_dry < min_gain_dry, min_gain_dry)
gain_dry = gamma * gain_dry + (1 - gamma) * new_gain_dry
new_gain_wet = 1 - gain_dry
gain_wet = gamma * gain_wet + (1 - gamma) * new_gain_wet
# calculatnig signals
dry_stft[i, :] = gain_dry * frame
wet_stft[i, :] = gain_wet * frame
# shifting previous frames
dry_frames[1:blocks, :] = dry_frames[0:blocks - 1, :]
dry_frames[0, :] = np.power(np.abs(dry_stft[i, :]), 2)
raw_frames[1:blocks, :] = raw_frames[0:blocks - 1, :]
raw_frames[0, :] = frame_power
if estimate_execution_time:
loop_times[0] = round(1000 * (time.time() - loop_times[0]))
if show_progress_bar:
printProgressBar(i,
frame_count,
prefix='Progress',
suffix='Complete',
length=30)
window = optimal_synth_window(window, win_ovlap)
if TEST_SCOPE:
t = (np.arange(frame_count) * (win_len_ms *
(1 - win_ovlap_p))).astype(int)
f = np.linspace(0, fs / 2, frequency_count).astype(int)
txx, fxx = np.meshgrid(t, f)
fig, axes = plt.subplots(3, 1, figsize=(10, 10))
axes[0].pcolormesh(txx,
fxx,
np.log10(np.power(np.abs(sig_stft.T), 2)),
cmap=plt.get_cmap('plasma'))
axes[0].set_title("Original signal")
axes[1].pcolormesh(txx,
fxx,
np.log10(np.power(np.abs(dry_stft.T), 2)),
cmap=plt.get_cmap('plasma'))
axes[1].set_title("Dry signal")
axes[2].pcolormesh(txx,
fxx,
np.log10(np.power(np.abs(wet_stft.T), 2)),
cmap=plt.get_cmap('plasma'))
axes[2].set_title("Reverberant signal")
fig.show()
wave_dry = reconstruct(dry_stft, window, win_ovlap)
wave_wet = reconstruct(wet_stft, window, win_ovlap)
return h_stft_pow, wave_dry, wave_wet
def calc_distance_df(locs,
data_norm,
cellGraph,
gmmDict,
fdr_df,
tissue_mat_new,
sort_tissue=False):
hamming = list()
jaccard = list()
hausdorff = list()
recal_genes = []
genelist = []
for geneID in fdr_df.index:
exp = data_norm.loc[:, geneID].values
gmm = gmmDict[geneID]
target_seed_mat_list = list()
temp_factor = int(fdr_df.loc[geneID].smooth_factor)
newLabels = fdr_df.loc[geneID][4:].values.astype(int)
node = fdr_df.loc[geneID, 'nodes']
p = fdr_df.loc[geneID, 'p_value']
num_isolate = count_isolate(locs, cellGraph, newLabels)
num_noise = int((num_isolate[0] + num_isolate[1]) * 1 +
(num_isolate[2] + num_isolate[3]) * 2 +
(num_isolate[4] + num_isolate[5]) * 3)
# print(num_noise,len(node[0]),(locs.shape[0]-num_noise),p)
if len(p) == 1 and len(node[0]) == locs.shape[0]:
recal_genes.append(
geneID) ## no cuts genes, can't calculate hausdorff
# elif len(p)==1 and min(p)>0.1 and len(node[0])==(locs.shape[0]-num_noise): ## all noise,like
# genelist.append(geneID)
# total_tissue_mat=np.sum(tissue_mat_new,axis=0)
# total_target_mat=newLabels
# hamming.append(compute_diff_vs_common_const10(total_tissue_mat , total_target_mat))
# jaccard.append(jaccard_dist(total_tissue_mat , total_target_mat))
# target_locs = locs[total_target_mat == 1]
# tissue_locs = locs[total_tissue_mat == 1]
# hausdorff.append(compute_hausdorff(target_locs,tissue_locs))
else:
genelist.append(geneID)
gmm_pred = gmm.predict(exp.reshape(-1, 1))
a = exp[exp > 0]
zero_label = gmm.predict(np.min(a).reshape(-1, 1))[0]
if sum(exp == 0) > 0:
np.place(gmm_pred, exp == 0, zero_label)
# print(major_label, unique_com, counts_com)
# plot_voronoi_boundary(geneID, locs, exp, newLabels, min(p))
# get major label in min(p) segment
target_seed_mat_list = list()
node_list = node[np.argmin(p)]
temp_vec = np.zeros(len(exp))
temp_vec[node_list] = 1
target_seed_mat_list.append(temp_vec)
# for nn in node[np.argmin(p)]:
# plt.text(locs[nn,0],locs[nn,1],str(nn))
# plot_voronoi_boundary(geneID, locs,exp,newLabels,min(p))
com_size = len(node_list)
temp_exp = exp[np.array(list(node_list))]
gmm_pred_com = gmm.predict(temp_exp.reshape(-1, 1))
if sum(temp_exp == 0) > 0:
# a= temp_exp[temp_exp > 0]
# zero_label = gmm.predict(np.min(a).reshape(-1,1))[0]
np.place(gmm_pred_com, temp_exp == 0, zero_label)
unique_com, counts_com = np.unique(gmm_pred_com,
return_counts=True)
major_label = unique_com[np.where(
counts_com == counts_com.max())[0][0]]
max_size = 0
size = [len(node[nn]) for nn in range(len(node))]
p_sort = [p[i] for i in np.argsort(size)]
node_sort = [node[i] for i in np.argsort(size)]
#p_sort
for p_index in np.arange(
len(p)): #np.where(np.array(p) < 0.01)[0]:
if p_sort[p_index] > min(p): #p[p_index] <= p_cutoff and
temp_vec = np.zeros(len(exp))
node_list = node_sort[p_index]
temp_exp = exp[np.array(list(node_list))]
gmm_pred_com = gmm.predict(temp_exp.reshape(-1, 1))
if sum(temp_exp == 0) > 0:
# a= temp_exp[temp_exp > 0]
# zero_label = gmm.predict(np.min(a).reshape(-1,1))[0]
np.place(gmm_pred_com, temp_exp == 0, zero_label)
unique_com, counts_com = np.unique(gmm_pred_com,
return_counts=True)
temp_major_label = unique_com[np.where(
counts_com == counts_com.max())[0][0]]
# print(temp_major_label)
if temp_major_label == major_label: ## get other pattern with major label
#['Psap'] in Rep8
if len(node_list) + sum(
np.sum(target_seed_mat_list,
axis=0)) < (2 / 3) * len(exp) + 10:
temp_vec[node_list] = 1
target_seed_mat_list.append(temp_vec)
if len(node_list) > max_size:
max_size = len(node_list)
# print(max_size)
else:
if sum(target_seed_mat_list[0]) > max_size:
max_size = sum(target_seed_mat_list[0])
else:
if sum(target_seed_mat_list[0]) > max_size:
max_size = sum(
target_seed_mat_list[0]
) ## sum(np.sum(target_seed_mat_list,axis=0))
else:
if sum(target_seed_mat_list[0]) > max_size:
max_size = sum(target_seed_mat_list[0])
temp_list = list()
if max_size > 5:
for nn in target_seed_mat_list:
if sum(nn == 1) > 5:
temp_list.append(nn)
else:
for nn in target_seed_mat_list:
if sum(nn == 1) == max_size:
temp_list.append(nn)
target_mat = np.array(temp_list)
total_target_mat = np.sum(target_mat, axis=0)
## effect target mat: merge target mat as same label with min(p)
if sum(total_target_mat) > len(exp) / 2 + 10:
total_target_mat = abs(
1 - total_target_mat) ## 1-total_target_mat, for 'Hk3'
# now find all tissue mat that with 50% of the tissue mat seg are in target
temp_tissue_seg_list = list()
best_overlap = list()
best_overlap_val = 0
match_index = []
for ts_index in np.arange(len(tissue_mat_new)):
# overlap = max(sum((tissue_mat_new[kk,:] + total_target_mat) == 2)/sum(tissue_mat_new[kk,:]),
# sum((tissue_mat_new[kk,:] + total_target_mat) == 2)/sum(total_target_mat))
overlap = compute_inclusion_min(tissue_mat_new[ts_index],
total_target_mat)
# print(ts_index, overlap)
if overlap > 0.45:
match_index.append(ts_index)
temp_tissue_seg_list.append(tissue_mat_new[ts_index])
if overlap >= best_overlap_val: #and best_overlap_val>=0.1 :
# match_index.append(ts_index)
best_overlap_val = overlap
best_overlap = tissue_mat_new[ts_index].reshape(1, -1)
if len(temp_tissue_seg_list) > 0:
if sort_tissue == True:
if len(temp_tissue_seg_list) == 1 and match_index[0] > 3:
if match_index[0] == 4 or match_index[0] == 6:
temp_tissue_seg_list.append(tissue_mat_new[5])
if match_index[0] == 5:
temp_tissue_seg_list.append(tissue_mat_new[4])
temp_tissue_seg_list.append(tissue_mat_new[6])
match_tissue_mat = np.array(temp_tissue_seg_list)
else:
match_tissue_mat = np.array(temp_tissue_seg_list)
else:
match_tissue_mat = np.array(temp_tissue_seg_list)
else:
if best_overlap_val > 0:
match_tissue_mat = np.array(best_overlap)
## 2. some points are missed by tissue mat, using min(hausdorff) to match tissue mat.['Postn'] in Rep8
else: ## all(overlap==0)
dist_temp = []
temp_vec = np.zeros(locs.shape[0])
temp_vec[:] = 1
temp_vec[node[0]] = 0 ## mostly nodes
u = locs[np.where(temp_vec == 1)]
for ts_index in range(len(tissue_mat_new)):
# ja_dist=jaccard_dist(temp_vec , tissue_mat_new[ts_index]) ## single noise not in any tissue mat.
# ja_dist_temp.append(ja_dist)
v = locs[np.where(tissue_mat_new[ts_index] == 1)]
dist = compute_hausdorff(u, v)
dist_temp.append(dist)
temp_index = np.argmin(dist_temp)
match_tissue_mat = tissue_mat_new[temp_index].reshape(
1, -1)
total_tissue_mat = np.sum(
match_tissue_mat,
axis=0) ## get tissue_mat that matched target
hamming.append(
compute_diff_vs_common_const10(total_tissue_mat,
total_target_mat))
jaccard.append(jaccard_dist(total_tissue_mat, total_target_mat))
target_locs = locs[total_target_mat == 1]
tissue_locs = locs[total_tissue_mat == 1]
hausdorff.append(compute_hausdorff(target_locs, tissue_locs))
# print(geneID, len(target_seed_mat_list), sum(total_target_mat),
# len(temp_tissue_seg_list), sum(total_tissue_mat), jaccard_dist(total_tissue_mat , total_target_mat),
# max(directed_hausdorff(tissue_locs, target_locs)[0],
# directed_hausdorff(target_locs, tissue_locs)[0]))
# plot_voronoi_boundary(geneID, locs, exp, total_target_mat, min(p))
#plot_voronoi_boundary(geneID, locs, exp, total_tissue_mat, min(p))
# print('------------------')
data_array = np.array((hamming, jaccard, hausdorff), dtype=float).T
c_labels = ['Hamming', 'Jaccard', 'Hausdorff']
dist_df = pd.DataFrame(data_array, index=genelist, columns=c_labels)
# print out the target sum mat, and matched tissue sum mat to see whether they make sense
return dist_df, recal_genes
def cross_jaccard(input_rasters, thresholds, verbose=False, logger=None):
""" Calculate Jaccard coefficients between all the inpur rasters.
This is a utility function that is intented to be used to compare
fractions of the landscape.
:param input_rasters list of input raster paths.
:param thresholds vector of numeric tuples (x_min, x_max, y_min, y_max) values of thresholds.
:param verbose: Boolean indicating how much information is printed out.
:param logger: logger object to be used.
:param ... additional arguments passed on to jaccard().
:return Pandas Dataframe with Jaccard coefficients between all rasters.
"""
# 1. Setup --------------------------------------------------------------
all_start = timer()
if not logger:
logging.basicConfig()
llogger = logging.getLogger('cross_jaccard')
llogger.setLevel(logging.DEBUG if verbose else logging.INFO)
else:
llogger = logger
# Check the inputs
assert len(input_rasters) > 1, "More than one input rasters are needed"
assert len(thresholds) >= 1, "At least one tuple of thresholds is needed"
# 2. Calculations --------------------------------------------------------
llogger.info(" [** COMPUTING JACCARD INDICES **]")
all_jaccards = pd.DataFrame({
"feature1": [],
"feature2": [],
"threshold": [],
"coef": []
})
n_rasters = len(input_rasters)
# Generate counter information for all the computations. The results
# matrix is always diagonally symmetrical.
n_computations = int(
(n_rasters * n_rasters - n_rasters) / 2 * len(thresholds))
no_computation = 1
for threshold in thresholds:
if len(threshold) != 4:
llogger.error("Threshold tuple needs 4 values")
next
for i in range(0, n_rasters):
x_min, x_max, y_min, y_max = threshold
raster1 = rasterio.open(input_rasters[i])
# To calculate the Jaccard index we are dealing with binary data
# only. Avoid using masked arrays and replace NoData values with
# zeros.
raster1_nodata = raster1.nodata
raster1_src = raster1.read(1)
np.place(raster1_src, np.isclose(raster1_src, raster1_nodata), 0.0)
for j in range(i + 1, n_rasters):
raster2 = rasterio.open(input_rasters[j])
raster2_nodata = raster2.nodata
raster2_src = raster2.read(1)
np.place(raster2_src, np.isclose(raster2_src, raster2_nodata),
0.0)
prefix = utils.get_iteration_prefix(no_computation,
n_computations)
llogger.info(("{} Calculating Jaccard ".format(prefix) +
"index for [{0}, {1}] ".format(x_min, x_max) +
"in {} ".format(input_rasters[i]) +
"and, [{0}, {1}] ".format(y_min, y_max) +
"in {}".format(input_rasters[j])))
coef = compute_jaccard(raster1_src,
raster2_src,
x_min=x_min,
x_max=x_max,
y_min=y_min,
y_max=y_max)
jaccards = pd.DataFrame({
"feature1": [input_rasters[i]],
"feature2": [input_rasters[j]],
"threshold": [threshold],
"coef": [coef]
})
all_jaccards = pd.concat([all_jaccards, jaccards])
no_computation += 1
all_jaccards.index = np.arange(0, len(all_jaccards.index), 1)
all_end = timer()
all_elapsed = round(all_end - all_start, 2)
llogger.info(" [TIME] All processing took {} sec".format(all_elapsed))
return all_jaccards
filter_data = shuffle(np.loadtxt('spambase.data.csv', delimiter=','))
X_train, X_test, y_train, y_test = train_test_split(filter_data[:, np.arange(57)], filter_data[:, 57], test_size=0.50)
indices = [np.nonzero(y_train==0)[0], np.nonzero(y_train)[0]]
mean = np.transpose([np.mean(X_train[indices[0], :], axis=0), np.mean(X_train[indices[1], :], axis=0)])
std = np.transpose([np.std(X_train[indices[0], :], axis=0), np.std(X_train[indices[1], :], axis=0)])
###################################################################################################################################################################
# Experiment-1 : Classification with Gaussian Naive Bayes #
# Compute prior probabilities for both classes and take their log.Find the feature indices which have zero standard deviation. #
# If any such indices exist, Replace the zero standard deviation value with a small number #
###################################################################################################################################################################
logP_class = np.log([1-np.mean(y_train), np.mean(y_train)])
zero_std = np.nonzero(std==0)[0]
if (np.any(zero_std)):
np.place(std, std==0, 0.0001)
predict = []
for i in range(0, X_test.shape[0]):
# Compute independent probabilities of features w.r.t. class
P_X = np.divide(np.exp(-1*(np.divide(np.power(np.subtract(X_test[i, :].reshape(X_test.shape[1], 1), mean), 2), 2*np.power(std, 2)))), math.sqrt(2*np.pi)*std)
# Compute class prediction for the test sample
Class_X = np.argmax(logP_class+np.sum(np.nan_to_num(np.log(P_X)), axis=0))
predict.append(Class_X)
print ("Experiment-1: Gaussian Naive Bayes")
print ("Accuracy: ",metrics.accuracy_score(predict,y_test))
print ("Precision: ",metrics.precision_score(y_test, predict))
print ("Recall: ",metrics.recall_score(y_test, predict))
print ("Confusion Matrix:\n",metrics.confusion_matrix(y_test, predict))
###################################################################################################################################################################
def JPEG_DCT(imgfile):
# 3 channels x 8 bits, indexed-colors
img = cv2.imread(imgfile, cv2.CV_LOAD_IMAGE_COLOR)
wndName = "original indexed-color"
cv2.namedWindow(wndName, cv2.WINDOW_AUTOSIZE)
cv2.imshow(wndName, img)
iHeight, iWidth = img.shape[:2]
print "size:", iWidth, "x", iHeight
# set size to multiply of 8
if (iWidth % 8) != 0:
filler = img[:, iWidth - 1:, :]
#print "width filler size", filler.shape
for i in range(8 - (iWidth % 8)):
img = np.append(img, filler, 1)
if (iHeight % 8) != 0:
filler = img[iHeight - 1:, :, :]
#print "height filler size", filler.shape
for i in range(8 - (iHeight % 8)):
img = np.append(img, filler, 0)
iHeight, iWidth = img.shape[:2]
print "new size:", iWidth, "x", iHeight
# convert to YCrCb, Y=luminance, Cr/Cb=red/blue-difference chrominance
img = cv2.cvtColor(img, cv2.COLOR_BGR2YCR_CB)
# array as storage for DCT + quant.result
img2 = np.empty(shape=(iHeight, iWidth, 3))
# FORWARD ----------------
# do calc. for each 8x8 non-overlapping blocks
cv2.imshow("dctimage", img2)
for startY in range(0, iHeight, 8):
for startX in range(0, iWidth, 8):
for c in range(0, 3):
block = img[startY:startY + 8, startX:startX + 8,
c:c + 1].reshape(8, 8)
# apply DCT for a block
blockf = np.float32(block) # float conversion
dst = cv2.dct(blockf) # dct
# quantization of the DCT coefficients
blockq = np.around(np.divide(dst, std_quant_tbl[c]))
blockq = np.multiply(blockq, std_quant_tbl[c])
# store the result
for y in range(8):
for x in range(8):
img2[startY + y, startX + x, c] = blockq[y, x]
# INVERSE ----------------
for startY in range(0, iHeight, 8):
for startX in range(0, iWidth, 8):
for c in range(0, 3):
block = img2[startY:startY + 8, startX:startX + 8,
c:c + 1].reshape(8, 8)
blockf = np.float32(block) # float conversion
dst = cv2.idct(blockf) # inverse dct
np.place(dst, dst > 255.0, 255.0) # saturation
np.place(dst, dst < 0.0, 0.0) # grounding
block = np.uint8(np.around(dst))
# store the results
for y in range(8):
for x in range(8):
img[startY + y, startX + x, c] = block[y, x]
print img.dtype
# convert to BGR
img = cv2.cvtColor(img, cv2.COLOR_YCR_CB2BGR)
wndName1 = "DCT + quantization + inverseDCT"
cv2.namedWindow(wndName1, cv2.WINDOW_AUTOSIZE)
cv2.imshow(wndName1, img)
cv2.waitKey(0)
cv2.destroyWindow(wndName1)
cv2.destroyWindow(wndName)
param_path = "../results/keep/fig031921/ber_3/params.json"
params = settings.load_param(param_path)
# n_sum = params["test_bits"] * params['SNR_AVERAGE'] * load_files
n_sum = params['test_bits'] * params['trials']
snrs_db = np.linspace(params['graph_x_min'], params['graph_x_max'], params['graph_x_num'])
fig, ax = graph.new_snr_ber_canvas(params['graph_x_min'], params['graph_x_max'], -4)
ax.set_yticks([10**0, 10**-1, 10**-2, 10**-3, 10**-4])
previous_n_sum = params['previous_test_bits'] * params['trials']
pkl_path = "../results/keep/fig031921/ber_3/result.pkl"
result = load_pkl_file(pkl_path)
errors_sum = np.sum(result.non_cancell_error_array, axis=1)
bers = errors_sum / previous_n_sum
np.place(bers, bers == 0, None)
ax.plot(snrs_db, bers, color='k', marker='x', linestyle=':', label="w/o canceller " + r"$(I=1)$", ms=10)
errors_sum = np.sum(result.previous_errors, axis=1)
bers = errors_sum / previous_n_sum
np.place(bers, bers == 0, None)
ax.plot(snrs_db, bers, color='k', marker='o', linestyle='--', label="Conventional " + r"$(I=1)$" + " [5]" , ms=10, markerfacecolor='None')
pkl_path = "../results/keep/fig031921/ber_1/result.pkl"
result = load_pkl_file(pkl_path)
errors_sum = np.sum(result.errors, axis=1)
bers = errors_sum / n_sum
np.place(bers, bers == 0, None)
ax.plot(snrs_db, bers, color='k', marker='^', linestyle='-', label="Proposed " + r"$(I=1)$", ms=10)
# pkl_path = "../results/ofdm_fde_system_model/2021/01/21/22_24_29/result.pkl"
assert(magic_num==2049)
raw_labels=f.read(num_labs)
test_labs=[i for i in raw_labels]
ncols=20
nrows=20
rand_indices=(10000*np.random.random([ncols*nrows])).astype("int")
img_mat=np.zeros((img_height*nrows, img_width*ncols))
mat_rselector=np.matlib.repmat(np.floor(np.arange(0, nrows*img_height)/img_height), ncols*img_width, 1).transpose()
mat_cselector=np.matlib.repmat(np.floor(np.arange(0, ncols*img_width)/img_width), nrows*img_height, 1)
for i in range(0, nrows):
labels=[]
for j in range(0, ncols):
np.place(img_mat, (mat_rselector==i) & (mat_cselector==j), train_imgs[rand_indices[i*ncols+j]])
labels.append(train_labs[rand_indices[i*ncols+j]])
print(labels)
plt.imshow(img_mat, cmap="Greys")
plt.show()
num_classes=10
test_labs=keras.utils.to_categorical(test_labs)
train_labs=keras.utils.to_categorical(train_labs)
test_imgs=test_imgs.reshape(test_imgs.shape[0], img_height, img_width, 1)
train_imgs=train_imgs.reshape(train_imgs.shape[0], img_height, img_width, 1)
test_imgs=test_imgs.astype('float32')
train_imgs=train_imgs.astype('float32')
test_imgs/=255
def broadcast_to(
value,
index: Index,
grouper: "Grouper" = None,
) -> Series:
"""Broastcast value to expected dimension, the result is a series with
the given index
Before calling this function, make sure that the index is already
broadcasted based on the value. This means that index is always the wider
then the value if it has one. Also the size for each group is larger then
the length of the value if value doesn't have an index.
Examples:
>>> # 1. broadcast scalars/arrays(no index)
>>> broadcast_to(1, Index(range(3)))
>>> # Series: [1,1,1], index: range(3)
>>> broadcast_to([1, 2], Index([0, 0, 1, 1]))
>>> # ValueError
>>>
>>> # grouper: size() # [1, 1]
>>> # grouper: result_index # ['a', 'b']
>>> broadcast_to([1, 2], Index([0, 0, 1, 1]), grouper)
>>> # Series: [1, 2, 1, 2], index: [0, 0, 1, 1]
>>>
>>> # 2. broadcast a Series
>>> broadcast(Series([1, 2]), index=[0, 0, 1, 1])
>>> # Series: [1, 1, 2, 2], index: [0, 0, 1, 1]
>>> broadcast(
>>> Series([1, 2], index=['a', 'b']),
>>> index=[4, 4, 5, 5],
>>> grouper,
>>> )
>>> # Series: [1, 1, 2, 2], index: [4, 4, 5, 5]
Args:
value: Value to be broadcasted
index: The index to broadcasted to
grouper: The grouper of the original value
Returns:
The series with the given index
"""
if not is_list_like(value):
return value
if len(value) == 1 and not is_list_like(value[0]):
return value[0]
if not grouper:
if len(value) == 0:
return Series(value, index=index, dtype=object)
# Series will raise the length problem
return Series(value, index=index)
gsizes = grouper.size()
if gsizes.size == 0:
return Series(value, index=index, dtype=object)
# broadcast value to each group
# length of each group is checked in _broadcast_base
# A better way to distribute the value to each group?
idx = np.concatenate(
[grouper.groups[gdata] for gdata in grouper.result_index])
# make np.tile([[3, 4]], 2) to be [[3, 4], [3, 4]],
# instead of [[3, 4, 3, 4]]
repeats = (grouper.ngroups, ) + (1, ) * (np.ndim(value) - 1)
# Can't do reindex for this case:
# >>> df = tibble(a=[1, 1, 2, 2, 3, 3], _index=[1, 1, 2, 2, 3, 3])
# >>> df["b"] = [4, 5] # error
out = Series(np.tile(value, repeats).tolist(), index=idx)
if out.index.is_unique:
return out.reindex(index)
# recode index
index_dict = out.index.to_frame().groupby(out.index).grouper.indices
new_idx = Index(range(out.size))
new_index = np.ones(out.size, dtype=int)
for ix, rowids in index_dict.items():
np.place(new_index, index == ix, rowids)
out.index = new_idx
out = out.reindex(new_index)
new_idx = np.ones(out.size, dtype=int)
for ix, rowids in index_dict.items():
new_idx[np.isin(new_index, rowids)] = ix
out.index = new_idx
return out
def voxel_samples3(image_path, patch_sizes):
"""
:param image_path: path to image generated by get_paths
:param patch_sizes: a list or array of patch sizes desired for convolution. Must be odd > 1.
:param total_sample_size: sampled patches for convolution, and corresponding labels
:return: sampled patches for convolution, and corresponding labels
"""
# Load image from path
Image = io.loadmat(image_path)
# Extract total number of voxels
Dims = np.prod(Image['scan'].shape)
# Create 1 mask image
Label = (Image['Tibia'] + 2 * Image['Femur'])
np.place(Label, Label > 2, [2, 1])
if Image['isright'] == 0:
Scan = np.fliplr(Image["scan"])
Label = np.fliplr(Label)
else:
Scan = Image["scan"]
Index = np.array([])
Label2 = (Image['Tibia'] + Image['Femur'])
B = morphology.distance_transform_cdt(np.absolute(Label2 - 1))
B = B.reshape(Dims)
X = np.float32(range(np.max(B), 0, -1)) / range(1, np.max(B) + 1, 1)
#print max(X)
#print np.unique(B)
#print min(X)
No_samples = 36000
Index = np.array([])
samples = np.divide(X, sum(X)) * No_samples
#print len(samples)
for i in range(1, np.max(B) + 1):
# print i
Index = np.int32(
np.append(
Index,
np.random.choice(np.squeeze(np.where(B == i)),
(samples[i - 1]),
replace=False)))
Label = Label.reshape(Dims)
# Find indexes of voxels to sample
Gold_ind = np.random.choice(np.squeeze(np.where(Label == 1)),
12000,
replace=False)
Silv_ind = np.random.choice(np.squeeze(np.where(Label == 2)),
12000,
replace=False)
# Edge_ind = np.squeeze(np.where((B>0) & (B<=3)))
# print Edge_ind.shape
#Neg_Samp_ind = np.random.choice(np.squeeze(np.where(Label == 0)), 3*N_samp, replace=False)
Index = np.hstack((Gold_ind, Silv_ind, Index))
np.random.shuffle(Index)
Patch_List = []
# for each desired patch size, create patches
for size in patch_sizes:
PS = (size - 1) / 2
npad = ((PS, PS), (PS, PS), (PS, PS))
window_size = (size, size, size)
image_pad = np.pad(Scan,
pad_width=npad,
mode='constant',
constant_values=0)
Patches = (view_as_windows(image_pad, window_size)).reshape(
(Dims, size, size, size))
Patch_List.append(
np.float32(Patches[Index, :, :, :].reshape(Index.shape[0], 1, size,
size, size)))
Inds = np.asarray(np.unravel_index(Index, Scan.shape, order='C'))
Positions = Inds.reshape(3, 1, Inds.shape[1]).T
Labels = Label[Index]
Patch_List = tuple(Patch_List)
return Patch_List, np.float32(Positions), np.int32(Labels)
