def splitSpectra(self,rawSpectra,inputMask):
assert((np.shape(inputMask)[1] == 3) or (np.shape(inputMask)[1] == 1))
print('socket.gethostname() = ', socket.gethostname())
if (socket.gethostname() == "dalibornb"):
rawSpectra[:,:,0] = ma.array(rawSpectra[:,:,0],mask=inputMask)
for i in range(len(rawSpectra)):
rawSpectra[i] = ma.array(ma.mask_rows(rawSpectra[i],axis=1))
if (np.shape(inputMask)[1] == 3): # masks for forward
if (socket.gethostname() == "dalibornb"):
self.dataEast = rawSpectra[:,0,:][~np.all(np.isnan(rawSpectra[:,0,:]),axis=1)]
self.dataCenter = rawSpectra[:,1,:][~np.all(np.isnan(rawSpectra[:,1,:]),axis=1)]
self.dataWest = rawSpectra[:,2,:][~np.all(np.isnan(rawSpectra[:,2,:]),axis=1)]
else:
for i in range(len(rawSpectra)):
if (inputMask[i][0].all() == False):
self.dataEast.append(rawSpectra[i,0,:])
if (inputMask[i][1].all() == False):
self.dataCenter.append(rawSpectra[i,1,:])
if (inputMask[i][2].all() == False):
self.dataWest.append(rawSpectra[i,2,:])
elif (np.shape(inputMask)[1] == 1): # for backscan
if (socket.gethostname() == "dalibornb"):
self.dataBack = rawSpectra[:,0,:][~np.all(np.isnan(rawSpectra[:,0,:]),axis=1)]
else:
for i in range(len(rawSpectra)):
if (inputMask[i].all() == False):
self.dataBack.append(rawSpectra[i,0,:])
return
def get_fence(self, xyfence):
"""Get surface values along fence."""
cxarr = xyfence[:, 0]
cyarr = xyfence[:, 1]
czarr = xyfence[:, 2].copy()
# czarr will be updated "inplace":
istat = _cxtgeo.surf_get_zv_from_xyv(
cxarr,
cyarr,
czarr,
self.ncol,
self.nrow,
self.xori,
self.yori,
self.xinc,
self.yinc,
self.yflip,
self.rotation,
self.get_values1d(),
)
if istat != 0:
logger.warning("Seem to be rotten")
xyfence[:, 2] = czarr
xyfence = ma.masked_greater(xyfence, xtgeo.UNDEF_LIMIT)
xyfence = ma.mask_rows(xyfence)
return xyfence
def maskData(df, mask):
dfMask = ma.MaskedArray(df)
dfMask = ma.masked_where(dfMask == mask, dfMask)
dfMask = ma.mask_rows(dfMask)
for i in dfMask:
if not i.mask.all():
print(i.data)
return dfMask
def format_and_clean_data_main(self):
"""
Main function to format and clean data based on choices by the user.
"""
# Check if over missing_bound percent or missing_bound number of values are missing
too_many_missing = self.has_too_many_missing(self.init_perc_remove)
if ma.any(too_many_missing):
idx, = ma.where(too_many_missing)
self.xs[idx] = ma.mask_rows(self.xs[idx])
# Check array to see if it is filled with values or empty
if ma.all(self.check_for_all()):
return self.xs
# Clean outliers
self.clean_outliers()
# Take average of neighbor values to fill up to a given missing value gap length
self.clean_gaps_w_linspace(fill_gap_length=self.max_gap_length)
if ma.all(ma.count_masked(self.xs[:, :-self.keep_n_values], axis=1)[np.newaxis,:] == 0):
return self.xs # if no masked values remain in values before recent ones
# Remove values if they start the array and are then followed by too many masked values
start_idx = self.find_new_starting_value()
# If there are over x% blank values left in the original data after above changes,
# check to see if x% of the blanks fall after the new start year
too_many_missing = self.has_too_many_missing(self.second_perc_remove) # boolean array
if ma.any(too_many_missing):
n_masked = np.array([ma.count_masked(self.xs[i,s_idx:])
for i, s_idx in enumerate(start_idx)]) / self.N > self.perc_remove_after_start_idx
if ma.any(n_masked):
idx, = ma.where(n_masked)
self.xs[idx] = ma.mask_rows(self.xs[idx])
# To fill in remaining values, run linear regression on non-zero values
self.clean_gaps_w_lin_regress(start_idx)
# If linear regression left negative or zero values, then use linear space to fill in middle gaps
if ma.any(ma.masked_less_equal(self.xs, 0.)):
self.clean_gaps_w_linspace()
def updateStructOptPB(self):
print self.ControlledTank
#--- Matrix A -----------------------#
self.VolMaxGPC = np.sum(np.array([self.CSOT_cf[x]['Volume'] for x in self.ControlledTank]))
A_temp = self.A.copy()
A_temp[:self.NbXPart , :self.NbXPart] /= float(self.VolMaxGPC)
A_temp[:self.NbXPart , :self.NbXPart] += np.identity(self.NbXPart)
# a_mask = ma.array(cvxpy.matrix(A_temp*self.CF_Weight),mask=False)
A_temp = np.nan_to_num(A_temp)
a_mask = ma.array(A_temp*self.CF_Weight,mask=False)
self.Uncont_TF = np.array([x not in self.ControlledTank for x in self.ConfigTanks])
self.Uncont_TF = np.kron(self.Uncont_TF,np.ones(self.MaxLag))
Index_UnCont = np.nonzero(self.Uncont_TF)[0]
a_mask.mask[:self.NbXPart,0] = self.Uncont_TF # Hom
a_mask.mask[-self.NbXPart:,0] = self.Uncont_TF # Ov
a_mask = ma.mask_rows(a_mask)
a_mask[np.ix_(np.arange(self.NbXPart), Index_UnCont)] = ma.masked# Hom
Vect = ~a_mask.any(axis=1).mask
a_mask = a_mask.compress(Vect.flatten(),axis=0)
self.AX = a_mask.filled(0)*self.x # other method : AvX and concatenate with self AoutAovX
self.maskB = ~Vect
self.constr = self.constr_cf # need deep copy ?
# Mask the constraints where only uncontrolled tanks are involved
self.constr.mask[:5*self.NbXPart] = np.tile(self.Uncont_TF,(1,5))
UnControlledTank = set(self.ConfigTanks)-set(self.ControlledTank)
Constr31TF = [not(x-UnControlledTank) for x in self.TanksName31]
self.constr.mask[5*self.NbXPart:] = Constr31TF
self.constr = list(self.constr.compressed())
TS1 = np.zeros((len(Index_UnCont),self.NbXPart))
Z = zip(np.arange(len(Index_UnCont)),Index_UnCont)
for tup in Z :
TS1[tup]=1
# The uncontrolled tanks outflows are assumed to be constant and equals to the QMaintenace flows over the prediction horizon
self.constr.append( TS1*self.x[self.NbXPart:2*self.NbXPart,0] == self.QMaint_extend[Index_UnCont] .T )
# The uncontrolled tanks overflows are assumed to be 0 over the prediction horizon
self.constr.append( TS1*self.x[2*self.NbXPart:,0] == np.zeros((np.sum(TS1),1)) )
def reduce_by_params(catalogs, params, galaxies=None, retro_reduce=True):
new_catalogs = []
for j, param in enumerate(params):
reduced = catalogs[0].reduce(catalogs, param, galaxies=galaxies)
if (retro_reduce):
reduced.data = ma.mask_rows(reduced.data.T).T
new_catalogs.append(reduced)
if len(new_catalogs) < 2:
return new_catalogs[0]
return new_catalogs
# In[41]:
print(thresholds)
# In[42]:
precision, recall, thresholds_2 = precision_recall_curve(y_test, y_score)
a = np.column_stack((recall,precision))
a = ma.masked_less_equal(a,0)
a = ma.mask_rows(a)
f1 = hmean(a,axis=1)
# In[43]:
threshold_maximizing_F1 = thresholds[np.argmax(f1)]
print('f1 optimizing threshold: {}'.format(threshold_maximizing_F1))
# In[ ]:
import numpy as np import numpy.ma as ma a = np.zeros((3, 3), dtype=np.int) a[1, 1] = 1 a a = ma.masked_equal(a, 1) a ma.mask_rows(a)
import numpy as np
import numpy.ma as ma
import pandas as pd
#get data
df = pd.read_csv('https://archive.ics.uci.edu/ml/'
'machine-learning-databases/iris/iris.data', header = None)
#randomize data
dfRand = df.iloc[0:-1].values
np.random.shuffle(dfRand)
#apply mask
dfMask = ma.MaskedArray(dfRand)
dfMask = ma.masked_where(dfMask == 'Iris-setosa', dfMask)
dfMask = ma.mask_rows(dfMask)
#get non-masked data
# for i in dfMask:
# if not i.mask.all():
# print(i.data)
#in method format
def maskData(df, mask):
dfMask = ma.MaskedArray(df)
dfMask = ma.masked_where(dfMask == mask, dfMask)
dfMask = ma.mask_rows(dfMask)
for i in dfMask:
if not i.mask.all():
print(i.data)
return dfMask