def getCompArray(self, datasetA, datasetB, plot):
warnings.filterwarnings("error")
aggregateDataA, offScreen = datasetA.getAggregateData()
aggregateDataB, offScreen = datasetB.getAggregateData()
results = []
# get x, y magnitude of difference between sets, and significance
for i in range(self.params['gridWidth']):
for j in range(self.params['gridHeight']):
# get two arrays for given plot
setA = aggregateDataA[i][j].getResult(plot)
setB = aggregateDataB[i][j].getResult(plot)
# only compare if mean counts of both are greater than one
if st.nanmean(aggregateDataA[i][j].getResult(0)) > 1 or st.nanmean(aggregateDataB[i][j].getResult(0)) > 1:
# print str(i) + ", " + str(j) + ": " + str(st.nanmean(setA))
try:
mww_z, p = stats.ranksums(setA, setB)
except UserWarning:
p = numpy.nan
results.append((st.nanmean(setA), st.nanmean(setB), p))
else:
# print str(i) + ", " + str(j) + ": " + str(0)
results.append((numpy.nan, numpy.nan, numpy.nan))
return results
def getCompArray(self, datasetA, datasetB, plot):
warnings.filterwarnings("error")
aggregateDataA, offScreen = datasetA.getAggregateData()
aggregateDataB, offScreen = datasetB.getAggregateData()
results = []
# get x, y magnitude of difference between sets, and significance
for i in range(self.params['gridWidth']):
for j in range(self.params['gridHeight']):
# get two arrays for given plot
setA = aggregateDataA[i][j].getResult(plot)
setB = aggregateDataB[i][j].getResult(plot)
# only compare if mean counts of both are greater than one
if st.nanmean(
aggregateDataA[i][j].getResult(0)) > 1 or st.nanmean(
aggregateDataB[i][j].getResult(0)) > 1:
# print str(i) + ", " + str(j) + ": " + str(st.nanmean(setA))
try:
mww_z, p = stats.ranksums(setA, setB)
except UserWarning:
p = numpy.nan
results.append((st.nanmean(setA), st.nanmean(setB), p))
else:
# print str(i) + ", " + str(j) + ": " + str(0)
results.append((numpy.nan, numpy.nan, numpy.nan))
return results
def generateMplotStats(self,
dataSetA,
dataSetB,
plot,
pairwise=False,
sig=0.05):
# catch warnings - eg we may not have enough samples for wilcoxon test
warnings.filterwarnings("error")
aggregateDataA, offScreen = dataSetA.getAggregateData()
aggregateDataB, offScreen = dataSetB.getAggregateData()
differenceFound = False
xvalues = []
yvalues = []
sizes = []
pvals = []
differenceNum = 0
# get x, y magnitude of difference between sets, and significance
for j in range(self.params['gridHeight']):
for i in range(self.params['gridWidth']):
# get two arrays for given plot
setA = aggregateDataA[i][j].getResult(plot)
setB = aggregateDataB[i][j].getResult(plot)
# only compare if mean counts of both are greater than one
if st.nanmean(
aggregateDataA[i][j].getResult(0)) > 0.5 or st.nanmean(
aggregateDataB[i][j].getResult(0)) > 0.5:
try:
if pairwise:
wilcoxon_t, p = stats.wilcoxon(setA, setB)
else:
mww_z, p = stats.ranksums(setA, setB)
except UserWarning:
p = 1
print("can't do stats on " + str(i) + " " + str(j))
xvalues.append(i)
yvalues.append(j)
# now work out difference to illustrate scale of difference
# given as proportion of the bigger number
if st.nanmean(setA) > st.nanmean(setB):
size = (st.nanmean(setA) -
st.nanmean(setB)) / st.nanmean(setA)
sizes.append(500 * size * size)
pvals.append(1 - p)
else:
size = (st.nanmean(setB) -
st.nanmean(setA)) / st.nanmean(setB)
sizes.append(500 * size * size)
pvals.append(p - 1)
# print str(i) + " " + str(j) + " " + str(p)
if p < sig:
differenceFound = True
differenceNum += 1
return differenceFound, {
'x': xvalues,
'y': yvalues,
's': sizes,
'p': pvals
}, differenceNum
def printBoxData(self, datasets, boxCoord, plot):
print "Box " + str(boxCoord)
means = []
print "Mean, StdDev, n"
for ds in datasets:
alldata = ds.getAggregateDataAsArray(plot)
boxdata = alldata[boxCoord[0]][boxCoord[1]]
means.append(st.nanmean(boxdata))
print str(st.nanmean(boxdata)) + ", " + str(numpy.std(boxdata)) + ", " + str(len(boxdata))
print "-----"
print str(st.nanmean(means)) + ", " + str(numpy.std(means)) + ", " + str(len(means))
for i in range(len(datasets)):
dsA = datasets[i]
alldata = dsA.getAggregateDataAsArray(plot)
boxdata = alldata[boxCoord[0]][boxCoord[1]]
for j in range(len(datasets))[i+1:]:
dsB = datasets[j]
alldataB = dsB.getAggregateDataAsArray(plot)
boxdataB = alldataB[boxCoord[0]][boxCoord[1]]
try:
mww_z, p = stats.ranksums(boxdata, boxdataB)
except UserWarning:
p = 1
if p <= 0.05:
print "Difference between " + dsA.label + " and " + dsB.label + ". p = " + str(p)
else:
print "Nothing between " + dsA.label + " and " + dsB.label + "(p=" + str(p) + ")"
def printBoxData(self, datasets, boxCoord, plot):
print "Box " + str(boxCoord)
means = []
print "Mean, StdDev, n"
for ds in datasets:
alldata = ds.getAggregateDataAsArray(plot)
boxdata = alldata[boxCoord[0]][boxCoord[1]]
means.append(st.nanmean(boxdata))
print str(st.nanmean(boxdata)) + ", " + str(
numpy.std(boxdata)) + ", " + str(len(boxdata))
print "-----"
print str(st.nanmean(means)) + ", " + str(
numpy.std(means)) + ", " + str(len(means))
for i in range(len(datasets)):
dsA = datasets[i]
alldata = dsA.getAggregateDataAsArray(plot)
boxdata = alldata[boxCoord[0]][boxCoord[1]]
for j in range(len(datasets))[i + 1:]:
dsB = datasets[j]
alldataB = dsB.getAggregateDataAsArray(plot)
boxdataB = alldataB[boxCoord[0]][boxCoord[1]]
try:
mww_z, p = stats.ranksums(boxdata, boxdataB)
except UserWarning:
p = 1
if p <= 0.05:
print "Difference between " + dsA.label + " and " + dsB.label + ". p = " + str(
p)
else:
print "Nothing between " + dsA.label + " and " + dsB.label + "(p=" + str(
p) + ")"
def reportCreate(data, paramDict):
report = copy.deepcopy(paramDict)
setKeys = data["DataSets"].keys()
# Order all Mod first, then all Org
setKeys.sort()
bestRes = ""
start = 0
end = len(setKeys)
middle = end / 2
i = start
while i < end / 2:
# Calculate Score
modBs = np.array(data["DataSets"][setKeys[i]])
obsBs = np.array(data["DataSets"][setKeys[middle]])
modBsmean = nanmean(modBs)
obsBsmean = nanmean(obsBs)
obsBsMinModBs = obsBs - modBs
obsBsMinMean = obsBs - obsBsmean
SSres = np.nansum(obsBsMinModBs ** 2)
SStot = np.nansum(obsBsMinMean ** 2)
ResNorm = SSres ** 0.5
if i == 0:
bestRes = copy.copy(ResNorm)
report[(setKeys[i] + "_RN")] = ResNorm # Norm of residuals
i = i + 1
middle = middle + 1
return report, bestRes
def team_rank(prev_rank_o,prev_rank_d,raw_o,raw_d,A,gp):
alpha = 0.2
raw_avg_all_o = st.nanmean(raw_o)
raw_avg_all_d = st.nanmean(raw_d)
raw_o = xnan(raw_o)
raw_d = xnan(raw_d)
#initialize to raw efficiency
adj_avg_o = raw_o
adj_avg_d = raw_d
#print adj_avg_o
#print raw_avg_all_o
#return None
#print dot(A,gp).T
cnt = 0
r_off = 1
r_def = 1
while cnt < 100 and not (r_off < 0.001 and r_def < 0.001):
adj_avg_o_prev = adj_avg_o
adj_avg_d_prev = adj_avg_d
adj_avg_o_opp = np.divide(np.dot(A,adj_avg_o_prev),gp)
adj_avg_d_opp = np.divide(np.dot(A,adj_avg_d_prev),gp)
'''d_off = np.divide(adj_avg_o_prev,gp)
d_def = np.divide(adj_avg_d_prev,gp)
adj_avg_o_opp = np.dot(A,xnan(d_off))
adj_avg_d_opp = np.dot(A,xnan(d_def))'''
#print 'adj opp',adj_avg_o_opp, adj_avg_d_opp
adj_o = raw_avg_all_o*(np.divide(raw_o,adj_avg_d_opp))
adj_d = raw_avg_all_d*(np.divide(raw_d,adj_avg_o_opp))
#print 'adj', adj_o, adj_d
adj_avg_o = prev_rank_o*(1-alpha)+adj_o*alpha
adj_avg_d = prev_rank_d*(1-alpha)+adj_d*alpha
r_off = np.linalg.norm(xnan(adj_avg_o_prev - adj_avg_o))
r_def = np.linalg.norm(xnan(adj_avg_d_prev - adj_avg_d))
#print r_off, r_def
#print 'count = %i' % cnt,
#print adj_avg_o, adj_avg_d
cnt += 1
#print adj_avg_o, adj_avg_d
for k in range(len(adj_avg_o)):
if np.isnan(adj_avg_o[k]):
adj_avg_o[k] = prev_rank_o[k]
if np.isnan(adj_avg_d[k]):
adj_avg_d[k] = prev_rank_d[k]
return adj_avg_o, adj_avg_d
def average_ams(model, X_train, X_test, Y_train, Y_test, W_train, W_test, cuts,
prob, bm):
ams_train_list = []
ams_test_list = []
for i in cuts:
ams_train, ams_test = ams_score(model, X_train, X_test, Y_train,
Y_test, W_train, W_test, i, prob, bm)
ams_train_list.append(ams_train)
ams_test_list.append(ams_test)
return nanmean(ams_train_list), nanmean(ams_test_list)
def all_means(cp): """Function accepts a cp object and returns a mean and standard deviation of all parameters for all fingerprints in this form: list of (feature, mu, std) for each feature """ data = [] for parameter_pair in zip(*[finger.items() for finger in cp.fingerprints()]): parameter, values = zip(*parameter_pair) print parameter, values print st.nanmean(values), st.nanstd(values) data.append([parameter[0], st.nanmean(values), st.nanstd(values)]) return data
def team_rank2(alpha_raw,beta_raw, ind_mat, alpha_pre, beta_pre ,W):
#alpha = 0.2
#initialize to raw efficiency
alpha_adj = np.array([np.average(alpha_raw[:,k]) for k in range(alpha_raw.shape[1])])
beta_adj = np.array([np.average(beta_raw[:,k]) for k in range(beta_raw.shape[1])])
#alpha_adj_all = st.nanmean(alpha_adj)
#beta_adj_all = st.nanmean(beta_adj)
alpha_raw_all = st.nanmean(alpha_adj)
beta_raw_all = st.nanmean(beta_adj)
print alpha_raw_all, beta_raw_all
#print raw_avg_all_o
#return None
#print dot(A,gp).T
cnt = 0
r_off = 1
r_def = 1
while cnt < 10 and not (r_off < 0.001 and r_def < 0.001):
alpha_adj_prev = alpha_adj
beta_adj_prev = beta_adj
#test1 = [sum(np.append(alpha_pre[k],np.multiply(np.true_divide(alpha_raw[:,k],beta_adj_prev[np.array([0,1])]),1))*W[:,k]) for k in range(alpha_raw.shape[1])]
#print test1
alpha_adj = [sum(np.append(alpha_pre[k],np.multiply(np.true_divide(alpha_raw[:,k],beta_adj_prev[ind_mat[:,k]]),alpha_raw_all))*W[:,k]) for k in range(alpha_raw.shape[1])]
#alpha_adj = [sum(np.multiply(np.true_divide(alpha_raw[:,k],beta_adj_prev[ind_mat[:,k]]),alpha_adj_all)*W[:,k]) for k in range(alpha_raw.shape[1])]
alpha_adj = np.array(alpha_adj)
#print alpha_adj
beta_adj = [sum(np.append(beta_pre[k],np.multiply(np.true_divide(beta_raw[:,k],alpha_adj_prev[ind_mat[:,k]]),beta_raw_all))*W[:,k]) for k in range(alpha_raw.shape[1])]
#beta_adj = [sum(np.multiply(np.true_divide(beta_raw[:,k],alpha_adj_prev[ind_mat[:,k]]),beta_adj_all)*W[:,k]) for k in range(alpha_raw.shape[1])]
beta_adj = np.array(beta_adj)
r_off = np.linalg.norm(xnan(alpha_adj_prev - alpha_adj))
r_def = np.linalg.norm(xnan(beta_adj_prev - beta_adj))
print alpha_adj, beta_adj
cnt += 1
#print adj_avg_o, adj_avg_d
'''for k in range(len(adj_avg_o)):
if np.isnan(adj_avg_o[k]):
adj_avg_o[k] = prev_rank_o[k]
if np.isnan(adj_avg_d[k]):
adj_avg_d[k] = prev_rank_d[k]'''
return alpha_adj, beta_adj
def plot_ensemble_mean_vectors(adcp,
fig=None,
title=None,
n_vectors=50,
return_panel=False):
"""
Generates a QPanel, plotting mean uv velocity vectors in the x-y plane.
Inputs:
adcp = ADCPData object
fig = input figure number [integer or None]
title = figure title text [string or None]
n_vectors = desired number of vectors [integer]
return_panel = optinally return the QPanel instead of the figure
Returns:
fig = matplotlib figure object, or
vectors = QPanel object
"""
dude = np.zeros((adcp.n_ensembles, 2), np.float64)
velocity = adcp.get_unrotated_velocity()
# this doesn't factor in depth, may integrate bad values if the have not been filtered into NaNs somehow
dude[:, 0] = sp.nanmean(velocity[:, :, 0], axis=1)
dude[:, 1] = sp.nanmean(velocity[:, :, 1], axis=1)
vectors = QPanel(velocity=dude,
u_vecs=n_vectors,
arrow_color='k',
title=title,
units='m/s')
if adcp.xy is not None:
vectors.x = adcp.xy[:, 0]
vectors.y = adcp.xy[:, 1]
vectors.xlabel = 'm'
vectors.ylabel = 'm'
vectors.equal_axes = True
elif adcp.lonlat is not None:
vectors.x = adcp.lonlat[:, 0]
vectors.y = adcp.lonlat[:, 1]
vectors.xy_is_lonlat = True
else:
vectors.x = adcp.mtime
vectors.y = np.zeros(np.size(vectors.x))
vectors.x_is_mtime = True
if return_panel:
return vectors
else:
fig = get_fig(fig)
vectors.plot()
plt.tight_layout()
return fig
def getBoxData(self, dataset, plot, coords):
alldata = dataset.getAggregateDataAsArray(plot)
box = alldata[coords[0]][coords[1]]
mean = st.nanmean(box)
median = st.nanmedian(box)
sigma = numpy.std(box)
return box, (mean, median, sigma)
def writeBoxesToCsvFile(self, dataset, filename, plot):
boxArray = dataset.getAggregateDataAsArray(plot)
outFile = open(filename, 'w')
outFile.write("Means:\n")
#print("Means:\n")
for j in reversed(range(self.params['gridHeight'])):
dataline = ""
for i in range(self.params['gridWidth']):
dataline += str(st.nanmean(boxArray[i][j]))
dataline += ","
#print dataline
dataline += "\n"
outFile.write(dataline)
outFile.write("\nStandardDeviations:\n")
#print("\nStandardDeviations:\n")
for j in reversed(range(self.params['gridHeight'])):
dataline = ""
for i in range(self.params['gridWidth']):
dataline += str(numpy.std(boxArray[i][j]))
dataline += ","
#print dataline
dataline += "\n"
outFile.write(dataline)
outFile.close()
def cinfo(CL,param):
""" This property returns information on the parameter in the cloud (all given in the units of the parameter). Note that the parameter is averaged over the entire cloud time at the altitude required (bottom, top or in-cloud) - not the case using vpinfo(CL,param).
CloudObj.cinfo["bottom"]: param at the cloud base
CloudObj.cinfo["top"]: param at the cloud top
CloudObj.cinfo["mean"]: mean param through the cloud (in cloud)
CloudObj.cinfo["median"]: median param through the cloud (in cloud)
CloudObj.cinfo["stdev"]: standard deviation of the param through the cloud (in cloud)
CloudObj.cinfo["delta"]: difference of param between the bottom and the top
CloudObj.cinfo["slope"]: delta divided by the mean thickness
The property can be accessed as e.g. CloudObj.cinfo["bottom"] or CloudObj.cinfo (dictionary) """
H=dict()
H["bottom"]=list(); H["top"]=list(); H["mean"]=list(); H["median"]=list(); H["stdev"]=list(); H["delta"]=list(); H["slope"]=list(); H["units"]=list();
alt=[i for i,x in enumerate(CL.dttl) if x == 'altitude'][0]
T=[i for i,x in enumerate(CL.dttl) if x == param][0]
try:
for i in range(len(CL.props["height"])):
ix=nonzero((CL.data[alt]>=CL.props["height"][i][1])*(CL.data[alt]<=CL.props["height"][i][2]))
H["bottom"].append(float(st.nanmedian(CL.data[T][nonzero((CL.data[alt]>=CL.props["height"][i][0])*(CL.data[alt]<=CL.props["height"][i][1]))])))
H["top"].append(float(st.nanmedian(CL.data[T][nonzero((CL.data[alt]>=CL.props["height"][i][2])*(CL.data[alt]<=CL.props["height"][i][3]))])))
H["mean"].append(float(st.nanmean(CL.data[T][ix])))
H["median"].append(float(st.nanmedian(CL.data[T][ix])))
H["stdev"].append(float(st.nanstd(CL.data[T][ix])))
H["delta"].append(H["bottom"][i]-H["top"][i])
H["slope"].append(H["delta"][i]/(np.mean([CL.props["height"][i][2], CL.props["height"][i][3]])-np.mean([CL.props["height"][i][0], CL.props["height"][i][1]]))) # units/meter
H["units"].append(CL.dunit[T])
del ix
except: print("[cinfo] Height properties must be defined first using the defheight method.")
return H
def getBoxData(self, dataset, plot, coords):
alldata = dataset.getAggregateDataAsArray(plot)
box = alldata[coords[0]][coords[1]]
mean = st.nanmean(box)
median = st.nanmedian(box)
sigma = numpy.std(box)
return box, (mean, median, sigma)
def load_roi(nii, thresh=6000, standardize=True, outlen=None, padding=np.mean, meantc=False):
"""Load a nifti file and apply several functions to it.
nii -- name of nifti to load or ndarray
standardize -- return all tcs with mean = 0 and stdev = 1
outlen -- specify how long resultant tcs should be, pad if necessary
padding -- function to apply to ndarray, must take axis=-1 argument
meantc -- return the meantc over all voxels (before evaluating standardize)
TODO: fix default outlen arg
"""
if not hasattr(nii, 'shape'): nii = nib.load(nii).get_data()
nii[nii.mean(axis=-1) < thresh] = np.nan
if meantc: out = nanmean(nii.reshape([-1, nii.shape[-1]]), axis=0) #USE apply_over_axes instead?
else: out = nii
if standardize: out = (out - out.mean(axis=-1)[..., np.newaxis]) / out.std(axis=-1, ddof=1)[..., np.newaxis] #this is unruly, use the current standardize func
outdiff = outlen - out.shape[-1]
if outdiff >= 0:
if hasattr(padding, 'func_name'): padding = padding(out, axis=-1)[..., np.newaxis] #call padding if it is a function
out = np.concatenate([out] + [padding]*outdiff, axis=-1)
else:
out = out[..., :outlen]
return out
def writeBoxesToCsvFile(self, dataset, filename, plot):
boxArray = dataset.getAggregateDataAsArray(plot)
outFile = open(filename, 'w')
outFile.write("Means:\n")
#print("Means:\n")
for j in reversed(range(self.params['gridHeight'])):
dataline = ""
for i in range(self.params['gridWidth']):
dataline += str(st.nanmean(boxArray[i][j]))
dataline += ","
#print dataline
dataline += "\n"
outFile.write(dataline)
outFile.write("\nStandardDeviations:\n")
#print("\nStandardDeviations:\n")
for j in reversed(range(self.params['gridHeight'])):
dataline = ""
for i in range(self.params['gridWidth']):
dataline += str(numpy.std(boxArray[i][j]))
dataline += ","
#print dataline
dataline += "\n"
outFile.write(dataline)
outFile.close()
def generateMplotStats(self, dataSetA, dataSetB, plot, pairwise=False, sig=0.05):
# catch warnings - eg we may not have enough samples for wilcoxon test
warnings.filterwarnings("error")
aggregateDataA, offScreen = dataSetA.getAggregateData()
aggregateDataB, offScreen = dataSetB.getAggregateData()
differenceFound = False
xvalues = []
yvalues = []
sizes = []
pvals = []
differenceNum=0
# get x, y magnitude of difference between sets, and significance
for j in range(self.params['gridHeight']):
for i in range(self.params['gridWidth']):
# get two arrays for given plot
setA = aggregateDataA[i][j].getResult(plot)
setB = aggregateDataB[i][j].getResult(plot)
# only compare if mean counts of both are greater than one
if st.nanmean(aggregateDataA[i][j].getResult(0)) > 0.5 or st.nanmean(aggregateDataB[i][j].getResult(0)) > 0.5:
try:
if pairwise:
wilcoxon_t, p = stats.wilcoxon(setA, setB)
else:
mww_z, p = stats.ranksums(setA, setB)
except UserWarning:
p = 1
print("can't do stats on " + str(i) + " " + str(j))
xvalues.append(i)
yvalues.append(j)
# now work out difference to illustrate scale of difference
# given as proportion of the bigger number
if st.nanmean(setA) > st.nanmean(setB):
size = (st.nanmean(setA) - st.nanmean(setB))/st.nanmean(setA)
sizes.append(500*size*size)
pvals.append(1-p)
else:
size = (st.nanmean(setB) - st.nanmean(setA))/st.nanmean(setB)
sizes.append(500*size*size)
pvals.append(p-1)
# print str(i) + " " + str(j) + " " + str(p)
if p < sig:
differenceFound = True
differenceNum+=1
return differenceFound, {'x': xvalues, 'y':yvalues, 's':sizes, 'p':pvals} , differenceNum
def plot_ensemble_mean_vectors(adcp,fig=None,title=None,n_vectors=50,return_panel=False):
"""
Generates a QPanel, plotting mean uv velocity vectors in the x-y plane.
Inputs:
adcp = ADCPData object
fig = input figure number [integer or None]
title = figure title text [string or None]
n_vectors = desired number of vectors [integer]
return_panel = optinally return the QPanel instead of the figure
Returns:
fig = matplotlib figure object, or
vectors = QPanel object
"""
dude = np.zeros((adcp.n_ensembles,2),np.float64)
velocity = adcp.get_unrotated_velocity()
# this doesn't factor in depth, may integrate bad values if the have not been filtered into NaNs somehow
dude[:,0] = sp.nanmean(velocity[:,:,0],axis=1)
dude[:,1] = sp.nanmean(velocity[:,:,1],axis=1)
vectors = QPanel(velocity = dude,
u_vecs = n_vectors,
arrow_color = 'k',
title = title,
units = 'm/s')
if adcp.xy is not None:
vectors.x = adcp.xy[:,0]
vectors.y = adcp.xy[:,1]
vectors.xlabel = 'm'
vectors.ylabel = 'm'
vectors.equal_axes = True
elif adcp.lonlat is not None:
vectors.x = adcp.lonlat[:,0]
vectors.y = adcp.lonlat[:,1]
vectors.xy_is_lonlat = True
else:
vectors.x = adcp.mtime
vectors.y = np.zeros(np.size(vectors.x))
vectors.x_is_mtime = True
if return_panel:
return vectors
else:
fig = get_fig(fig)
vectors.plot()
plt.tight_layout()
return fig
def fit_spec_poly5(xData, yData, dyData, order=5):
"""
Fit a 5th order polynomial to a spectrum. To avoid overflow errors the
X-axis data should not be large numbers (e.g.: x10^9 Hz; use GHz instead).
"""
# Lower order limit is a line with slope
if order < 1:
order = 1
if order > 5:
order = 5
# Estimate starting coefficients
C1 = nanmean(np.diff(yData)) / nanmedian(np.diff(xData))
ind = int(np.median(np.where(~np.isnan(yData))))
C0 = yData[ind] - (C1 * xData[ind])
C5 = 0.0
C4 = 0.0
C3 = 0.0
C2 = 0.0
inParms = [{
'value': C5,
'parname': 'C5'
}, {
'value': C4,
'parname': 'C4'
}, {
'value': C3,
'parname': 'C3'
}, {
'value': C2,
'parname': 'C2'
}, {
'value': C1,
'parname': 'C1'
}, {
'value': C0,
'parname': 'C0'
}]
# Set the polynomial order
for i in range(len(inParms)):
if len(inParms) - i - 1 > order:
inParms[i]['fixed'] = True
else:
inParms[i]['fixed'] = False
# Function to evaluate the difference between the model and data.
# This is minimised in the least-squared sense by the fitter
def errFn(p, fjac=None):
status = 0
return status, (poly5(p)(xData) - yData) / dyData
# Use mpfit to perform the fitting
mp = mpfit(errFn, parinfo=inParms, quiet=True)
return mp
def produce_stats(data):
m = {}
for (server, size, conns, it, time, rate) in data:
map_add(m, (server, size, conns), rate)
data = []
for k, v in m.items():
mean = stats.nanmean(v)
stddev = stats.nanstd(v)
data += [k + (mean, stddev)]
return data
def compute_average(nh=10, lr_num=10, lr_denum=1000, prefix='rbm', smoothing=True):
cmd = "grep -rl --include='orig.conf' 'lr_num = %i$' . |" % lr_num +\
"xargs grep 'lr_denum = %i$' " % lr_denum
print cmd
p = os.popen(cmd)
numseeds = len([pi for pi in enumerate(p)])
p = os.popen(cmd)
x = numpy.ones((numseeds, 20)) * numpy.nan
y = numpy.ones((numseeds, 20)) * numpy.nan
for i, match in enumerate(p):
jid = match.split('/')[1]
rfname = '%s/%s_train_callback.hdf5' % (jid, prefix)
if not os.path.exists(rfname):
continue
fp = tables.openFile(rfname)
_x = fp.root.train_ll.col('n')
_y = fp.root.train_ll.col('train_ll')
_vlogz = fp.root.var_logz.col('var_logz')
fp.close()
if smoothing:
idx = numpy.where(_vlogz < 50.)[0]
x[i, idx] = _x[idx]
y[i, idx] = _y[idx]
else:
x[i, :len(_x)] = _x
y[i, :len(_y)] = _y
print '**** prefix=%s nh=%i lr_num=%s lr_denum=%s ******' % (prefix, nh, lr_num, lr_denum)
print nanmean(y, axis=0)
xmean = nanmean(x, axis=0)
ymean = nanmean(y, axis=0)
ystd = nanstd(y, axis=0)
ystd[numpy.isnan(ystd)] = 0.
idx = ~numpy.isnan(xmean)
return [xmean[idx], ymean[idx], ystd[idx]]
def aggregate_ftr_matrix(self, ftr_matrix):
sig = []
for ftr in ftr_matrix:
median = stats.nanmedian(ftr)
mean = stats.nanmean(ftr)
std = stats.nanstd(ftr)
# Invalid double scalars warning appears here
skew = stats.skew(ftr) if any(ftr) else 0.0
kurtosis = stats.kurtosis(ftr)
sig.extend([median, mean, std, skew, kurtosis])
return sig
def aggregate_ftr_matrix(self, ftr_matrix):
sig = []
for ftr in ftr_matrix:
median = stats.nanmedian(ftr)
mean = stats.nanmean(ftr)
std = stats.nanstd(ftr)
# Invalid double scalars warning appears here
skew = stats.skew(ftr) if any(ftr) else 0.0
kurtosis = stats.kurtosis(ftr)
sig.extend([median, mean, std, skew, kurtosis])
return sig
def dmso_means(cp): """Function accepts a cp object and returns a mean and standard deviation of all parameters for the dmso fingerprints in this form: list of (feature, mu, std) for each feature """ dmsos = [dmso.items() for dmso in get_dmsos(cp)] data = [] for parameter_pair in zip(*dmsos): parameter, values = zip(*parameter_pair) data.append((parameter[0], st.nanmean(values), st.nanstd(values))) return data
def getBoxArray(self, dataset, plot):
alldata = dataset.getAggregateDataAsArray(plot)
boxes = []
for row in alldata:
for resultArray in row:
# print box
# resultArray = box.getResults(plot)
mean = st.nanmean(resultArray)
sd = numpy.std(resultArray)
count = len(resultArray) - numpy.isnan(resultArray).sum()
boxes.append((mean, sd, count))
return boxes
def getBoxArray(self, dataset, plot):
alldata = dataset.getAggregateDataAsArray(plot)
boxes = []
for row in alldata:
for resultArray in row:
# print box
# resultArray = box.getResults(plot)
mean = st.nanmean(resultArray)
sd = numpy.std(resultArray)
count = len(resultArray) - numpy.isnan(resultArray).sum()
boxes.append((mean, sd, count))
return boxes
def addDescriptives(self):
"""Adds averages and errors to the PivotMatrix"""
# Determine the row averages and std
self.rowMeans = []
self.rowStds = []
for rowIndex in range(self.nRows):
row = self.m[self.rowHeaders+rowIndex][self.colHeaders:-2]
self.rowMeans.append(nanmean(row, axis=None))
self.rowStds.append(nanstd(row, axis=None))
self.m[self.rowHeaders+rowIndex][-2] = nanmean(row, axis=None)
self.m[self.rowHeaders+rowIndex][-1] = nanstd(row, axis=None)
# Determine the column averages and std
_m = self.m.swapaxes(0,1)
self.colMeans = []
self.colErrs = []
for colIndex in range(self.nCols):
col = _m[self.colHeaders+colIndex][self.rowHeaders:-2]
_m[self.colHeaders+colIndex][-2] = nanmean(col, axis=None)
if self.err == '95ci':
e = nanstd(col, axis=None)/np.sqrt(col.size)*1.96
elif self.err == 'se':
e = nanstd(col, axis=None)/np.sqrt(col.size)
elif self.err == 'std':
e = nanstd(col, axis=None)
else:
raise Exception('Err keyword must be "95ci", "se", or "std"')
_m[self.colHeaders+colIndex][-1] = e
self.colMeans.append(nanmean(col, axis=None))
self.colErrs.append(e)
# Determine the grand average and std
self.m[-2,-2] = nanmean(self.m[self.rowHeaders:-2, self.colHeaders:-2], \
axis=None)
self.m[-1,-1] = nanstd(self.m[self.rowHeaders:-2, self.colHeaders:-2], \
axis=None)
def avsizedist(CL,prof='belowcloud',scan=0,inst='PCASP',filler=0):
""" Returning method. This method returns the average size distribution for a given instrument and scan type and number. The returned average size distribution is an array with the mean concentration of particles in the first row and the size of the particles in the second.
Example: R=CloudObj.avsizedist(prof='abovecloud',scan=1,inst='PCASP') would return in R the average size distribution for the above cloud #1 for the PCASP.
The defaults are prof='belowcloud',scan=0,inst='PCASP', filler=0
filler: filling with a given time step (useful for 2d type data) =1 ON, =0 OFF. Time step can be adjusted in the call of the method filler
Special profile "belowvert" can also be used, this we return the average size distribution below cloud, where a vertical profile was measured."""
cancel=0
instn=[i for i,x in enumerate(CL.sd) if x["Distname"].upper()==inst.upper()]
if len(instn)<=0: cancel=1; raise ValueError("This instrument name has not been found (inst=%s)." % inst)
if filler==1: sd=samac.filler(CL,CL.times["cloud"][0],inst=inst)
else: sd=copy.deepcopy(CL.sd[instn[0]])
if type(sd["time"])==float: cancel=1
elif len(sd["time"])==0: cancel=1
if cancel==1: avsd=[]; return avsd
if prof=='belowvert':
if len(sd["time"])<=1: # interpolation cannot take place if there is not at least 2 points to interpolate from
print("[avsizedist] Only 1 point available: special profile belowvert cannot be used.")
avsd=[]; return avsd
ctime=[i for i,x in enumerate(CL.dttl) if x == 'time']; ctime=ctime[0]
basictime=CL.data[ctime]
if sum(basictime==sd["time"])==len(basictime): pass # the instrument share the timeline of the basic data from which belowvert is calculated
else: # the instrument does not share the basic timeline and must be interpolated
if sum(sd["data"])>0: M=np.min(sd["data"][sd["data"]>0.0]) # minimum non-zero concentration
else: M=0
f=interpolate.RectBivariateSpline(sd["bins"],sd["time"],sd["data"])
sd["data"]=f(sd["bins"],basictime); sd["time"]=basictime;
sd["data"][sd["data"]<0.01*M]=0.0 # all interpolated data smaller than 1% of the minimum.
R=belowprof(CL)
try:
if np.size(R)>0:
Tix=R[scan]['areaindex']
concs=sd["data"][:,nonzero(Tix)[0]]
else: print("[avsizedist] There is no data below cloud under vertical profile #%d." % scan)
except: print("[avsizedist] The instrument number (%d) may be wrong, or no data corresponds to the profile." % instn[0])
else:
try:
T=CL.times[prof][scan]
except: print("[avsizedist] The profile %s [%d] doesn't exist." % (prof,scan))
try:
concs=sd["data"][:,nonzero((sd["time"]>=T[0])*(sd["time"]<=T[1]))[0]]
except: print("[avsizedist] The instrument number (%d) may be wrong, or no data corresponds to the profile." % instn[0])
if "concs" in locals():
if len(sd["time"])==1: avsd=[concs,sd["bins"]]
else: avsd=np.array([st.nanmean(concs,axis=1), sd["bins"]])
return avsd
else:
print("[avsizedist] Failed to return the averaged size distribution.")
avsd=[]; return avsd
def reportCreate(data, paramDict): report = copy.deepcopy(paramDict) setKeys = data['DataSets'].keys() # Order all Mod first, then all Org setKeys.sort() start = 0 end = len(setKeys) middle = end/2 i = start while i < end/2: # Calculate Score modBs = np.array(data['DataSets'][setKeys[i]]) obsBs = np.array(data['DataSets'][setKeys[middle]]) modBsmean = nanmean(modBs) obsBsmean = nanmean(obsBs) obsBsMinModBs = obsBs - modBs obsBsMinMean = obsBs - obsBsmean SSres = (np.nansum(obsBsMinModBs**2)) SStot = (np.nansum(obsBsMinMean**2)) ResNorm = SSres**0.5 report[(setKeys[i]+'_RN')] = ResNorm # Norm of residuals version i = i+1 middle = middle+1 return report, ResNorm
def collapse(self, keys, vName): """ desc: Collapse the data by a (list of) keys and get statistics on a dependent variable. arguments: keys: desc: A key or list of keys to collapse the data on. type: [list, str, unicode] vName: desc: The dependent variable to collapse. Alternative, you can specifiy a function, in which case the error will be 0. type: [str, unicode, function] returns: desc: A DataMatrix with the collapsed data, with the descriptives statistics on `vName`. type: DataMatrix """ if isinstance(keys, basestring): keys = [keys] m = [keys + ['mean', 'median', 'std', 'se', '95ci', 'count']] for g in self.group(keys): l = [] for key in keys: l.append(g[key][0]) if type(vName) == types.FunctionType: l.append(vName(g)) l.append(np.nan) l.append(np.nan) l.append(np.nan) l.append(np.nan) l.append(len(g)) else: a = g[vName] l.append(nanmean(a)) l.append(nanmedian(a)) l.append(nanstd(a)) l.append(nanstd(a)/np.sqrt(a.size)) l.append(1.96*nanstd(a)/np.sqrt(a.size)) l.append(a.size) m.append(l) return DataMatrix(m)
def roimask(data, roi, filter_func = None, proc_func = None, mean_ts = False):
"""Mask data using values in roi > 0
Parameters:
data -- numpy array to be masked
roi -- array to be checked for nonzero values
filter_func -- function to further subset roi (e.g. remove time courses with mean < 6000)
proc_func -- TODO
"""
if type(roi) == str: roi = nib.load(roi).get_data()
if len(roi.shape) > 3: roi = roi[:,:,:,0]
shapes = roi.shape, data.shape
if roi.shape[:3] != data.shape[:3]: raise BaseException('roi shape: %s \ndata shape: %s'%shapes)
roi_indx = np.nonzero(roi)
roi = data[roi_indx]
if filter_func: roi[filter_func(roi)] = np.nan
if mean_ts: roi = nanmean(roi, axis=0)
return roi
def run_stats(x,n):
"""runstats(x,n). Calculates and returns the running mean, median, standard deviation, and median absolute deviation (MAD). This function handles NaNs and masked values (masked arrays) by ignoring them.
x (input) is the array on which the running statistics are calculated (only one dimension, 1D array).
n is the number of points taken in the running statistics window."""
x=copy.deepcopy(x)
try: x.mask
except:
x=np.ma.array(x,mask=False)
if len(np.shape(x))>2: raise ValueError("The array provided has more than 2 dimensions, at most 1 or 2 dimensions can be handled.")
try: [ro,co]=np.shape(x)
except: ro=np.shape(x)[0]; co=1
if ro==1 or co==1:
ro=max(ro,co)
x=x.reshape(ro,)
else: raise ValueError("The array must be a vector (one column or row)")
# initializing matrix
M=ones([ro,n])*NaN;
M=ma.asanyarray(M)
# building matrix
if n%2==1: # if n is odd
for j in range(int(n/2),0,-1):
posi=int(n/2)-j # current position
M[0:ro-j,posi]=x[j:]
for j in range(1,2+int(n/2),1):
posi=int(n/2)+j-1;
M[j-1:,posi]=x[0:(ro+1)-j]
elif n%2==0: # if n is even
for j in range(n/2,0,-1):
posi=n/2-j
M[0:ro-j,posi]=x[j:]
for j in range(1,n/2+1):
posi=n/2+j-1;
M[j-1:,posi]=x[0:(ro+1)-j]
else: print("Well, that's pretty weird. Are you sure n is an integer?")
M.data[M.mask]=nan
ave=st.nanmean(M, axis=1);
med=st.nanmedian(M, axis=1);
stde=st.nanstd(M, axis=1);
mad=medabsdev(M,axis=1)
return [ave, med, stde, mad]
def rasSubras(flist, flistfolder, Afile, outfolder):
"""Subtract one file from others in list
To use: filnm = rasSubras(flist,flistfolder,Afile,outfolder) """
# time_one = datetime.now()
# print time_one.strftime('Start raster subtraction at:%j %H:%M:%S')
# Get first raster file
Adata, AXg, AYg, AXg1, AYg1, Arx, Ary, Ademmeta = DemImport(Afile)
Amean = nanmean(Adata)
# Get second for creating mask to deal with NaN
# Bfile = os.path.join(flistfolder,flist[1])
# Bdata,BXg,BYg,BXg1,BYg1,Brx,Bry,Bdemmeta = DemImport(Bfile)
# Adata[Adata==Ademmeta[6]]=np.nan
# Bdata[Bdata==Bdemmeta[6]]=np.nan
# AdataMasked = np.ma.masked_array(np.nan_to_num(Adata), mask=np.isnan(Adata) & np.isnan(Bdata))
counter = 1
print "\n" * 3
print "*" * 20
# Go through list of other raster files
for B in flist:
print "file ", counter + 1, " of ", len(flist)
Bfile = os.path.join(flistfolder, B)
Bdata, BXg, BYg, BXg1, BYg1, Brx, Bry, Bdemmeta = DemImport(Bfile)
print B, " data type: ", type(Bdata)
print "Rows (A,B): ", Arx, Brx, " Columns (A,B): ", Ary, Bry
print "xres (A,B): ", Ademmeta[2][1], Bdemmeta[2][1], " yres (A,B): ", Ademmeta[2][5], Bdemmeta[2][5]
print "No data (A): ", Ademmeta[6], " No data (B): ", Bdemmeta[6]
# Check matching resolution
if Arx != Brx or Ary != Bry:
print B, " resolution mismatch with ", Afile
continue
elif Ademmeta[4] != Bdemmeta[4] or Ademmeta[5] != Bdemmeta[5]:
print "Size mismatch between ", B, " and ", Afile
continue
# Add current file to sum
Bdata[Bdata == Bdemmeta[6]] = np.nan
AdataMasked = np.ma.masked_array(np.nan_to_num(Adata), mask=np.isnan(Adata) & np.isnan(Bdata))
BdataMasked = np.ma.masked_array(np.nan_to_num(Bdata), mask=AdataMasked.mask)
outdata = (BdataMasked - AdataMasked).filled(np.nan)
outdata = np.ma.masked_array(np.nan_to_num(outdata), mask=np.isnan(outdata))
counter = counter + 1
outname = namer(B)[0] + "Sub" + namer(Afile)[0]
filnm = datawrite(outdata, Adata, Ademmeta, outname, outfolder)
return filnm
def generatePlotArray(self, boxArray, plot, sampleSize):
resultArray = numpy.empty((self.gridy, self.gridx))
#Spot the artwork : changed from nan to zeros
resultArray[:] = numpy.nan
means = []
for i in range(self.gridx):
for j in range(self.gridy):
box = boxArray[i][j]
count = box.count
try:
count = sum(count)
except:
print "count exception"
pass
if count > sampleSize / 4: # only plot if averaged at least one fixation per participant
# Compute the mean over the given axis ignoring nans.
z = st.nanmean(box.getResult(plot))
resultArray[j][i] = z
means.append(z)
resultArrayStatistics = {}
resultArrayStatistics.update(
{"Minimum value": str(numpy.nanmin(resultArray))})
resultArrayStatistics.update(
{"Max value": str(numpy.nanmax(resultArray))})
resultArrayStatistics.update(
{"Mean value": str(numpy.nanmean(resultArray))})
resultArrayStatistics.update({"Std": str(numpy.nanstd(means))})
print "Min :" + str(numpy.nanmin(resultArray)) + " Box index : " + str(
numpy.nanargmin(resultArray))
print "Max :" + str(numpy.nanmax(resultArray)) + " Box index : " + str(
numpy.nanargmax(resultArray))
print "Mean:" + str(numpy.nanmean(resultArray))
print "Standard deviation" + str(numpy.nanstd(means))
print "Area of interest size : " + str(
np.count_nonzero(~np.isnan(resultArray))) + " " + str(
size(resultArray))
#Spot the artwork : print array to check inconsistencies
#print resultArray
return resultArray, resultArrayStatistics
def fit_spec_poly5(xData, yData, dyData, order=5):
xData = np.array(xData, dtype='f8')
yData = np.array(yData, dtype='f8')
# Estimate starting coefficients
C1 = nanmean(np.diff(yData)) / nanmedian(np.diff(xData))
ind = int(np.median(np.where(~np.isnan(yData))))
C0 = yData[ind] - (C1 * xData[ind])
if order<1:
order=1
p0 = [0.0, 0.0, 0.0, 0.0, C1, C0]
# Set the order
p0 = p0[(-order-1):]
def chisq(p, x, y):
return np.sum( ((poly5(p)(x) - y)/ dyData)**2.0 )
# Use minimize to perform the fit
return op.fmin_bfgs(chisq, p0, args=(xData, yData), full_output=1)
def runstats(x,n):
# Stephanie Gagne, UHel, 2010
# converted to Python, Dal, 2012
# x is an array of 1 dimension.
# n is the number of point taken in the running statistic
"""takes data, number of points for the running mean/standard deviation and returns the running mean and running standard deviation."""
try: x.mask
except:
x=ma.asanyarray(x);
x.mask=ones(np.shape(x))*False
try: [ro,co]=np.shape(x)
except: ro=np.shape(x)[0]; co=1
if ro==1 or co==1: x=x.reshape(max(ro,co),)
else: print("The array must be a vector (one column or row)")
# initializing matrix
ro=max(ro,co)
M=ones([ro,n])*NaN;
M=ma.asanyarray(M)
# building matrix
if n%2==1: # if n is odd
for j in range(int(n/2),0,-1):
posi=int(n/2)-j # current position
M[0:ro-j,posi]=x[j:]
for j in range(1,2+int(n/2),1):
posi=int(n/2)+j-1;
M[j-1:,posi]=x[0:(ro+1)-j]
elif n%2==0: # if n is even
for j in range(n/2,0,-1):
posi=n/2-j
M[0:ro-j,posi]=x[j:]
for j in range(1,n/2+1):
posi=n/2+j-1;
M[j-1:,posi]=x[0:(ro+1)-j]
else: print("Well, that's pretty weird. Are you sure n is an integer?")
M.data[M.mask]=NaN
ave=st.nanmean(M, axis=1);
stde=st.nanstd(M, axis=1);
return [ave, stde]
def multilooking(src, xscale, yscale, thresh=0):
"""
Implementation of Matlab Pirate looks.m function.
src: numpy array of phase data
thresh: min number of non-NaNs required for a valid tile resampling
"""
thresh = int(thresh)
num_cells = xscale * yscale
if thresh > num_cells or thresh < 0:
msg = "Invalid threshold: %s (need 0 <= thr <= %s" % (thresh,
num_cells)
raise ValueError(msg)
rows, cols = src.shape
rows_lowres = int(floor(rows / yscale))
cols_lowres = int(floor(cols / xscale))
dest = ones((rows_lowres, cols_lowres)) * nan
size = xscale * yscale
for row in range(rows_lowres):
for col in range(cols_lowres):
ys = row * yscale
ye = ys + yscale
xs = col * xscale
xe = xs + xscale
patch = src[ys:ye, xs:xe]
num_values = num_cells - npsum(isnan(patch))
if num_values >= thresh and num_values > 0:
# nanmean() only works on 1g axis
reshaped = patch.reshape(size)
dest[row, col] = nanmean(reshaped)
return dest
def generatePlotArray(self, boxArray, plot, sampleSize):
resultArray = numpy.empty((self.gridy, self.gridx))
#Spot the artwork : changed from nan to zeros
resultArray[:] = numpy.nan
means=[]
for i in range(self.gridx):
for j in range(self.gridy):
box = boxArray[i][j]
count = box.count
try:
count = sum(count)
except:
print "count exception"
pass
if count > sampleSize / 4 : # only plot if averaged at least one fixation per participant
# Compute the mean over the given axis ignoring nans.
z = st.nanmean(box.getResult(plot))
resultArray[j][i] = z
means.append(z)
resultArrayStatistics = {}
resultArrayStatistics.update({"Minimum value":str(numpy.nanmin(resultArray))})
resultArrayStatistics.update({"Max value":str(numpy.nanmax(resultArray))})
resultArrayStatistics.update({"Mean value":str(numpy.nanmean(resultArray))})
resultArrayStatistics.update({"Std":str(numpy.nanstd(means))})
print "Min :" + str(numpy.nanmin(resultArray)) + " Box index : " + str(numpy.nanargmin(resultArray))
print "Max :" + str(numpy.nanmax(resultArray)) + " Box index : " + str(numpy.nanargmax(resultArray))
print "Mean:" + str(numpy.nanmean(resultArray))
print "Standard deviation" + str(numpy.nanstd(means))
print "Area of interest size : " + str(np.count_nonzero(~np.isnan(resultArray))) + " " + str(size(resultArray))
#Spot the artwork : print array to check inconsistencies
#print resultArray
return resultArray, resultArrayStatistics
csv_writer = csv.writer(csvfile, delimiter=',',quotechar='|')
csv_writer.writerow(hot[beginning_time:end_time])
with open('volume all 394 gp data.csv', 'ab') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=',',quotechar='|')
csv_writer.writerow(gp[beginning_time:end_time])
#Append data
hot_total_volume.append(hot[beginning_time:end_time])
gp_total_volume.append(gp[beginning_time:end_time])
date_1 += timedelta(days=1)
#Average volume at each time across all days
average_hot_volume = sci.nanmean(hot_total_volume,axis=0)
variance_hot_volume = sci.nanstd(hot_total_volume,axis=0)
average_gp_volume = sci.nanmean(gp_total_volume,axis=0)
variance_gp_volume = sci.nanstd(gp_total_volume,axis=0)
#Group into 3 minute increments (6 x 30 seconds)
k=0
resolution = 6
while k in range(0,len(average_hot_volume)):
average_hot_volume[k:k+resolution] = sum(average_hot_volume[k:k+resolution])/resolution
variance_hot_volume[k:k+resolution] = sum(variance_hot_volume[k:k+resolution])/resolution
average_gp_volume[k:k+resolution] = sum(average_gp_volume[k:k+resolution])/resolution
variance_gp_volume[k:k+resolution] = sum(variance_gp_volume[k:k+resolution])/resolution
k+=resolution
#Write out averaged 3 minute data
def stack(stype='day'):
db = connect()
components_to_compute = get_components_to_compute(db)
output_folder = get_config(db, 'output_folder')
export_format = get_config(db, 'export_format')
if export_format == "BOTH":
mseed = True
sac = True
elif export_format == "SAC":
mseed = False
sac = True
elif export_format == "MSEED":
mseed = True
sac = False
if stype == "day":
start, end, datelist = build_daystack_datelist(db)
format = "stack"
elif stype == "mov":
start, end, datelist = build_movstack_datelist(db)
format = "matrix"
mov_stack = get_config(db, "mov_stack")
if mov_stack.count(',') == 0:
mov_stacks = [
int(mov_stack),
]
else:
mov_stacks = [int(mi) for mi in mov_stack.split(',')]
mov_stacks.remove(1) #remove 1 day stack, it should exist already
elif stype == "ref":
start, end, datelist = build_ref_datelist(db)
format = "stack"
for f in get_filters(db, all=False):
filterid = int(f.ref)
for components in components_to_compute:
for station1, station2 in get_station_pairs(db, used=True):
sta1 = "%s_%s" % (station1.net, station1.sta)
sta2 = "%s_%s" % (station2.net, station2.sta)
pair = "%s:%s" % (sta1, sta2)
if updated_days_for_dates(db,
start,
end,
pair.replace('_', '.'),
type='CC',
interval='1 DAY'):
logging.debug("New Data for %s-%s-%i" %
(pair, components, filterid))
if stype in ['mov', 'ref']:
nstack, stack_total = get_results(db,
sta1,
sta2,
filterid,
components,
datelist,
format=format)
updated_days = updated_days_for_dates(db,
start,
end,
pair.replace(
'_', '.'),
type='CC',
interval='1 DAY',
returndays=True)
if nstack > 0:
if stype == "mov":
for i, date in enumerate(datelist):
jobadded = False
for mov_stack in mov_stacks:
if i < mov_stack:
low = 0
high = mov_stack
else:
low = i - mov_stack + 1
high = i + 1
newdata = False
for uday in updated_days:
if uday in datelist[low:high]:
newdata = True
break
if newdata:
corr = stack_total[low:high]
if not np.all(np.isnan(corr)):
day_name = "%s_%s" % (sta1,
sta2)
logging.debug(
"%s %s [%s - %s] (%i day stack)"
% (day_name, date,
datelist[low],
datelist[i], mov_stack))
corr = nanmean(corr, axis=0)
corr = scipy.signal.detrend(
corr)
stack_path = os.path.join(
"STACKS",
"%02i" % filterid,
"%03i_DAYS" % mov_stack,
components, day_name)
filename = os.path.join(
stack_path, str(date))
if mseed:
export_mseed(
db, filename, pair,
components, filterid,
corr)
if sac:
export_sac(
db, filename, pair,
components, filterid,
corr)
day_name = "%s:%s" % (sta1,
sta2)
if not jobadded:
add_job(
db, date,
day_name.replace(
'_', '.'), 'DTT')
jobadded = True
del corr
elif stype == "ref":
stack_path = os.path.join(
"STACKS", "%02i" % filterid, "REF",
components)
ref_name = "%s_%s" % (sta1, sta2)
filename = os.path.join(stack_path, ref_name)
stack_total = scipy.signal.detrend(stack_total)
if mseed:
export_mseed(db, filename, pair,
components, filterid,
stack_total)
if sac:
export_sac(db, filename, pair, components,
filterid, stack_total)
ref_name = "%s:%s" % (sta1, sta2)
add_job(db, "REF", ref_name.replace('_', '.'),
'DTT')
del stack_total
elif stype == 'day':
updated_days = updated_days_for_dates(db,
start,
end,
pair.replace(
'_', '.'),
type='CC',
interval='1 DAY',
returndays=True)
for date in updated_days:
date = date
print "Stacking %s day=%s" % (pair, date)
daystack = os.path.join("STACKS",
"%02i" % filterid,
"001_DAYS", components,
"%s_%s" % (sta1, sta2),
str(date))
stack = np.zeros(get_maxlag_samples(db))
ncorr = 0
try:
os.makedirs(os.path.split(daystack)[0])
except:
pass
path = os.path.join(output_folder,
"%02i" % filterid, sta1, sta2,
components, str(date))
if os.path.isdir(path):
for file in os.listdir(path):
if len(file) == 8:
st = read(os.path.join(path, file))
if not np.any(np.isnan(
st[0].data)) and not np.any(
np.isinf(st[0].data)):
stack += st[0].data
ncorr += 1
if ncorr > 0:
if mseed:
export_mseed(db, daystack, pair,
components, filterid,
stack / ncorr, ncorr)
if sac:
export_sac(db, daystack, pair,
components, filterid,
stack / ncorr, ncorr)
del stack
def turbhrz(CL,MaxCA=2,MinDist=20000,PtNo=25,Pts=5,Pts2=50):
""" This method calculates the turbulence average over at least 20 km or more in a horizontal enough leg (the plane cannot be above an angle of 2 degrees using a running standard deviation (of 25 points) to calculate the turbulence. It is assumed that the TAS is in the basic data, along with other aircraft related data (e.g. extradata module not handled for TAS). Extradata module handled for updraft velocity.
Options:
MaxCA: maximum change in the combined angle (phi+theta) in degrees. Default is 2.
MinDist: minimum distance (in meters) an aircraft must be level enough for to calculated the turbulence (updraft). Default is 20000 or 20 km.
PtNo: number of points from which to calculate the turbulence=stdev(updraft vel). Default is 25
Pts: number of points taken in running statistics of angles. Default is 5.
Pts2: number of points taken in running statistics of the combined angle. Default is 50.
Returns a dictionary:
R["InCloud"]: List with one item for each stretch level enough for at least MinDist km in cloud (N, first=0). Each item contains an array with the turbulence value R["InCloud"][N][0] and the distance on which it was calculated R["InCloud"][N][1].
R["BlwCloud"]: List with one item for each stretch level enough for at least 20 km below cloud (N, first=0). Each item contains an array with the turbulence value R["InCloud"][N][0] and the distance on which it was calculated R["InCloud"][N][1].
R["InCloudStretches"]: List with one item for each stretch level enough during at least 20 km in cloud (N, first=0). Each item contains an array with the indexes corresponding to the stretch used to calculate the corresponding turbulence value.
R["BlwCloudStretches"]: List with one item for each stretch level enough during at least 20 km below cloud (N, first=0). Each item contains an array with the indexes corresponding to the stretch used to calculate the corresponding turbulence value. """
palt=[i for i,x in enumerate(CL.dttl) if x == 'altitude'][0]
ptim=[i for i,x in enumerate(CL.dttl) if x == 'time'][0]
ptas=[i for i,x in enumerate(CL.dttl) if 'TAS' in x.upper()][0]
pudv=[i for i,x in enumerate(CL.dttl) if x == 'udvel']
if len(pudv)==1: pudv=pudv[0]; udv=CL.data[pudv][0:-1]
elif len(pudv)>1: print("[turbhrz] multiple updraft velocities found in the basic data.")
else:
posx=[]
for i,ttl in enumerate(CL.extrattl): # for all extra datasets available
posx=posx+[[i,j] for j,x in enumerate(ttl) if x.lower() == 'udvel'] # check all titles matching with updraft velocity
if len(posx)==1:
udv=CL.extradata[posx[0][0]][posx[0][1]] # loading the updraft velocity data
j=[j for j,x in enumerate(CL.extrattl[i]) if x.lower() == 'time'][0]
udvt=CL.extradata[posx[0][0]][j] # loading associated time stamp
# padding so the interpolation works
if udvt[0]>CL.data[ptim][0]: udv=hstack((nan,udv)); udvt=hstack((CL.data[ptim][0],udvt));
if udvt[-1]<CL.data[ptim][-1]: udv=hstack((udv,nan)); udvt=hstack((udvt,CL.data[ptim][-1]));
fudv=interpolate.interp1d(udvt,udv,kind='linear')
udv=fudv(CL.data[ptim])[0:-1]
else: print("[turbhrz] No updraft velocity (or multiple) found in the basic or the extra data."); return dict()
Alt=CL.data[palt][0:-1]
A=samac.angles(CL)
theta=samac.runstats(A[1],Pts)[0] # change of altitude
phi=diff(A[2])
phi=np.ma.masked_where((phi>350)+(isnan(phi)), phi)
phi=np.ma.masked_where((phi<-350+(isnan(phi))), phi)
phi=hstack([NaN,samac.runstats(phi,Pts)[0]]) # change in direction
phi[(abs(phi)>=89)]=0 # accounts for jump from 0 to 360 degrees. After smoothing, the limit cannot be too high.
CAstats=samac.runstats(abs(theta)+abs(phi),Pts2) # combined angle
CA=CAstats[0]
# udv=CL.data[pudv][0:-1] # updraft
Sp=CL.data[ptas][0:-1]
dt=diff(CL.data[ptim]*24*60*60)
D=cumsum(Sp*dt)
# Initializing indexes for below cloud and in-cloud measurements.
bc=ones(np.shape(CL.data[ptim]))*False
ic=ones(np.shape(CL.data[ptim]))*False
for j in range(len(CL.times["belowcloud"])):
bctmp=(CL.data[ptim]>=CL.times["belowcloud"][j][0])*(CL.data[ptim]<=CL.times["belowcloud"][j][1])
try: bc=bc+bctmp;
except: bc=bctmp;
for j in range(len(CL.times["horicloud"])):
ictmp=(CL.data[ptim]>=CL.times["horicloud"][j][0])*(CL.data[ptim]<=CL.times["horicloud"][j][1])
try: ic=ic+ictmp;
except: ic=ictmp;
# Below Cloud #
Stret=list()
i=0
while i<len(CA):
ix=list()
if (CA[i]<=MaxCA and bc[i]==True):
while (i<len(CA) and CA[i]<=MaxCA and bc[i]==True):
ix.append(i)
i=i+1
Stret.append(ix)
else: i=i+1
BCStretches=list()
for n,leg in enumerate(Stret):
Dist=D[leg];
Dist=Dist[(Dist.mask==False)]
if (Dist[-1]-Dist[0]) >= MinDist: # if the stretch is longer than the minimum distance.
BCStretches.append(np.array(leg))
del Dist
# In-Cloud #
Stret=list()
i=0
while i<len(CA):
ix=list()
if (CA[i]<=MaxCA and ic[i]==True):
while (i<len(CA) and CA[i]<=MaxCA and ic[i]==True):
ix.append(i)
i=i+1
Stret.append(ix)
else: i=i+1
ICStretches=list()
for n,leg in enumerate(Stret):
Dist=D[leg]; Dist=Dist[(Dist.mask==False)]
if (Dist[-1]-Dist[0]) >= MinDist: # if the strecht is longer than the minimum distance.
ICStretches.append(np.array(leg))
del Dist
# Calculating the average turbulence.
ICturb=list(); BCturb=list()
for leg in ICStretches:
Dist=D[leg]; Dist=Dist[(Dist.mask==False)]
if (sum(leg)>PtNo and sum(~isnan(udv[leg]))>PtNo):
turb=st.nanmean(samac.runstats(udv[leg],PtNo)[1])
#print("In-Cloud %d: %.4f km. w' = %0.4f m/s" %(cn,(Dist[-1]-Dist[0])/1000,turb))
ICturb.append(np.array([turb,(Dist[-1]-Dist[0])]))
del Dist
for leg in BCStretches:
Dist=D[leg]; Dist=Dist[(Dist.mask==False)]
if (sum(leg)>PtNo and sum(~isnan(udv[leg]))>PtNo):
turb=st.nanmean(samac.runstats(udv[leg],PtNo)[1])
#print("Below-Cloud %d: %.4f km. w' = %0.4f m/s" %(cn,(Dist[-1]-Dist[0])/1000,turb))
BCturb.append(np.array([turb,(Dist[-1]-Dist[0])]))
else: BCturb.append(np.array([nan,(Dist[-1]-Dist[0])]))
del Dist
R=dict()
R["InCloud"]=ICturb; R["BlwCloud"]=BCturb; R["InCloudStretches"]=ICStretches; R["BlwCloudStretches"]=BCStretches
return R
data_array = np.array(map(parse_img, files))
data_array = data_array.astype(float) # convert to float to use nans
times = np.array(map(parse_time, files))
data_array = data_array[np.argsort(times)]
# median 3x3 filter
data_array = median_filter(data_array, size=(1, 3, 3))
# flatfield parsing
directory = './'+flat_field+'/'
files = [directory+str(i) for i in os.listdir(directory) if 'tif' in i]
print '...start parsing flatfields'
flat_field_array = np.array(map(parse_img, files))
mean = np.mean(flat_field_array, axis=(0))
mean = mean.astype(float)
ff = mean / nanmean(mean)
# print out delta time
print 'parsed all files and flatfields.'
print 'time: {0} minute'.format((time.time()-time_Start) / 60)
data_array = data_array / ff
# divide out background
bkg = data_array[:, 0:20, 0:20]
mean_bkg = np.ma.mean(bkg, axis=(1, 2))
data_array = data_array / mean_bkg[:, np.newaxis, np.newaxis]
data_array[data_array == 0.] = 1
# Save the data into a hdf5 file
while d < days:
minDTO = startDTO + dt.timedelta(days=d)
maxDTO = startDTO + dt.timedelta(days=d + 1)
criterion = (
(cdata[:, 0] >= calendar.timegm(minDTO.utctimetuple())) &
(cdata[:, 0] < calendar.timegm(maxDTO.utctimetuple())))
tind = np.array(np.where(criterion))[0]
ele_model = []
# check we have some data for each day
if np.size(tind) > 0:
# split the data for this test
blkm, blkmstd = esm.blockMedian(cdata[tind, 1:4])
for j in range(0, 181):
ele_model.append(nanmean(blkm[:, j]))
ele_model = np.array(ele_model)
eleMedians[d, :] = np.array(ele_model)
d += 1
elevation = []
for j in range(0, 181):
elevation.append(90. - j * 0.5)
#===========================================================
fig = plt.figure(figsize=(3.62, 2.76))
ax = fig.add_subplot(111)
for i in range(0, np.shape(eleMedians)[0]):
ax.plot(elevation, eleMedians[i, :], alpha=0.5)
# now compute the over all median
resultOptim[itr, ridx] = resultOpt
resultQuant[itr, ridx] = resultQu
resultBAful[itr, ridx] = resultBA
# Drop data points with more than x percent outage
outageThreshold = 1
resultOptim[:, (np.sum(np.isnan(resultOptim), 0) /
float(iterations) > outageThreshold)] = np.nan
resultQuant[:, (np.sum(np.isnan(resultQuant), 0) /
float(iterations) > outageThreshold)] = np.nan
resultBAful[:, (np.sum(np.isnan(resultBAful), 0) /
float(iterations) > outageThreshold)] = np.nan
# Average over iterations
resultOptim = scistats.nanmean(resultOptim, axis=0)
resultQuant = scistats.nanmean(resultQuant, axis=0)
resultBAful = scistats.nanmean(resultBAful, axis=0)
# Put into single array for storage
upperlimit = (bs.p0 + bs.m * cell.pMax) * np.ones(rateSteps)
result = np.array([rate, upperlimit, resultBAful, resultOptim, resultQuant])
filename = 'resultsJSAC'
from results import resultshandler
resultshandler.saveBin(filename, result)
# timeit
print 'Code time %.0f seconds' % (time.time() - start)
# lastly, try to generate the plot right away
def mean(x):
""" E\{%s\} := Average %s """
from scipy.stats.stats import nanmean
x = notnone(x)
return nanmean(x)
def DoSummaryInfo(obs, estimated) :
return({'rmse': numpy.sqrt(sss.nanmean((estimated - obs) ** 2.0, axis = 1)),
'mae': sss.nanmean(numpy.abs(estimated - obs), axis=1),
'corr': numpy.diag(numpy.corrcoef(estimated, obs), k=estimated.shape[0]),
'sse': numpy.sum((estimated - obs) ** 2.0, axis = 1)})
def oneIteration(rate, CSI_Optim, SINR_Quant):
"""Perform one iteration and return the supply powers"""
### Step 1 ###
# Optimization call
import pdb
pdb.set_trace()
pSupplyOptim, resourceAlloc, status = optimMinPow.optimizePCDTX(
CSI_Optim, PnoiseIf_Optim, rate, wrld.PHY.systemBandwidth, cell.pMax,
bs.p0, bs.m, bs.pS)
print '{0:50} {1:5.2f} W'.format('Real-valued optimization objective:',
pSupplyOptim)
## Plot ##
if plotting:
channelplotter.ChannelPlotter().bar(resourceAlloc,
'Resource Share Optim',
'rscshare.pdf')
channelplotter.ChannelPlotter().bar(PnoiseIf_Optim,
'Interference power Optim',
'ifpower.pdf')
channelplotter.ChannelPlotter().bar(
((np.abs(np.mean(np.mean(CSI_Optim, 1), 1)))**2) /
(PnoiseIf_Optim), 'SINR Optim', 'OptimSINR.pdf')
channelplotter.ChannelPlotter().OFDMAchannel(SINR_Quant, 'SINR Quant',
'sinrquant.pdf')
channelplotter.ChannelPlotter().OFDMAchannel(
noiseIfPQuant, 'Noise Interference Power Quant',
'noiseifquant.pdf')
import pdb
pdb.set_trace()
### Step 2 ###
# Map real valued solution to OFDMA frame
# QUANTMAP
resourcesPerTimeslot = quantmap.quantmap(resourceAlloc, N, T)
outmap = np.empty([N, T])
for t in np.arange(T):
# RCG
outmap[:, t], _ = rcg.rcg(SINR_Quant[:, t, :], resourcesPerTimeslot[
t, :]) # outmap.shape = (N,T) tells the user index
# Given allocation and rate target, we inverse waterfill channels for each user separately on the basis of full CSI
# IWF
powerlvls = np.empty([N, T, mob.antennas])
powerlvls[:] = np.nan
for idx, obj in enumerate(
wrld.consideredMobiles): # WRONG TODO: Only all mobiles of a BS
# grab user CSI
CSI_usr = obj.OFDMA_SINR[:, :, outmap ==
idx] # all CSI assigned to this user
noiseIfPower_usr = CSI_usr[0, 0, :].repeat(
2
) * 0 + 1 # remove later #(obj.baseStations[obj.BS].cells[obj.cell].OFDMA_interferencePower + obj.baseStations[obj.BS].cells[obj.cell].OFDMA_noisePower) * np.ones(CSI_user_all[0,0,:,:].shape)[outmap==idx].ravel().repeat(2) # one IF value per resource, so repeat once to match spatial channels
# create list of eigVals
eigVals = np.real([
linalg.eig(CSI_usr[:, :, i])[0]
for i in np.arange(CSI_usr.shape[2])
]).ravel() # two eigvals (spatial channels) per resource
targetLoad = rate * wrld.PHY.simulationTime
# inverse waterfill and fill back to OFDMA position
powlvl, waterlvl, cap = iwf.inversewaterfill(
eigVals, targetLoad, noiseIfPower_usr,
wrld.PHY.systemBandwidth / N, wrld.PHY.simulationTime / T)
powerlvls[outmap == idx, :] = powlvl.reshape(CSI_usr.shape[2],
obj.antennas)
ptx = np.array([
np.nansum(np.nansum(powerlvls[:, t, :], axis=0), axis=0)
for t in np.arange(T)
])
if ptx.any() > cell.pMax:
raise ValueError('Transmission power too high in IWF.')
psupplyPerSlot = bs.p0 + bs.m * ptx
psupplyPerSlot[np.isnan(psupplyPerSlot)] = bs.pS
pSupplyQuant = np.mean(psupplyPerSlot)
print '{0:50} {1:5.2f} W'.format('Integer-valued optimization objective:',
pSupplyQuant)
### SOTA comparison ###
pSupplyBA = np.nan
CSI_BA = np.empty([2, 2, N, T, users], dtype=complex)
CSI_BA[:] = np.nan
noiseIfPowerPerResource = np.empty([N, T, users])
noiseIfPowerPerResource[:] = np.nan
for idx, obj in enumerate(wrld.consideredMobiles):
CSI_BA[:, :, :, :,
idx] = obj.baseStations[obj.BS].cells[obj.BS.cells[0]].CSI_OFDMA
noiseIfPowerPerResource[:, :,
idx] = obj.noiseIfPower * np.ones([N, T]) / N
pTxBA = OFDMA_BA(
np.ones(users) * rate * wrld.PHY.simulationTime,
np.ones([N, T, users]) * cell.pMax / N, wrld.PHY.systemBandwidth,
wrld.PHY.simulationTime, noiseIfPowerPerResource, CSI_BA)
pSupplyBA = scistats.nanmean(bs.p0 + bs.m * pTxBA)
print '{0:50} {1:5.2f} W'.format('SOTA objective:', pSupplyBA)
print ' '
return pSupplyOptim, pSupplyQuant, pSupplyBA
def main(searchpath, outpath):
"""Search all directories in searchpath for settings files. Handle them according to the settings found in those files. Eventually build a csv file for plotting the sum rate agains the power consumption.
Input: search path, output path
"""
data_types = 9
rate = 1e6 * 2.0 # 2 Mbps
sweep_values = [] # we do not know beforehand how much data we have
depth = None # worst case number of data points
# enum
axis_index = 0
sequential_index = 1
random_shift_each_iter_index = 2
random_shift_once_index = 3
random_each_iter_index = 4
random_once_index = 5
sinr_index = 6
static_3_index = 7
dtx_segregation = 8
# initially, we only check how much data we have
for dirname, dirnames, filenames in os.walk(searchpath):
if depth is None:
depth = len(dirnames)
for subdirname in dirnames:
for filename in glob.glob(
os.path.join(dirname, subdirname) + '/*settings*'):
config = ConfigParser.RawConfigParser()
config.read(filename)
sweep_values.append(
int(config.getfloat(
'General',
'user_rate'))) # TODO: handle rate*users automatically
#sweep_values.append(config.getfloat('General', 'numcenterusers'))
sweep_values = sorted(set(sweep_values))
count = np.zeros([len(sweep_values), data_types])
result = np.empty([len(sweep_values), data_types, depth])
result[:] = np.nan
# now start filling the result
dep = 0
for dirname, dirnames, filenames in os.walk(searchpath):
for subdirname in dirnames:
for filename in glob.glob(
os.path.join(dirname, subdirname) + '/*settings*.cfg'):
config = ConfigParser.RawConfigParser()
config.read(filename)
#users = (config.getint('General', 'numcenterusers'))
users = int(config.getfloat('General', 'user_rate'))
sleep_alignment = config.get('General', 'sleep_alignment')
initial_power = config.get('General', 'initial_power')
pS = config.getint('General', 'pS')
p0 = config.getint('General', 'p0')
# seqDTX sequential
if ('DTX' in filename) and (sleep_alignment == 'none'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = sequential_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# random shift each iteration
elif ('DTX' in filename) and (sleep_alignment
== 'random_shift_iter'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = random_shift_each_iter_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# random shift once
elif ('DTX' in filename) and (sleep_alignment
== 'random_shift_once'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = random_shift_once_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# random each iter
elif ('DTX' in filename) and (sleep_alignment
== 'random_iter'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = random_each_iter_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# random once
elif ('DTX' in filename) and (sleep_alignment
== 'random_once'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = random_once_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# sinr ordering
elif ('DTX' in filename) and (sleep_alignment == 'sinr'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = sinr_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# static assignment, reuse 3
elif ('DTX' in filename) and (sleep_alignment == 'static'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = static_3_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# dtx segregation
elif ('DTX' in filename) and (sleep_alignment
== 'dtx_segregation'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = dtx_segregation
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
else:
print 'What is this folder?' + os.path.join(
dirname, subdirname) + '/' + filename
dep += 1
result = nanmean(result, axis=2)
result[:,
axis_index] = np.array(sweep_values) * 10 # provide x-axis in Mbps
result = result.T # expected format
np.savetxt(outpath + '/power_consumption_over_sumrates.csv',
result,
delimiter=',')
# print 'Count PF_ba: ' + str(count_PF_ba)
# print 'Count PF_dtx: ' + str(count_PF_dtx)
# print 'Count RAPS_old: ' + str(count_RAPS_naive)
# print 'Count RAPS_random: ' + str(count_RAPS_random)
# print 'Count RAPS_PC: ' + str(count_RAPS_PC)
# print 'Count seqDTX_rand: ' + str(count_seqDTX_rand)
# print 'Count seqDTX_noshift: ' + str(count_seqDTX_noshift)
print count.T
def ba(rate, wrld, cell, mobiles, users):
"""The considered State of the Art comparison. Allocated target bitload over resources at an equal power share until user target is fulfilled. If the load is too high, np.nan is returned.
Input:
bitload: array of bit load by user index
pPerResource: (N,T) power level
systemBandwidth: in Hz
totalTime: in seconds
noiseIfPowerPerResource: noise plus interference power (N,T)
CSI: (n_tx, n_rx, N,T)
Output:
lastUsedResource: Flattened CSI index where allocation finished."""
N = wrld.PHY.numFreqChunks
T = wrld.PHY.numTimeslots
K = users
bitloadPerUser = rate * wrld.PHY.simulationTime * np.ones(users)
pPerResource = np.ones([N, T, users]) * cell.pMax / N
systemBandwidth = wrld.PHY.systemBandwidth
totalTime = wrld.PHY.simulationTime
pSupplyBA = np.nan
CSI_BA = np.empty([2, 2, N, T, users], dtype=complex)
CSI_BA[:] = np.nan
noiseIfPowerPerResource = np.empty([N, T, users])
noiseIfPowerPerResource[:] = np.nan
for idx, obj in enumerate(mobiles):
CSI_BA[:, :, :, :,
idx] = obj.baseStations[obj.BS]['cells'][cell]['CSI_OFDMA']
noiseIfPowerPerResource[:, :,
idx] = obj.noiseIfPower * np.ones([N, T]) / N
resourceTime = totalTime / T
resourceBandwidth = systemBandwidth / N
baseSNR = pPerResource / (noiseIfPowerPerResource)
user = 0
usedRBCounter = np.zeros(T)
flag = False
for n in np.arange(N):
for t in np.arange(T):
usedRBCounter[t] = usedRBCounter[t] + 1
H = np.dot(CSI_BA[:, :, n, t, user], CSI_BA[:, :, n, t,
user].conj().T)
bitsInThisRB = resourceBandwidth * resourceTime * np.real(
utils.ergMIMOCapacityCDITCSIR(H, baseSNR[n, t, user]))
bitloadPerUser[user] = bitloadPerUser[user] - bitsInThisRB
if bitloadPerUser[user] <= 0:
user = user + 1
if user >= K:
flag = True
break
if flag:
break
if (bitloadPerUser < 0).all():
pTxBA = pPerResource[0, 0, 0] * usedRBCounter
pSupplyBA = scistats.nanmean(mobiles[0].BS.p0 +
mobiles[0].BS.m * pTxBA)
else:
logger.warning('SOTA overload.')
pSupplyBA = np.nan
logger.info('{0:50} {1:5.2f} W'.format('SOTA objective:', pSupplyBA))
return pSupplyBA
ax = fig.add_subplot(111)
[plot_ethogram(etho,'jpak',(len(ethograms)-1)*2-i*2) for i,etho in enumerate(ethograms)]
ytks = [i*2+0.5 for i in range(len(ethograms))]
rats = [39,37,29,27,25,23,21,38,36,28,26,24,22,20]
ytklabels = ['jpak' + str(r) for r in rats]
plt.yticks(ytks,ytklabels)
hline(13.5)
xlabel('Time from first contact (seconds)')
xlim([0,3])
lesionfreezetimes = [get_total_freezetime(etho) for etho in ethograms[0:7]]
lesioncrossingtimes = [get_crossing_time(etho) for etho in ethograms[0:7]]
lesioncrossingtimes = [x for x in lesioncrossingtimes if x >= 0]
lesionmeans = [np.mean(lesionfreezetimes),nanmean(lesioncrossingtimes)]
lesionerr= [[0,0],[sem(lesionfreezetimes),sem(lesioncrossingtimes)]]
shamfreezetimes = [get_total_freezetime(etho) for i,etho in enumerate(ethograms[7:])]
shamcrossingtimes = [get_crossing_time(etho) for etho in ethograms[7:]]
shamcrossingtimes = [x for x in shamcrossingtimes if x >= 0]
shammeans = [np.mean(shamfreezetimes),nanmean(shamcrossingtimes)]
shamerr = [[0,0],[sem(shamfreezetimes),sem(shamcrossingtimes)]]
fig = plt.figure()
ax = fig.add_subplot(111)
barwidth = 0.35
ind = np.arange(2)
rects1 = plt.bar(ind,lesionmeans,barwidth,color='r',yerr=lesionerr,ecolor='k')
rects2 = plt.bar(ind+barwidth,shammeans,barwidth,color='g',yerr=shamerr,ecolor='k')
plt.xticks(ind+barwidth,('Freeze time', 'Crossing time'))
def path3d(CL):
""" This method will make a 3D plot of the flightpath of the plane during measurement.
Colours correspond to the time tracking colours (colours may not work if using an old version of matplotlib)."""
from mpl_toolkits.mplot3d import Axes3D
plat=[i for i,x in enumerate(CL.dttl) if x == 'latitude'][0]
lat=copy.deepcopy(CL.data[plat])
plon=[i for i,x in enumerate(CL.dttl) if x == 'longitude'][0]
lon=copy.deepcopy(CL.data[plon])
palt=[i for i,x in enumerate(CL.dttl) if x == 'altitude'][0]
# Not sure why we used to copy only the data part (and not the mask). If someone knows why, feel free to expain.
#alt=copy.deepcopy(CL.data[palt].data) #t=copy.defrom pylab import *epcopy(CL.data[pt].data)
alt=copy.deepcopy(CL.data[palt])
pt=[i for i,x in enumerate(CL.dttl) if x == 'time'][0]
t=copy.deepcopy(CL.data[pt])
#FIND THE QUADRANT
if st.nanmean(lat)>0:
quad_NS = 'N'
else:
quad_NS = 'S'
if st.nanmean(lon)>0:
quad_EW = 'E'
else:
quad_EW = 'W'
# This piece used to take out anomalously deviant altitude data. We now decided to leave it to the user to go and mask the bad data.
#M=runstats(alt,20)
#alt=np.ma.masked_where((alt>(M[0]+M[1]*1.)+(isnan(alt))), alt)
norm = matplotlib.colors.Normalize(vmin=t[0],vmax=t[-1])
fig = figure()
ax = Axes3D(fig)
majorFormatter_lon = FormatStrFormatter('%.2f '+quad_EW)
majorFormatter_lat = FormatStrFormatter('%.2f '+quad_NS)
try:
if int(matplotlib.__version__[0])>0:
ax.scatter(abs(lat),abs(lon),alt,lw=0,alpha=1,cmap='spectral',norm=norm,c=t)
ax.view_init(28,145)
ax.yaxis.set_major_formatter(majorFormatter_lon)
ax.xaxis.set_major_formatter(majorFormatter_lat)
if quad_EW == 'E':
ax.set_ylim(ax.get_ylim()[::-1])
if quad_NS == 'S':
ax.set_xlim(ax.get_xlim()[::-1])
else: # old version of matplotlib that doesn't support the color tracker
ax.scatter(abs(lat),abs(lon),alt,lw=0,alpha=1,cmap='spectral',norm=norm)
ax.view_init(28,145)
ax.yaxis.set_major_formatter(majorFormatter_lon)
ax.xaxis.set_major_formatter(majorFormatter_lat)
if quad_EW == 'E':
ax.set_ylim(ax.get_ylim()[::-1])
if quad_NS == 'S':
ax.set_xlim(ax.get_xlim()[::-1])
except: print("[path3d] Error evaluating your version of matplotlib.")
ax.set_xlabel('Latitude')
ax.set_ylabel('Longitude')
ax.set_zlabel('Altitude')
plt.ion()
plt.show()
def min_mean_max(a):
""" b\{%s\} := Min, mean and max of %s """
a = np.asarray(a, dtype="float") # converts bool
from scipy.stats.stats import nanmean
return (np.nanmin(a), nanmean(a), np.max(a))
def main(searchpath, outpath):
"""Search all directories in searchpath for settings files. Handle them according to the settings found in those files. Eventually build a csv file for plotting the sum rate agains the power consumption.
Input: search path, output path
"""
data_types = 9
rate = 1e6 * 2.0 # 2 Mbps
iterations = None
sweep_values = [] # we do not know beforehand how much data we have
depth = None
# enum
axis_index = 0
sequential_index = 1
random_each_iter_index = 4
sinr_index = 6
dtx_segregation = 8
# initially, we only check how much data we have
for dirname, dirnames, filenames in os.walk(searchpath):
if depth is None:
depth = len(dirnames) # worst case amount of data
for subdirname in dirnames:
for filename in glob.glob(os.path.join(dirname, subdirname)+'/*settings*'):
config = ConfigParser.RawConfigParser()
config.read(filename)
iterations = int(config.getfloat('General', 'iterations'))
sweep_values.append(int(config.getfloat('General', 'user_rate')))
sweep_values = sorted(set(sweep_values))
result = np.empty([iterations, len(sweep_values), data_types, depth])
result[:] = np.nan
count = np.zeros([len(sweep_values), data_types])
# now start filling the result
dep = 0
for dirname, dirnames, filenames in os.walk(searchpath):
for subdirname in dirnames:
for filename in glob.glob(os.path.join(dirname, subdirname)+'/*settings*.cfg'):
config = ConfigParser.RawConfigParser()
config.read(filename)
rate = int(config.getfloat('General', 'user_rate')) # sum rate
sleep_alignment = config.get('General', 'sleep_alignment')
# seqDTX sequential
if ('DTX' in filename) and (sleep_alignment == 'none'):
filedata = np.genfromtxt(os.path.join(dirname, subdirname)+'/delivered_individual.csv', delimiter=',')
index = sequential_index
result[:, sweep_values.index(rate), index, dep] = filedata[1,1:]
count[sweep_values.index(rate), index] += 1
# random each iter
elif ('DTX' in filename) and (sleep_alignment == 'random_iter'):
filedata = np.genfromtxt(os.path.join(dirname, subdirname)+'/delivered_individual.csv', delimiter=',')
index = random_each_iter_index
result[:, sweep_values.index(rate), index, dep] = filedata[1,1:]
count[sweep_values.index(rate), index] += 1
# sinr ordering
elif ('DTX' in filename) and (sleep_alignment == 'sinr'):
filedata = np.genfromtxt(os.path.join(dirname, subdirname)+'/delivered_individual.csv', delimiter=',')
index = sinr_index
result[:, sweep_values.index(rate), index, dep] = filedata[1,1:]
count[sweep_values.index(rate), index] += 1
# dtx segregation
elif ('DTX' in filename) and (sleep_alignment == 'dtx_segregation'):
filedata = np.genfromtxt(os.path.join(dirname, subdirname)+'/delivered_individual.csv', delimiter=',')
index = dtx_segregation
result[:, sweep_values.index(rate), index, dep] = filedata[1,1:]
count[sweep_values.index(rate), index] += 1
else:
print 'What is this folder?'+os.path.join(dirname, subdirname)+'/'+filename
if (filedata[1,-1]>2*rate/10).any():
# find outliers
print filedata[1,-1], filename.split('/')[7]
dep += 1
####
result[np.where(result==0)]=np.nan # remove outage data
####
result = nanmean(result, axis=3)
for i, s in enumerate(sweep_values):
target = outpath + '/delivered_rate_' + str(s) + '.csv'
res = result[:, i, :]
res[:,axis_index] = np.arange(1, iterations+1) # provide x-axis
res = res.T # expected format
np.savetxt(target, res, delimiter=',')
print count.T
import scipy.stats.stats as st m=st.nanmean(vec)
with open('volume all 394 hot data.csv', 'ab') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=',', quotechar='|')
csv_writer.writerow(hot[beginning_time:end_time])
with open('volume all 394 gp data.csv', 'ab') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=',', quotechar='|')
csv_writer.writerow(gp[beginning_time:end_time])
#Append data
hot_total_volume.append(hot[beginning_time:end_time])
gp_total_volume.append(gp[beginning_time:end_time])
date_1 += timedelta(days=1)
#Average volume at each time across all days
average_hot_volume = sci.nanmean(hot_total_volume, axis=0)
variance_hot_volume = sci.nanstd(hot_total_volume, axis=0)
average_gp_volume = sci.nanmean(gp_total_volume, axis=0)
variance_gp_volume = sci.nanstd(gp_total_volume, axis=0)
#Group into 3 minute increments (6 x 30 seconds)
k = 0
resolution = 6
while k in range(0, len(average_hot_volume)):
average_hot_volume[k:k + resolution] = sum(
average_hot_volume[k:k + resolution]) / resolution
variance_hot_volume[k:k + resolution] = sum(
variance_hot_volume[k:k + resolution]) / resolution
average_gp_volume[k:k + resolution] = sum(
average_gp_volume[k:k + resolution]) / resolution
variance_gp_volume[k:k + resolution] = sum(
def main(searchpath, outpath):
"""Search all directories in searchpath for settings files. Handle them according to the settings found in those files. Eventually build a csv file for plotting the sum rate agains the power consumption.
Input: search path, output path
There are the following types of data:
- BA/SOTA
- DTX/SOTA
- PF-BA
- PF-DTX
- RAPS without sleep alignment
- RAPS with random sleep alignment
"""
data_types = 11
rate = 1e6 * 2.0 # 2 Mbps
sweep_values = [] # we do not know beforehand how much data we have
depth = None # worst case number of data points
# enum
axis_index = 0
BA_index = 1
seqDTX_noshift_index = 2
seqDTX_rand_index = 3
PF_ba_index = 4
PF_dtx_index = 5
RAPS_naive_index = 6
RAPS_random_index = 7
RAPS_PC_index = 8
RAPS_sinr_index = 9
RAPS_sinr_protect_index = 10
# initially, we only check how much data we have
for dirname, dirnames, filenames in os.walk(searchpath):
if depth is None:
depth = len(dirnames)
for subdirname in dirnames:
for filename in glob.glob(
os.path.join(dirname, subdirname) + '/*settings*'):
config = ConfigParser.RawConfigParser()
config.read(filename)
sweep_values.append(
int(config.getfloat(
'General',
'user_rate'))) # TODO: handle rate*users automatically
#sweep_values.append(config.getfloat('General', 'numcenterusers'))
sweep_values = sorted(set(sweep_values))
count = np.zeros([len(sweep_values), data_types])
result = np.empty([len(sweep_values), data_types, depth])
result[:] = np.nan
# now start filling the result
dep = 0
for dirname, dirnames, filenames in os.walk(searchpath):
for subdirname in dirnames:
for filename in glob.glob(
os.path.join(dirname, subdirname) + '/*settings*.cfg'):
config = ConfigParser.RawConfigParser()
config.read(filename)
#users = (config.getint('General', 'numcenterusers'))
users = int(config.getfloat('General', 'user_rate'))
sleep_alignment = config.get('General', 'sleep_alignment')
initial_power = config.get('General', 'initial_power')
pS = config.getint('General', 'pS')
p0 = config.getint('General', 'p0')
if initial_power == 'zero':
continue
# PF_BA
if 'PF_ba' in filename:
# there should be a file called resultPF.csv containing one line of
# comma separated entries. We only want the last one.
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultPF.csv',
delimiter=',')
index = PF_ba_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# PF_DTX
elif 'PF_dtx' in filename:
# there should be a file called resultPF.csv containing one line of
# comma separated entries. We only want the last one.
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultPF.csv',
delimiter=',')
index = PF_dtx_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# RAPS_naive
elif ('RAPS' in filename) and (sleep_alignment
== 'none') and pS < p0:
# there should be a file called resultRAPS.csv containing one line of
# comma separated entries. We only want the last one.
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultRAPS.csv',
delimiter=',')
index = RAPS_naive_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultBA.csv',
delimiter=',')
index = BA_index
result[sweep_values.index(users), index, dep] = filedata
# RAPS_random
elif ('RAPS' in filename) and (sleep_alignment
== 'random') and pS < p0:
# there should be a file called resultRAPS.csv containing one line of
# comma separated entries. We only want the last one.
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultRAPS.csv',
delimiter=',')
index = RAPS_random_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultBA.csv',
delimiter=',')
index = BA_index
result[sweep_values.index(users), index, dep] = filedata
# RAPS_PC
elif ('RAPS' in filename) and pS > p0:
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultRAPS.csv',
delimiter=',')
index = RAPS_PC_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultBA.csv',
delimiter=',')
index = BA_index
result[sweep_values.index(users), index, dep] = filedata
# seqDTX no shift
elif ('DTX' in filename) and (sleep_alignment == 'none'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = seqDTX_noshift_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# seqDTX shift
elif ('DTX' in filename) and (sleep_alignment == 'random'):
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultDTX.csv',
delimiter=',')
index = seqDTX_rand_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
# RAPS with sinr based sleep selection
elif ('RAPS' in filename) and (sleep_alignment
== 'sinr') and pS < p0:
# there should be a file called resultRAPS.csv containing one line of
# comma separated entries. We only want the last one.
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultRAPS.csv',
delimiter=',')
index = RAPS_sinr_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultBA.csv',
delimiter=',')
index = BA_index
result[sweep_values.index(users), index, dep] = filedata
# RAPS with sinr based sleep selection and protection
elif ('RAPS' in filename) and (sleep_alignment
== 'sinr_protect') and pS < p0:
# there should be a file called resultRAPS.csv containing one line of
# comma separated entries. We only want the last one.
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultRAPS.csv',
delimiter=',')
index = RAPS_sinr_protect_index
result[sweep_values.index(users), index,
dep] = filedata[1, -1]
if not np.isnan(filedata[1, -1]) and not filedata[1,
-1] == 0:
count[sweep_values.index(users), index] += 1
filedata = np.genfromtxt(
os.path.join(dirname, subdirname) + '/resultBA.csv',
delimiter=',')
index = BA_index
result[sweep_values.index(users), index, dep] = filedata
else:
print 'What is this folder?' + os.path.join(
dirname, subdirname) + '/' + filename
dep += 1
result = nanmean(result, axis=2)
result[:, axis_index] = np.array(
sweep_values) * rate # provide x-axis in Mbps
result = result.T # expected format
np.savetxt(outpath + '/sumrate.csv', result, delimiter=',')
# print 'Count PF_ba: ' + str(count_PF_ba)
# print 'Count PF_dtx: ' + str(count_PF_dtx)
# print 'Count RAPS_old: ' + str(count_RAPS_naive)
# print 'Count RAPS_random: ' + str(count_RAPS_random)
# print 'Count RAPS_PC: ' + str(count_RAPS_PC)
# print 'Count seqDTX_rand: ' + str(count_seqDTX_rand)
# print 'Count seqDTX_noshift: ' + str(count_seqDTX_noshift)
print count.T
