def mean_std():
activities= ['Entertainment','Cooking','Cleaning','Bathroom']
subjects = ['sub1','sub2']
dir_name = 'FullResults/'
for act in activities:
file1 = dir_name+'sub1/'+act+'.csv'
data1 = np.genfromtxt(file1, delimiter=",")
file2 = dir_name+'sub2/'+act+'.csv'
data2 = np.genfromtxt(file2, delimiter=",")
data = data1 + data2
data = data/2.0
row,col = data1.shape
data_mean = []
data_std = []
for i in range(row):
std = []
mn = []
for j in range(col):
a = np.array([data1[i][j],data2[i][j]])
mn.append(np.mean(a))
std.append(np.std(a))
data_mean+=[mn]
data_std+=[std]
data_mean = np.array(data_mean)
print data
print data_mean
data_std = np.array(data_std)
print data_std
np.savetxt(dir_name+'mean_std/'+act+'_mean.csv',data_mean,delimiter=',')
np.savetxt(dir_name+'mean_std/'+act+'_std.csv',data_std,delimiter=',')
def read_all_pot(nmodes, fc_init_pot,fc_decay_pot,fc_fin_pot):
R={};Vg={};Vd={};Vf={}
for i in range(nmodes):
R[i],Vg[i]=np.genfromtxt(fc_init_pot[i],skip_header=1,unpack=True,dtype=float)
R[i],Vd[i]=np.genfromtxt(fc_decay_pot[i],skip_header=1,unpack=True,dtype=float)
R[i],Vf[i]=np.genfromtxt(fc_fin_pot[i],skip_header=1,unpack=True,dtype=float)
return R,Vg,Vd,Vf
def _Load_InfData_LabView(file_path):
'''info from inf file '''
Cycles, dump, Frequency, LED_Voltage, dump, dump, dump, dump, DataPoints, dump = np.genfromtxt(
Directory + InfFile, skip_header=20, skip_footer=22, delimiter=':', usecols=(1), autostrip=True, unpack=True)
Waveform, LED_intensity = np.genfromtxt(
Directory + InfFile, skip_header=31, skip_footer=20, delimiter=':', usecols=(1), dtype=None, autostrip=True, unpack=True)
l = np.genfromtxt(
Directory + InfFile, skip_header=36, delimiter=':', usecols=(1))
Doping = l[9]
Ai = l[6]
Fs = l[7]
Thickness = l[12]
Quad = l[12]
Lin = l[12]
Const = 0
Binning = int(l[2])
Reflection = (1 - l[16]) * 100
dic = locals()
del dic['self']
del dic['l']
del dic['dump']
return dic
def compare(mooseCsv, nrnCsv):
mooseData = None
nrnData = None
with open(mooseCsv, "r") as f:
mooseTxt = f.read().split("\n")
mooseHeader, mooseData = mooseTxt[0].split(","), np.genfromtxt(mooseTxt[1:],
delimiter=',').T
with open(nrnCsv, "r") as f:
nrnTxt = f.read().split("\n")
nrnHeader, nrnData = nrnTxt[0].split(','), 1e-3*np.genfromtxt(nrnTxt[1:],
delimiter=',').T
nrnTimeVec, nrnData = nrnData[0], nrnData[1:]
mooseTimeVec, mooseData = mooseData[0], mooseData[1:]
for i, comptName in enumerate(nrnHeader[1:]):
if i == 1:
break
nrnComptName = comptName.replace("table_", "")
mooseComptId, mooseComptName = get_index(nrnComptName, mooseHeader[1:])
print("%s %s- moose equivalent %s %s" % (i, nrnComptName, mooseComptId,
mooseComptName))
pylab.plot(mooseTimeVec, mooseData[ mooseComptId ], label = "Neuron: " + nrnComptName)
pylab.plot(nrnTimeVec, nrnData[i], label = "MOOSE: " + mooseComptName)
pylab.legend(loc='best', framealpha=0.4)
pylab.show()
def learn(tuned_parameters,model):
# produceFeature(trainfile)
dataset = genfromtxt(open('Data/'+trainfile,'r'), delimiter=',',dtype='f8')[0:]
target = [x[0] for x in dataset]
train = [x[1:] for x in dataset]
# print train[1:10]
# print target
# print len(train)
# produceFeature(testfile)
test = genfromtxt(open('Data/'+testfile,'r'),delimiter=',',dtype='f8')[0:]
test_target = [x[1:] for x in test]
# X, y = digits.data, digits.target
trainnp = np.asarray(train)
targetnp = np.asarray(target)
# turn the data in a (samples, feature) matrix:
X, y = trainnp, targetnp
# X = digits.images.reshape((n_samples, -1))
# y = digits.target
# Split the dataset in two equal parts
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.5, random_state=0)
scores = ['precision', 'recall']
for score in scores:
print("# Tuning hyper-parameters for %s" % score)
print()
clf = GridSearchCV(model, tuned_parameters, cv=5,
scoring='%s_weighted' % score)
clf.fit(X_train, y_train)
print("Best parameters set found on development set:")
print()
print(clf.best_params_)
print()
print("Grid scores on development set:")
print()
for params, mean_score, scores in clf.grid_scores_:
print("%0.3f (+/-%0.03f) for %r"
% (mean_score, scores.std() * 2, params))
print()
print("Detailed classification report:")
print()
print("The model is trained on the full development set.")
print("The scores are computed on the full evaluation set.")
print()
y_true, y_pred = y_test, clf.predict(X_test)
print(classification_report(y_true, y_pred))
print()
def test_run(xTrainFile, yTrainFile):
X = np.genfromtxt(xTrainFile,delimiter=",")
Y = np.genfromtxt(yTrainFile,delimiter=",")
TEST = np.genfromtxt("/home/john/Downloads/kaggle.X1.test.txt",delimiter=",")
M = X.shape[0]
#maxDepth
########################
nBags = 125
rforest = [None] * nBags
maxDepth = 40
nFeatures = 100
minParent = 8
for l in range(nBags):
print "bags", l
Xi,Yi = ml.bootstrapData(X,Y, M)
rforest[l] = dtree.treeRegress()
rforest[l].train(Xi,Yi,maxDepth=maxDepth)
mTest = TEST.shape[0]
predict = np.zeros( (mTest, nBags) )
for i in range(nBags):
predict[:,i] = rforest[i].predict(TEST).T[0]
predict = predict[:,0]
_write_to_kaggle("treebag.csv",predict)
def setUp(self):
"""
"""
# Read initial dataset
filename = os.path.join(self.BASE_DATA_PATH,
'completeness_test_cat.csv')
test_data = np.genfromtxt(filename, delimiter=',', skip_header=1)
# Create the catalogue A
self.catalogueA = Catalogue.make_from_dict(
{'year': test_data[:,3], 'magnitude': test_data[:,17]})
# Read initial dataset
filename = os.path.join(self.BASE_DATA_PATH,
'recurrence_test_cat_B.csv')
test_data = np.genfromtxt(filename, delimiter=',', skip_header=1)
# Create the catalogue A
self.catalogueB = Catalogue.make_from_dict(
{'year': test_data[:,3], 'magnitude': test_data[:,17]})
# Read the verification table A
filename = os.path.join(self.BASE_DATA_PATH,
'recurrence_table_test_A.csv')
self.true_tableA = np.genfromtxt(filename, delimiter = ',')
# Read the verification table A
filename = os.path.join(self.BASE_DATA_PATH,
'recurrence_table_test_B.csv')
self.true_tableB = np.genfromtxt(filename, delimiter = ',')
def spin_stiffness(h,vector=np.array([0.,0.,1.]),
qrange = None,atoms = None):
""" Calculates the spin stiffness by performing a set
of selfconsistent calculations"""
from copy import deepcopy
if qrange == None:
qrange = np.arange(0.01,2.,0.02)
feq = open("EnergyVSq.OUT","w")
feq.write("# qx qy qz Energy\n")
for q in qrange: # loop over qvectors
hi = deepcopy(h) # copy the hamiltonian
hi.generate_spin_spiral(vector=vector,angle=q,atoms=atoms)
# generate spin spiral in the hamiltonian
hi.write(output_file="hamiltonian_0.in") # write te hamiltonian
os.system("tb90.x") # run the calculation
try:
energy = np.genfromtxt("ENERGY.OUT").transpose()[0][-1] # last energy
except: # only one iteration
energy = np.genfromtxt("ENERGY.OUT")[0] # last energy
iq = vector*q
feq.write(str(iq[0])+" ")
feq.write(str(iq[1])+" ")
feq.write(str(iq[2])+" ")
feq.write(str(energy)+"\n") # write in file
feq.close()
def wide_dataset_large():
print("Reading in Arcene training data for binomial modeling.")
trainDataResponse = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_train_labels.labels"), delimiter=' ')
trainDataResponse = np.where(trainDataResponse == -1, 0, 1)
trainDataFeatures = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_train.data"), delimiter=' ')
xtrain = np.transpose(trainDataFeatures).tolist()
ytrain = trainDataResponse.tolist()
trainData = h2o.H2OFrame.fromPython([ytrain]+xtrain)
trainData[0] = trainData[0].asfactor()
print("Run model on 3250 columns of Arcene with strong rules off.")
model = H2OGeneralizedLinearEstimator(family="binomial", lambda_search=False, alpha=1)
model.train(x=range(1,3250), y=0, training_frame=trainData)
print("Test model on validation set.")
validDataResponse = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_valid_labels.labels"), delimiter=' ')
validDataResponse = np.where(validDataResponse == -1, 0, 1)
validDataFeatures = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_valid.data"), delimiter=' ')
xvalid = np.transpose(validDataFeatures).tolist()
yvalid = validDataResponse.tolist()
validData = h2o.H2OFrame.fromPython([yvalid]+xvalid)
prediction = model.predict(validData)
print("Check performance of predictions.")
performance = model.model_performance(validData)
print("Check that prediction AUC better than guessing (0.5).")
assert performance.auc() > 0.5, "predictions should be better then pure chance"
def imp(*args):
global pathname
global filename
global first
tmp= []
tmp
Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
filename = askopenfilename() # show an "Open" dialog box and return the path to the selected file
pathname = filename.rsplit(".", 1)[0]
if filename.split(".")[-1] == "wfm":
os.system('c:/temp/ConvertTekWfm \"' + filename + '\" /CSV c:/temp/temp.csv')
tmp = genfromtxt('c:/temp/temp.csv', delimiter=',')
return import_tek(tmp)
if filename.split(".")[-1] == "csv":
f = open(filename, 'r')
first = f.readline().split(",")[0]
f.close()
tmp = genfromtxt(filename, delimiter=',')
print first
if first == '"Record Length"':
return import_tek(tmp)
else:
a = 5
def make_dtopo():
'''
Make geoclaw dtopo file
'''
from numpy import genfromtxt,zeros
#Run params
f='/Users/dmelgarm/Research/Slip_Inv/tohoku_tsunami/'
stafile='tohoku.sta'
dlon=0.033333
dlat=0.033333
dt=5
stat_or_dyn='s'
#Get station list
sta=genfromtxt(f+'data/station_info/'+stafile,usecols=0,dtype='S4')
s=genfromtxt(f+'data/station_info/'+stafile,usecols=[1,2])
lon=s[:,0]
lat=s[:,1]
if stat_or_dyn.lower()=='s':
n=zeros(len(sta))
e=n.copy()
u=n.copy()
for ksta in range(len(sta)):
print ksta
neu=genfromtxt(f+'output/forward_models/'+str(sta[ksta])+'.static.neu')
n[ksta]=neu[0]
e[ksta]=neu[1]
u[ksta]=neu[2]
print neu[2]
def load_lexicons():
"""
This function loads in Harvard's postivie/negative word lexicons.
"""
positive_words = np.genfromtxt('Positive_&_Negative_Words.csv', skip_header = 1, usecols = (0, ), delimiter = ',', dtype = 'str')
negative_words = np.genfromtxt('Positive_&_Negative_Words.csv', skip_header = 1, usecols = (1, ), delimiter = ',', dtype = 'str')
return positive_words, negative_words
def test_dtype_with_object(self):
"Test using an explicit dtype with an object"
from datetime import date
import time
data = """
1; 2001-01-01
2; 2002-01-31
"""
ndtype = [('idx', int), ('code', np.object)]
func = lambda s: strptime(s.strip(), "%Y-%m-%d")
converters = {1: func}
test = np.genfromtxt(StringIO.StringIO(data), delimiter=";", dtype=ndtype,
converters=converters)
control = np.array([(1, datetime(2001,1,1)), (2, datetime(2002,1,31))],
dtype=ndtype)
assert_equal(test, control)
#
ndtype = [('nest', [('idx', int), ('code', np.object)])]
try:
test = np.genfromtxt(StringIO.StringIO(data), delimiter=";",
dtype=ndtype, converters=converters)
except NotImplementedError:
pass
else:
errmsg = "Nested dtype involving objects should be supported."
raise AssertionError(errmsg)
def RunNBCShogun(q):
totalTimer = Timer()
Log.Info("Loading dataset", self.verbose)
try:
# Load train and test dataset.
trainData = np.genfromtxt(self.dataset[0], delimiter=',')
testData = np.genfromtxt(self.dataset[1], delimiter=',')
# Labels are the last row of the training set.
labels = MulticlassLabels(trainData[:, (trainData.shape[1] - 1)])
with totalTimer:
# Transform into features.
trainFeat = RealFeatures(trainData[:,:-1].T)
testFeat = RealFeatures(testData.T)
# Create and train the classifier.
nbc = GaussianNaiveBayes(trainFeat, labels)
nbc.train()
# Run Naive Bayes Classifier on the test dataset.
nbc.apply(testFeat).get_labels()
except Exception as e:
q.put(-1)
return -1
time = totalTimer.ElapsedTime()
q.put(time)
return time
def read(input_file="POSITIONS.OUT"):
""" Reads a geometry """
m = np.genfromtxt(input_file).transpose()
g = geometry() # cretae geometry
g.dimensionality = 0
g.x = m[0]
g.y = m[1]
g.x = g.x - sum(g.x)/len(g.x) # normalize
g.y = g.y - sum(g.y)/len(g.y) # normalize
g.z = m[2]
g.xyz2r() # create r coordinates
try: lat = np.genfromtxt("LATTICE.OUT") # read lattice
except: lat = None
try: # two dimensional
g.a1 = np.array([lat[0,0],lat[0,1],0.0])
g.a2 = np.array([lat[1,0],lat[1,1],0.0])
g.dimensionality=2
return g
except: pass
try:
g.celldis = lat
g.dimensionality = 1
return g
except: pass
g.dimensionality = 0
return g
def RunExp(StrModel:str, Param:str, FeaUsed:list, DataPath:str, Label:str, StrMeasure:str, std:bool = False, N:int = 0): Data = np.genfromtxt(DataPath + Label, delimiter = ',', dtype = int) Data = Data[:, np.newaxis] for f in FeaUsed: T = (np.genfromtxt(DataPath + Features[f], delimiter = ',' , dtype = float)) if len(T.shape) < 2: T = T[:, np.newaxis] Data = np.concatenate((Data, T), axis = 1) if N > 0: Data = Data[:N, :] Lbl = Data[:, 0] Fea = Data[:,1:] if std: scaler = preprocessing.StandardScaler() Fea = scaler.fit_transform(Fea) Model = base.clone(Models[StrModel]) SetParam(Model, Param) Model.fit(Fea, Lbl) Pred = Model.predict(Fea) st = Measures[StrMeasure](Lbl, Pred) sv = cross_validation.cross_val_score(base.clone(Models[StrModel]), Fea, Lbl, metrics.make_scorer(Measures[StrMeasure]), cv = 5, n_jobs = 5) return st, np.mean(sv)
def main():
trainset = np.genfromtxt(open('train.csv','r'), delimiter=',')[1:]
X = np.array([x[1:8] for x in trainset])
y = np.array([x[8] for x in trainset])
#print X,y
import math
for i, x in enumerate(X):
for j, xx in enumerate(x):
if(math.isnan(xx)):
X[i][j] = 26.6
testset = np.genfromtxt(open('test.csv','r'), delimiter = ',')[1:]
test = np.array([x[1:8] for x in testset])
for i, x in enumerate(test):
for j, xx in enumerate(x):
if(math.isnan(xx)):
test[i][j] = 26.6
X, test = decomposition_pca(X, test)
bdt = AdaBoostClassifier(base_estimator = KNeighborsClassifier(n_neighbors=20, algorithm = 'auto'), algorithm="SAMME", n_estimators = 200)
bdt.fit(X, y)
print 'PassengerId,Survived'
for i, t in enumerate(test):
print '%d,%d' % (i + 892, int(bdt.predict(t)[0]))
def load_nanoZInFolder(dirname):
#function that loads all the nanoZ measurements in a folder.
#returns a list of the names of the txt files.
#make sure there was an input directory, otherwise use a default one
if not dirname:
dirname = '/Users/danieljdenman/Academics/allen/BlancheLab/electrodeMeasurements/imec/data/impTesting_r37_stability/'
#go through each file in the folder, open it, determine if it is nanoZ data
#allow the user to skip, force load, or abort loading altogether
nms=[]; rL=[]
for file_name in os.listdir(dirname):
fullPath = dirname+file_name;
tmp = np.genfromtxt(fullPath,skip_header=3,skip_footer=1,filling_values = '-1',invalid_raise=False,usecols=(1,2))
#if no exception (or user forces), put this file in the tuple of open files
#and the name of this file in the list of filenames that corresponds to the tuple.
if not tmp.any():
print "------------------------------------------------"
print "found a bad one!: "+fullPath
print "------------------------------------------------"
else:
rL.append(np.genfromtxt(fullPath,skip_header=3,skip_footer=1,filling_values = '-1',invalid_raise=False, usecols=(1,2)))
nms.append(file_name)
return [nms,rL]
def wide_dataset_large():
print("Reading in Arcene training data for binomial modeling.")
trainDataResponse = np.genfromtxt(tests.locate("smalldata/arcene/arcene_train_labels.labels"), delimiter=' ')
trainDataResponse = np.where(trainDataResponse == -1, 0, 1)
trainDataFeatures = np.genfromtxt(tests.locate("smalldata/arcene/arcene_train.data"), delimiter=' ')
trainData = h2o.H2OFrame(np.column_stack((trainDataResponse, trainDataFeatures)).tolist())
print("Run model on 3250 columns of Arcene with strong rules off.")
model = h2o.glm(x=trainData[1:3250], y=trainData[0].asfactor(), family="binomial", lambda_search=False, alpha=[1])
print("Test model on validation set.")
validDataResponse = np.genfromtxt(tests.locate("smalldata/arcene/arcene_valid_labels.labels"), delimiter=' ')
validDataResponse = np.where(validDataResponse == -1, 0, 1)
validDataFeatures = np.genfromtxt(tests.locate("smalldata/arcene/arcene_valid.data"), delimiter=' ')
validData = h2o.H2OFrame(np.column_stack((validDataResponse, validDataFeatures)).tolist())
prediction = model.predict(validData)
print("Check performance of predictions.")
performance = model.model_performance(validData)
print("Check that prediction AUC better than guessing (0.5).")
assert performance.auc() > 0.5, "predictions should be better then pure chance"
def loadTables(self, filesDir, varianceTbl = True):
"""
Read in and store in dictionary the IGM Lookup Tables that contain IGM transmission
for a given redshift and must be formatted in two columns:
(wavelength (nm), IGM Transmission %) or for variance
(wavelength (nm), IGM Transmission % Variance). Variance tables are not required and
can be turned off as a requirement. Names in directory formatted as
'MeanLookupTable_zSourceX.X.tbl' or 'VarLookupTable_zSourceX.X.tbl' where X.X is the redshift
of the given lookup table.
@param [in] filesDir is the location of the directory where lookup table are stored
@param [in] varianceTbl is a boolean that is True if variance tables are present in dir
for loading.
"""
self.meanLookups = {}
self.varLookups = {}
for zValue in self.zRange:
self.meanLookups['%.1f' % zValue] = np.genfromtxt(str(filesDir + '/MeanLookupTable_zSource' +
'%.1f' % zValue + '.tbl.gz'))
if varianceTbl == True:
try:
self.varLookups['%.1f' % zValue] = np.genfromtxt(str(filesDir + '/VarLookupTable_zSource' +
'%.1f' % zValue + '.tbl.gz'))
except IOError:
raise IOError("Cannot find variance tables.")
self.tablesPresent = True
def cvalidate():
from sklearn import cross_validation
trainset = np.genfromtxt(open('train','r'), delimiter=' ')
targetset = np.genfromtxt(open('target','r'), delimiter=' ')
X = np.array([x[0:64] for x in trainset])
y = np.array([x for x in targetset])
#print X,y
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size = 0.3, random_state = 0)
#X_train, X_test = decomposition_pca(X_train, X_test)
X_train, X_test = decomposition_pca(X_train, X_test)
c_range = 10.0 ** np.arange(6.5,7.5,1)
gamma_range = 10.0 ** np.arange(-2.5,0.5,1)
#parameters = {'kernel':['rbf'], 'C':c_range}
parameters = {'kernel':['rbf'], 'C':c_range, 'gamma':gamma_range}
svr = SVC(kernel = 'rbf', C = 0.72, gamma = 0.299)
#clf = grid_search.GridSearchCV(svr, parameters)
#print clf.estimator
##clf = Pipeline([('scale', Scaler()), ('svm', SVC())])
svr.fit(X_train, y_train)
print svr.score(X_test, y_test)
def RunQDAShogun(q):
totalTimer = Timer()
Log.Info("Loading dataset", self.verbose)
try:
# Load train and test dataset.
trainData = np.genfromtxt(self.dataset[0], delimiter=',')
trainFeat = modshogun.RealFeatures(trainData[:,:-1].T)
if len(self.dataset) == 2:
testSet = np.genfromtxt(self.dataset[1], delimiter=',')
testFeat = modshogun.RealFeatures(testData.T)
# Labels are the last row of the training set.
labels = modshogun.MulticlassLabels(trainData[:, (trainData.shape[1] - 1)])
with totalTimer:
model = modshogun.QDA(trainFeat, labels)
model.train()
if len(self.dataset) == 2:
model.apply(testFeat).get_labels()
except Exception as e:
q.put(-1)
return -1
time = totalTimer.ElapsedTime()
q.put(time)
return time
def plotme(typeid,num,h):
yMax = '6'
cuibmFolder = '/scratch/src/cuIBM'
caseFolder = cuibmFolder + '/validation/error/cylinder/'+typeid+num
validationData = cuibmFolder + '/validation-data/cylinderRe40-KL95.txt'
print caseFolder+'/forces'
my_data = genfromtxt(caseFolder+'/forces',dtype=float,delimiter='\t')
time = [my_data[i][0] for i in xrange(1,len(my_data))]
force = [my_data[i][1]*2 for i in xrange(1,len(my_data))]
validation_data = genfromtxt(validationData, dtype=float, delimiter='\t')
validation_time=[validation_data[i][0]*0.5 for i in xrange(1,len(validation_data))]
validation_force=[validation_data[i][1] for i in xrange(1,len(validation_data))]
plt.plot(validation_time,validation_force, 'o', color = 'red', markersize = 8, label = 'Koumoutsakos and Leonard, 1995')
plt.plot(time,force, '-', color='blue', linewidth=2, label='Present Work')
plt.title('Flow over an impulsively started cylinder at Reynolds number 40')
plt.legend(loc='upper right', numpoints=1, fancybox=True)
plt.xlabel('Non-dimensional time')
plt.ylabel('Drag Coefficient')
plt.xlim([0,3])
plt.ylim([0,int(yMax)])
plt.savefig('/scratch/src/cuIBM/validation/error/cylinder/'+typeid+h+'.pdf')
plt.clf()
def load(self):
# load data
values = []
if verbose: print()
if verbose: print("visualization: loading chains ...")
f = "prob-chain0.dump"
if not os.path.exists(f):
raise Exception("visualization: chains not available yet.")
try:
# I think the first column is the probabilities, the second is without prior
probabilities = numpy.genfromtxt(f, skip_footer=1, dtype='f')[:,0]
except Exception as e:
raise Exception("visualization: chains couldn't be loaded; perhaps no data yet: " + str(e))
for p in self.params:
f = "%s-chain-0.prob.dump" % p['name']
if verbose: print(" loading chain %s" % f)
if not os.path.exists(f):
raise Exception("visualization: chains not available yet.")
try:
v = numpy.genfromtxt(f, skip_footer=1, dtype='f')
except Exception as e:
raise Exception("visualization: chains couldn't be loaded; perhaps no data yet: " + str(e))
values.append(v)
nvalues = min(map(len, values))
if verbose: print("visualization: loading chains finished; %d values" % nvalues)
self.values = [v[:nvalues][-self.nlast::nevery] for v in values]
self.probabilities = probabilities[:nvalues][-self.nlast::nevery]
def read_gf_from_txt(block_txtfiles, block_name):
"""
Read a GfReFreq from text files with the format (w, Re(G), Im(G)) for a single block.
Notes
-----
A BlockGf must be constructed from multiple GfReFreq objects if desired.
The mesh must be the same for all files read in.
Non-uniform meshes are not supported.
Parameters
----------
block_txtfiles: Rank 2 square np.array(str) or list[list[str]]
The text files containing the GF data that need to read for the block.
e.g. [['up_eg1.dat']] for a one-dimensional block and
[['up_eg1_1.dat','up_eg2_1.dat'],
['up_eg1_2.dat','up_eg2_2.dat']] for a 2x2 block.
block_name: str
Name of the block.
Returns
-------
g: GfReFreq
The real frequency Green's function read in.
"""
block_txtfiles = np.array(block_txtfiles) # Must be an array to use certain functions
N1,N2 = block_txtfiles.shape
mesh = np.genfromtxt(block_txtfiles[0,0],usecols=[0]) # Mesh needs to be the same for all blocks
g = GfReFreq(indices=range(N1),window=(np.min(mesh),np.max(mesh)),n_points=len(mesh),name=block_name)
for i,j in product(range(N1),range(N2)):
data = np.genfromtxt(block_txtfiles[i,j],usecols=[1,2])
g.data[:,i,j] = data[:,0] + 1j*data[:,1]
return g
def main():
print "Solve small matrix..."
R = array([0, 0, 1, 1, 1, 2, 2])
C = array([0, 1, 0, 1, 2, 1, 2])
V = array([4.0, -1.0, -1.0, 4.0, -1.0, -1.0, 4.0])
b = array([3.0, 2.0, 3.0])
A = coo_matrix((V, (R, C)), shape=(3, 3))
# convert to csr format for efficiency
x = spsolve(A.tocsr(), b)
print "x = ", x
print "Solve psd matrix..."
# skip the first row (n, nnz)
A = numpy.genfromtxt('../data/psd.txt', skiprows=1)
b = numpy.genfromtxt('../data/b.txt')
coo = coo_matrix((A[:, 2], (A[:, 0], A[:, 1])))
x = spsolve(coo.tocsr(), b)
print 'x = ', x
print "Solve big matrix..."
A = numpy.genfromtxt('../data/mat_helmholtz.txt', skiprows=1)
coo = coo_matrix((A[:, 2], (A[:, 0], A[:, 1])))
n = coo.shape[0]
b = numpy.ones(n)
x = spsolve(coo.tocsr(), b)
print 'x = ', x
def load_hop(s, hop=hop_script_path):
"""
Loads the hop catalog for the given RAMSES snapshot. If the
catalog doesn't exist, it tries to run hop to create one via the
'script_hop.sh' script found in the RAMSES distribution. The hop
output should be in a 'hop' directory in the base directory of the
simulation.
**Input**:
*s* : loaded RAMSES snapshot
**Optional Keywords**:
*hop* : path to `script_hop.sh`
"""
if s.filename[-1] == '/' :
name = s.filename[-6:-1]
filename = s.filename[:-13]+'hop/grp%s.pos'%name
else:
name = s.filename[-5:]
filename = s.filename[:-12]+'hop/grp%s.pos'%name
try :
data = np.genfromtxt(filename,unpack=True)
except IOError :
import os
dir = s.filename[:-12] if len(s.filename[:-12]) else './'
os.system('cd %s;/home/itp/roskar/ramses/galaxy_formation/script_hop.sh %d;cd ..'%(dir,int(name)))
data = np.genfromtxt(filename,unpack=True)
return data
def co():
import numpy as np
import os
home = os.path.expanduser('~')
band = raw_input('Select the band:')
if band == 'pacs':
upper, lower = 200, 54
if band == 'spire':
upper, lower = 671, 200
level = np.genfromtxt(home+'/data/co_level.txt', dtype='str')
ref = np.genfromtxt(home+'/data/co_ref.txt','str')
for i in range(0,len(level[0:])):
for j in range(0, len(ref[0:])):
if ref[j,1] == level[i,0]:
ref[j,0] = level[i,2]
ref[j,1] = level[i,3]
if ref[j,2] == level[i,0]:
ref[j,2] = level[i,3]
c = 2.998e8
ref[:,4] = c/ref[:,4].astype(float)/1e9*1e6
ref = ref[np.argsort(ref[:,4].astype(float))]
ref_sort = ref
dummy = np.copy(ref[:,0])
ref_sort[:,0],ref_sort[:,1],ref_sort[:,2],ref_sort[:,3],ref_sort[:,4] = ref[:,1],ref[:,2],ref[:,4],ref[:,3],ref[:,5]
ref_sort[:,5] = dummy
ind = np.where((ref_sort[:,2].astype(float) >= lower) & (ref_sort[:,2].astype(float) <= upper))
slt_trans = ref_sort[ind,:]
print slt_trans
print len(slt_trans[0,:,0])
foo = open(home+'/data/co_ref_sort.txt','w')
np.savetxt(foo,ref_sort, fmt='%s')
foo.close()
def main(options):
freq_range=range(options["from"], options["to"]+1)
gt_file=gzip.open(options["gt_file"], "r")
pos_file=gzip.open(options["pos_file"], "r")
out_haps=gzip.open(options["out_root"]+"/haps.gz", "w")
out_haps_fn=[gzip.open(options["out_root"]+"/haps.f"+str(x)+".gz", "w") for x in freq_range]
out_samples=open(options["out_root"]+"/samples.txt", "w")
gt=np.genfromtxt(gt_file, delimiter=1)
pos=np.genfromtxt(pos_file)
pos=np.floor(pos*options["chr_len"]).astype(int)
gt=gt.transpose().astype(int)
# This is because on some platforms the np.genfromtxt tries to import the line endings...
gt=gt[range(len(pos)),]
(nsnp,nind)=gt.shape
ACs=np.sum(gt, axis=1)
MACs=np.minimum(ACs, nind-ACs)
for i in range(nsnp):
out_haps.write(("\t".join(["%d"]*(nind+1))+"\n")%((pos[i],)+tuple(gt[i,])))
if MACs[i]>=options["from"] and MACs[i]<= options["to"]:
idx=MACs[i]-options["from"]
out_haps_fn[idx].write(("\t".join(["%d"]*(nind+1))+"\n")%((pos[i],)+tuple(gt[i,])))
for i in range(int(nind/2)):
out_samples.write("SIM%d\n"%(i+1,))
for fil in [gt_file, pos_file, out_haps]+out_haps_fn:
fil.close()
def get_spec(file, nfreq=300, mu=-1, nmu=10):
#Read file assumed to be in the format of tlusty uni 14 file. File has lines with wavelengths and h followed by blocks with
#intensities and polariztions
spec=np.genfromtxt(file, skip_header=1, invalid_raise=False)
wlh=np.genfromtxt(file, invalid_raise=False)
spec=spec.flatten()
wlh=wlh.flatten()
spec=spec[::2]
wl=wlh[::2]
#mu is and index for the polar angle to use. -1 corresponds to returning emergent flux from the disk
if mu == -1:
h=wlh[1::2]
spec=4*np.pi*h
spec=np.vstack((wl, spec))
return spec
spec=spec.reshape(( nfreq, nmu))
spec=spec[:, mu]
spec=np.vstack((wl,spec))
return spec
def get_cellmap_tensors(mesh,subdomains_assigned,Domains,MRI_param,DTI_param,default_material):
#Tissue_array_get=read_csv('MRI_DTI_derived_data/Tissue_array_MRI.csv', delimiter=' ', header=None)
#Tissue_array=Tissue_array_get.values
Tissue_array=np.load('MRI_DTI_derived_data/Tissue_array_MRI.npy')
x_vect=np.genfromtxt('MRI_DTI_derived_data/x_vector_MRI_Box.csv', delimiter=' ')
y_vect=np.genfromtxt('MRI_DTI_derived_data/y_vector_MRI_Box.csv', delimiter=' ')
z_vect=np.genfromtxt('MRI_DTI_derived_data/z_vector_MRI_Box.csv', delimiter=' ')
#DTI_array_get=read_csv('MRI_DTI_derived_data/Tensor_array_DTI.csv', delimiter=' ', header=None)
#DTI_array=DTI_array_get.values
DTI_array=np.load('MRI_DTI_derived_data/Tensor_array_DTI.npy')
x_vect_DTI=np.genfromtxt('MRI_DTI_derived_data/x_vector_DTI_Box.csv', delimiter=' ')
y_vect_DTI=np.genfromtxt('MRI_DTI_derived_data/y_vector_DTI_Box.csv', delimiter=' ')
z_vect_DTI=np.genfromtxt('MRI_DTI_derived_data/z_vector_DTI_Box.csv', delimiter=' ')
#[Mx_dti,My_dti,Mz_dti]=map_DTI()
#voxel_array_get=read_csv('voxel_array_ROI.csv', delimiter=' ', header=None)
#print voxel_array[0:3,:]
'''structure of voxel array'''
'''rows: x,y,z,material index'''
'''first voxel has to have coordinates: 0+voxelsize_x,0+voxelsize_y,0+voxelsize_z'''
'''Indeed, such a structure of the array is not optimal, since x,y,z coordinates can be simply stored in coordinate vectors'''
'''Any suggestions (incl. minimal working example) will be appreciated'''
'''ROUND DTI DATA!'''
#[aa,ab,ac,ad,ae,af,ag,x_start_DTI,y_start_DTI,z_start_DTI]=np.genfromtxt('MRI_DTI_files/DTI_misc.csv', delimiter=' ')
voxel_size_x=MRI_param.x_vox_size
voxel_size_y=MRI_param.y_vox_size
voxel_size_z=MRI_param.z_vox_size
Mx_mri=MRI_param.M_x
My_mri=MRI_param.M_y
Mz_mri=MRI_param.M_z
voxel_size_x_DTI=DTI_param.x_vox_size
voxel_size_y_DTI=DTI_param.y_vox_size
voxel_size_z_DTI=DTI_param.z_vox_size
Mx_dti=DTI_param.M_x
My_dti=DTI_param.M_y
Mz_dti=DTI_param.M_z
x_vect_DTI[:]=x_vect_DTI[:]+DTI_param.x_start
y_vect_DTI[:]=y_vect_DTI[:]+DTI_param.y_start
z_vect_DTI[:]=z_vect_DTI[:]+DTI_param.z_start
# subdomains =MeshFunctionSizet(mesh, 3, 0)
subdomains = MeshFunction('size_t',mesh,3)
subdomains.set_all(0)
#Create mesh functions for c00, c01, c11
c00 = MeshFunction("double", mesh, 3, 1.0)
c01 = MeshFunction("double", mesh, 3, 0.0)
c02 = MeshFunction("double", mesh, 3, 0.0)
c11 = MeshFunction("double", mesh, 3, 1.0)
c12 = MeshFunction("double", mesh, 3, 0.0)
c22 = MeshFunction("double", mesh, 3, 1.0)
for cell in cells(mesh):
x_coord=cell.midpoint().x()
y_coord=cell.midpoint().y()
z_coord=cell.midpoint().z()
xv_mri=int(x_coord/voxel_size_x-0.000000001) #defines number of steps to get to the voxels containing x[0] coordinate
yv_mri=xv_mri+(int(y_coord/voxel_size_y-0.000000001))*Mx_mri #defines number of steps to get to the voxels containing x[0] and x[1] coordinates
zv_mri=yv_mri+(int(z_coord/voxel_size_z-0.000000001))*Mx_mri*My_mri #defines number of steps to get to the voxels containing x[0], x[1] and x[2] coordinates
k_mri=zv_mri
k_mri=int(k_mri)
if k_mri>=Mx_mri*My_mri*Mz_mri or k_mri<0:
k_mri=0 #cell is outside MRI data, assign first voxel of MRI, which will not fullfil the following condition, and the cell will be remain marked with 0 (default)
x_vect_index=(int((x_coord)/voxel_size_x-0.000000001))
y_vect_index=(int((y_coord)/voxel_size_y-0.000000001))
z_vect_index=(int((z_coord)/voxel_size_z-0.000000001))
if x_vect_index>=Mx_mri or y_vect_index>=My_mri or z_vect_index>=Mz_mri or x_coord<0.0 or y_coord<0.0 or z_coord<0.0:
x_vect_index,y_vect_index,z_vect_index=0,0,0 #cell is outside MRI data, assign first voxel of MRI, which will not fullfil the following condition, and the cell will be remain marked with 0 (default)
xv_dti=int((x_coord-DTI_param.x_start)/voxel_size_x_DTI-0.00000001)
yv_dti=xv_dti+(int((y_coord-DTI_param.y_start)/voxel_size_y_DTI-0.00000001))*Mx_dti
zv_dti=yv_dti+(int((z_coord-DTI_param.z_start)/voxel_size_z_DTI-0.00000001))*Mx_dti*My_dti
k_dti=zv_dti
k_dti=int(k_dti)
if k_dti>=Mx_dti*My_dti*Mz_dti or k_dti<0:
k_dti=0 #cell is outside DTI data, assign first voxel of DTI, which will not fullfil the following condition, and the cell will be considered as isotropic
''' otherwise comment it out'''
x_vect_DTI_index=(int((x_coord-DTI_param.x_start)/voxel_size_x_DTI-0.000000001))
y_vect_DTI_index=(int((y_coord-DTI_param.y_start)/voxel_size_y_DTI-0.000000001))
z_vect_DTI_index=(int((z_coord-DTI_param.z_start)/voxel_size_z_DTI-0.000000001))
if x_vect_DTI_index>=Mx_dti or y_vect_DTI_index>=My_dti or z_vect_DTI_index>=Mz_dti or x_coord<0.0 or y_coord<0.0 or z_coord<0.0 or z_vect_DTI_index<0 or y_vect_DTI_index<0 or x_vect_DTI_index<0:
x_vect_DTI_index,y_vect_DTI_index,z_vect_DTI_index=0,0,0 #cell is outside MRI data, assign first voxel of MRI, which will not fullfil the following condition, and the cell will be remain marked with 0 (default)
'''First, find it in DTI'''
if round(y_vect_DTI[y_vect_DTI_index]-y_coord,8)<=voxel_size_y_DTI and round(y_vect_DTI[y_vect_DTI_index]-y_coord,8)>=0 and round(x_vect_DTI[x_vect_DTI_index]-x_coord,8)<=voxel_size_x_DTI and round(x_vect_DTI[x_vect_DTI_index]-x_coord,8)>=0 and (round(z_vect_DTI[z_vect_DTI_index]-z_coord,8)<=voxel_size_z_DTI and round(z_vect_DTI[z_vect_DTI_index]-z_coord,8)>=0):
if int(Tissue_array[k_mri])==3.0 and (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
subdomains[cell]=3
if int(Tissue_array[k_mri])==2.0 and (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
subdomains[cell]=2
if int(Tissue_array[k_mri])==1.0 and (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
subdomains[cell]=1
if int(Tissue_array[k_mri])==0.0 and (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
#print "Unknown material, assigning GM marker"
subdomains[cell]=int(default_material)
c00[cell] = DTI_array[k_dti,0] #we don't scale tensor with cond. values, because they depend on frequency, will do it directly in EQS_function2
c01[cell] = DTI_array[k_dti,1]
c02[cell] = DTI_array[k_dti,2]
c11[cell] = DTI_array[k_dti,3]
c12[cell] = DTI_array[k_dti,4]
c22[cell] = DTI_array[k_dti,5]
'''if MRI data set is not for the full computational domain, the following condition will be triggered (cell does not correspond to MRI voxel)'''
if not (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
subdomains[cell]=int(default_material)
'''If DTI voxel wasn't found (we might have DTI data only for a part of MRI data)'''
#else:
if not(round(y_vect_DTI[y_vect_DTI_index]-y_coord,8)<=voxel_size_y_DTI and round(y_vect_DTI[y_vect_DTI_index]-y_coord,8)>=0 and round(x_vect_DTI[x_vect_DTI_index]-x_coord,8)<=voxel_size_x_DTI and round(x_vect_DTI[x_vect_DTI_index]-x_coord,8)>=0 and (round(z_vect_DTI[z_vect_DTI_index]-z_coord,8)<=voxel_size_z_DTI and round(z_vect_DTI[z_vect_DTI_index]-z_coord,8)>=0)):
if int(Tissue_array[k_mri])==3.0 and (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
subdomains[cell]=3
if int(Tissue_array[k_mri])==2.0 and (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
subdomains[cell]=2
if int(Tissue_array[k_mri])==1.0 and (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
subdomains[cell]=1
if int(Tissue_array[k_mri])==0.0 and (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
subdomains[cell]=int(default_material)
c00[cell] = 1.0 #no anisotropy defined for this voxel
c01[cell] = 0.0
c02[cell] = 0.0
c11[cell] = 1.0
c12[cell] = 0.0
c22[cell] = 1.0
#'''if MRI data set is not for the full computational domain, the following condition will be triggered (cell does not correspond to MRI voxel)'''
#if not (round(voxel_array[k_mri,1]-y_coord,8)<=voxelsize and round(voxel_array[k_mri,1]-y_coord,8)>=0) and (round(voxel_array[k_mri,0]-x_coord,8)<=voxelsize and round(voxel_array[k_mri,0]-x_coord,8)>=0) and (round(voxel_array[k_mri,2]-z_coord,8)<=voxelsize and round(voxel_array[k_mri,2]-z_coord,8)>=0):
# subdomains[cell]=int(default_material)
for i in range(len(Domains.Encup_index)):
subdomains.array()[subdomains_assigned.array()==Domains.Encup_index[i]]=4 #mark every cell with value 3 from xml to index
c00.array()[subdomains_assigned.array()==Domains.Encup_index[i]]=1.0 #no anisotropy in encap, index 3 is from xml
c01.array()[subdomains_assigned.array()==Domains.Encup_index[i]]=0.0 #no anisotropy in encap
c02.array()[subdomains_assigned.array()==Domains.Encup_index[i]]=0.0 #no anisotropy in encap
c11.array()[subdomains_assigned.array()==Domains.Encup_index[i]]=1.0 #no anisotropy in encap
c12.array()[subdomains_assigned.array()==Domains.Encup_index[i]]=0.0 #no anisotropy in encap
c22.array()[subdomains_assigned.array()==Domains.Encup_index[i]]=1.0 #no anisotropy in encap
if Domains.Float_contacts!=-1:
if isinstance(Domains.Float_contacts,int):
subdomains.array()[subdomains_assigned.array()==Domains.Float_contacts]=5 #4 is index of encap
c00.array()[subdomains_assigned.array()==Domains.Float_contacts]=1.0 #no anisotropy in encap, index 3 is from xml
c01.array()[subdomains_assigned.array()==Domains.Float_contacts]=0.0 #no anisotropy in encap
c02.array()[subdomains_assigned.array()==Domains.Float_contacts]=0.0 #no anisotropy in encap
c11.array()[subdomains_assigned.array()==Domains.Float_contacts]=1.0 #no anisotropy in encap
c12.array()[subdomains_assigned.array()==Domains.Float_contacts]=0.0 #no anisotropy in encap
c22.array()[subdomains_assigned.array()==Domains.Float_contacts]=1.0 #no anisotropy in encap
else:
for i in range(len(Domains.Float_contacts)):
subdomains.array()[subdomains_assigned.array()==Domains.Float_contacts[i]]=5
c00.array()[subdomains_assigned.array()==Domains.Float_contacts[i]]=1.0 #no anisotropy in encap, index 3 is from xml
c01.array()[subdomains_assigned.array()==Domains.Float_contacts[i]]=0.0 #no anisotropy in encap
c02.array()[subdomains_assigned.array()==Domains.Float_contacts[i]]=0.0 #no anisotropy in encap
c11.array()[subdomains_assigned.array()==Domains.Float_contacts[i]]=1.0 #no anisotropy in encap
c12.array()[subdomains_assigned.array()==Domains.Float_contacts[i]]=0.0 #no anisotropy in encap
c22.array()[subdomains_assigned.array()==Domains.Float_contacts[i]]=1.0 #no anisotropy in encap
#subdomains.array()[subdomains_assigned.array()==3]=4 #mark every cell with value 3 from xml to index encap as 4
#here we also keep a reference to the number of elements as the mesh identifier
hdf = HDF5File(mesh.mpi_comm(), 'Results_adaptive/Tensors_to_solve_num_el_'+str(mesh.num_cells())+'.h5', 'w')
hdf.write(c00, "/c00")
hdf.write(c01, "/c01")
hdf.write(c02, "/c02")
hdf.write(c11, "/c11")
hdf.write(c12, "/c12")
hdf.write(c22, "/c22")
hdf.close()
file=File('Tensors/c00_unscaled.pvd')
file<<c00,mesh
file=File('Tensors/c01_unscaled.pvd')
file<<c01,mesh
file=File('Tensors/c02_unscaled.pvd')
file<<c02,mesh
file=File('Tensors/c11_unscaled.pvd')
file<<c11,mesh
file=File('Tensors/c12_unscaled.pvd')
file<<c12,mesh
file=File('Tensors/c22_unscaled.pvd')
file<<c22,mesh
#Store to file
#mesh_file = File("mesh.xml.gz")
#c00_file = File("Tensors/c00.xml.gz")
#c01_file = File("Tensors/c01.xml.gz")
#c02_file = File("Tensors/c02.xml.gz")
#c11_file = File("Tensors/c11.xml.gz")
#c12_file = File("Tensors/c12.xml.gz")
#c22_file = File("Tensors/c22.xml.gz")
##mesh_file << mesh
#c00_file << c00
#c01_file << c01
#c02_file << c02
#c11_file << c11
#c12_file << c12
#c22_file << c22
return (subdomains) #cond, perm are cell functions and can be directly used
def read_file(file_name):
return np.genfromtxt('data/' + file_name, delimiter=',')
sys.stdout.flush()
# reset seed the same each sample k
if (seed > 0):
np.random.seed(seed)
# put everything in "try", so error on one sample run won't crash whole job. But this makes it hard to debug, so may want to comment this out when debugging.
# try:
######################################################################################
if model_mode == 'sensitivity':
#####FLOW GENERATOR#####
print(k)
# read in k'th sample from sensitivity sample file
sensitivity_sample = np.genfromtxt(sensitivity_sample_file,
delimiter='\t',
skip_header=k + 1,
max_rows=1)[1:]
with open(sensitivity_sample_file, 'r') as f:
sensitivity_sample_names = f.readlines()[0].split('\n')[0]
np.savetxt(results_folder + '/sample' + str(k) + '.txt',
sensitivity_sample.reshape(1, len(sensitivity_sample)),
delimiter=',',
header=sensitivity_sample_names)
sensitivity_sample_names = sensitivity_sample_names.split('\t')[1:]
#Initialize flow input scenario based on sensitivity sample
new_inputs = Inputter(base_data_file,
expected_release_datafile,
model_mode,
results_folder,
k,
subj_corrs = []
for net in nets:
for hemi in ['L', 'R']:
# figure out all ROIs for which to compute correlation
if age=='adults':
rois_fname = data_dir + '/regionsIn%d_%s.1D'%(net,hemi)
else:
rois_fname = data_dir + '/regionsIn%d_%s.1D'%(net,hemi)
fid = open(rois_fname, 'r')
rois = [line.rstrip() for line in fid]
fid.close()
# calculate the pairwise correlations within the network
net_dir = data_dir + '/net%d/'%net
for rcomb in itertools.combinations(rois,2):
seed1 = np.genfromtxt('%s/%s_%sin%d.1D'%(net_dir,subj,rcomb[0],net))
seed2 = np.genfromtxt('%s/%s_%sin%d.1D'%(net_dir,subj,rcomb[1],net))
# if there were problems creating the seed average
if len(seed1)==0 or len(seed2)==0:
val = np.nan
else:
val = np.corrcoef(seed1,seed2)[0,1]
subj_corrs.append(val)
all_corrs.append(subj_corrs)
# construct first column
rows = [[i] + k for i,k in zip(subjs, all_corrs)]
# construct header (only for adults), slim version
if age=='adults':
header = ['maskids']
cs = 0
i = 0
ms = 0
mf = 0
df = []
ds = []
l = []
lf = 0
ls = 0
lcf = 0
lcs = 0
fs = 0
ff = 0
train_label = []
test_label = []
train_dat = np.genfromtxt('P2_train.csv', delimiter=',')
test_dat = np.genfromtxt('P2_test.csv', delimiter=',')
train_data = np.array(train_dat)
train_label = train_data[:, 2]
train_data = train_data[:, (0, 1)]
test_data = np.array(test_dat)
test_label = test_data[:, 2]
test_data = test_data[:, (0, 1)]
for i in range(0, 310):
if (train_label[i] == 1):
mf = mf + train_data[i]
df.append(train_data[i])
cf = cf + 1
else:
ms = ms + train_data[i]
ds.append(train_data[i])
# plt.scatter(angles_cut[phase_mask], cosqm_obs[:,9][phase_mask],s=0.1, label='Y',c='y')
# plt.legend()
#%%--------------------
#
# AOD Values from AERONET network
# Values are from level 1.0, meaning there are no corrections
#
# COSQM Day format changed from 06-11: full night to split at midnight, so beware
# read from columns 4 to 25 (aod 1640 to 340nm)
cols = np.arange(4, 26)
path = 'cosqm_santa/20190611_20191231_Santa_Cruz_Tenerife.lev10'
header = 7
data_aod = np.genfromtxt(path, delimiter=',', skip_header=header, usecols=cols)
data_aod[data_aod < 0] = 0
# DATES
dates_str = np.genfromtxt(path,
delimiter=',',
skip_header=header,
usecols=[0, 1],
dtype=str)
dates_aod = np.array([
dt.strptime(dates + times, '%d:%m:%Y%H:%M:%S').timestamp()
for dates, times in dates_str
])
# BANDS AOD_1640nm', 'AOD_1020nm', 'AOD_870nm', 'AOD_865nm', 'AOD_779nm',
# 'AOD_675nm', 'AOD_667nm', 'AOD_620nm', 'AOD_560nm', 'AOD_555nm',
def get_cellmap(mesh,subdomains_assigned,Domains,MRI_param,default_material):
#Tissue_array_get=read_csv('MRI_DTI_derived_data/Tissue_array_MRI.csv', delimiter=' ', header=None)
#Tissue_array=Tissue_array_get.values
Tissue_array=np.load('MRI_DTI_derived_data/Tissue_array_MRI.npy')
x_vect=np.genfromtxt('MRI_DTI_derived_data/x_vector_MRI_Box.csv', delimiter=' ')
y_vect=np.genfromtxt('MRI_DTI_derived_data/y_vector_MRI_Box.csv', delimiter=' ')
z_vect=np.genfromtxt('MRI_DTI_derived_data/z_vector_MRI_Box.csv', delimiter=' ')
#[Mx_mri,My_mri,Mz_mri]=map_MRI() #don't need to run know, the voxel_array already formed
#voxel_array_get=read_csv('voxel_array_ROI.csv', delimiter=' ', header=None)
#voxel_array_get=read_csv('MRI_DTI_files/voxel_array_'+MRI_param.name[:-4]+'.csv', delimiter=' ', header=None)
#voxel_array=voxel_array_get.values
#print voxel_array[0:3,:]
'''structure of voxel array'''
'''rows: x,y,z,material index'''
'''first voxel has to have coordinates: 0+voxelsize_x,0+voxelsize_y,0+voxelsize_z'''
'''Indeed, such a structure of the array is not optimal, since x,y,z coordinates can be simply stored in coordinate vectors'''
'''Any suggestions (incl. minimal working example) will be appreciated'''
voxel_size_x=MRI_param.x_vox_size
voxel_size_y=MRI_param.y_vox_size
voxel_size_z=MRI_param.z_vox_size
#print MRI_param.M_x,MRI_param.M_y,MRI_param.M_z
Mx_mri=MRI_param.M_x
My_mri=MRI_param.M_y
Mz_mri=MRI_param.M_z
subdomains = MeshFunction('size_t',mesh,3)
subdomains.set_all(0)
#subdomains = MeshFunctionSizet(mesh, 3, 0)
for cell in cells(mesh):
x_coord=cell.midpoint().x()
y_coord=cell.midpoint().y()
z_coord=cell.midpoint().z()
xv_mri=int((x_coord)/voxel_size_x-0.000000001) #defines number of steps to get to the voxels containing x[0] coordinate
yv_mri=(int((y_coord)/voxel_size_y-0.000000001))*Mx_mri #defines number of steps to get to the voxels containing x[0] and x[1] coordinates
zv_mri=(int((z_coord)/voxel_size_z-0.000000001))*Mx_mri*My_mri #defines number of steps to get to the voxels containing x[0], x[1] and x[2] coordinates
k_mri=xv_mri+yv_mri+zv_mri
#print k_mri
k_mri=int(k_mri)
x_vect_index=(int((x_coord)/voxel_size_x-0.000000001))
y_vect_index=(int((y_coord)/voxel_size_y-0.000000001))
z_vect_index=(int((z_coord)/voxel_size_z-0.000000001))
if x_vect_index>=Mx_mri or y_vect_index>=My_mri or z_vect_index>=Mz_mri or x_coord<0.0 or y_coord<0.0 or z_coord<0.0:
x_vect_index,y_vect_index,z_vect_index=0,0,0 #cell is outside MRI data, assign first voxel of MRI, which will not fullfil the following condition, and the cell will be remain marked with 0 (default)
if k_mri>=Mx_mri*My_mri*Mz_mri or k_mri<0:
k_mri=0 #cell is outside MRI data, assign first voxel of MRI, which will not fullfil the following condition, and the cell will be remain marked with 0 (default)
'''First, find the corresponding voxel in MRI, then check the value of the voxel and assign a corresponding conductivity'''
if (round(y_vect[y_vect_index]-y_coord,8)<=voxel_size_y and round(y_vect[y_vect_index]-y_coord,8)>=0 and (round(x_vect[x_vect_index]-x_coord,8)<=voxel_size_x and round(x_vect[x_vect_index]-x_coord,8)>=0) and round(z_vect[z_vect_index]-z_coord)<=voxel_size_z and round(z_vect[z_vect_index]-z_coord,8)>=0):
if int(Tissue_array[k_mri])==3:
#cell.mark(subdomains, 1)
subdomains[cell]=3
if int(Tissue_array[k_mri])==2:
subdomains[cell]=2
if int(Tissue_array[k_mri])==1:
subdomains[cell]=1
if int(Tissue_array[k_mri])==0:
subdomains[cell]=default_material
#old way
# if Domains.Float_contacts!=-1:
# subdomains.array()[subdomains_assigned.array()==Domains.Float_contacts]=5 #5 is index for float contact (very high cond and perm)
if Domains.Float_contacts!=-1:
if isinstance(Domains.Float_contacts,int):
subdomains.array()[subdomains_assigned.array()==Domains.Float_contacts]=5
else:
for i in range(len(Domains.Float_contacts)):
subdomains.array()[subdomains_assigned.array()==Domains.Float_contacts[i]]=5 #5 is index for float contact (very high cond and perm)
for i in range(len(Domains.Encup_index)):
subdomains.array()[subdomains_assigned.array()==Domains.Encup_index[i]]=4 #4 is index of encap
# boundaries = MeshFunction('size_t',mesh,'Meshes/Mesh_unref_facet_region.xml')
#
# for cell in cells(mesh):
# for f in facets(cell):
# #print(boundaries[f]!=0)
# if boundaries[f]!=0:
# subdomains[cell]=4 #cells on the interface should be marked with encap, otherwise no current on the floating contact
# for v in vertices(f): #also all cells which vertices are on the active boundaries should be marked
# for c in cells(v):
# subdomains[c]=4
# #cell_markers[cell] = True
#file=File('subdomains.pvd')
#file<<subdomains
del Tissue_array
return (subdomains) #cond, perm are cell functions and can be directly used
inferredNetworkFileName = str(sys.argv[3])
N = int(sys.argv[4])
plot = False
if dataset != 0:
filename = 'CRCNS/data/DataSet' + str(dataset) + '.mat'
N = scipy.io.loadmat(filename)['data']['nNeurons'][0][0][0][0]
x_array = scipy.io.loadmat(filename)['data']['x'][0][0][0]
y_array = scipy.io.loadmat(filename)['data']['y'][0][0][0]
S_hat = np.zeros((N,N))
for i in range(N):
filename = aHatFileName + str(i+1) + '.csv'
S_hat[i] = np.genfromtxt(filename, delimiter=',')
# Transpose the inferred matrix
S_hat = S_hat.T
# Convert NaN to zeros
S_hat[np.where(np.isnan(S_hat))] = 0
# Remove values very close to zero
S_hat[np.where(S_hat <= np.percentile(S_hat,99, interpolation='lower'))] = 0
S_hat = np.interp(S_hat, np.linspace(S_hat.min(),S_hat.max(),1000), np.linspace(0,30,1000))
# S_hat[np.where(S_hat < 5)] = 0
if dataset != 0 and plot:
print new_file
old_file = iOld.next()
newD = sio.loadmat(new_file)
print newD
newD = newD['fibergraph']
newD = newD.todense()
print newD
#newD = np.delete(newD,0,1)
#newD = np.delete(newD,0,0)
oldD = genfromtxt(old_file, delimiter=' ')
oldD = np.delete(oldD, 0, 1)
oldD = np.delete(oldD, 0, 0)
diff = np.subtract(newD, oldD)
absdiff = np.absolute(diff)
result = np.divide(absdiff, oldD)
resultArray = np.add(diff, resultArray)
#np.savetxt((outputDir + "analysis" + str(counter) + ".csv"), result, delimiter=",")
#counter = counter + 1
tempRow = list()
def main():
"""
Train the TensorFlow models.
"""
# Get hyper-parameters
if os.path.exists(FLAGS.checkpoint_path) == False:
os.makedirs(FLAGS.checkpoint_path)
checkpoint_file_path = FLAGS.checkpoint_path + "/checkpoint.ckpt"
latest_checkpoint_file_path = tf.train.latest_checkpoint(
FLAGS.checkpoint_path)
if os.path.exists(FLAGS.output_path) == False:
os.makedirs(FLAGS.output_path)
# Step 1: Construct the dataset op
epoch_number = FLAGS.epoch_number
if epoch_number <= 0:
epoch_number = -1
train_buffer_size = FLAGS.train_batch_size * 3
validation_buffer_size = FLAGS.train_batch_size * 3
train_filename_list = [filename for filename in FLAGS.train_files.split(",")]
train_filename_placeholder = tf.placeholder(tf.string, shape=[None])
if FLAGS.file_format == "tfrecords":
train_dataset = tf.data.TFRecordDataset(train_filename_placeholder)
train_dataset = train_dataset.map(parse_tfrecords_function).repeat(
epoch_number).batch(FLAGS.train_batch_size).shuffle(
buffer_size=train_buffer_size)
elif FLAGS.file_format == "csv":
# Skip the header or not
train_dataset = tf.data.TextLineDataset(train_filename_placeholder)
train_dataset = train_dataset.map(parse_csv_function).repeat(
epoch_number).batch(FLAGS.train_batch_size).shuffle(
buffer_size=train_buffer_size)
train_dataset_iterator = train_dataset.make_initializable_iterator()
train_features_op, train_label_op = train_dataset_iterator.get_next()
validation_filename_list = [
filename for filename in FLAGS.validation_files.split(",")
]
validation_filename_placeholder = tf.placeholder(tf.string, shape=[None])
if FLAGS.file_format == "tfrecords":
validation_dataset = tf.data.TFRecordDataset(
validation_filename_placeholder)
validation_dataset = validation_dataset.map(
parse_tfrecords_function).repeat(epoch_number).batch(
FLAGS.validation_batch_size).shuffle(
buffer_size=validation_buffer_size)
elif FLAGS.file_format == "csv":
validation_dataset = tf.data.TextLineDataset(
validation_filename_placeholder)
validation_dataset = validation_dataset.map(parse_csv_function).repeat(
epoch_number).batch(FLAGS.validation_batch_size).shuffle(
buffer_size=validation_buffer_size)
validation_dataset_iterator = validation_dataset.make_initializable_iterator(
)
validation_features_op, validation_label_op = validation_dataset_iterator.get_next(
)
# Step 2: Define the model
input_units = FLAGS.feature_size
output_units = FLAGS.label_size
logits = inference(train_features_op, input_units, output_units, True)
if FLAGS.loss == "sparse_cross_entropy":
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=train_label_op)
loss = tf.reduce_mean(cross_entropy, name="loss")
elif FLAGS.loss == "cross_entropy":
#train_label_op =
#validation_label_op =
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=train_label_op)
loss = tf.reduce_mean(cross_entropy, name="loss")
elif FLAGS.loss == "mean_square":
msl = tf.square(logits - train_label_op, name="msl")
loss = tf.reduce_mean(msl, name="loss")
global_step = tf.Variable(0, name="global_step", trainable=False)
learning_rate = FLAGS.learning_rate
if FLAGS.enable_lr_decay:
logging.info(
"Enable learning rate decay rate: {}".format(FLAGS.lr_decay_rate))
starter_learning_rate = FLAGS.learning_rate
learning_rate = tf.train.exponential_decay(
starter_learning_rate,
global_step,
100000,
FLAGS.lr_decay_rate,
staircase=True)
optimizer = get_optimizer_by_name(FLAGS.optimizer, learning_rate)
train_op = optimizer.minimize(loss, global_step=global_step)
# Need to re-use the Variables for training and validation
tf.get_variable_scope().reuse_variables()
# Define accuracy op and auc op for train
train_accuracy_logits = inference(train_features_op, input_units,
output_units, False)
train_softmax_op, train_accuracy_op = model.compute_softmax_and_accuracy(
train_accuracy_logits, train_label_op)
train_auc_op = model.compute_auc(train_softmax_op, train_label_op,
FLAGS.label_size)
# Define accuracy op and auc op for validation
validation_accuracy_logits = inference(validation_features_op, input_units,
output_units, False)
validation_softmax_op, validation_accuracy_op = model.compute_softmax_and_accuracy(
validation_accuracy_logits, validation_label_op)
validation_auc_op = model.compute_auc(validation_softmax_op,
validation_label_op, FLAGS.label_size)
# Define inference op
inference_features = tf.placeholder(
"float", [None, FLAGS.feature_size], name="features")
inference_logits = inference(inference_features, input_units, output_units,
False)
inference_softmax_op = tf.nn.softmax(
inference_logits, name="inference_softmax")
inference_prediction_op = tf.argmax(
inference_softmax_op, 1, name="inference_prediction")
keys_placeholder = tf.placeholder(tf.int32, shape=[None, 1], name="keys")
keys_identity = tf.identity(keys_placeholder, name="inference_keys")
model_signature = signature_def_utils.build_signature_def(
inputs={
"keys": utils.build_tensor_info(keys_placeholder),
"features": utils.build_tensor_info(inference_features)
},
outputs={
"keys": utils.build_tensor_info(keys_identity),
"prediction": utils.build_tensor_info(inference_prediction_op),
"softmax": utils.build_tensor_info(inference_softmax_op),
},
method_name=signature_constants.PREDICT_METHOD_NAME)
# Initialize saver and summary
saver = tf.train.Saver()
tf.summary.scalar("loss", loss)
if FLAGS.scenario == "classification":
tf.summary.scalar("train_accuracy", train_accuracy_op)
tf.summary.scalar("train_auc", train_auc_op)
tf.summary.scalar("validate_accuracy", validation_accuracy_op)
tf.summary.scalar("validate_auc", validation_auc_op)
summary_op = tf.summary.merge_all()
init_op = [
tf.global_variables_initializer(),
tf.local_variables_initializer()
]
# Step 3: Create session to run
with tf.Session() as sess:
writer = tf.summary.FileWriter(FLAGS.output_path, sess.graph)
sess.run(init_op)
sess.run(
[
train_dataset_iterator.initializer,
validation_dataset_iterator.initializer
],
feed_dict={
train_filename_placeholder: train_filename_list,
validation_filename_placeholder: validation_filename_list
})
if FLAGS.mode == "train":
if FLAGS.resume_from_checkpoint:
restore_from_checkpoint(sess, saver, latest_checkpoint_file_path)
try:
start_time = datetime.datetime.now()
while True:
if FLAGS.enable_benchmark:
sess.run(train_op)
else:
_, global_step_value = sess.run([train_op, global_step])
# Step 4: Display training metrics after steps
if global_step_value % FLAGS.steps_to_validate == 0:
if FLAGS.scenario == "classification":
loss_value, train_accuracy_value, train_auc_value, validate_accuracy_value, validate_auc_value, summary_value = sess.run(
[
loss, train_accuracy_op, train_auc_op,
validation_accuracy_op, validation_auc_op, summary_op
])
end_time = datetime.datetime.now()
logging.info(
"[{}] Step: {}, loss: {}, train_acc: {}, train_auc: {}, valid_acc: {}, valid_auc: {}".
format(end_time - start_time, global_step_value,
loss_value, train_accuracy_value, train_auc_value,
validate_accuracy_value, validate_auc_value))
elif FLAGS.scenario == "regression":
loss_value, summary_value = sess.run([loss, summary_op])
end_time = datetime.datetime.now()
logging.info("[{}] Step: {}, loss: {}".format(
end_time - start_time, global_step_value, loss_value))
writer.add_summary(summary_value, global_step_value)
saver.save(
sess, checkpoint_file_path, global_step=global_step_value)
start_time = end_time
except tf.errors.OutOfRangeError:
if FLAGS.enable_benchmark:
logging.info("Finish training for benchmark")
else:
# Step 5: Export the model after training
save_model(
FLAGS.model_path,
FLAGS.model_version,
sess,
model_signature,
is_save_graph=False)
elif FLAGS.mode == "savedmodel":
if restore_from_checkpoint(sess, saver,
latest_checkpoint_file_path) == False:
logging.error("No checkpoint for exporting model, exit now")
return
save_model(
FLAGS.model_path,
FLAGS.model_version,
sess,
model_signature,
is_save_graph=False)
elif FLAGS.mode == "inference":
if restore_from_checkpoint(sess, saver,
latest_checkpoint_file_path) == False:
logging.error("No checkpoint for inference, exit now")
return
# Load test data
inference_result_file_name = FLAGS.inference_result_file
inference_test_file_name = FLAGS.inference_data_file
inference_data = np.genfromtxt(inference_test_file_name, delimiter=",")
inference_data_features = inference_data[:, 0:9]
inference_data_labels = inference_data[:, 9]
# Run inference
start_time = datetime.datetime.now()
prediction, prediction_softmax = sess.run(
[inference_prediction_op, inference_softmax_op],
feed_dict={inference_features: inference_data_features})
end_time = datetime.datetime.now()
# Compute accuracy
label_number = len(inference_data_labels)
correct_label_number = 0
for i in range(label_number):
if inference_data_labels[i] == prediction[i]:
correct_label_number += 1
accuracy = float(correct_label_number) / label_number
# Compute auc
y_true = np.array(inference_data_labels)
y_score = prediction_softmax[:, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_score, pos_label=1)
auc = metrics.auc(fpr, tpr)
logging.info("[{}] Inference accuracy: {}, auc: {}".format(
end_time - start_time, accuracy, auc))
# Save result into the file
np.savetxt(inference_result_file_name, prediction_softmax, delimiter=",")
logging.info(
"Save result to file: {}".format(inference_result_file_name))
def preprocess(path):
n_features = 18
data = np.genfromtxt(path, delimiter=',', encoding='big5')[1:, 3:]
# original delete 722 ~ 1441 (march and april)
# data = np.delete(data, range(721,1441), axis=0)
# data = np.delete(data, range(10,data.shape[0], n_features), axis=0) # delete RAINFALL (NR)
for i in range(10, data.shape[0], n_features):
# data[i] = np.zeros(24) # convert RAINFALL to 0
for j in range(data[i].shape[0]): # convert nan(NR) to 0
if np.isnan(data[i][j]):
data[i][j] = 0
n_data = data.shape[0] // 18
# interpolation for negative value
# for i in range(data.shape[0]):
# for j in range(data.shape[1]):
# if data[i][j] < 0:
# if j == 0 or data[i][j-1]<0:
# if data[i][j+1] > 0:
# data[i][j] = data[i][1]
# else:
# data[i][j] = 0
# elif j == 23 or data[i][j+1]<0:
# if data[i][j-1] > 0:
# data[i][j] = data[i][22]
# else:
# data[i][j] = 0
# else:
# data[i][j] = (data[i][j-1] + data[i][j+1]) / 2
# data.shape = (240*18,24)
# data indexed 9 is PM2.5
# x_train = []
# y_train = []
# for i in range(0, n_data):
# for j in range(0, 24-10):
# x_train.append(data[i*n_features:(i+1)*n_features, j:j+9]) # get previous 9 hours
# y_train.append(data[i*n_features+9][j+9]) # predict the 10-th hour
# return np.array(x_train), np.array(y_train)
# contact data into (12,18,480)
concat_data = []
data = np.array_split(data, n_data)
for i in range(12):
concat_data.append(
np.concatenate(data[(i * 20):((i + 1) * 20)], axis=1))
concat_data = np.array(concat_data)
# linear interpolation
for i in range(12):
for j in range(18):
for k in range(480):
if concat_data[i][j][k] < 0:
l, r = 0, 0
if k == 0 or k == 479:
concat_data[i][j][k] = 0
continue
else:
l = k - 1
r = k + 1
while (concat_data[i][j][l] < 0):
l -= 1
if l < 0:
concat_data[i][j][0] = 0
l = 1
break
while (concat_data[i][j][r] < 0):
r += 1
if r > 479:
concat_data[i][j][479] = 0
r = 478
break
delta = (concat_data[i][j][r] -
concat_data[i][j][l]) / (r - l)
concat_data[i][j][l:r] = [
concat_data[i][j][l] + (x - l) * delta
for x in range(l, r)
]
x_train = []
y_train = []
for i in range(0, 12):
for j in range(0, 480 - 9):
x_train.append(concat_data[i][:, j:j + 9]) # get previous 9 hours
y_train.append(concat_data[i][9, j + 9]) # predict the 10-th hour
print(len(x_train))
return np.array(x_train), np.array(y_train)
from sklearn.preprocessing import Imputer
def calculateErrorRates(clf, testset, testresult):
errorRates = 0;
totalPoints = 0;
for data, realresult in zip(testset,testresult):
prediction = clf.predict([data])
errorRates += math.sqrt((realresult-prediction)*(realresult-prediction))
totalPoints += 1;
errorRates = errorRates / totalPoints;
print(errorRates)
return(errorRates)
print('start')
data = np.genfromtxt('modifiedData.csv', delimiter= ",")
imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
imp.fit(data)
for i in data:
if math.isnan(i[3]):
i[3] = 0.0
#data = preprocessing.scale(data)
pca = PCA(n_components=3)
data = pca.fit_transform(data)
#/usr/local/bin/python
import matplotlib
from matplotlib import rc
from matplotlib.font_manager import FontProperties
from matplotlib.ticker import MaxNLocator, FormatStrFormatter
import matplotlib.pyplot as plt
import numpy as np
matplotlib.rcParams['mathtext.fontset'] = 'cm'
matplotlib.rcParams['font.family'] = 'STIXGeneral'
majorFormatter = FormatStrFormatter('%d')
data = np.genfromtxt('banditMeanAERVsTimeSummaryPostProcessed.csv',
delimiter=',',
names=True)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_title(
r"$Ensemble\ Algorithm\ Comparison\ -\ Mean\ Relative\ AER\ vs.\ Time$",
fontsize='16',
y=1.02)
ax.set_xlabel(r"$Time\ (Seconds)$")
ax.set_ylabel(r"$Mean\ Relative\ AER$")
ax.plot(data['TimeBin'],
data['EnsembleRandom'] / data['Random'],
lw='1.25',
from astropy.io import fits
import numpy as np
from astropy.table import Table
from collections import OrderedDict
data = fits.open("catalogue.fits")
snData = fits.open("../line_fluxes/sn_R1000_lines_combined.fits")
lineArr = ["C4_1548","C4_1551","O3_1661","O3_1666","C3_1907_C3_1910","O2_3726_O2_3729","Ne3_3869","O3_4363","HBaB_4861","O3_4959","O3_5007","HBaA_6563","N2_6584","S2_6716","S2_6731"]
eMPTout1 = np.genfromtxt("output_dither_MDC_00.txt", dtype=None, names=True)
eMPTout2 = np.genfromtxt("output_dither_MDC_01.txt", dtype=None, names=True)
eMPTout3 = np.genfromtxt("output_dither_MDC_02.txt", dtype=None, names=True)
outputDict = OrderedDict()
outputDict["ID"] = snData[1].data["ID"]
id_for_peter = np.zeros_like(snData[1].data["ID"])-1
for i in range(len(id_for_peter)):
tempIdx = np.where(eMPTout1["ID_cat"] == snData[1].data["ID"][i])[0]
if len(tempIdx) > 0:
id_for_peter[i] = eMPTout1["ID_emma"][tempIdx]-1
else:
tempIdx = np.where(eMPTout2["ID_cat"] == snData[1].data["ID"][i])[0]
if len(tempIdx) > 0:
id_for_peter[i] = eMPTout2["ID_emma"][tempIdx]-1
else:
tempIdx = np.where(eMPTout3["ID_cat"] == snData[1].data["ID"][i])[0]
if len(tempIdx) > 0:
id_for_peter[i] = eMPTout3["ID_emma"][tempIdx]-1
outputDict["ID_for_peter"] = id_for_peter
outputDict["redshift"] = data['GALAXY PROPERTIES'].data['redshift']
rand_col_verif = 100*np.random.rand(len(x_verif))
x_verif = np.insert(x_verif, 2, rand_col_verif, axis=1)
N = 100
#Select noise
iter_noise = 2.1
#Select training set
train_filename = 'samples-'+str(N)+'/'+str(N)+'_'+str(iter_noise).replace('.','-')+'_1-train'#+str(case)+'-train'
#Select test set
test_filename = 'samples-'+str(N)+'/'+str(N)+'_'+str(iter_noise).replace('.','-')+'_1-test'#+str(case)+'-test'
metric_array = []
mse_array = []
train_data = np.genfromtxt(train_filename, delimiter=",")
test_data = np.genfromtxt(test_filename, delimiter=",")
x_train = train_data[:,:-1]
y_train = train_data[:,-1]
x_test = test_data[:,:-1]
y_test = test_data[:,-1]
#Add a column of random values - these should be bad for performance
rand_col_train = 100*np.random.rand(len(x_train))
x_train = np.insert(x_train, 2, rand_col_train, axis=1)
rand_col_test = 100*np.random.rand(len(x_test))
x_verif = np.insert(x_test, 2, rand_col_test, axis=1)
#3 nested loops - generate scaled weights
lin_combos = []
from uncertainties.unumpy import nominal_values as noms
from uncertainties.unumpy import std_devs as stds
def E_00(d,I_0,d_0, omega):
return I_0*np.exp( -2*( (d-d_0)/omega )**2 )
def E_10(d, I_01, d_01, omega_1, I_02, d_02, omega_2):
return I_01 * np.exp( -2 * ( (d-d_01) / omega_1 )**2 )+ I_02 * np.exp( -2 * ( ( d-d_02) / omega_2 )**2 )
def polarisation( phi, I_0, phi_0):
return I_0*( np.cos(phi-phi_0) )**2
###################
# Auswertung T_00 #
###################
print('\n\n\n-----------------------------------------------------------\n','Auswertung der Grundmode T_00', '\n', '-----------------------------------------------------------\n\n\n')
r_00, I_00=np.genfromtxt('Messwerte\Grundmode.txt',unpack=True)
params_I_00, cov_I_00 = curve_fit(E_00,r_00,I_00 )
error_I_00= np.sqrt(np.diag(cov_I_00))
amplitude_I_00 = ufloat(params_I_00[0],error_I_00[0])
verschiebung_I_00 = ufloat( params_I_00[1], error_I_00[1])
omega_I_00 = ufloat( params_I_00[2], error_I_00[2])
print('Amplitude: ', amplitude_I_00,'\n')
print('Verschiebung ', verschiebung_I_00,'\n')
print('Frequenz ', omega_I_00, '\n')
# Plotten
d_00=np.linspace(r_00[0],r_00[-1],10000)
import numpy as np
import matplotlib.pylab as plt
import sys
def exp(x,Lambda=1):
return Lambda*np.exp(-Lambda*x)
datos=sys.argv[1]
Lambda = float(sys.argv[2])
archivo=np.genfromtxt(datos)
a=archivo
x = np.linspace(0,10.0*1.0/Lambda**2)
plt.plot(x,exp(x,Lambda))
plt.hist(a,bins=50,density=True)
# plt.savefig("Histogram.pdf")
plt.show()
def read(filename, delimiter=','):
return np.genfromtxt(filename, delimiter=delimiter)
#Ben Moreau
import numpy as np
from math import *
from matplotlib import pyplot as plt
import os
Num_exp = np.genfromtxt("param.txt", skip_header=1)[101, 0]
Num_rolls = np.genfromtxt("param.txt", skip_header=1)[101, 1]
Num_sides = np.genfromtxt("param.txt", skip_header=1)[101, 2]
Side_number = np.genfromtxt("param.txt", skip_header=1)[101, 3]
Dice_power = np.genfromtxt("param.txt", skip_header=1)[101, 4]
LogLikeRatio0I = np.genfromtxt("log_like0.txt", skip_header=0)[:]
LogLikeRatio1I = np.genfromtxt("log_like1.txt", skip_header=0)[:]
LogLikeRatio0 = []
LogLikeRatio1 = []
for i in range(len(LogLikeRatio0I)):
LogLikeRatio0.append((LogLikeRatio0I[i]))
LogLikeRatio1.append((LogLikeRatio1I[i]))
Num_rolls = int(Num_rolls)
# make LLR figure
plt.hist(LogLikeRatio1,
Num_rolls + 1,
density=True,
facecolor='green',
alpha=0.7,
label="assuming $\\mathbb{H}_1$")
plt.hist(LogLikeRatio0,
def optimum_rotor():
#====INPUT starts here================
#setup_file=input("Enter the setup file: ")
setup_file='setup/optimum_rotor.inp'
#---Read input setup file
with open(setup_file,'r') as f:
lines = f.readlines()
#---save file related
save_folder= lines[3].split(' ')[0]
save_file= lines[4].split(' ')[0]
save_file=save_folder+'/'+save_file
#---Computation related
wake_rotation=str2bool(lines[6].split(' ')[0]) # Flag to include wake rotation in optimum rotor calculation. True or False (logic)
#---Rotor parameters
Drotor =float(lines[11].split(' ')[0]) # Rotor diameter
Rad_rotor=Drotor/2.0 # Radius of rotor
Nblade =int(lines[12].split(' ')[0]) # Number of blades
tsr =float(lines[13].split(' ')[0]) # Tip speed ratio
#---Airfoil properties
Nairfoil =int(lines[17].split(' ')[0]) # Number of airfoils
Airfoilfile=np.empty((Nairfoil),dtype=object)
Airfoil_no =np.empty((Nairfoil),dtype=int)
AoA_col =np.empty((Nairfoil),dtype=int) # Column with angle of attack in Cl Cd file
Cl_col =np.empty((Nairfoil),dtype=int)
Cd_col =np.empty((Nairfoil),dtype=int)
header =np.empty((Nairfoil),dtype=int)
footer =np.empty((Nairfoil),dtype=int)
#---Airoifl file information
for n in range(Nairfoil):
Airfoilfile[n]=lines[19+n].split(',')[0]
Airfoil_no[n] =int(lines[19+n].split(',')[1])-1
AoA_col[n] =int(lines[19+n].split(',')[2])-1
Cl_col[n] =int(lines[19+n].split(',')[3])-1
Cd_col[n] =int(lines[19+n].split(',')[4])-1
header[n] =int(lines[19+n].split(',')[5]) #No. lines as header in airfoil dat file
footer[n] =int(lines[19+n].split(',')[6])
#---Blade properties
Nnode =int(lines[30].split(' ')[0])
Rnode =np.empty((Nnode),dtype=float) # Center of node
dRnode =np.empty((Nnode),dtype=float) # Span of node
Airfoil_id=np.empty((Nnode),dtype=int) # Airfoil number to use at the node
for n in range(Nnode):
Rnode[n]=float(lines[32+n].split(',')[0])
dRnode[n]=float(lines[32+n].split(',')[1])
Airfoil_id[n]=int(lines[32+n].split(',')[2])-1
#====INPUT ends here==================
#---Some checks
# Check if Rnode values in setup files are correct.
if np.abs(Rnode[-1]-Rad_rotor)/Rad_rotor*100>5:
print("WARNING: The last Rnode value ",str(Rnode[-1]), "m is different from Radius of the rotor", str(Rad_rotor),"m by more than",str(5),"%.\nThe calculation will terminate here.")
sys.exit()
# Check if dRnode values in setup files are correct.
if np.abs(np.sum(dRnode)-Rad_rotor)/Rad_rotor*100>5:
print("WARNING: Sum of dRnode values of",str(np.sum(dRnode)), "m is different from Radius of the rotor", str(Rad_rotor),"m by more than",str(5),"%.\nThe calculation will terminate here.")
sys.exit()
Chord =np.empty((Nnode),dtype=float) # Chord length of the section
BlAngleRelWind=np.empty((Nnode),dtype=float) # Angle of relative wind at node position
BlSectionPitch=np.empty((Nnode),dtype=float) # Pitch/twist angle at node position
BlAoA =np.empty((Nnode),dtype=float) # Angle of attack with min(Cd/Cl)
#---Run the iteration for each blade node
for n in range(Nnode):
# read airfoil file
data_clcd=np.genfromtxt(Airfoilfile[Airfoil_id[n]],skip_header=header[Airfoil_id[n]],skip_footer=footer[Airfoil_id[n]],delimiter="",dtype=float,encoding="ISO-8859-1") # read data file
AoA=data_clcd[:,AoA_col[Airfoil_id[n]]]
Cl =data_clcd[:,Cl_col[Airfoil_id[n]]]
Cd =data_clcd[:,Cd_col[Airfoil_id[n]]]
imin=np.argmin(np.abs(Cd/Cl))
imax=np.argmax(Cl/Cd)
BlAoA[n]=AoA[imax] # AoA with maximum Cd/Cl value
Clopt = Cl[imax] # Cl with maximum Cd/Cl value
Cdopt = Cd[imax] # Cd with maximum Cd/Cl value
# BlAoA[n]=7.0;Clopt=1.0; # just for checking
print('AoA for max Cl/Cd',BlAoA[n],' Clopt',Clopt,' Cdopt',Cdopt)
tsr_node= tsr*Rnode[n]/Rad_rotor
#---Calculate angle of relative wind and chord lenght
if wake_rotation:
#---Wake rotation included
BlAngleRelWind[n]=2.0/3.0*np.arctan(1.0/tsr_node) # Eq. (3.105) of reference
Chord[n] = 8*np.pi*Rnode[n]*(1.0-np.cos(BlAngleRelWind[n]))/(Nblade*Clopt) # Eq. (3.106)
else:
# Wake rotation ignored
BlAngleRelWind[n]=np.arctan(2.0/(3.0*tsr_node)) # Eq. (3.78) of reference
Chord[n] = 8*np.pi*Rnode[n]*np.sin(BlAngleRelWind[n])/(3.0*Nblade*Clopt*tsr_node) # Eq. (3.79)
BlAngleRelWind[n] =BlAngleRelWind[n]*180.0/np.pi
BlSectionPitch[n]=BlAngleRelWind[n]-BlAoA[n]
del AoA,Cl,Cd,imin,Clopt,Cdopt
save_dat = np.array([Rnode[:],dRnode[:],Chord[:],BlSectionPitch[:],Airfoil_id[:]+1]).T
with open(save_file,'w') as f:
f.write('Parameters for Betz Optimum blade'+'\n')
f.write('Rotor diameter:'+str(Drotor)+' Tip speed ratio: '+str(tsr)+'\n')
f.write('Rnode,dRnode, Chord,Twist, Afoil no.'+'\n')
f.write(' [m] , [m] , [m] ,[deg], [-] '+'\n')
with open(save_file,'ab') as f:
np.savetxt(f,save_dat[:,:],fmt='%s,%s,%s,%s,%s')
return
import keras
import keras.preprocessing.text as kpt
from keras.preprocessing.text import Tokenizer
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
#find most popular words and use those
max_words = 750
tokenizer = Tokenizer(num_words=max_words)
#extract data from csv
#only use cols 1,3
training = np.genfromtxt('data.csv', delimiter=',', skip_header=1, usecols=(
1, 3), dtype=None, encoding=None)
train_x = [x[1] for x in training]
# index all the sentiment labels
train_y = np.asarray([x[0] for x in training])
#sort data into dictionary with most frequent words having lower index
tokenizer.fit_on_texts(train_x)
#tokenizer comes with its own dictionary of words with IDs
dictionary = tokenizer.word_index
#save it to json
with open('dic.json','w') as dictionary_file:
json.dump(dictionary,dictionary_file)
import numpy as np
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
import csv
import tensorflow as tf
import os
import argparse
import re
import matplotlib.pyplot as plt
with open('/home/daanvir/gg/project/SRM_project/x.csv') as f:
shape = tuple(int(num) for num in re.findall(r'\d+', f.readline()))
X = np.loadtxt('/home/daanvir/gg/project/SRM_project/x.csv').reshape(shape)
Y = np.genfromtxt(
r'/home/daanvir/gg/project/SRM_project/y.csv', delimiter=',')
yp=np.zeros((1024,8))
print(Y[0])
for i in range(1024):
yp[i][int(Y[i])]=1
print(X,Y)
X, Y = shuffle(X, Y, random_state=1)
train_x, test_x, train_y, test_y = train_test_split(
X, yp, test_size=0.2, random_state=415)
x_test = X
n_dim = 48
import operator
import math
import random
import numpy
import pygraphviz as pgvusing
from deap import algorithms
from deap import base
from deap import creator
from deap import tools
from deap import gp
data = numpy.genfromtxt('training.csv', delimiter=",")
def protectedDiv(left, right):
if right == 0:
right = 1
return left / right
pset = gp.PrimitiveSet("MAIN", 4)
pset.addPrimitive(operator.add, 2)
pset.addPrimitive(operator.sub, 2)
pset.addPrimitive(operator.mul, 2)
pset.addPrimitive(protectedDiv, 2)
pset.addPrimitive(operator.neg, 1)
pset.addEphemeralConstant("rand101", lambda: random.randrange(-1, 1))
pset.renameArguments(ARG0='Column1')
import numpy as np
import sys
sys.path.insert(
0, '/home/david/Work/DFT-Surface-Science/sublattice-method/funcs')
from funcs import rep, rot, lsave, genPOSCAR
lvecs = np.genfromtxt('../in/_lvecs')
bases = np.genfromtxt('../in/_bases')
zn1, zn2, o1, o2 = bases
a, b, c = lvecs
R = 12. #Angstroms
arange = int(np.ceil(R / np.linalg.norm(a)))
brange = int(np.ceil(R / np.linalg.norm(b)))
crange = int(np.ceil(R / np.linalg.norm(c)))
comb = []
lstzn1 = []
lstzn2 = []
lsto1 = []
lsto2 = []
i1 = 0
i2 = 0
for na in range(-arange, arange):
for nb in range(-brange, brange):
for nc in range(-crange, crange):
dist = np.linalg.norm(na * a + nb * b + nc * c)
i1 += 1
if dist < R:
i2 += 1
comb.append([na, nb, nc])
def read_data():
train_set = np.genfromtxt(train_data_name, comments='#', delimiter=';')
return (train_set, train_set)
def main():
print(
'-------------------------------------------------------------------------------'
)
print('Training...')
random.seed(256)
pop = toolbox.population(n=50)
hof = tools.HallOfFame(5)
stats_fit = tools.Statistics(lambda ind: ind.fitness.values)
stats_size = tools.Statistics(len)
mstats = tools.MultiStatistics(fitness=stats_fit, size=stats_size)
mstats.register("avg", numpy.mean)
mstats.register("std", numpy.std)
mstats.register("min", numpy.min)
mstats.register("max", numpy.max)
pop, log = algorithms.eaSimple(pop,
toolbox,
0.8,
0.2,
50,
stats=mstats,
halloffame=hof,
verbose=True)
best = hof.__getitem__(0)
print(best)
func = toolbox.compile(expr=best)
expr = tools.selBest(pop, 1)[0]
nodes, edges, labels = gp.graph(expr)
# Plot the tree
g = pgv.AGraph(nodeSep=1.0)
g.add_nodes_from(nodes)
g.add_edges_from(edges)
g.layout(prog="dot")
for i in nodes:
n = g.get_node(i)
n.attr["label"] = [i]
g.draw("tree.pdf")
print(
'-------------------------------------------------------------------------------'
)
print('Testing...')
print(''.join(
row.ljust(15) for row in ['Row', 'Predicted', 'Actual', 'Error']))
i = 0
for row in numpy.genfromtxt('testing.csv', delimiter=","):
result = func(row[0], row[1], row[2], row[3])
actual = row[4]
diff = (100 * abs((result - actual)) / actual)
i += 1
result = '%f' % result
actual = '%f' % actual
diff = '%.3f%%' % diff
print(''.join(row.ljust(15) for row in [str(i), result, actual, diff]))
return pop, log, hof
# Claire Nolan 24Apr2019
# PANDS Project 2019
# AIM: Small module to analyse a data set and output the standard deviation
# Data set to be analysed is Fisher's Iris Data Set
# code copied from initial work done for mean values (Mean.py).
# Step 1: Format data set for analysis
import numpy
data = numpy.genfromtxt('DataSet.csv', delimiter=',')
# this line of code states that the data to be analysed is generarted from the DataSet.csv file and that the data should be separated into columns using the comma as a delimiter.
SepLeng = data[:, 0]
SepWid = data[:, 1]
PetLeng = data[:, 2]
PetWid = data[:, 3]
Species = data[:, 4]
# SepLen is used to identify all Sepal length data, SepWid identifies Sepal Width, PetLeng identifes petal length, PetWid identifes petal width and species defines the iris species
# First code: Get the standard deviation of each column with numerical values
# Using ipython i found that "std" is used to calculate the standard deviation
StdSepLeng = numpy.std(data[:, 0])
print("Sepal length standard deviation is ", StdSepLeng)
StdSepWid = numpy.std(data[:, 1])
print("Sepal Width standard deviation is ", StdSepWid)
projdir = mygithub + 'CIM-work/Mayak lung/'
modinfodir = hdf5path + 'mayak/mwds16lung/'
# rlzdir = hdf5path + 'mayak/mwds2016/'
os.chdir(projdir)
sys.path.append('../pythonutils')
# import utility functions for this applicaiton
from psvdict2 import modeleval, getmodinfo, calc_fvalues, showfittedcases
from doseintfun import setdoseintinfo, doseinterp
from cimfuncs import mkcim, mkcimdr, getCI, dispcimbnds, mkQmat, modcimprmbnds
# open dose replocatom and modcov files
intdrep = np.genfromtxt(modinfodir + intdrepfn, delimiter=',', names=True)
mwcmc = np.genfromtxt(modinfodir + mcfname, delimiter=',', names=True)
print "Ungrouped record count:", len(mwcmc)
# open psave file and get model information
psavef = open(modinfodir + psvname, 'r')
models = getmodinfo(psavef)
for mno in range(len(models)):
print "model ", mno, models[mno]['title']
psavef.close()
# ## Set model form and analysis subset
'''
umod = 0 | 3 # male + F:M sex ratio model
umod = 1 | 4 # female + M"F sex ratio model
umod = 2 | 5 # male +_ female dose effect model
import sensor_msgs.point_cloud2 as pc2
import PyKDL as kd
import math
import cPickle
import pickle
from PIL import Image
import threading
from std_msgs.msg import String
import random
from mychallenge.msg import MyOption
heightmaps = []
bridge = CvBridge()
time_stamp_count = [0]
tracklet = np.genfromtxt("/home/xiaoyu/Documents/Tracklet.csv", delimiter=",")
DEFAULT_FILE_DIR = '/home/xiaoyu/Documents/data/release/train'
#blog.csdn.net/hehe123456zxc/article/details/52264829
#www.2cto.com/kf/201504/395335.html
class HrdSemaphore:
def __init__(self):
self.hrd_img = None
self.stamp = None
self.event = threading.Event()
self.event.clear()
def set(self, im, stamp):
self.hrd_img = im
self.stamp = stamp
import numpy as np
import matplotlib.pyplot as plt
from collections import OrderedDict
filename = './befkbhalderstatkode.csv'
data = np.genfromtxt(filename, delimiter=',', dtype=np.uint, skip_header=1)
neighb = {
1: 'Indre By',
2: 'Østerbro',
3: 'Nørrebro',
4: 'Vesterbro/Kgs. Enghave',
5: 'Valby',
6: 'Vanløse',
7: 'Brønshøj-Husum',
8: 'Bispebjerg',
9: 'Amager Øst',
10: 'Amager Vest',
99: 'Udenfor'
}
norseCountryCodes = {
5104: 'Finland',
5106: 'Island',
5110: 'Norge',
5120: 'Sverige'
}
def getSomething(hood):
deezMask = (data[:, 0] == 2015) & (data[:, 1] == hood)
return np.sum(data[deezMask][:, 4])