def result():
project_id = "598330041668"
credentials = GoogleCredentials.get_application_default()
bigquery_service = build('bigquery', 'v2', credentials=credentials)
# This shopList will change after when we get this data from a database
shoplist = ['001', '002']
for name in shoplist:
dataset_id = 'recommendation_' + name
data = create(bigquery_service, dataset_id)
data = normalize(data)
data = predict(data)
data = createData(data)
# value k will change after
k = 10
f = open('result.json', 'w')
check = False
for user in data.keys():
tmp = recommend(data, user, k)
if tmp != []:
check = True
for (sku, distance) in tmp:
res = '{\'customer_id\': \'' + user + '\', \'sku\': \'' + sku + '\', \'distance\': ' + str(
distance) + '}'
f.write(res)
f.write('\n')
f.close()
if check:
createTable(bigquery_service, dataset_id)
insertValues(bigquery_service, dataset_id)
def BCSH1(weight, bits=64, iters=50, lambd=0.1):
X = weight.transpose()
X[X > 0] = 1
X1 = X.copy()
X0 = 1 - X
X1 = normalize(X1)
X0 = normalize(X0)
[n, m] = X.shape
B = np.random.randint(0, 2, (bits, m))
B[B == 0] = -1
#PB1 = np.zeros((bits,1))
for i in range(iters):
tempB1 = B.copy()
tempB0 = -B
tempB1[tempB1 < 0] = 0
tempB0[tempB0 < 0] = 0
PX1_B1 = (np.dot(tempB1, X.transpose()) + 1) / (m + 2)
PX1_B0 = (np.dot(tempB0, X.transpose()) + 1) / (m + 2)
PB1 = np.sum(tempB1, 1, keepdims=True) / m
PX0_B1 = 1 - PX1_B1
PX0_B0 = 1 - PX1_B0
PB0 = 1 - PB1
logPX1_B1 = np.log2(PX1_B1)
logPX1_B0 = np.log2(PX1_B0)
logPX0_B1 = np.log2(PX0_B1)
logPX0_B0 = np.log2(PX0_B0)
logPB1 = np.dot(logPX1_B1, X1) + np.dot(logPX0_B1, X0)
logPB0 = np.dot(logPX1_B0, X1) + np.dot(logPX0_B0, X0)
tmp = logPB1 - logPB0
tmp[tmp > 32] = 32
PXB1 = np.power(2, tmp)
PXB1 = PXB1 / (1 + PXB1)
Fx = PXB1 * 2 - 1
Y = Update(B, bits)
old_B = B.copy()
for i in range(bits):
for j in range(m):
if ((np.power((1 - Fx[i, j]), 2) + lambd * np.power(
(1 - Y[i, j]), 2)) <= (np.power(
(-1 - Fx[i, j]), 2) + lambd * np.power(
(-1 - Y[i, j]), 2))):
B[i, j] = 1
else:
B[i, j] = -1
updateB = sum(sum(B != old_B))
print('update-------------------')
print(updateB)
scipy.io.savemat(
'./argfile/arg1.mat', {
'B': B,
'logPX1_B1': logPX1_B1,
'logPX1_B0': logPX1_B0,
'logPX0_B1': logPX0_B1,
'logPX0_B0': logPX0_B0
})
return B
# Print it:
print("Predicted label = {} (confidence={})".format(p_label, p_confidence))
# Cool! Finally we'll plot the Eigenfaces, because that's
# what most people read in the papers are keen to see.
# Just like in C++ you have access to all model internal
# data, because the cv::FaceRecognizer is a cv::Algorithm.
# You can see the available parameters with getParams():
#print( model.getParams())
# Now let's get some data:
mean = model.getMean()
eigenvectors = model.getEigenVectors()
# We'll save the mean, by first normalizing it:
mean_norm = normalize(mean, 0, 255, dtype=np.uint8)
mean_resized = mean_norm.reshape(X[0].shape)
if out_dir is None:
cv2.imshow("mean", mean_resized)
else:
cv2.imwrite("{}/mean.png".format(out_dir), mean_resized)
# Turn the first (at most) 16 eigenvectors into grayscale
# images. You could also use cv::normalize here, but sticking
# to NumPy is much easier for now.
# Note: eigenvectors are stored by column:
for i in range(min(len(X), 16)):
eigenvector_i = eigenvectors[:, i].reshape(X[0].shape)
eigenvector_i_norm = normalize(eigenvector_i, 0, 255, dtype=np.uint8)
# Show or save the images:
def main():
print("Starting application..\n")
traffic = normalize(filename_adj)
print("1 - Display the information gain list")
print("2 - Display the Chi-squared list")
print("3 - Display the Recursive Feature Elimination list")
print("4 - Display the WRFS list")
selection = input("Enter your selection: ")
#########################
#Information Gain list
#########################
if (selection == 1):
# sorted_gain_list = InfoGain.ig_list(traffic)
sorted_gain_list = [
'land', 'urgent', 'wrong_fragment', 'rerror_rate',
'srv_rerror_rate', 'count', 'dst_host_rerror_rate',
'dst_host_srv_rerror_rate', 'duration', 'flag', 'serror_rate',
'srv_serror_rate', 'dst_host_serror_rate',
'dst_host_srv_serror_rate', 'diff_srv_rate', 'same_srv_rate',
'dst_host_same_srv_rate', 'srv_diff_host_rate',
'dst_host_diff_srv_rate', 'dst_host_srv_count',
'dst_host_srv_diff_host_rate', 'dst_host_count', 'protocol_type',
'srv_count', 'dst_host_same_src_port_rate', 'dst_bytes', 'service',
'src_bytes'
]
print("Features selected using Information Gain.")
#########################
#Chi-Squared list
#########################
elif (selection == 2):
# sorted_chi2_list = Chi2.chi2_list()
sorted_chi2_list = [
'dst_host_same_srv_rate', 'count', 'rerror_rate',
'srv_rerror_rate', 'same_srv_rate', 'wrong_fragment',
'dst_host_rerror_rate', 'diff_srv_rate',
'dst_host_srv_rerror_rate', 'dst_host_diff_srv_rate',
'serror_rate', 'srv_serror_rate', 'dst_host_serror_rate',
'dst_host_srv_serror_rate', 'flag', 'dst_host_same_src_port_rate',
'protocol_type', 'srv_diff_host_rate',
'dst_host_srv_diff_host_rate', 'dst_host_srv_count', 'service',
'land', 'urgent', 'dst_host_count', 'srv_count', 'duration',
'src_bytes', 'dst_bytes'
]
print("Features selected using Chi-squared.")
######################################
#Recursive Feature Elimination list
######################################
elif (selection == 3):
sorted_rfe = [
'diff_srv_rate', 'same_srv_rate', 'dst_host_srv_serror_rate',
'srv_serror_rate', 'rerror_rate', 'srv_rerror_rate',
'protocol_type', 'dst_host_serror_rate', 'wrong_fragment',
'dst_host_same_src_port_rate', 'dst_host_srv_diff_host_rate',
'dst_host_same_srv_rate', 'dst_host_srv_rerror_rate',
'serror_rate', 'flag', 'dst_host_diff_srv_rate', 'count',
'service', 'dst_host_srv_count', 'srv_count',
'dst_host_rerror_rate', 'dst_host_count', 'src_bytes',
'srv_diff_host_rate', 'urgent', 'dst_bytes', 'land', 'duration'
]
print("Features selected using Recursive Feature Elimination.")
#########################
#WRFS list
#########################
elif (selection == 4):
sorted_WRFS_list = [
'dst_bytes', 'src_bytes', 'srv_count', 'dst_host_count', 'service',
'duration', 'srv_diff_host_rate', 'dst_host_srv_count',
'dst_host_srv_diff_host_rate', 'dst_host_same_src_port_rate',
'urgent', 'land', 'protocol_type', 'dst_host_diff_srv_rate',
'flag', 'serror_rate', 'dst_host_rerror_rate',
'dst_host_serror_rate', 'dst_host_srv_serror_rate',
'dst_host_same_srv_rate', 'dst_host_srv_rerror_rate',
'srv_serror_rate', 'count', 'diff_srv_rate', 'same_srv_rate',
'wrong_fragment', 'srv_rerror_rate', 'rerror_rate'
]
print("Features selected using WRFS.")
else:
print("Invalid selection")
print("\n")
print("*****************")
print("Classification")
print("*****************")
print("1 - Naive Bayes ")
print("2 - SVM")
print("3 - Decision Tree")
print("4 - Random Forest")
clx_selection = input("Enter your selection: ")
book = xlwt.Workbook(encoding="utf-8")
sheet1 = book.add_sheet("Sheet 1")
sheet1.write(0, 2, "Accuracy Values")
for i in range(4, 5):
num_features = i
print("Number of features selected - ", i)
################
#Naive Bayes
################
if (selection == 1 and clx_selection == 1):
print("Classifying using Naive Bayes...")
accuracy = Classifiers.NaiveBayes.naive_bayes(
sorted_gain_list, num_features)
sheet1.write(i, 2, accuracy)
elif (selection == 2 and clx_selection == 1):
print("Classifying using Naive Bayes...")
accuracy = Classifiers.NaiveBayes.naive_bayes(
sorted_chi2_list, num_features)
sheet1.write(i, 5, accuracy)
elif (selection == 3 and clx_selection == 1):
print("Classifying using Naive Bayes...")
accuracy = Classifiers.NaiveBayes.naive_bayes(
sorted_rfe, num_features)
sheet1.write(i, 2, accuracy)
elif (selection == 4 and clx_selection == 1):
print("Classifying using Naive Bayes...")
accuracy = Classifiers.NaiveBayes.naive_bayes(
sorted_WRFS_list, num_features)
sheet1.write(i, 2, accuracy)
################
#SVM
################
elif (selection == 1 and clx_selection == 2):
print("Classifying using SVM...")
accuracy = Classifiers.SVM.svm(sorted_gain_list, num_features)
sheet1.write(i, 3, accuracy)
elif (selection == 2 and clx_selection == 2):
print("Classifying using SVM...")
accuracy = Classifiers.SVM.svm(sorted_chi2_list, num_features)
sheet1.write(i, 3, accuracy)
elif (selection == 3 and clx_selection == 2):
print("Classifying using SVM...")
accuracy = Classifiers.SVM.svm(sorted_rfe, num_features)
sheet1.write(i, 3, accuracy)
elif (selection == 4 and clx_selection == 2):
print("Classifying using SVM...")
accuracy = Classifiers.SVM.svm(sorted_WRFS_list, num_features)
sheet1.write(i, 3, accuracy)
###################
#Decision Trees
###################
elif (selection == 1 and clx_selection == 3):
print("Classifying using Decision Trees...")
accuracy = Classifiers.DecisionTree.decision_tree(
sorted_gain_list, num_features)
sheet1.write(i, 3, accuracy)
elif (selection == 2 and clx_selection == 3):
print("Classifying using Decision Trees...")
accuracy = Classifiers.DecisionTree.decision_tree(
sorted_chi2_list, num_features)
sheet1.write(i, 3, accuracy)
elif (selection == 3 and clx_selection == 3):
print("Classifying using Decision Trees...")
accuracy = Classifiers.DecisionTree.decision_tree(
sorted_rfe, num_features)
sheet1.write(i, 3, accuracy)
elif (selection == 4 and clx_selection == 3):
print("Classifying using Decision Trees...")
accuracy = Classifiers.DecisionTree.decision_tree(
sorted_WRFS_list, num_features)
sheet1.write(i, 3, accuracy)
################
#Random Forest
################
elif (selection == 1 and clx_selection == 4):
print("Classifying using Random Forest...")
accuracy = Classifiers.RandomForest.randomForest(
sorted_gain_list, num_features)
sheet1.write(i, 6, accuracy)
elif (selection == 2 and clx_selection == 4):
print("Classifying using Random Forest...")
accuracy = Classifiers.RandomForest.randomForest(
sorted_chi2_list, num_features)
sheet1.write(i, 6, accuracy)
elif (selection == 3 and clx_selection == 4):
print("Classifying using Random Forest...")
accuracy = Classifiers.RandomForest.randomForest(
sorted_rfe, num_features)
sheet1.write(i, 6, accuracy)
elif (selection == 4 and clx_selection == 4):
print("Classifying using Random Forest...")
accuracy = Classifiers.RandomForest.randomForest(
sorted_WRFS_list, num_features)
sheet1.write(i, 6, accuracy)
else:
print("Invalid Selection")
print("End")
book.save("results.xls")
def go():
num_papers = normalize('input', 'output')
#make_pairs_by_type('output')
print('complete')
def DesOpt(SysEq, x0, xU, xL, xDis=[], gc=[], hc=[], SensEq=[], Alg="SLSQP", SensCalc="FD", DesVarNorm=True,
deltax=1e-3, StatusReport=False, ResultReport=False, Video=False, DoE=False, SBDO=False,
Debug=False, PrintOut=True, OptNameAdd="", AlgOptions=[], Alarm=True):
#-----------------------------------------------------------------------------------------------------------------------
# Define optimization problem and optimization options
#-----------------------------------------------------------------------------------------------------------------------
"""
:type OptNode: object
"""
if Debug is True:
StatusReport = False
if StatusReport is True:
print "Debug is set to True; overriding StatusReport"
if ResultReport is True:
print "Debug is set to True; overriding ResultReport"
ResultReport = False
computerName = platform.uname()[1]
operatingSystem = platform.uname()[0]
architecture = platform.uname()[4]
nProcessors = str(multiprocessing.cpu_count())
userName = getpass.getuser()
OptTime0 = time.time()
OptNodes = "all"
MainDir = os.getcwd()
if operatingSystem != 'Windows':
DirSplit = "/"
homeDir = "/home/"
else:
DirSplit = "\\"
homeDir = "c:\\Users\\"
OptModel = os.getcwd().split(DirSplit)[-1]
try:
OptAlg = eval("pyOpt." + Alg + '()')
pyOptAlg = True
except:
OptAlg = Alg
pyOptAlg = False
if hasattr(SensEq, '__call__'):
SensCalc = "OptSensEq"
print "Function for sensitivity analysis has been provided, overriding SensCalc to use function"
else:
pass
StartTime = datetime.datetime.now()
loctime = time.localtime()
today = time.strftime("%B", time.localtime()) + ' ' + str(loctime[2]) + ', ' + str(loctime[0])
if SBDO is True:
OptNameAdd = OptNameAdd + "_SBDO"
OptName = OptModel + OptNameAdd + "_" + Alg + "_" + StartTime.strftime("%Y%m%d%H%M%S")
global nEval
nEval = 0
LocalRun = True
ModelDir = os.getcwd()[:-(len(OptModel) + 1)]
ModelFolder = ModelDir.split(DirSplit)[-1]
DesOptDir = ModelDir[:-(len(ModelFolder) + 1)]
ResultsDir = DesOptDir + os.sep + "Results"
RunDir = DesOptDir + os.sep + "Run"
try:
inform
except NameError:
inform = ["Running"]
if LocalRun is True and Debug is False:
try: os.mkdir(ResultsDir)
except: pass
os.mkdir(ResultsDir + DirSplit + OptName)
os.mkdir(ResultsDir + os.sep + OptName + os.sep + "ResultReport" + os.sep)
shutil.copytree(os.getcwd(), RunDir + os.sep + OptName)
#if SensCalc == "ParaFD":
# import OptSensParaFD
# os.system("cp -r ParaPythonFn " + homeDir + userName + "/DesOptRun/" + OptName)
if LocalRun is True and Debug is False:
os.chdir("../../Run/" + OptName + "/")
sys.path.append(os.getcwd())
#-----------------------------------------------------------------------------------------------------------------------
# Print start-up splash to output screen
#-----------------------------------------------------------------------------------------------------------------------
if PrintOut is True:
print("--------------------------------------------------------------------------------")
PrintDesOptPy()
print("")
print("Optimization model: " + OptModel)
try: print("Optimization algorithm: " + Alg)
except: pass
print("Optimization start: " + StartTime.strftime("%Y%m%d%H%M"))
print("Optimization name: " + OptName)
print("--------------------------------------------------------------------------------")
#-----------------------------------------------------------------------------------------------------------------------
# Optimization problem
#-----------------------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------------------
# Define functions: system equation, normalization, etc.
#-----------------------------------------------------------------------------------------------------------------------
def OptSysEq(x):
x = np.array(x) # NSGA2 gives a list back, this makes a float! TODO Inquire why it does this!
f, g = SysEq(x, gc)
fail = 0
global nEval
nEval += 1
if StatusReport == True:
OptHis2HTML.OptHis2HTML(OptName, OptAlg, DesOptDir, xL, xU, DesVarNorm, inform[0], OptTime0)
if len(xDis) > 0:
nD = len(xDis)
gDis = [[]]*2*nD
for ii in range(nD):
gDis[ii+0] = np.sum(x[-1*xDis[ii]:])-1
gDis[ii+1] = 1-np.sum(x[-1*xDis[ii]:])
gNew = np.concatenate((g, gDis), 0)
g = copy.copy(gNew)
# TODO add print out for optimization development!!
return f, g, fail
def OptSysEqNorm(xNorm):
xNorm = np.array(xNorm) # NSGA2 gives a list back, this makes a float! TODO Inquire why it does this!
x = denormalize(xNorm, xL, xU, DesVarNorm)
f, g, fail = OptSysEq(x)
return f, g, fail
def OptPenSysEq(x):
f, g, fail = OptSysEq(x)
fpen = f
return fpen
def OptSensEq(x, f, g):
dfdx, dgdx = SensEq(x, f, g, gc)
dfdx = dfdx.reshape(1,len(x))
fail = 0
return dfdx, dgdx, fail
def OptSensEqNorm(xNorm, f, g):
x = denormalize(xNorm, xL, xU, DesVarNorm)
dfxdx, dgxdx, fail = OptSensEq(x, f, g)
dfdx = dfxdx * (xU - xL)
# TODO not general for all normalizations! needs to be rewritten
if dgxdx != []:
dgdx = dgxdx * np.tile((xU - xL), [len(g), 1])
# TODO not general for all normalizations! needs to be rewritten
else:
dgdx = []
return dfdx, dgdx, fail
def OptSensEqParaFD(x, f, g):
global nEval
dfdx, dgdx, nb = OptSensParaFD.Para(x, f, g, deltax, OptName, OptNodes)
nEval += nb
fail = 0
return dfdx, dgdx, fail
def OptSensEqParaFDNorm(xNorm, f, g):
x = denormalize(xNorm, xL, xU, DesVarNorm)
dfxdx, dgxdx, fail = OptSensEqParaFD(x, f, g)
dfdx = dfxdx * (xU - xL)
# TODO not general for all normalizations! needs to be rewritten
dgdx = dgxdx * (np.tile((xU - xL), [len(g), 1]))
# TODO not general for all normalizations! needs to be rewritten
return dfdx, dgdx, fail
#-----------------------------------------------------------------------------------------------------------------------
# Surrogate-based optimization (not fully functioning yet!!!!)
#-----------------------------------------------------------------------------------------------------------------------
# TODO SBDO in a separate file???
if SBDO is not False:
if DoE > 0:
import pyDOE
try:
n_gc = len(gc)
except:
n_gc = 1
SampleCorners = True
if SampleCorners is True:
xTemp = np.ones(np.size(xL)) * 2
xSampFF = pyDOE.fullfact(np.array(xTemp, dtype=int)) # Kriging needs boundaries too!!
xSampLH = pyDOE.lhs(np.size(xL), DoE)
xDoE_Norm = np.concatenate((xSampFF, xSampLH), axis=0)
else:
xDoE_Norm = pyDOE.lhs(np.size(xL), DoE)
xDoE = np.zeros(np.shape(xDoE_Norm))
fDoE = np.zeros([np.size(xDoE_Norm, 0), 1])
gDoE = np.zeros([np.size(xDoE_Norm, 0), n_gc])
for ii in range(np.size(xDoE_Norm, 0)):
xDoE[ii] = denormalize(xDoE_Norm[ii], xL, xU, DesVarNorm)
fDoEii, gDoEii, fail = OptSysEqNorm(xDoE_Norm[ii])
fDoE[ii] = fDoEii
gDoE[ii, :] = gDoEii
n_theta = np.size(x0) + 1
ApproxObj = "QuadReg"
ApproxObj = "GaussianProcess"
if ApproxObj == "GaussianProcess":
from sklearn.gaussian_process import GaussianProcess
approx_f = GaussianProcess(regr='quadratic', corr='squared_exponential',
normalize=True, theta0=0.1, thetaL=1e-4, thetaU=1e+1,
optimizer='fmin_cobyla')
elif ApproxObj == "QuadReg":
# from PolyReg import *
approx_f = PolyReg()
approx_f.fit(xDoE, fDoE)
from sklearn.gaussian_process import GaussianProcess
gDoEr = np.zeros(np.size(xDoE_Norm, 0))
approx_g = [[]] * n_gc
gpRegr = ["quadratic"] * n_gc
gpCorr = ["squared_exponential"] * n_gc
for ii in range(n_gc):
for iii in range(np.size(xDoE_Norm, 0)):
gDoEii = gDoE[iii]
gDoEr[iii] = gDoEii[ii]
approx_g[ii] = GaussianProcess(regr=gpRegr[ii], corr=gpCorr[ii], theta0=0.01,
thetaL=0.0001, thetaU=10., optimizer='fmin_cobyla')
approx_g[ii].fit(xDoE, gDoEr)
DoE_Data = {}
DoE_Data['xDoE_Norm'] = xDoE_Norm
DoE_Data['gDoE'] = gDoE
DoE_Data['fDoE'] = fDoE
output = open(OptName + "_DoE.pkl", 'wb')
pickle.dump(DoE_Data, output)
output.close()
else:
Data = pickle.load(open("Approx.pkl"))
approx_f = Data["approx_f"]
approx_g = Data["approx_g"]
def ApproxOptSysEq(x):
f = approx_f.predict(x)
g = np.zeros(len(gc))
for ii in range(len(gc)):
# exec("g[ii], MSE = gp_g"+str(ii)+".predict(x, eval_MSE=True)")
g[ii] = approx_g[ii].predict(x)
# sigma = np.sqrt(MSE)
fail = 0
return f, g, fail
def ApproxOptSysEqNorm(xNorm):
xNorm = xNorm[0:np.size(xL), ]
x = denormalize(xNorm, xL, xU, DesVarNorm)
f = approx_f.predict(x)
g = np.zeros(len(gc))
for ii in range(len(gc)):
# exec("g[ii], MSE = gp_g"+str(ii)+".predict(x, eval_MSE=True)")
g[ii] = approx_g[ii].predict(x)
# sigma = np.sqrt(MSE)
fail = 0
return f, g, fail
if xDis is not []:
for ii in range(np.size(xDis, 0)):
xExpand0 = np.ones(xDis[ii]) * 1./xDis[ii] # Start at uniform of all materials etc.
xNew0 = np.concatenate((x0, xExpand0), 0)
xExpandL = np.ones(xDis[ii]) * 0.0001
xNewL = np.concatenate((xL, xExpandL), 0)
xExpandU = np.ones(xDis[ii])
xNewU = np.concatenate((xU, xExpandU), 0)
x0 = copy.copy(xNew0)
xL = copy.copy(xNewL)
xU = copy.copy(xNewU)
gcNew = np.concatenate((gc, np.ones(2,)), 0)
gc = copy.copy(gcNew)
if DesVarNorm in ["None", None, False]:
x0norm = x0
xLnorm = xL
xUnorm = xU
DefOptSysEq = OptSysEq
else:
[x0norm, xLnorm, xUnorm] = normalize(x0, xL, xU, DesVarNorm)
DefOptSysEq = OptSysEqNorm
nx = np.size(x0)
ng = np.size(gc)
#-----------------------------------------------------------------------------------------------------------------------
# pyOpt optimization
#-----------------------------------------------------------------------------------------------------------------------
if pyOptAlg is True:
if SBDO is not False and DesVarNorm in ["xLxU", True, "xLx0", "x0", "xU"]: #in ["None", None, False]:
OptProb = pyOpt.Optimization(OptModel, ApproxOptSysEqNorm, obj_set=None)
elif SBDO is not False and DesVarNorm in ["None", None, False]:
OptProb = pyOpt.Optimization(OptModel, ApproxOptSysEq, obj_set=None)
else:
OptProb = pyOpt.Optimization(OptModel, DefOptSysEq)
if np.size(x0) == 1:
OptProb.addVar('x', 'c', value=x0norm, lower=xLnorm, upper=xUnorm)
elif np.size(x0) > 1:
for ii in range(np.size(x0)):
OptProb.addVar('x' + str(ii + 1), 'c', value=x0norm[ii], lower=xLnorm[ii], upper=xUnorm[ii])
OptProb.addObj('f')
if np.size(gc) == 1:
OptProb.addCon('g', 'i')
ng = 1
elif np.size(gc) > 1:
for ii in range(len(gc)):
OptProb.addCon('g' + str(ii + 1), 'i')
ng = ii + 1
if np.size(hc) == 1:
OptProb.addCon('h', 'i')
elif np.size(hc) > 1:
for ii in range(ng):
OptProb.addCon('h' + str(ii + 1), 'i')
if AlgOptions == []:
AlgOptions = OptAlgOptions.setDefault(Alg)
OptAlg = OptAlgOptions.setUserOptions(AlgOptions, Alg, OptName, OptAlg)
#if AlgOptions == []:
# OptAlg = OptAlgOptions.setDefaultOptions(Alg, OptName, OptAlg)
#else:
# OptAlg = OptAlgOptions.setUserOptions(AlgOptions, Alg, OptName, OptAlg)
if PrintOut is True:
print(OptProb)
if Alg in ["MMA", "FFSQP", "FSQP", "GCMMA", "CONMIN", "SLSQP", "PSQP", "KSOPT", "ALGENCAN", "NLPQLP", "IPOPT"]:
if SensCalc == "OptSensEq":
if DesVarNorm not in ["None", None, False]:
[fOpt, xOpt, inform] = OptAlg(OptProb, sens_type=OptSensEqNorm, store_hst=OptName)
else:
[fOpt, xOpt, inform] = OptAlg(OptProb, sens_type=OptSensEq, store_hst=OptName)
elif SensCalc == "ParaFD": # Michi Richter
if DesVarNorm not in ["None", None, False]:
[fOpt, xOpt, inform] = OptAlg(OptProb, sens_type=OptSensEqParaFDNorm, store_hst=OptName)
else:
[fOpt, xOpt, inform] = OptAlg(OptProb, sens_type=OptSensEqParaFD, store_hst=OptName)
else: # Here FD (finite differencing)
[fOpt, xOpt, inform] = OptAlg(OptProb, sens_type=SensCalc, sens_step=deltax, store_hst=OptName)
elif Alg in ["SDPEN", "SOLVOPT"]:
[fOpt, xOpt, inform] = OptAlg(OptProb)
else:
[fOpt, xOpt, inform] = OptAlg(OptProb, store_hst=OptName)
if PrintOut is True:
try: print(OptProb.solution(0))
except: pass
if Alg not in ["PSQP", "SOLVOPT", "MIDACO", "SDPEN", "ralg"] and PrintOut is True:
print(OptAlg.getInform(0))
#-----------------------------------------------------------------------------------------------------------------------
# OpenOpt optimization -- not fully implemented in this framework and not yet working...
#-----------------------------------------------------------------------------------------------------------------------
elif Alg == "ralg":
from openopt import NLP
f, g = lambda x: OptSysEq(x)
# g = lambda x: OptSysEq(x)[1][0]
p = NLP(f, x0, c=g, lb=xL, ub=xU, iprint=50, maxIter=10000, maxFunEvals=1e7, name='NLP_1')
r = p.solve(Alg, plot=0)
print(OptAlg.getInform(1))
#-----------------------------------------------------------------------------------------------------------------------
# pyCMAES
#-----------------------------------------------------------------------------------------------------------------------
elif Alg == "pycmaes":
print "CMA-ES == not fully implemented in this framework"
print " no constraints"
import cma
def CMA_ES_ObjFn(x):
f, g, fail = OptSysEq(x)
return f
OptRes = cma.fmin(CMA_ES_ObjFn, x0, sigma0=1)
xOpt = OptRes[0]
fOpt = OptRes[1]
nEval = OptRes[4]
nIter = OptRes[5]
#-----------------------------------------------------------------------------------------------------------------------
# MATLAB fmincon optimization -- not fully implemented in this framework and not yet working...
#-----------------------------------------------------------------------------------------------------------------------
elif Alg == "fmincon": # not fully implemented in this framework
def ObjFn(x):
f, g, fail = OptSysEqNorm(xNorm)
return f, []
from mlabwrap import mlab
mlab._get(ObjFn)
mlab.fmincon(mlab._get("ObjFn"), x) # g,h, dgdx = mlab.fmincon(x.T,cg,ch, nout=3)
#-----------------------------------------------------------------------------------------------------------------------
# PyGMO optimization
#-----------------------------------------------------------------------------------------------------------------------
elif Alg[:5] == "PyGMO":
DesVarNorm = "None"
#print nindiv
dim = np.size(x0)
# prob = OptSysEqPyGMO(dim=dim)
prob = OptSysEqPyGMO(SysEq=SysEq, xL=xL, xU=xU, gc=gc, dim=dim, OptName=OptName, Alg=Alg, DesOptDir=DesOptDir,
DesVarNorm=DesVarNorm, StatusReport=StatusReport, inform=inform, OptTime0=OptTime0)
# prob = problem.death_penalty(prob_old, problem.death_penalty.method.KURI)
if AlgOptions == []:
AlgOptions = OptAlgOptions.setDefault(Alg)
OptAlg = OptAlgOptions.setUserOptions(AlgOptions, Alg, OptName, OptAlg)
#algo = eval("PyGMO.algorithm." + Alg[6:]+"()")
#de (gen=100, f=0.8, cr=0.9, variant=2, ftol=1e-06, xtol=1e-06, screen_output=False)
#NSGAII (gen=100, cr=0.95, eta_c=10, m=0.01, eta_m=10)
#sga_gray.__init__(gen=1, cr=0.95, m=0.02, elitism=1, mutation=PyGMO.algorithm._algorithm._gray_mutation_type.UNIFORM, selection=PyGMO.algorithm._algorithm._gray_selection_type.ROULETTE, crossover=PyGMO.algorithm._algorithm._gray_crossover_type.SINGLE_POINT)
#nsga_II.__init__(gen=100, cr=0.95, eta_c=10, m=0.01, eta_m=10)
#emoa (hv_algorithm=None, gen=100, sel_m=2, cr=0.95, eta_c=10, m=0.01, eta_m=10)
#pade (gen=10, max_parallelism=1, decomposition=PyGMO.problem._problem._decomposition_method.BI, solver=None, T=8, weights=PyGMO.algorithm._algorithm._weight_generation.LOW_DISCREPANCY, z=[])
#nspso (gen=100, minW=0.4, maxW=1.0, C1=2.0, C2=2.0, CHI=1.0, v_coeff=0.5, leader_selection_range=5, diversity_mechanism=PyGMO.algorithm._algorithm._diversity_mechanism.CROWDING_DISTANCE)
#corana: (iter=10000, Ts=10, Tf=0.1, steps=1, bin_size=20, range=1)
#if Alg[6:] in ["de", "bee_colony", "nsga_II", "pso", "pso_gen", "cmaes", "py_cmaes",
# "spea2", "nspso", "pade", "sea", "vega", "sga", "sga_gray", "de_1220",
# "mde_pbx", "jde"]:
# algo.gen = ngen
#elif Alg[6:] in ["ihs", "monte_carlo", "sa_corana"]:
# algo.iter = ngen
#elif Alg[6:] == "sms_emoa":
# print "sms_emoa not working"
#else:
# sys.exit("improper PyGMO algorithm chosen")
#algo.f = 1
#algo.cr=1
#algo.ftol = 1e-3
#algo.xtol = 1e-3
#algo.variant = 2
#algo.screen_output = False
#if Alg == "PyGMO_de":
# algo = PyGMO.algorithm.de(gen=ngen, f=1, cr=1, variant=2,
# ftol=1e-3, xtol=1e-3, screen_output=False)
#else:
# algo = PyGMO.algorithm.de(gen=ngen, f=1, cr=1, variant=2,
# ftol=1e-3, xtol=1e-3, screen_output=False)
#pop = PyGMO.population(prob, nIndiv)
#pop = PyGMO.population(prob, nIndiv, seed=13598) # Seed fixed for random generation of first individuals
#algo.evolve(pop)
isl = PyGMO.island(OptAlg, prob, AlgOptions.nIndiv)
isl.evolve(1)
isl.join()
xOpt = isl.population.champion.x
# fOpt = isl.population.champion.f[0]
nEval = isl.population.problem.fevals
nGen = int(nEval/AlgOptions.nIndiv) # currently being overwritten and therefore not being used
StatusReport = False # turn off status report, so not remade (and destroyed) in following call!
fOpt, gOpt, fail = OptSysEq(xOpt) # verification of optimal solution as values above are based on penalty!
#-----------------------------------------------------------------------------------------------------------------------
# SciPy optimization
#-----------------------------------------------------------------------------------------------------------------------
elif Alg[:5] == "scipy":
import scipy.optimize as sciopt
bounds = [[]]*len(x0)
for ii in range(len(x0)):
bounds[ii] = (xL[ii], xU[ii])
print bounds
if Alg[6:] == "de":
sciopt.differential_evolution(DefOptSysEq, bounds, strategy='best1bin',
maxiter=None, popsize=15, tol=0.01, mutation=(0.5, 1),
recombination=0.7, seed=None, callback=None, disp=False,
polish=True, init='latinhypercube')
#-----------------------------------------------------------------------------------------------------------------------
# Simple optimization algorithms to demonstrate use of custom algorithms
#-----------------------------------------------------------------------------------------------------------------------
#TODO: add history to these
elif Alg == "SteepestDescentSUMT":
from CustomAlgs import SteepestDescentSUMT
fOpt, xOpt, nIter, nEval = SteepestDescentSUMT(DefOptSysEq, x0, xL, xU)
elif Alg == "NewtonSUMT":
from CustomAlgs import NewtonSUMT
fOpt, xOpt, nIter, nEval = NewtonSUMT(DefOptSysEq, x0, xL, xU)
#-----------------------------------------------------------------------------------------------------------------------
#
#-----------------------------------------------------------------------------------------------------------------------
else:
sys.exit("Error on line 694 of __init__.py: algorithm misspelled or not supported")
#-----------------------------------------------------------------------------------------------------------------------
# Optimization post-processing
#-----------------------------------------------------------------------------------------------------------------------
if StatusReport == 1:
OptHis2HTML.OptHis2HTML(OptName, OptAlg, DesOptDir, xL, xU, DesVarNorm, inform.values()[0], OptTime0)
OptTime1 = time.time()
loctime0 = time.localtime(OptTime0)
hhmmss0 = time.strftime("%H", loctime0)+' : '+time.strftime("%M", loctime0)+' : '+time.strftime("%S", loctime0)
loctime1 = time.localtime(OptTime1)
hhmmss1 = time.strftime("%H", loctime1)+' : '+time.strftime("%M", loctime1)+' : '+time.strftime("%S", loctime1)
diff = OptTime1 - OptTime0
h0, m0, s0 = (diff // 3600), int((diff / 60) - (diff // 3600) * 60), diff % 60
OptTime = "%02d" % (h0) + " : " + "%02d" % (m0) + " : " + "%02d" % (s0)
#-----------------------------------------------------------------------------------------------------------------------
# Read in results from history files
#-----------------------------------------------------------------------------------------------------------------------
OptHist = pyOpt.History(OptName, "r")
fAll = OptHist.read([0, -1], ["obj"])[0]["obj"]
xAll = OptHist.read([0, -1], ["x"])[0]["x"]
gAll = OptHist.read([0, -1], ["con"])[0]["con"]
if Alg == "NLPQLP":
gAll = [x * -1 for x in gAll]
gGradIter = OptHist.read([0, -1], ["grad_con"])[0]["grad_con"]
fGradIter = OptHist.read([0, -1], ["grad_obj"])[0]["grad_obj"]
failIter = OptHist.read([0, -1], ["fail"])[0]["fail"]
if Alg == "COBYLA" or Alg == "NSGA2" or Alg[:5] == "PyGMO":
fIter = fAll
xIter = xAll
gIter = gAll
else:
fIter = [[]] * len(fGradIter)
xIter = [[]] * len(fGradIter)
gIter = [[]] * len(fGradIter)
# SuIter = [[]] * len(fGradIter)
for ii in range(len(fGradIter)):
Posdg = OptHist.cues["grad_con"][ii][0]
Posf = OptHist.cues["obj"][ii][0]
iii = 0
while Posdg > Posf:
iii = iii + 1
try:
Posf = OptHist.cues["obj"][iii][0]
except:
Posf = Posdg + 1
iii = iii - 1
fIter[ii] = fAll[iii]
xIter[ii] = xAll[iii]
gIter[ii] = gAll[iii]
OptHist.close()
#-----------------------------------------------------------------------------------------------------------------------
# Convert all data to numpy arrays
#-----------------------------------------------------------------------------------------------------------------------
fIter = np.asarray(fIter)
xIter = np.asarray(xIter)
gIter = np.asarray(gIter)
gGradIter = np.asarray(gGradIter)
fGradIter = np.asarray(fGradIter)
#-----------------------------------------------------------------------------------------------------------------------
# Denormalization of design variables
#-----------------------------------------------------------------------------------------------------------------------
xOpt = np.resize(xOpt[0:np.size(xL)], np.size(xL))
if DesVarNorm in ["None", None, False]:
x0norm = []
xIterNorm = []
xOptNorm = []
else:
xOpt = np.resize(xOpt, [np.size(xL), ])
xOptNorm = xOpt
xOpt = denormalize(xOptNorm.T, xL, xU, DesVarNorm)
try:
xIterNorm = xIter[:, 0:np.size(xL)]
xIter = np.zeros(np.shape(xIterNorm))
for ii in range(len(xIterNorm)):
xIter[ii] = denormalize(xIterNorm[ii], xL, xU, DesVarNorm)
except:
x0norm = []
xIterNorm = []
xOptNorm = []
nIter = np.size(fIter)
if np.size(fIter) > 0:
if fIter[0] != 0:
fIterNorm = fIter / fIter[0] # fIterNorm=(fIter-fIter[nEval-1])/(fIter[0]-fIter[nEval-1])
else:
fIterNorm = fIter
else:
fIterNorm = []
#-----------------------------------------------------------------------------------------------------------------------
# Active constraints for use in the calculation of the Lagrangian multipliers and optimality criterion
#-----------------------------------------------------------------------------------------------------------------------
epsActive = 1e-3
xL_ActiveIndex = (xOpt - xL) / xU < epsActive
xU_ActiveIndex = (xU - xOpt) / xU < epsActive
xL_Grad = -np.eye(nx)
xU_Grad = np.eye(nx)
xL_GradActive = xL_Grad[:, xL_ActiveIndex]
xU_GradActive = xU_Grad[:, xU_ActiveIndex] # or the other way around!
xGradActive = np.concatenate((xL_GradActive, xU_GradActive), axis=1)
# TODO change so that 1D optimization works!
try:
xL_Active = xL[xL_ActiveIndex]
except:
xL_Active = np.array([])
try:
xU_Active = xU[xU_ActiveIndex]
except:
xU_Active = np.array([])
if len(xL_Active)==0:
xActive = xU_Active
elif len(xU_Active)==0:
xActive = xL_Active
else:
xActive = np.concatenate((xL_Active, xU_Active))
if np.size(xL) == 1:
if xL_ActiveIndex == False:
xL_Active = np.array([])
else:
xL_Active = xL
if xU_ActiveIndex == False:
xU_Active = np.array([])
else:
xU_Active = xU
else:
xL_Active = xL[xL_ActiveIndex]
xU_Active = np.array(xU[xU_ActiveIndex])
if len(xL_Active)==0:
xLU_Active = xU_Active
elif len(xU_Active)==0:
xLU_Active = xL_Active
else:
xLU_Active = np.concatenate((xL_Active, xU_Active))
#TODO needs to be investigated for PyGMO!
# are there nonlinear constraints active, in case equality constraints are added later, this must also be added
if np.size(gc) > 0 and Alg[:5] != "PyGMO":
gMaxIter = np.zeros([nIter])
for ii in range(len(gIter)):
gMaxIter[ii] = max(gIter[ii])
gOpt = gIter[nIter - 1]
gOptActiveIndex = gOpt > -epsActive
gOptActive = gOpt[gOpt > -epsActive]
elif np.size(gc) == 0:
gOptActiveIndex = [[False]] * len(gc)
gOptActive = np.array([])
gMaxIter = np.array([] * nIter)
gOpt = np.array([])
else:
gMaxIter = np.zeros([nIter])
for ii in range(len(gIter)):
gMaxIter[ii] = max(gIter[ii])
gOptActiveIndex = gOpt > -epsActive
gOptActive = gOpt[gOpt > -epsActive]
if len(xLU_Active)==0:
g_xLU_OptActive = gOptActive
elif len(gOptActive)==0:
g_xLU_OptActive = xLU_Active
else:
if np.size(xLU_Active) == 1 and np.size(gOptActive) == 1:
g_xLU_OptActive = np.array([xLU_Active, gOptActive])
else:
g_xLU_OptActive = np.concatenate((xLU_Active, gOptActive))
if np.size(fGradIter) > 0:
#fGradOpt = fGradIter[nIter - 1]
fGradOpt = fGradIter[-1]
if np.size(gc) > 0:
gGradOpt = gGradIter[nIter - 1]
gGradOpt = gGradOpt.reshape([ng, nx]).T
gGradOptActive = gGradOpt[:, gOptActiveIndex == True]
try:
cOptActive = gc[gOptActiveIndex == True]
cActiveType = ["Constraint"]*np.size(cOptActive)
except:
cOptActive = []
cActiveType = []
if np.size(xGradActive) == 0:
g_xLU_GradOptActive = gGradOptActive
c_xLU_OptActive = cOptActive
c_xLU_ActiveType = cActiveType
elif np.size(gGradOptActive) == 0:
g_xLU_GradOptActive = xGradActive
c_xLU_OptActive = xActive
c_xLU_ActiveType = ["Bound"]*np.size(xActive)
else:
g_xLU_GradOptActive = np.concatenate((gGradOptActive, xGradActive), axis=1)
c_xLU_OptActive = np.concatenate((cOptActive, xActive))
xActiveType = ["Bound"]*np.size(xActive)
c_xLU_ActiveType = np.concatenate((cActiveType, xActiveType))
else:
g_xLU_GradOptActive = xGradActive
gGradOpt = np.array([])
c_xLU_OptActive = np.array([])
g_xLU_GradOptActive = np.array([])
c_xLU_ActiveType = np.array([])
else:
fGradOpt = np.array([])
gGradOpt = np.array([])
g_xLU_GradOptActive = np.array([])
c_xLU_OptActive = np.array([])
c_xLU_ActiveType = np.array([])
#-----------------------------------------------------------------------------------------------------------------------
# § Post-processing of optimization solution
#-----------------------------------------------------------------------------------------------------------------------
lambda_c, SPg, OptRes, Opt1Order, KKTmax = OptPostProc(fGradOpt, gc, gOptActiveIndex, g_xLU_GradOptActive,
c_xLU_OptActive, c_xLU_ActiveType, DesVarNorm)
#-----------------------------------------------------------------------------------------------------------------------
# § Save optimizaiton solution to file
#-----------------------------------------------------------------------------------------------------------------------
OptSolData = {}
OptSolData['x0'] = x0
OptSolData['xOpt'] = xOpt
OptSolData['xOptNorm'] = xOptNorm
OptSolData['xIter'] = xIter
OptSolData['xIterNorm'] = xIterNorm
OptSolData['fOpt'] = fOpt
OptSolData['fIter'] = fIter
OptSolData['fIterNorm'] = fIterNorm
OptSolData['gIter'] = gIter
OptSolData['gMaxIter'] = gMaxIter
OptSolData['gOpt'] = gOpt
OptSolData['fGradIter'] = fGradIter
OptSolData['gGradIter'] = gGradIter
OptSolData['fGradOpt'] = fGradOpt
OptSolData['gGradOpt'] = gGradOpt
OptSolData['OptName'] = OptName
OptSolData['OptModel'] = OptModel
OptSolData['OptTime'] = OptTime
OptSolData['loctime'] = loctime
OptSolData['today'] = today
OptSolData['computerName'] = computerName
OptSolData['operatingSystem'] = operatingSystem
OptSolData['architecture'] = architecture
OptSolData['nProcessors'] = nProcessors
OptSolData['userName'] = userName
OptSolData['Alg'] = Alg
OptSolData['DesVarNorm'] = DesVarNorm
OptSolData['KKTmax'] = KKTmax
OptSolData['lambda_c'] = lambda_c
OptSolData['nEval'] = nEval
OptSolData['nIter'] = nIter
OptSolData['SPg'] = SPg
OptSolData['gc'] = gc
#OptSolData['OptAlg'] = OptAlg
OptSolData['SensCalc'] = SensCalc
OptSolData['xIterNorm'] = xIterNorm
OptSolData['x0norm'] = x0norm
OptSolData['xL'] = xL
OptSolData['xU'] = xU
OptSolData['ng'] = ng
OptSolData['nx'] = nx
OptSolData['Opt1Order'] = Opt1Order
OptSolData['hhmmss0'] = hhmmss0
OptSolData['hhmmss1'] = hhmmss1
#-----------------------------------------------------------------------------------------------------------------------
# § Save in Python format
#-----------------------------------------------------------------------------------------------------------------------
output = open(OptName + "_OptSol.pkl", 'wb')
pickle.dump(OptSolData, output)
output.close()
np.savez(OptName + "_OptSol", x0, xOpt, xOptNorm, xIter, xIterNorm, xIter, xIterNorm, fOpt, fIter, fIterNorm, gIter,
gMaxIter, gOpt, fGradIter, gGradIter,
fGradOpt, gGradOpt, OptName, OptModel, OptTime, loctime, today, computerName, operatingSystem,
architecture, nProcessors, userName, Alg, DesVarNorm, KKTmax)
#-----------------------------------------------------------------------------------------------------------------------
# §5.2 Save in MATLAB format
#-----------------------------------------------------------------------------------------------------------------------
#OptSolData['OptAlg'] = []
spio.savemat(OptName + '_OptSol.mat', OptSolData, oned_as='row')
#-----------------------------------------------------------------------------------------------------------------------
#
#-----------------------------------------------------------------------------------------------------------------------
os.chdir(MainDir)
if LocalRun is True and Debug is False:
try:
shutil.move(RunDir + os.sep + OptName,
ResultsDir + os.sep + OptName + os.sep + "RunFiles" + os.sep)
# except WindowsError:
except:
print "Run files not deleted from " + RunDir + os.sep + OptName
shutil.copytree(RunDir + os.sep + OptName,
ResultsDir + os.sep + OptName + os.sep + "RunFiles" + os.sep)
#-----------------------------------------------------------------------------------------------------------------------
# § Graphical post-processing
#-----------------------------------------------------------------------------------------------------------------------
if ResultReport is True:
print("Entering preprocessing mode")
OptResultReport.OptResultReport(OptName, OptAlg, DesOptDir, diagrams=1, tables=1, lyx=1)
# try: OptResultReport.OptResultReport(OptName, diagrams=1, tables=1, lyx=1)
# except: print("Problem with generation of Result Report. Check if all prerequisites are installed")
if Video is True:
OptVideo.OptVideo(OptName)
#-----------------------------------------------------------------------------------------------------------------------
# § Print out
#-----------------------------------------------------------------------------------------------------------------------
if PrintOut is True:
print("")
print("--------------------------------------------------------------------------------")
print("Optimization results - DesOptPy")
print("--------------------------------------------------------------------------------")
print("Optimization with " + Alg)
print("g* = " + str(gOpt))
print("x* = " + str(xOpt.T))
print("f* = " + str(fOpt))
print("Lagrangian multipliers = " + str(lambda_c))
print("Shadow prices = " + str(SPg))
try:
print("nGen = " + str(nGen))
except:
print("nIter = " + str(nIter))
print("nEval = " + str(nEval))
print("Time of optimization [h:m:s] = " + OptTime)
if Debug is False:
print("See results directory: " + ResultsDir + os.sep + OptName + os.sep)
else:
print("Local run, no results saved to results directory")
print("--------------------------------------------------------------------------------")
if operatingSystem == "Linux" and Alarm is True:
t = 1
freq = 350
os.system('play --no-show-progress --null --channels 1 synth %s sine %f' % (t, freq))
return xOpt, fOpt, SPg
def OptHis2HTML(OptName, Alg, DesOptDir, xL, xU, DesVarNorm, inform, starttime, StatusDirectory=""):
# ----------------------------------------------------------------------------------------------------
# General calculations for the uppermost information table are computed like time running, algorithm
# name, optimization problem name etc.
# ----------------------------------------------------------------------------------------------------
StartTime = str(starttime)[0:10] + "000"
EndTime = ""
RefRate = '2000'
if inform != "Running":
EndTime = str(time())[0:10] + "000"
RefRate = '1000000'
Iteration = 'Iteration' # Label and Legends may be Iteration, Generation or Evaluations depending on Algorithm
if StatusDirectory == "": # Change the target directory for the status report files if the user wants to
StatusDirectory = DesOptDir
# Variables for the data extraction
pos_of_best_ind = [] # position of the best individual if a GA or ES is used as algorithm
fIter = [] # objective function array
xIter = [] # design vector array
gIter = [] # constraint vector array
# template_directory= DesOpt_Base + "/.DesOptPy/_OptStatusReport/" # directory with the html files etc.
template_directory = os.path.dirname(
os.path.realpath(__file__)) + "/StatusReportFiles/" # directory with the html files etc.
OptHist = History(OptName, "r") # open the history file generated by pyOpt or own algorithm
# read the obj fct, design and constraint vector values into the array
fAll = OptHist.read([0, -1], ["obj"])[0]["obj"]
xAll = OptHist.read([0, -1], ["x"])[0]["x"]
gAll = OptHist.read([0, -1], ["con"])[0]["con"]
# ----------------------------------------------------------------------------------------------------
# The next lines manipulate the data corresponding to the used algorithm. pyOpt Histories are different
# depending on the used algorithm
# ----------------------------------------------------------------------------------------------------
if Alg.name == "NLPQLP":
gAll = [x * -1 for x in gAll]
fGradIter = OptHist.read([0, -1], ["grad_obj"])[0]["grad_obj"]
if Alg.name == "COBYLA":
fIter = fAll
xIter = xAll
gIter = gAll
# If NSGA2 is used the best individual of a population is considered the objective fct, otherwise every
# individual would be shown in the graphs
elif Alg.name == "NSGA-II":
Iteration = 'Generation'
if inform == 0:
inform = 'Optimization terminated successfully'
PopSize = Alg.options['PopSize'][1]
for i in range(0, fAll.__len__() / PopSize): # Iteration trough the Populations
best_fitness = 9999999
max_violation_of_all_g = np.empty(PopSize)
max_violation_of_all_g.fill(99999999)
for u in range(0, PopSize): # Iteration trough the Individuals of the actual population
if np.max(gAll[i * PopSize + u]) < max_violation_of_all_g[u]:
max_violation_of_all_g[u] = np.max(gAll[i * PopSize + u])
pos_smallest_violation = np.argmin(max_violation_of_all_g)
# only not feasible designs, so choose the less violated one as best
if max_violation_of_all_g[pos_smallest_violation] > 0:
fIter.append(fAll[i * PopSize + pos_smallest_violation])
xIter.append(xAll[i * PopSize + pos_smallest_violation])
gIter.append(gAll[i * PopSize + pos_smallest_violation])
else: # find the best feasible one
# Iteration trough the Individuals of the actual population
for u in range(0, PopSize):
if np.max(fAll[i * PopSize + u]) < best_fitness:
if np.max(gAll[i * PopSize + u]) <= 0:
best_fitness = fAll[i * PopSize + u]
pos_of_best_ind = i * PopSize + u
fIter.append(fAll[pos_of_best_ind])
xIter.append(xAll[pos_of_best_ind])
gIter.append(gAll[pos_of_best_ind])
else:
fIter = [[]] * len(fGradIter)
xIter = [[]] * len(fGradIter)
gIter = [[]] * len(fGradIter)
for ii in range(len(fIter)):
Posdg = OptHist.cues["grad_con"][ii][0]
Posf = OptHist.cues["obj"][ii][0]
iii = 0
while Posdg > Posf:
iii = iii + 1
try:
Posf = OptHist.cues["obj"][iii][0]
except:
Posf = Posdg + 1
iii = iii - 1
fIter[ii] = fAll[iii]
xIter[ii] = xAll[iii]
gIter[ii] = gAll[iii]
if Alg.name != "NSGA-II":
if len(fGradIter) == 0: # first calculation
fIter = fAll
xIter = xAll
gIter = gAll
OptHist.close()
# convert the arrays to numpy arrays
fIter = np.asarray(fIter)
xIter = np.asarray(xIter)
gIter = np.asarray(gIter)
niter = len(fIter) - 1
# ----------------------------------------------------------------------------------------------------
# The design variables are normalized or denormalized so both can be displayed in the graphs and tables
# ----------------------------------------------------------------------------------------------------
if xIter.size != 0:
if DesVarNorm == False:
xIter_denormalized = np.zeros((niter + 1, len(xIter[0])))
for y in range(0, niter + 1):
xIter_denormalized[y] = xIter[y]
for y in range(0, niter + 1):
[xIter[y, :], xLnorm, xUnorm] = normalize(xIter_denormalized[y, :], xL, xU, "xLxU")
else:
xIter_denormalized = np.zeros((niter + 1, len(xIter[0])))
for y in range(0, niter + 1):
xIter_denormalized[y, :] = denormalize(xIter[y, :], xL, xU, DesVarNorm)
time_now = strftime("%Y-%b-%d %H:%M:%S", localtime()) # update the time for the information table
number_des_vars = "0"
number_constraints = "0"
# ----------------------------------------------------------------------------------------------------
# The .csv files are created and the first row is filled with the correct labels. Those .csv files
# are loaded by the javascript library. Afterwards the files are closed.
# ----------------------------------------------------------------------------------------------------
with open('objFct_maxCon.csv', 'wb') as csvfile:
datawriter = csv.writer(csvfile, dialect='excel')
datawriter.writerow(['Iteration', 'Objective function', 'Constraint'])
csvfile.close()
with open('desVarsNorm.csv', 'wb') as csvfile:
datawriter = csv.writer(csvfile, delimiter=',', escapechar=' ', quoting=csv.QUOTE_NONE)
labels = ['Iteration']
if xIter.size != 0:
for i in range(1, xIter.shape[1] + 1):
labels = labels + ['x' + str(i)]
datawriter.writerow(labels)
csvfile.close()
with open('desVars.csv', 'wb') as csvfile:
datawriter = csv.writer(csvfile, delimiter=',', escapechar=' ', quoting=csv.QUOTE_NONE)
labels = ['Iteration']
if xIter.size != 0:
for i in range(1, xIter.shape[1] + 1):
labels = labels + ['x' + str(i)]
datawriter.writerow(labels)
csvfile.close()
with open('constraints.csv', 'wb') as csvfile:
datawriter = csv.writer(csvfile, delimiter=',', escapechar=' ', quoting=csv.QUOTE_NONE)
labels = ['Iteration']
if gIter.size != 0:
for i in range(1, gIter.shape[1] + 1):
labels = labels + ['g' + str(i)]
datawriter.writerow(labels)
csvfile.close()
# ----------------------------------------------------------------------------------------------------
# Now the real data like obj fct value and constraint values are writen into the .csv files
# ----------------------------------------------------------------------------------------------------
# Objective function and maximum constraint values
for x in range(0, niter + 1):
with open('objFct_maxCon.csv', 'ab') as csvfile:
datawriter = csv.writer(csvfile, dialect='excel')
datawriter.writerow([x, str(float(fIter[x])), float(np.max(gIter[x]))])
csvfile.close()
# Normalized design variables
if xIter.size != 0:
for x in range(0, niter + 1):
datasets = str(xIter[x][:].tolist()).strip('[]')
with open('desVarsNorm.csv', 'ab') as csvfile:
datawriter = csv.writer(csvfile, dialect='excel', quotechar=' ')
datawriter.writerow([x, datasets])
csvfile.close()
# non normalized design variables
if xIter.size != 0:
for x in range(0, niter + 1):
datasets_denorm = str(xIter_denormalized[x][:].tolist()).strip('[]')
with open('desVars.csv', 'ab') as csvfile:
datawriter = csv.writer(csvfile, dialect='excel', quotechar=' ')
datawriter.writerow([x, datasets_denorm])
csvfile.close()
# constraint variables
if gIter.size != 0:
for x in range(0, niter + 1):
datasetsg = str(gIter[x][:].tolist()).strip('[]')
with open('constraints.csv', 'ab') as csvfile:
datawriter = csv.writer(csvfile, dialect='excel', quotechar=' ')
datawriter.writerow([x, datasetsg])
csvfile.close()
# ----------------------------------------------------------------------------------------------------
# The data for the graphs is generated, now follows the table generation routine
# ----------------------------------------------------------------------------------------------------
# Objective function table generation
ObjFct_table = "<td></td>"
if xIter.size != 0:
if gIter.size != 0:
for x in range(0, niter + 1):
ObjFct_table += "<tr>\n<td>" + str(x) + "</td>\n<td>" + str(round(fIter[x], 4)) + "</td>\n<td>" + str(
round(np.max(gIter[x]), 4)) + "</td>\n</tr>"
else:
for x in range(0, niter + 1):
ObjFct_table += "<tr>\n<td>" + str(x) + "</td>\n<td>" + str(
round(fIter[x], 4)) + "</td>\n<td> no constraints </td>\n</tr>"
# Design Variable table generation
DesVar_table = "<td></td>"
if xIter.size != 0:
number_des_vars = str(len(xIter[0]))
for x in range(0, len(xIter[0])):
DesVar_table += "<td>" + "x̂<sub>" + str(x + 1) + "</sub></td>" + "<td>" + "x<sub>" + str(
x + 1) + " </sub></td>"
for y in range(0, niter + 1):
DesVar_table += "<tr>\n<td>" + str(y) + "</td>"
for x in range(0, len(xIter[0])):
DesVar_table += "<td>" + str(round(xIter[y][x], 4)) + "</td><td>" + str(
round(xIter_denormalized[y][x], 4)) + "</td>"
DesVar_table += "</tr>"
# Constraint table generation
Constraint_table = "<td></td>"
if gIter.size != 0:
number_constraints = str(len(gIter[0]))
for x in range(0, len(gIter[0])):
Constraint_table += "<td>" + "g<sub>" + str(x + 1) + "</sub></td>"
for y in range(0, niter + 1):
Constraint_table += "<tr>\n<td>" + str(y) + "</td>"
for x in range(0, len(gIter[0])):
if (round(gIter[y][x], 4) > 0):
Constraint_table += "<td class=\"negativ\">" + str(round(gIter[y][x], 4)) + "</td>"
else:
Constraint_table += "<td class=\"positiv\">" + str(round(gIter[y][x], 4)) + "</td>"
Constraint_table += "</tr>"
# ----------------------------------------------------------------------------------------------------
# Everything is computed, now the html master template is opened and the placeholders are replaced
# with the right values
# ----------------------------------------------------------------------------------------------------
html = open(template_directory + '/initial.html', 'r') # open template
hstr = html.read()
html.close()
# replace the placeholder values with the true values
if gIter.size != 0 or gIter.size > 100:
hstrnew = hstr.replace('xxxxName', OptName)
hstrnew = hstrnew.replace('xxxxTime', time_now)
hstrnew = hstrnew.replace('xxxxtableObjFct', ObjFct_table)
hstrnew = hstrnew.replace('xxxxtableDesVar', DesVar_table)
hstrnew = hstrnew.replace('xxxxnumber_des_var', number_des_vars * 2)
hstrnew = hstrnew.replace('xxxxtableConstr', Constraint_table)
hstrnew = hstrnew.replace('xxxxnumber_constraints', number_constraints)
hstrnew = hstrnew.replace('xxxxAlg', Alg.name)
hstrnew = hstrnew.replace('xxxxStatus', str(inform))
hstrnew = hstrnew.replace('xxxxRefRate', RefRate)
hstrnew = hstrnew.replace('xxxxStartTime', StartTime)
hstrnew = hstrnew.replace('xxxxEndTime', EndTime)
hstrnew = hstrnew.replace('xxxxIteration', Iteration)
else:
hstrnew = hstr.replace('xxxxName', OptName)
hstrnew = hstrnew.replace('xxxxTime', time_now)
hstrnew = hstrnew.replace('xxxxtableObjFct', ObjFct_table)
hstrnew = hstrnew.replace('xxxxtableDesVar', DesVar_table)
hstrnew = hstrnew.replace('xxxxAlg', Alg.name)
hstrnew = hstrnew.replace('xxxxStatus', inform)
hstrnew = hstrnew.replace('xxxxRefRate', RefRate)
hstrnew = hstrnew.replace('xxxxStartTime', StartTime)
hstrnew = hstrnew.replace('xxxxEndTime', EndTime)
hstrnew = hstrnew.replace('xxxxIteration', Iteration)
# remove the hmtl parts which are only needed for constrained problems
try:
for i in range(0, 10):
hstrnew = hstrnew[0:hstrnew.find("<!--Start of constraint html part-->")] + hstrnew[hstrnew.find(
"<!--End of constraint html part-->") + 34:-1]
except:
print ""
# generate a new html file which is filled with the actual content
html = open('initial1.html', 'w')
html.write(hstrnew)
html.close()
# copy everything needed to the result directory
if not os.path.exists(StatusDirectory + os.sep + "Results" + os.sep + OptName):
os.makedirs(StatusDirectory + os.sep + "Results" + os.sep + OptName)
shutil.copy("initial1.html",
StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep + OptName + "_Status.html")
shutil.copy("objFct_maxCon.csv",
StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep + "objFct_maxCon.csv")
shutil.copy("desVars.csv",
StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep + "desVars.csv")
shutil.copy("desVarsNorm.csv",
StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep + "desVarsNorm.csv")
shutil.copy("constraints.csv",
StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep + "constraints.csv")
for file in glob.glob(template_directory + "*.png"):
shutil.copy(file, StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep)
for file in glob.glob(template_directory + "*.js"):
shutil.copy(file, StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep)
for file in glob.glob(template_directory + "*.css"):
shutil.copy(file, StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep)
for file in glob.glob(template_directory + "*.ico"):
shutil.copy(file, StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep)
for file in glob.glob(template_directory + "view_results.py"):
shutil.copy(file, StatusDirectory + os.sep + "Results" + os.sep + OptName + os.sep)
return 0