def main():
oob_scores = []
pred = []
y_test_sets =[]
for i in range(1,6):
print(i)
# model = SVC(kernel='rbf')
# model = LogisticRegression()
# model = RandomForestClassifier(n_estimators=20)
model=SVC(gamma='auto', kernel='poly', degree=1, max_iter=-1, probability=True)
# model = DecisionTreeClassifier(max_depth=9,random_state=66)
# model = BaggingClassifier(SVC(kernel='rbf'), max_samples=0.7, max_features=0.8)
#读入训练集
data = pd.read_csv('A/'+str(i)+'/train.csv',header=None)
# print(type(idx))
array = data.values
dataInfo = data.shape[1]
X_train = array[:, 1:dataInfo]
y_train = array[:, 0]
scaler = StandardScaler()
scaler.fit(X_train)
trainX_scaled = scaler.transform(X_train)
model.fit(trainX_scaled,y_train)
oob_score = metrics.precision_score(y_train, model.predict(trainX_scaled))
oob_scores.append(oob_score)
#读入测试集
data = pd.read_csv('A/'+str(i)+'/test.csv',header=None)
array = data.values
dataInfo2 = data.shape[1]
X_test = array[:, 1:dataInfo2]
y_test= array[:, 0]
scaler.fit(X_test)
testX_scaled = scaler.transform(X_test)
y_test_sets.append(y_test)
pred.append(model.predict(testX_scaled))
print(metrics.accuracy_score(y_test, model.predict(testX_scaled)))
y_test_sets_mean = np.mean(y_test_sets,axis=0)
p=np.shape(y_test_sets_mean)[0]
for i in range(p):
if y_test_sets_mean[i]>0.5 or y_test_sets_mean[i]==0.5:
y_test_sets_mean[i]=1
else :
y_test_sets_mean[i]=0
W = weights(oob_scores)
pred = np.vstack(pred)
pred = np.transpose(pred)
result = []
# 加权投票
L = len(pred)
M = len(pred[0])
result = []
for j in range(L):
tmp = {0: 0, 1: 0}
for k in range(M):
tmp[pred[j][k]]+=W[k]
if(tmp[0]>tmp[1]):
result.append(0)
elif(tmp[0]<tmp[1]):
result.append(1)
else:
result.append(random.choice([0,1]))
#硬投票
result1=[]
for item in pred:
result1.append(np.argmax(np.bincount(item.astype('int'))))
pred = np.array(result1).astype('float')
score = indicator(pred,y_test_sets_mean)
Accuracy,Precision,Recall,F_meature = score.getMetrics()
Specific = score.getSpecific()
TPR,FPR = score.getfprtpr()
AUC,x,y = score.getAuc()
print("Accuracy:", Accuracy)
print("Precison:", Precision)
print("Recall:", Recall)
print("F-meature:", F_meature)
print("Specific:", Specific)
print("MCC:",metrics.matthews_corrcoef(y_true=y_test_sets_mean,y_pred=pred))
print("AUC:",AUC)
print("TPR:",TPR)
print("FPR:",FPR)
dt_data2 = pd.read_csv(dt_filename5, header=None)
array2 = dt_data2.values
dataInfo2 = dt_data2.shape[1]
dt_testX = array2[:, 1:dataInfo2]
dt_testy = array2[:, 0]
scaler.fit(dt_testX)
dt_testX_scaled = scaler.transform(dt_testX)
dt_model = DecisionTreeClassifier(max_depth=9, random_state=66)
print("DT")
dt_model.fit(dt_trainX, dt_trainy)
print(dt_model.score(dt_testX, dt_testy))
pred = dt_model.predict(dt_testX)
score = indicator(pred, dt_testy)
Accuracy, Precision, Recall, F_meature = score.getMetrics()
Specific = score.getSpecific()
TPR, FPR = score.getfprtpr()
AUC, x, y = score.getAuc()
sumx += x
sumy += y
MCC = score.getMCC()
sumAccuracy += Accuracy
print(classification_report(dt_testy, pred))
print("Accuracy:", Accuracy)
print("Precison:", Precision)
print("Recall:", Recall)
print("F-meature:", F_meature)
print("Specific:", Specific)
print("MCC:", MCC)
tmp[pred[j][k]] += W[k]
if (tmp[0] > tmp[1]):
result.append(0)
elif (tmp[0] < tmp[1]):
result.append(1)
else:
result.append(random.choice([0, 1]))
#硬投票
result1 = []
for item in pred:
result1.append(np.argmax(np.bincount(item.astype('int'))))
pred = np.array(result1).astype('float')
score = indicator(pred, y_test_sets_mean)
Accuracy, Precision, Recall, F_meature = score.getMetrics()
Specific = score.getSpecific()
TPR, FPR = score.getfprtpr()
AUC, x, y = score.getAuc()
MCC = score.getMCC()
print("Accuracy:", Accuracy)
print("Precison:", Precision)
print("Recall:", Recall)
print("F-meature:", F_meature)
print("Specific:", Specific)
print("MCC:",
metrics.matthews_corrcoef(y_true=y_test_sets_mean, y_pred=pred))
print("AUC:", AUC)
print("TPR:", TPR)
def initUI(self):
if cfg('saveLayout', True):
self.layout = Layout(True)
if self.layout['running']:
answer = utils.yesNoDialog(
'Warning',
'Another RybaFish is already running, all the layout and autosave features will be disabled.\n\nExit now?',
ignore=True)
#answer = utils.yesNoDialog('Warning', 'RybaFish is already running or crashed last time, all the layout and autosave features will be disabled.\n\nExit now?', ignore = True)
if answer == True or answer is None:
exit(0)
if answer == 'ignore':
log('Ignoring the layout')
else:
self.layout = None
else:
self.layout['running'] = True
self.layout.dump()
else:
self.layout = Layout()
# bottom left frame (hosts)
hostsArea = QFrame(self)
self.hostTable = hostsTable.hostsTable()
# bottom right frame (KPIs)
self.kpisTable = kpiTable.kpiTable()
kpisTable = self.kpisTable
# top (main chart area)
self.chartArea = chartArea.chartArea()
ind = indicator()
self.chartArea.indicator = ind
# establish hard links:
kpisTable.kpiScales = self.chartArea.widget.nscales
self.chartArea.widget.hosts = self.hostTable.hosts
kpisTable.hosts = self.chartArea.widget.hosts #why do we have hosts inside widget? because we have all data there...
kpisTable.hostKPIs = self.chartArea.hostKPIs
kpisTable.srvcKPIs = self.chartArea.srvcKPIs
kpisTable.nkpis = self.chartArea.widget.nkpis
# bottm part left+right
self.kpiSplitter = QSplitter(Qt.Horizontal)
self.kpiSplitter.addWidget(self.hostTable)
self.kpiSplitter.addWidget(kpisTable)
self.kpiSplitter.setSizes([200, 380])
self.tabs = QTabWidget()
# self.tabs.currentChanged.connect(self.tabChanged)
# main window splitter
self.mainSplitter = QSplitter(Qt.Vertical)
kpisWidget = QWidget()
lo = QVBoxLayout(kpisWidget)
lo.addWidget(self.kpiSplitter)
self.mainSplitter.addWidget(self.chartArea)
self.mainSplitter.addWidget(kpisWidget)
if self.layout is not None:
if self.layout['mainSplitter']:
self.mainSplitter.setSizes(self.layout['mainSplitter'])
else:
self.mainSplitter.setSizes([300, 90])
if self.layout['kpiSplitter']:
self.kpiSplitter.setSizes(self.layout['kpiSplitter'])
else:
self.kpiSplitter.setSizes([200, 380])
if self.layout['hostTableWidth']:
hostTableWidth = self.layout['hostTableWidth']
for i in range(self.hostTable.columnCount()):
if i > len(hostTableWidth) - 1:
break
self.hostTable.setColumnWidth(i, hostTableWidth[i])
if self.layout['KPIsTableWidth']:
KPIsTableWidth = self.layout['KPIsTableWidth']
for i in range(self.kpisTable.columnCount()):
if i > len(KPIsTableWidth) - 1:
break
self.kpisTable.setColumnWidth(i, KPIsTableWidth[i])
else:
self.mainSplitter.setSizes([300, 90])
self.kpiSplitter.setSizes([200, 380])
self.mainSplitter.setAutoFillBackground(True)
# central widget
#self.setCentralWidget(mainSplitter)
kpisWidget.autoFillBackground = True
self.tabs.addTab(self.mainSplitter, 'Chart')
self.chartArea.selfRaise.connect(self.raiseTab)
ind.iClicked.connect(self.chartArea.indicatorSignal)
self.setCentralWidget(self.tabs)
# service stuff
self.statusbar = self.statusBar()
self.statusbar.addPermanentWidget(ind)
#menu
iconPath = resourcePath('ico\\favicon.ico')
exitAct = QAction('&Exit', self)
exitAct.setShortcut('Alt+Q')
exitAct.setStatusTip('Exit application')
exitAct.triggered.connect(self.menuQuit)
dummyAct = QAction('&Dummy', self)
dummyAct.setShortcut('Alt+D')
dummyAct.setStatusTip('Dummy Data provider')
dummyAct.triggered.connect(self.menuDummy)
configAct = QAction('&Connect', self)
configAct.setShortcut('Alt+C')
configAct.setStatusTip('Configure connection')
configAct.triggered.connect(self.menuConfig)
importAct = QAction('&Import nameserver history trace', self)
importAct.setShortcut('Ctrl+I')
importAct.setStatusTip('Import nameserver.trc')
importAct.triggered.connect(self.menuImport)
sqlConsAct = QAction('New &SQL Console', self)
sqlConsAct.setShortcut('Alt+S')
sqlConsAct.setStatusTip('Create SQL Console')
sqlConsAct.triggered.connect(self.menuSQLConsole)
openAct = QAction('&Open file in new sql console', self)
openAct.setShortcut('Ctrl+O')
openAct.setStatusTip('Open new console with the file')
openAct.triggered.connect(self.menuOpen)
saveAct = QAction('&Save sql to a file', self)
saveAct.setShortcut('Ctrl+S')
saveAct.setStatusTip('Saves sql from current console to a file')
saveAct.triggered.connect(self.menuSave)
menubar = self.menuBar()
fileMenu = menubar.addMenu('&File')
fileMenu.addAction(configAct)
fileMenu.addAction(importAct)
fileMenu.addAction(sqlConsAct)
fileMenu.addAction(openAct)
fileMenu.addAction(saveAct)
if cfg('experimental'):
fileMenu.addAction(dummyAct)
fileMenu.addAction(exitAct)
actionsMenu = menubar.addMenu('&Actions')
if cfg('experimental'):
# fileMenu.addAction(aboutAct) -- print not sure why its here
fontAct = QAction('&Adjust Fonts', self)
fontAct.setStatusTip(
'Adjust margins after font change (for example after move to secondary screen)'
)
fontAct.triggered.connect(self.menuFont)
actionsMenu.addAction(fontAct)
# issue #255
reloadConfigAct = QAction('Reload &Config', self)
reloadConfigAct.setStatusTip(
'Reload configuration file. Note: some values used during the connect or other one-time-actions (restart required).'
)
reloadConfigAct.triggered.connect(self.menuReloadConfig)
actionsMenu.addAction(reloadConfigAct)
reloadCustomKPIsAct = QAction('Reload Custom &KPIs', self)
reloadCustomKPIsAct.setStatusTip('Reload definition of custom KPIs')
reloadCustomKPIsAct.triggered.connect(self.menuReloadCustomKPIs)
actionsMenu.addAction(reloadCustomKPIsAct)
# help menu part
aboutAct = QAction(QIcon(iconPath), '&About', self)
aboutAct.setStatusTip('About this app')
aboutAct.triggered.connect(self.menuAbout)
confHelpAct = QAction('Configuration', self)
confHelpAct.setStatusTip('Configuration options description')
confHelpAct.triggered.connect(self.menuConfHelp)
confCustomHelpAct = QAction('Custom KPIs', self)
confCustomHelpAct.setStatusTip('Short manual on custom KPIs')
confCustomHelpAct.triggered.connect(self.menuCustomConfHelp)
confTipsAct = QAction('Tips and tricks', self)
confTipsAct.setStatusTip('Tips and tricks description')
confTipsAct.triggered.connect(self.menuTips)
helpMenu = menubar.addMenu('&Help')
helpMenu.addAction(confHelpAct)
helpMenu.addAction(confCustomHelpAct)
helpMenu.addAction(confTipsAct)
helpMenu.addAction(aboutAct)
# finalization
if self.layout is not None and self.layout['pos'] and self.layout[
'size']:
pos = self.layout['pos']
size = self.layout['size']
#print('screen number', QApplication.desktop().screenNumber())
#print('number of screens', QApplication.desktop().screenCount())
#print('available geometry:', QApplication.desktop().availableGeometry())
#print('screen geometry:', QApplication.desktop().screenGeometry())
r = QRect(pos[0], pos[1], size[0], size[1])
if QApplication.desktop().screenCount() == 1:
# only when just one screen is available...
if not QApplication.desktop().screenGeometry().contains(
r) and not cfg('dontAutodetectScreen'):
#the window will not be visible so jump to the main screen:
(pos[0], pos[1]) = (100, 50)
#self.setGeometry(pos[0] + 8, pos[1] + 31, size[0], size[1])
#self.setGeometry(pos[0], pos[1], size[0], size[1])
self.move(pos[0], pos[1])
self.resize(size[0], size[1])
else:
self.setGeometry(200, 200, 1400, 800)
self.setWindowTitle('RybaFish Charts')
self.setWindowIcon(QIcon(iconPath))
scrollPosition = []
if cfg('saveOpenTabs',
True) and self.layout is not None and self.layout['tabs']:
for t in self.layout['tabs']:
if len(t) != 4:
continue
console = sqlConsole.sqlConsole(self, None, '?')
console.nameChanged.connect(self.changeActiveTabName)
console.cons.closeSignal.connect(self.closeTab)
self.tabs.addTab(console, console.tabname)
ind = indicator()
console.indicator = ind
console.selfRaise.connect(self.raiseTab)
ind.iClicked.connect(console.reportRuntime)
self.statusbar.addPermanentWidget(ind)
self.tabs.setCurrentIndex(self.tabs.count() - 1)
if t[0] is not None or t[1] is not None:
# such a tab should not ever be saved (this call will just open fileOpen dialog), anyway...
# should we even create such a tab?
console.openFile(t[0], t[1])
pos = t[2]
block = t[3]
scrollPosition.append(block)
if isinstance(pos, int) and isinstance(block, int):
cursor = console.cons.textCursor()
cursor.setPosition(pos, cursor.MoveAnchor)
console.cons.setTextCursor(cursor)
indx = self.layout['currentTab']
if isinstance(indx, int):
self.tabs.setCurrentIndex(indx)
w = self.tabs.widget(indx)
if isinstance(w, sqlConsole.sqlConsole):
w.cons.setFocus()
else:
self.tabs.setCurrentIndex(0)
self.show()
#scroll everything to stored position
for i in range(self.tabs.count() - 1, 0, -1):
w = self.tabs.widget(i)
if isinstance(w, sqlConsole.sqlConsole):
if i - 1 < len(scrollPosition):
block = scrollPosition[i - 1]
w.cons.edit.verticalScrollBar().setValue(block)
else:
log('[w] scroll position list out of range, ignoring scrollback...'
)
'''
set up some interactions
'''
# bind kpi checkbox signal
kpisTable.checkboxToggle.connect(self.chartArea.checkboxToggle)
# bind change scales signal
kpisTable.adjustScale.connect(self.chartArea.adjustScale)
kpisTable.setScale.connect(self.chartArea.setScale)
# host table row change signal
self.hostTable.hostChanged.connect(kpisTable.refill)
# to fill hosts
self.chartArea.hostsUpdated.connect(self.hostTable.hostsUpdated)
# refresh
self.chartArea.kpiToggled.connect(kpisTable.refill)
# update scales signal
self.chartArea.scalesUpdated.connect(kpisTable.updateScales)
log('self.scalesUpdated.emit() #0', 5)
self.chartArea.scalesUpdated.emit(
) # it really not supposed to have any to update here
#bind statusbox updating signals
self.chartArea.statusMessage_.connect(self.statusMessage)
self.chartArea.widget.statusMessage_.connect(self.statusMessage)
self.chartArea.connected.connect(self.setTabName)
log('init finish()')
# offline console tests
if cfg('developmentMode'):
#tname = sqlConsole.generateTabName()
#idx = self.tabs.count()
self.sqlTabCounter += 1
idx = self.sqlTabCounter
if idx > 1:
tname = 'sql' + str(idx)
else:
tname = 'sql'
console = sqlConsole.sqlConsole(self, None, tname)
console.nameChanged.connect(self.changeActiveTabName)
from SQLSyntaxHighlighter import SQLSyntaxHighlighter
self.tabs.addTab(console, tname)
console.selfRaise.connect(self.raiseTab)
self.tabs.setCurrentIndex(self.tabs.count() - 1)
self.SQLSyntax = SQLSyntaxHighlighter(console.cons.document())
#console.cons.setPlainText('select * from dummy;\n\nselect \n *\n from dummy;\n\nselect * from m_host_information;');
ind = indicator()
console.indicator = ind
self.statusbar.addPermanentWidget(ind)
ind.iClicked.connect(console.reportRuntime)
if cfg('developmentMode'):
console.cons.setPlainText('''select 0 from dummy;
create procedure ...
(
(as begin)
select * from dummy);
end;
where timestamp between '2020-02-10 00:00:00' and '2020-02-16 23:59:59' -- test comment
where not "NAME1" = '' and "DOKST" in ('D0', 'D2') and (1 = 2)
select 1 from dummy;
select 2 from dummy;
select 3 from dummy;''')
console.dummyResultTable()
self.statusMessage('', False)
if self.chartArea.dp:
self.chartArea.initDP()
def menuSQLConsole(self):
conf = self.connectionConf
if conf is None:
self.statusMessage('No configuration...', False)
return
self.statusMessage('Connecting...', True)
ind = indicator()
self.statusbar.addPermanentWidget(ind)
ind.status = 'sync'
ind.repaint()
log('menuSQLConsole...')
noname = True
while noname:
self.sqlTabCounter += 1
idx = self.sqlTabCounter
if idx > 1:
tname = 'sql' + str(idx)
else:
tname = 'sql'
for i in range(self.tabs.count() - 1, 0, -1):
w = self.tabs.widget(i)
if isinstance(w, sqlConsole.sqlConsole):
if w.tabname == tname or w.tabname == tname + ' *': # so not nice...
break
else:
noname = False
# console = sqlConsole.sqlConsole(self, conf, tname) # self = window
try:
console = sqlConsole.sqlConsole(self, conf, tname) # self = window
log('seems connected...')
except dbException as e:
log('[!] failed to open console expectedly')
self.statusMessage('Connection error', True)
self.statusbar.removeWidget(ind)
return
'''
except Exception as e:
log('[!] failed to open console unexpectedly: ' + str(e))
self.statusMessage('Connection error?', True)
return
'''
console.indicator = ind
ind.iClicked.connect(console.reportRuntime)
console.nameChanged.connect(self.changeActiveTabName)
console.cons.closeSignal.connect(self.closeTab)
self.tabs.addTab(console, tname)
console.selfRaise.connect(self.raiseTab)
self.tabs.setCurrentIndex(self.tabs.count() - 1)
if console.unsavedChanges:
# if autoloaded from backup
# cannot be triggered from inside as signal not connected on __init__
self.changeActiveTabName(console.tabname + ' *')
if self.layout == None:
# no backups to avoid conflicts...
console.noBackup = True
self.statusMessage('', False)
console.indicator.status = 'idle'
console.indicator.repaint()
def menuOpen(self):
'''
so much duplicate code with menuSqlConsole
and with dumpLayout!
'''
fname = QFileDialog.getOpenFileNames(self, 'Open file', '', '*.sql')
openfiles = {}
for i in range(self.tabs.count()):
w = self.tabs.widget(i)
if isinstance(w, sqlConsole.sqlConsole):
fn = w.fileName
if fn is not None:
openfiles[fn] = i
for filename in fname[0]:
if filename in openfiles:
# the file is already open
idx = openfiles[filename]
self.tabs.setCurrentIndex(idx)
continue
conf = self.connectionConf
self.statusMessage('Connecting console...', True)
try:
console = sqlConsole.sqlConsole(self, conf, 'sqlopen')
except:
self.statusMessage('Failed', True)
log('[!] error creating console for the file')
return
self.statusMessage('', False)
console.nameChanged.connect(self.changeActiveTabName)
console.cons.closeSignal.connect(self.closeTab)
self.tabs.addTab(console, console.tabname)
console.selfRaise.connect(self.raiseTab)
ind = indicator()
console.indicator = ind
ind.iClicked.connect(console.reportRuntime)
self.statusbar.addPermanentWidget(ind)
self.tabs.setCurrentIndex(self.tabs.count() - 1)
console.openFile(filename)
if self.layout == None:
# no backups to avoid conflicts...
console.noBackup = True
verbose=2)
# model2=getGoogleNet(input_shape2)
# adam=keras.optimizers.Adam(lr=0.001)
# model2.compile(optimizer=adam, loss='sparse_categorical_crossentropy', metrics=['acc'])
# model2.build(input_shape=(None, size2, size2))
# model2.fit(x_train2, y_train2, batch_size=64, epochs=20, verbose=2)
# EnsembelNet_model=EnsembelNet([model,model2],1)
# EnsembelNet_model.compile(optimizer=adam, loss='sparse_categorical_crossentropy', metrics=['acc'])
# EnsembelNet_model.build(input_shape=(None, size2, size2))
# EnsembelNet_model.fit(x_train2, y_train2, batch_size=64, epochs=50, verbose=2)
pred = model.predict([x_test, x_test2, x_test3])
pred = np.argmax(pred, axis=1)
score = indicator(pred, y_test)
Accuracy, Precision, Recall, F_meature = score.getMetrics()
Specific = score.getSpecific()
TPR, FPR = score.getfprtpr()
AUC, x, y = score.getAuc()
MCC = score.getMCC()
print("Accuracy:", Accuracy)
print("Precison:", Precision)
print("Recall:", Recall)
print("F-meature:", F_meature)
print("Specific:", Specific)
print("MCC:", MCC)
print("AUC:", AUC)
print("TPR:", TPR)
print("FPR:", FPR)
svm_testy = array2[:, 0]
scaler.fit(svm_testX)
svm_testX_scaled = scaler.transform(svm_testX)
svm_model = SVC(gamma='auto',
kernel='poly',
degree=1,
max_iter=-1,
probability=True)
print("svm")
svm_model.fit(svm_trainX_scaled, svm_trainy)
print(svm_model.score(svm_testX_scaled, svm_testy))
pred = svm_model.predict(svm_testX_scaled)
score = indicator(pred, svm_testy)
Accuracy, Precision, Recall, F_meature = score.getMetrics()
Specific = score.getSpecific()
TPR, FPR = score.getfprtpr()
AUC, x, y = score.getAuc()
sumx += x
sumy += y
MCC = score.getMCC()
sumAccuracy += Accuracy
print(classification_report(svm_testy, pred))
print("Accuracy:", Accuracy)
print("Precison:", Precision)
print("Recall:", Recall)
print("F-meature:", F_meature)
print("Specific:", Specific)
print("MCC:", MCC)