def processedData(go):
'''
generate necessary data for visualization
Args:
a list of GO terms
Return:
matrix:a matrix M where M(i,j) represents the number of intersected genes betweeen GO[i] and GO[j]
go_index_reord:an array representing the position change of GO terms after hieracical clustering
go_hier:a list of GO that are ancesters of enriched GO terms.
go_inf_reord:an array of enriched GO terms
clusterHierData:an array storing hierarcical data use to generated hierarcical tree
'''
dataProcess = DataProcess()
try:
preProcessedData = dataProcess.dataProcess(go)
except Exception as e:
logger.error(str(e))
else:
matrix = preProcessedData["matrix"]["matrix_count"]
go_index_reord = preProcessedData["go_index_reord"]
go_hier = preProcessedData["go_hier"]
go_inf_reord = preProcessedData["go_inf"]
clusterHierData = preProcessedData["clusterHierData"]
return matrix, go_index_reord, go_hier, go_inf_reord, clusterHierData
def __init__(self):
#super(Main,self).__init__()
#全局变量字典定义
Mygol._init()
MyFilegol._init()
#读取配置信息至全局变量
FileHandle.readMyfile()
ConfigHandle.readConfig()
self.businessTask = BusinessProcess()
self.dataTask = DataProcess()
def OpenFile(self):
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
self.fileName, _ = QFileDialog.getOpenFileName(
self,
"QFileDialog.getOpenFileName()",
"",
"All Files (*);;Python Files (*.nc)",
options=options)
if self.fileName:
print(self.fileName)
self.APP_dataprocess = DataProcess(self.fileName)
def recommendxx(p, q, carid, cp):
rank = dict()
list = DataProcess.getNeighbor(
cp) # some grid may not be visited by any carid.
for i in list: # i should get from file.
pui = visitPosibility(carid, i, p, q)
rank[i] = pui #记录对该网格的兴趣度
rank = sorted(rank.items(), key=lambda x: x[1], reverse=True)
return rank
class Main():
def __init__(self):
#super(Main,self).__init__()
#全局变量字典定义
Mygol._init()
MyFilegol._init()
#读取配置信息至全局变量
FileHandle.readMyfile()
ConfigHandle.readConfig()
self.businessTask = BusinessProcess()
self.dataTask = DataProcess()
#数据接收线程
def DataRecvProcessTask(self):
global is_exit
while not is_exit:
self.dataTask.OnRecvDataProcess()
#数据发送线程
def DataSendProcessTask(self):
global is_exit
while not is_exit:
self.dataTask.OnSendDataProcess()
def index():
# Main page
if request.method == 'POST' and 'data' in request.files:
for filename in request.files.getlist('data'):
datas.save(filename)
dataset = DataProcess(os.getcwd() + "/base_test_sum.csv",
os.getcwd() + "/knowledge_test_sum.csv",
os.getcwd() + "/money_report_test_sum.csv",
os.getcwd() + "/year_report_test_sum.csv")
#print(Inference(dataset))
Inference(dataset).to_csv(os.getcwd() + "/result.csv",
encoding="utf-8",
index=False)
return render_template('index.html')
def main():
net = load_net(21, 256, [0.0216, 0.0470, 0.0711, 0.1185, 1.],
'./model/patient_group_net.pth')
p_arrive, p_leave, delta, p_die = DataProcess(net,
time_interval=10).process()
reward = [a + b for (a, b) in zip(delta, p_die)]
reward = [0] + reward
# p_arrive = [0.8988013698630137, 0.012994672754946727, 0.01293759512937595, 0.01221461187214612, 0.013032724505327244, 0.013089802130898021, 0.011910197869101979, 0.012271689497716894, 0.012747336377473363]
# p_leave = [0.010825796872474158, 0.0038442559604472373, 0.00335442100653245, 0.0027426663004428303, 0.0026808956939254076, 0.0026285227575753496, 0.002433672892274736, 0.0018672590670831529]
# delta = [0, 0.00749376480848956, 0.023252303608424292, 0.38966471920753987, 0.1, 0.04492930254495119, 0.019348297623417, 1.1000634771910827, 1.080419000471227]
print(p_arrive, p_leave, reward)
env = Environment(5, 10, p_arrive, p_leave, reward, max_leave=3)
env.calc_dynamics()
rl = ValueIterationRL(env)
x, a = rl.value_iteration_sparse()
save_env(env)
def PostProcess(self):
app = DataProcess() # which read the data only
app.PlotFrames()
# calculated the first order which will be used at photon distribution, high order imaging, so do it firstly.
# and plot it
app.PlotFirstOdr()
app.PlotPhotonDistribution()
app.PlotSecondOdr()
def __init__(self, Vocab, hidden_size):
super(CnnEnc, self).__init__()
# embedding layer
self.embeddings = nn.Embedding(Args.args.vocab_size + 4,
Args.args.embed_dim)
self.feat_size = int((Args.args.hidden_size) / 3)
self.embed_dim = Args.args.embed_dim
self.max_sent = Args.args.max_sent
self.hidden_size = hidden_size
if Args.args.predefined == True:
wordEmbedding = DP.load_predefined_embedding(
Args.args.embed_path, Vocab)
self.embeddings.weight.data.copy_(
torch.from_numpy(wordEmbedding)) # pretrained vocabulary used.
# Kernel Operation
self.conv1 = nn.Conv2d(1, self.feat_size, (3, self.embed_dim))
self.conv2 = nn.Conv2d(1, self.feat_size, (4, self.embed_dim))
self.conv3 = nn.Conv2d(1, self.feat_size, (5, self.embed_dim))
# Dropout & Linear
self.drp = nn.Dropout(p=0.2)
self.fc = nn.Linear(self.feat_size * 3, self.hidden_size)
self.Tanh = nn.Tanh()
if __name__ == '__main__':
# 导入数据
info = DataInput()
df_train_data = info.load_train_data()
df_test_data = info.load_test_data()
del info
# 添加标签，以便分隔
df_train_data['train_or_test'] = '1'
df_test_data['train_or_test'] = '2'
#合并生成总集，方便编码
df_all = df_train_data.append(df_test_data, ignore_index=True)
# 数据预处理:
# 数据初处理
pro = DataProcess()
df_all = pro.prd(pro.removingInvalid(df_all))
del pro
print("数据初处理完成！")
# 数据编码
enc = Encoder()
df_all = enc.encoder(enc.prepareEncoder(df_all))
need_one_hot = enc.get_need_one_hot()
del enc
print("数据编码完成！")
# 重新拆分训练集与测试集
all_list = SplitTrOrTe().split(df_all,need_one_hot)
x_train,y_train,x_test,y_test,test_user_id = all_list[0],all_list[1],all_list[2],all_list[3],all_list[4]
del all_list
####solve data file1 workpath = r"F:\Users\JianyingLi\Desktop\S1P0.05E1" indexname = "eRHo" extname = ".dat" timeCoe = 10e-7 lengthCoe = 0.0003125 instance = Env() print instance.get_workingPath() fileReader = filereader(instance.get_fileList(workpath), workpath) fileReader.select_files(extname, indexname) fileReader.read_data() # print fileReader.data dp = DataProcess() x0,y0 = dp.data1D(fileReader.data, extname, indexname,timeCoe) ##################################################################### ####solve data file2 workpath = r"F:\Users\JianyingLi\Desktop\S1P0.05E0.4" indexname = "eRHo" extname = ".dat" timeCoe = 10e-7 lengthCoe = 0.0003125 instance = Env() print instance.get_workingPath() fileReader = filereader(instance.get_fileList(workpath), workpath)
from DataProcess import DataProcess from keras.optimizers import Adam from keras.models import Sequential, Model from keras.layers import Dense, Activation, Dropout, Conv1D, Flatten import keras import numpy as np from keras.utils import np_utils import time # get and process data data = DataProcess() # x_train, y_train, x_test, y_test, x_test_21, y_test_21 = data.return_processed_data_multiclass() x_train, y_train, x_test, y_test, x_test_21, y_test_21 = data.return_processed_data_binary() # reshape input to be [samples, timesteps, features] x_train = x_train.reshape(x_train.shape[0], 1, x_train.shape[1]) x_test = x_test.reshape(x_test.shape[0], 1, x_test.shape[1]) x_test_21 = x_test_21.reshape(x_test_21.shape[0], 1, x_test_21.shape[1]) # multiclass # y_train=np_utils.to_categorical(y_train) # y_test=np_utils.to_categorical(y_test) # y_test_21=np_utils.to_categorical(y_test_21) start = time.time() model = Sequential() model.add(Conv1D(60,kernel_size=1, activation='relu', input_shape=(x_train.shape[1],x_train.shape[2]))) model.add(Dropout(0.1))
db1 = grads['db1']
dW2 = grads['dW2']
db2 = grads['db2']
# 参数更新
self.w1 -= dW1 * learning_rate
self.b1 -= db1 * learning_rate
self.w2 -= dW2 * learning_rate
self.b2 -= db2 * learning_rate
if __name__ == '__main__':
start = time.time()
# 加载数据
# 获取训练集及标签
trainData, trainLabel = DataProcess.load_data('./data/Mnist/mnist_train.csv', binary_classification=False)
# 获取测试集及标签
# testData, testLabel = DataProcess.load_data('./data/Mnist/mnist_test.csv',binary_classification=False)
# 实例化感知机，输入特征为4
dnn = DeepNeuralNetwork(trainData, trainLabel, 88, 10)
for i in range(500):
grads = dnn.backward(trainData, trainLabel)
# dnn.update_parameters(grads, learning_rate=0.01)
# print(dnn.w1.max())
label_pre, cache_test = dnn.forward(trainData)
# loss_ce = -np.mean(np.sum(np.multiply(one_hot(10, trainLabel), np.log(label_pre)), axis=1))
loss_ce = cross_entropy(label_pre, trainLabel)
acc = np.sum(trainLabel == np.argmax(label_pre, axis=1)) / trainLabel.shape[0]
if not i % 50:
print("Iter", i, "Loss", loss_ce, "Acc", acc)
end = time.time()
i] * feature_array[i]
self.bias = self.bias + lr * label_array[i]
loss_flag = np.multiply(
-label_array, (np.dot(feature_array, self.weight) + self.bias))
loss = np.sum(loss_flag[loss_flag > 0])
acc = 1.0 - len(loss_flag[loss_flag > 0]) / len(loss_flag)
print("Iteration:", iter, "Loss:", loss, "Acc:", acc)
return self.weight, self.bias
def cal_acc(self, y_acc, y_hat):
pass
if __name__ == '__main__':
start = time.time()
# 加载数据
# 获取训练集及标签
trainData, trainLabel = DataProcess.load_data(
'./data/Mnist/mnist_train.csv')
# 获取测试集及标签
testData, testLabel = DataProcess.load_data('./data/Mnist/mnist_test.csv')
# 实例化感知机，输入特征为4
mlp = Perceptron(num_input=784)
# 训练模型，获得计算参数
weight, bias = mlp.fit(trainData, trainLabel, iteration=1000)
# 预测结果
y_pre = mlp.predict(testData)
end = time.time()
# 显示用时时长
print('time span:', end - start)
from single.tranformer_HAN_single import HAN_model
from DataProcess import DataProcess
import numpy as np
import config as cf
import tensorflow as tf
from tqdm import trange
DP = DataProcess()
train_x_all, train_y_input_float, train_y_input_int, train_y_output_all, test_x_all, test_y_input_float, test_y_input_int, test_y_output_all = DP.read_file(
'./data_multi')
print('1、load data完成')
Model = HAN_model()
print('2、构造模型完成')
# # with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
# opt_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(Model.all_loss)
saver = tf.train.Saver(max_to_keep=10)
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
merged = tf.summary.merge_all()
writer_train = tf.summary.FileWriter("single/logs_single/train",
sess.graph)
writer_test = tf.summary.FileWriter("single/logs_single/test")
sess.run(init_op)
# ckpt_single_extra = tf.train.get_checkpoint_state('./ckpt_single_extra/')
# saver.restore(sess, save_path=ckpt_single_extra.model_checkpoint_path)
print('3、初始化完成')
print('4、开始训练')
rule_all_acc = 0
zm_all_acc = 0
xq_all_acc = 0
epoch_number = int(len(train_y_output_all) // cf.batch_size)
# Author: Wenfeng Du
# Create Date: 2019-08-01
# Description: 基于百度的人工智能API，把图片识别成文字，
# 并把相应的营养成分信息提取出来，保存到数据库。
# -*- coding: utf-8 -*-
from DataProcess import DataProcess
from AILogic import AILogic
get_filename = 'scm_ProductQRCode_piclinks.xlsx'
client_id = 'XXXXXXXXXXXXXXXXXXXX'
client_secret = 'XXXXXXXXXXXXXXXXXXXXX'
save_table = 'scm_nutrition_ocr'
if __name__ == '__main__':
dp = DataProcess()
df = dp.get_from_excel(get_filename)
ai = AILogic()
access_token = ai.get_baidutoken(client_id,client_secret)
df = df.apply(ai.ocr_action, axis=1)
dp.save_to_db(df,save_table)
The TF-IDF transformed numpy.ndarray.
"""
transformer = TfidfTransformer()
tfidf = transformer.fit_transform(X)
return tfidf.toarray()
#Test
if __name__ == '__main__':
from DataProcess import DataProcess
import numpy as np
x = np.array([[1.0,2,3],[4.1,5,6]])
print x
print 'Test norm'
clf = DataProcess(norm='l2', scale=None)
print clf.transform(x)
print 'Test std scale'
clf = DataProcess(norm=None, scale='std')
print clf.transform(x)
print 'Test minmax scale'
clf = DataProcess(norm=None, scale='minmax')
print clf.transform(x)
print 'Test tfidf'
print clf.tfidf(x)
# -*- coding: utf-8 -*-
from ENV import Env
from FileReader import filereader
#import math
from DataProcess import DataProcess
from graph_maker import graph_maker
workpath = r"F:\Users\JianyingLi\Desktop\6"
indexname = "eRHo"
extname = ".dat"
timeCoe = 10e-7
lengthCoe = 0.0003125
instance = Env()
print instance.get_workingPath()
fileReader = filereader(instance.get_fileList(workpath), workpath)
fileReader.select_files(extname, indexname)
fileReader.read_data()
# print fileReader.data
dp = DataProcess()
Z, extent = dp.data2D(fileReader.data, extname, indexname, timeCoe, lengthCoe)
gm = graph_maker('title', 'xtitle', 'ytitle', 'test')
gm.plot2d(Z, extent)
print 'finished'
from multi_extra.transformer_HAN_multi_extra import HAN_model
from DataProcess import DataProcess
import numpy as np
import config as cf
import tensorflow as tf
from tqdm import trange
from validation import validation
valid = validation()
DP = DataProcess()
train_x_all, train_y_input_float, train_y_input_int, train_y_output_all, test_x_all, test_y_input_float, test_y_input_int, test_y_output_all = DP.read_file_multi(
r'D:\Project\transformer_model\data_multi')
print('1、load data完成')
Model = HAN_model()
print('2、构造模型完成')
# # with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
# opt_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(Model.all_loss)
saver = tf.train.Saver(max_to_keep=10)
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
merged = tf.summary.merge_all()
writer_train = tf.summary.FileWriter(
"./multi_extra/logs_multi_extra/train", sess.graph)
writer_test = tf.summary.FileWriter("./multi_extra/logs_multi_extra/test")
sess.run(init_op)
# ckpt_single_extra = tf.train.get_checkpoint_state('./ckpt_single_extra/')
# saver.restore(sess, save_path=ckpt_single_extra.model_checkpoint_path)
print('3、初始化完成')
print('4、开始训练')
rule_all_acc = 0
zm_all_acc = 0
xq_all_acc = 0
from DataProcess import DataProcess
from graph_maker import graph_maker
workpath = r"C:\Users\jianYing\Desktop\targetdata\P0.01E2P"
indexname = "eRHo"
extname = ".dat"
timeCoe = 10e-7
lengthCoe = 0.0003125
instance = Env()
print instance.get_workingPath()
fileReader = filereader(instance.get_fileList(workpath), workpath)
fileReader.select_files(extname, indexname)
fileReader.read_data()
# print fileReader.data
dp = DataProcess()
Z,extent = dp.data2D(fileReader.data, extname, indexname,timeCoe,lengthCoe)
gm = graph_maker('title','xtitle','ytitle','test')
gm.plot3d(Z,extent)
print 'finished'
def process_data():
print("Training Data:")
train_data = DataProcess(train_file)
print("Has missing data? " + str(train_data.has_missing()))
train_data.handle_missing_data()
print("Has missing data? " + str(train_data.has_missing()))
# train_data.plot_data()
# train_data.normalize_data()
print("\nTest Data:")
test_data = DataProcess(test_file)
print("Has missing data? " + str(test_data.has_missing()))
test_data.handle_missing_data()
print("Has missing data? " + str(test_data.has_missing()))
# test_data.plot_data()
# test_data.normalize_data()
return train_data, test_data
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import (precision_score, recall_score,f1_score, accuracy_score,mean_squared_error,mean_absolute_error)
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import Normalizer
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix
from sklearn.metrics import (precision_score, recall_score,f1_score, accuracy_score,mean_squared_error,mean_absolute_error, roc_curve, classification_report,auc)
# get and process data
data = DataProcess()
x_train, y_train, x_test, y_test = data.return_processed_cicids_data_binary()
model = LogisticRegression()
model.fit(x_train, y_train.ravel())
# make predictions
expected = y_test.ravel()
predicted = model.predict(x_test)
accuracy = accuracy_score(expected, predicted)
recall = recall_score(expected, predicted, average="binary")
precision = precision_score(expected, predicted , average="binary")
f1 = f1_score(expected, predicted , average="binary")
print("******** LogisticRegression ***********\n")
cm = metrics.confusion_matrix(expected, predicted)
print(cm)
tpr = float(cm[0][0])/np.sum(cm[0])
from DataProcess import DataProcess from keras.optimizers import Adam from keras.models import Sequential, Model from keras.layers import Dense, Activation, Dropout, LSTM, TimeDistributed import keras import numpy as np from keras.utils import np_utils import time from sklearn.metrics import classification_report, confusion_matrix import winsound # get and process data data = DataProcess() x_train, y_train, x_test, y_test = data.return_processed_cicids_data_multiclass() # x_train, y_train, x_test, y_test = data.return_processed_cicids_data_binary() # reshape input to be [samples, timesteps, features] x_train = x_train.reshape(x_train.shape[0], 1, x_train.shape[1]) x_test = x_test.reshape(x_test.shape[0], 1, x_test.shape[1]) # multiclass y_train=np_utils.to_categorical(y_train) y_test=np_utils.to_categorical(y_test) model = Sequential() model.add(LSTM(120, input_shape = (x_train.shape[1],x_train.shape[2]), return_sequences=True)) model.add(Dropout(0.2)) model.add(LSTM(120, return_sequences=True))
import torch
import Train
import Evaluation
import etc.Utils as U
import pickle
stime = [
None
] # elements of list in python which is passed to a function as a parameter can be modified within the function.
referenced_id = {'trained': [], 'closed': [], 'tested': []}
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
# Create Vocab
print("Reading Vocabulary .... ", end='')
Vocab = DP.Vocab(Args.args.voca_path, Args.args.vocab_size)
print("Done !!!")
if True:
# Call DataSet
print('before calling dataset allocated (dictionary): ', end=' ')
print(torch.cuda.memory_allocated(1))
print("Obtaining Dataset .... ", end='')
U.timeCheck('s', stime)
data_examples_gen = DP.form_examples(
Args.args.data_path,
Args.args.train_size) # generator of article, abstract pair
DataManager = DS.NewsDataset(data_examples_gen, Vocab,
False) # dataset defined
# DataManager = DS.NewsDataset(data_examples_gen, Vocab) # dataset defined
# U.saveExamples(DataManager.formed_examples)
class MyTableWidget(QWidget):
def __init__(self, parent):
super(QWidget, self).__init__(parent)
self.fileName = ''
self.layout = QVBoxLayout(
self) # create a Layout, which will be setted for self
# Initialize tab screen
self.tabs = QTabWidget()
self.tab1 = QWidget()
self.tab2 = QWidget()
self.tab3 = QWidget()
# self.tabs.resize(800,600)
self.tabs.addTab(self.tab1, "Auto-Image") # Add tabs
self.tabs.addTab(self.tab2, "Save Multi-frames")
self.tabs.addTab(self.tab3, "Data Processing")
self.layout.addWidget(self.tabs) # Add tabs to widget
self.setLayout(self.layout)
self.initalUI_tab_1()
self.initalUI_tab_2()
self.initalUI_tab_3()
def initalUI_tab_1(self):
#################### Create Plot cavas widget ###################################################
# Create first tab
self.tab1_layout = QVBoxLayout(
self) # create a Layout, which will be setted for tab_1
self.tab1_layout_Up = QHBoxLayout(self)
# left side of Table 1
self.tab1_layout_L = QVBoxLayout(self)
self.createGridLayout("Para setting panel")
# show the info of setup
self.infotextshow = QPlainTextEdit(self)
self.infotextshow.insertPlainText("...")
infobox = QVBoxLayout()
infobox.addWidget(self.infotextshow)
self.infotextGroupBox = QGroupBox("The experiment setup parameters: ")
self.infotextGroupBox.setLayout(infobox)
self.tab1_layout_L.addWidget(self.horizontalGroupBox)
self.tab1_layout_L.addWidget(self.infotextGroupBox)
## right side of Table 1
self.tab1_layout_R = QVBoxLayout(self)
self.plotUp = plt.figure("Background")
axes_Up = self.plotUp.add_subplot(111)
self.add_plotfigure(self.plotUp, self.tab1_layout_R)
#self.axes_Up.plot([0, 1, 2, 3], [1, 2, 0, 4], 'r')
self.buttonPlotUp = QPushButton(
'Start Plot') # Just some button connected to `plot` method
self.buttonPlotUp.clicked.connect(self.SingleImagefun(axes_Up))
self.tab1_layout_R.addWidget(self.buttonPlotUp)
self.buttonStopPlotUp = QPushButton(
'Stop Plot') # Just some button connected to `plot` method
self.buttonStopPlotUp.clicked.connect(self.StopPlotUp)
self.tab1_layout_R.addWidget(self.buttonStopPlotUp)
self.tab1_layout_Up.addLayout(self.tab1_layout_L)
# self.tab1_layout.addStretch()
self.tab1_layout_Up.addLayout(self.tab1_layout_R)
self.tab1_layout.addLayout(self.tab1_layout_Up)
self.tab1.setLayout(
self.tab1_layout) # set tab1.layout to be the layout of tab_1
def initalUI_tab_2(self):
#################### Create Plot cavas widget ###################################################
# Create first tab
self.tab2_layout = QHBoxLayout(
self) # create a Layout, which will be setted for tab_2
# left side of Table 2
self.tab2_layout_L = QHBoxLayout(self)
self.SaveFrameLayout("SaveFrame panel")
self.tab2_layout_L.addWidget(self.SaveFrameGroupBox)
# right side of Table 2
self.tab2_layout_R = QHBoxLayout(self)
# show the databox of experiment
self.datatextshow = QPlainTextEdit(self)
self.datatextshow.insertPlainText("...")
databox = QVBoxLayout()
databox.addWidget(self.datatextshow)
self.datatextGroupBox = QGroupBox("The experiment data info: ")
self.datatextGroupBox.setLayout(databox)
# add the note for experiment
self.notetextshow = QPlainTextEdit(self)
self.notetextshow.insertPlainText("")
notebox = QVBoxLayout()
notebox.addWidget(self.notetextshow)
self.notetextGroupBox = QGroupBox("The experiment notes: ")
self.notetextGroupBox.setLayout(notebox)
self.tab2_layout_R.addWidget(self.datatextGroupBox)
self.tab2_layout_R.addWidget(self.notetextGroupBox)
self.tab2_layout.addLayout(self.tab2_layout_L)
self.tab2_layout.addLayout(self.tab2_layout_R)
self.tab2.setLayout(
self.tab2_layout) # set tab1.layout to be the layout of tab_1
def initalUI_tab_3(self):
#################### Create Plot cavas widget ###################################################
# Create first tab
self.tab3_layout = QHBoxLayout(
self) # create a Layout, which will be setted for tab_1
###################### left side of Table 3
self.tab3_layout_L = QVBoxLayout(self)
self.buttonopenfile = QPushButton(
'Open File') # Just some button connected to `plot` method
self.buttonopenfile.clicked.connect(self.OpenFile)
self.buttonPostProcess = QPushButton(
'Post Process Data') # Just some button connected to `plot` method
self.buttonPostProcess.clicked.connect(self.PostProcess)
self.buttonCoherent = QPushButton(
'Coherent Plot') # Just some button connected to `plot` method
self.buttonCoherent.clicked.connect(self.CoherentPlot)
self.tab3_layout_L.addWidget(self.buttonopenfile)
self.tab3_layout_L.addWidget(self.buttonCoherent)
self.tab3_layout_L.addWidget(self.buttonPostProcess)
######################## right side of Table 3
self.tab3_layout_R = QVBoxLayout(self)
###### right top
self.tab3_layout_R_top = QHBoxLayout(self)
self.tab3_layout_R_top_L = QVBoxLayout(self)
self.tab3_layout_R_top_R = QVBoxLayout(self)
self.plotFirstOrd = plt.figure("First Order Imaging")
#axesTopL = self.plotFirstOrd.add_subplot(111)
self.add_plotfigure(self.plotFirstOrd, self.tab3_layout_R_top_L)
self.buttonplotFirstOrd = QPushButton(
'First Order Imaging Plot'
) # The button connected to `plot` method
self.buttonplotFirstOrd.clicked.connect(
self.PlotFirstOrdrfun(self.plotFirstOrd))
self.tab3_layout_R_top_L.addWidget(self.buttonplotFirstOrd)
self.plotHighOrd = plt.figure("High Order Imaging")
#axesTopR = self.plotHighOrd.add_subplot(111)
self.add_plotfigure(self.plotHighOrd, self.tab3_layout_R_top_R)
self.buttonplotHighOrd = QPushButton(
'High Order Imaging Plot') # The button connected to `plot` method
self.buttonplotHighOrd.clicked.connect(
self.PlotSecondOrdrfun(self.plotHighOrd))
self.NOrder_spinbox = QSpinBox()
self.NOrder_spinbox.setRange(2, 10)
self.NOrder_spinbox.setValue(2)
gridLayout_TopR = QGridLayout()
gridLayout_TopR.addWidget(self.NOrder_spinbox, 0, 0)
gridLayout_TopR.addWidget(self.buttonplotHighOrd, 0, 1)
self.tab3_layout_R_top_R.addLayout(gridLayout_TopR)
self.tab3_layout_R_top.addLayout(self.tab3_layout_R_top_L)
self.tab3_layout_R_top.addLayout(self.tab3_layout_R_top_R)
####### right bottom
self.tab3_layout_R_bot = QVBoxLayout(self)
self.plotDistri = plt.figure("Distribution Plot")
#axesBot = self.plotDistri.add_subplot(111)
self.add_plotfigure(self.plotDistri, self.tab3_layout_R_bot)
self.buttonplotDistri = QPushButton(
'Distribution Plot') # Just some button connected to `plot` method
self.buttonplotDistri.clicked.connect(
self.PlotDistrifun(self.plotDistri))
self.xpixel_spinbox = QSpinBox()
self.xpixel_spinbox.setRange(0, 1280)
self.xpixel_spinbox.setValue(1)
self.ypixel_spinbox = QSpinBox()
self.ypixel_spinbox.setRange(0, 1024)
self.ypixel_spinbox.setValue(1)
gridLayout_Bot = QGridLayout()
gridLayout_Bot.addWidget(QLabel('X pixel'), 0, 0)
gridLayout_Bot.addWidget(self.xpixel_spinbox, 0, 1)
gridLayout_Bot.addWidget(QLabel('y pixel'), 0, 2)
gridLayout_Bot.addWidget(self.ypixel_spinbox, 0, 3)
gridLayout_Bot.addWidget(self.buttonplotDistri, 0, 4)
self.tab3_layout_R_bot.addLayout(gridLayout_Bot)
self.tab3_layout_R.addLayout(self.tab3_layout_R_top)
self.tab3_layout_R.addLayout(self.tab3_layout_R_bot)
self.tab3_layout.addLayout(self.tab3_layout_L)
# self.tab3_layout.addStretch()
self.tab3_layout.addLayout(self.tab3_layout_R)
self.tab3.setLayout(
self.tab3_layout) # set tab1.layout to be the layout of tab_1
def add_plotfigure(self, figureName, plot_layout):
# self.figureName = plt.figure() # a figure instance to plot on
# if put "plt.ion" on the head, which will make two more figures idependently.
# this is the Canvas Widget that displays the `figure`, it takes the `figure` instance as a parameter to __init__
canvas_figureName = FigureCanvas(figureName)
toolbar_figureName = NavigationToolbar(
canvas_figureName, self
) # this is the Navigation widget, it takes the Canvas widget and a parent
plot_layout.addWidget(
toolbar_figureName
) # this also needed to show the Navigation of plot
plot_layout.addWidget(
canvas_figureName) # add Canvas Widget(plot widget) onto tab_2
def createGridLayout(self, layout_name):
self.horizontalGroupBox = QGroupBox(layout_name)
layout = QGridLayout()
# layout.setColumnStretch(1, 4)
# layout.setColumnStretch(2, 4)
self.ConnectCamera_bt = QPushButton('Connect Camera', self)
self.ConnectCamera_bt.clicked.connect(self.ConnectCamera)
self.cmos_para_bt = QPushButton('Set Camera', self)
self.cmos_para_bt.clicked.connect(self.SetCamera)
self.expose_spinbox = QSpinBox()
self.expose_spinbox.setRange(1, 66000)
self.expose_spinbox.setValue(1)
self.xpixs_spinbox = QSpinBox()
self.xpixs_spinbox.setRange(1, 1280)
self.xpixs_spinbox.setValue(1280)
self.xoffset_spinbox = QSpinBox()
self.xoffset_spinbox.setRange(0, 1280)
self.xoffset_spinbox.setValue(0)
self.ypixs_spinbox = QSpinBox()
self.ypixs_spinbox.setRange(1, 1024)
self.ypixs_spinbox.setValue(1024)
self.yoffset_spinbox = QSpinBox()
self.yoffset_spinbox.setRange(0, 1024)
self.yoffset_spinbox.setValue(0)
layout.addWidget(QLabel('Expose time(us)'), 0, 0)
layout.addWidget(self.expose_spinbox, 0, 1)
layout.addWidget(QLabel('X pixels'), 1, 0)
layout.addWidget(self.xpixs_spinbox, 1, 1)
layout.addWidget(QLabel('X offset'), 1, 2)
layout.addWidget(self.xoffset_spinbox, 1, 3)
layout.addWidget(QLabel('Y pixels'), 2, 0)
layout.addWidget(self.ypixs_spinbox, 2, 1)
layout.addWidget(QLabel('Y offset'), 2, 2)
layout.addWidget(self.yoffset_spinbox, 2, 3)
layout.addWidget(self.ConnectCamera_bt, 5, 0)
layout.addWidget(self.cmos_para_bt, 5, 1)
self.horizontalGroupBox.setLayout(layout)
def SaveFrameLayout(self, layout_name):
self.SaveFrameGroupBox = QGroupBox(layout_name)
layout = QGridLayout()
self.cmos_run_bt = QPushButton('Run to Get Multi-Image Data', self)
self.cmos_run_bt.clicked.connect(self.MultiImagefun)
self.cmos_save_bt = QPushButton('Save Data', self)
self.cmos_save_bt.clicked.connect(self.Savedata)
self.FrameNumber_spinbox = QSpinBox()
self.FrameNumber_spinbox.setRange(1, 100000)
self.FrameNumber_spinbox.setValue(100)
self.SegmentNumber_spinbox = QSpinBox()
self.SegmentNumber_spinbox.setRange(1, 1000)
self.SegmentNumber_spinbox.setValue(50)
self.TotleTime_spinbox = QSpinBox()
self.TotleTime_spinbox.setRange(0, 100)
self.TotleTime_spinbox.setValue(0)
layout.addWidget(QLabel('Frame Number'), 0, 0)
layout.addWidget(self.FrameNumber_spinbox, 0, 1)
layout.addWidget(QLabel('Segment Number'), 1, 0)
layout.addWidget(self.SegmentNumber_spinbox, 1, 1)
layout.addWidget(QLabel('Totle Time'), 2, 0)
layout.addWidget(self.TotleTime_spinbox, 2, 1)
layout.addWidget(self.cmos_run_bt, 3, 0)
layout.addWidget(self.cmos_save_bt, 4, 0)
self.SaveFrameGroupBox.setLayout(layout)
def ConnectCamera(self):
"""
connect the camera
:return:
"""
#try:
self.Cam = ThorlabsCamer()
self.Cam.ConnectCamera()
self.infotextshow.appendPlainText("Camera connected!")
def SetCamera(self):
"""
set the camera parameters
:return:
"""
xshift = self.xoffset_spinbox.value()
width = self.xpixs_spinbox.value()
yshift = self.yoffset_spinbox.value()
high = self.ypixs_spinbox.value()
exposeTime = self.expose_spinbox.value()
#self.expose_spinbox.value()
print("xshift, yshift, width , high, exposeTime", xshift, yshift,
width, high, exposeTime)
self.infotextshow.appendPlainText("xshift, yshift: (" + str(xshift) +
"," + str(yshift) + ")")
self.infotextshow.appendPlainText("width, high: (" + str(width) + "," +
str(high) + ")")
self.infotextshow.appendPlainText("exposeTime: " + str(exposeTime) +
" us")
self.Cam.SetCamera(yshift=yshift,
xshift=xshift,
high=high,
width=width,
exposeTime=exposeTime)
def SingleImagefun(self, ax):
"""
1) get each frame and plot it
2) auto-update the plot
:param ax: plt.addsubplot(111)
:return: auto-update the plot
"""
def inner_SingleImage_fun():
def update_data():
#print("AutoFresh the figure !")
#print("begin to get SingleImageData data ... ")
singledata = self.Cam.SingleImageData(self.infotextshow)
#print("The image data shape is: ", np.shape(singledata))
cax.set_data(singledata)
#We need to draw *and* flush
self.plotUp.canvas.draw()
self.plotUp.canvas.flush_events()
self.timer = QtCore.QTimer(self)
#print("begin to plot !")
self.infotextshow.appendPlainText("Plot start!")
try:
self.cbar.remove()
#print("clear self.cbar !")
except:
pass
#print("fail to clear self.cbar !")
ax.cla()
singledata = self.Cam.SingleImageData(self.infotextshow)
cax = ax.imshow(singledata, interpolation='nearest')
ax.set_title('CMOS Camera')
self.cbar = self.plotUp.colorbar(cax, orientation='vertical')
#cbar.ax.set_xticklabels(['Low', 'Medium', 'High']) # horizontal colorbar
self.timer.timeout.connect(update_data)
self.timer.start(500)
return inner_SingleImage_fun
def StopPlotUp(self):
"""
Stop the auto-plot
"""
self.timer.stop()
self.infotextshow.appendPlainText("Plot stop!")
def MultiImagefun(self):
"""
get the multi-fram image data, which are saved into *.npy files
"""
frameNumber = self.FrameNumber_spinbox.value()
segmentNumber = self.SegmentNumber_spinbox.value()
print("frameNumber, segmentNumber is: ", frameNumber, segmentNumber)
print("begin to get MultiImageData data ... ")
self.Cam.MultiImageData(infoObj=self.datatextshow,
frame_number_expected=frameNumber,
segment_frame=segmentNumber)
def Savedata(self):
"""
1, transfer *.npy to *.nc data, which includes the data dimension info and experiment notes
2, delete *.npy file
2, save the *.nc file
"""
frameNumber = self.FrameNumber_spinbox.value()
segmentNumber = self.SegmentNumber_spinbox.value()
width = self.xpixs_spinbox.value()
high = self.ypixs_spinbox.value()
print("frameNumber, segmentNumber, width, high is: ", frameNumber,
segmentNumber, width, high)
app = ReadData(noteObj=self.notetextshow,
frameNumber=frameNumber,
segmentFrame=segmentNumber,
width=width,
high=high)
self.multiFrameData = app.ImageData()
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
# it just provides the name of file that you want to write into
fileName, _ = QFileDialog.getSaveFileName(
self,
"QFileDialog.getSaveFileName()",
"",
"All Files (*);;NC Files (*.nc)",
options=options)
if fileName:
print(fileName)
self.multiFrameData.to_netcdf(fileName + '.nc')
self.datatextshow.appendPlainText("the data has saved as .nc file! ")
def OpenFile(self):
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
self.fileName, _ = QFileDialog.getOpenFileName(
self,
"QFileDialog.getOpenFileName()",
"",
"All Files (*);;Python Files (*.nc)",
options=options)
if self.fileName:
print(self.fileName)
self.APP_dataprocess = DataProcess(self.fileName)
def PlotFirstOrdrfun(self, fig):
def inner_PlotFirstOrdrfun():
# As self.firstOrdImaging will be used later, so we have to save as self.
self.firstOrdImaging = self.APP_dataprocess.Average_Fluctuation()
#try:
# self.firstOrdImaging = self.APP_dataprocess.Average_Fluctuation()
# print(np.shape(self.firstOrdImaging))
#except Exception:
# print("self.firstOrdImaging error!")
# plot the Averge imaging
ax = fig.add_subplot(111)
try:
self.cbar_FOrder.remove()
ax.cla()
#print("clear self.cbar !")
except:
pass
#print("fail to clear self.cbar !")
im = ax.imshow(self.firstOrdImaging)
# create an axes on the right side of ax. The width of cax will be 5%
# of ax and the padding between cax and ax will be fixed at 0.05 inch.
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
self.cbar_FOrder = plt.colorbar(im, cax=cax)
#plt.colorbar(im, cax=cax, ticks=[0, 5, 10])
ax.set_title('1th Order')
plt.savefig('1th Order Imaing.eps', format='eps', dpi=100)
plt.close()
return inner_PlotFirstOrdrfun
def PlotSecondOrdrfun(self, fig):
def inner_PlotSecondOrdrfun():
NorderValue = self.NOrder_spinbox.value()
#secondOrdImaging = self.APP_dataprocess.SecondOrder()
print("The ", self.NOrder_spinbox.value(), "th Order Imaging")
secondOrdImaging = self.APP_dataprocess.NOrder(NorderValue)
print(np.shape(secondOrdImaging))
#try:
# self.firstOrdImaging = self.APP_dataprocess.Average_Fluctuation()
# print(np.shape(self.firstOrdImaging))
#except Exception:
# print("self.firstOrdImaging error!")
# plot the Averge imaging
ax = fig.add_subplot(111)
try:
self.cbar_NOrder.remove()
ax.cla()
#print("clear self.cbar !")
except:
pass
#print("fail to clear self.cbar !")
im = ax.imshow(secondOrdImaging)
# create an axes on the right side of ax. The width of cax will be 5%
# of ax and the padding between cax and ax will be fixed at 0.05 inch.
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
self.cbar_NOrder = plt.colorbar(im, cax=cax)
#plt.colorbar(im, cax=cax, ticks=[0, 5, 10])
ax.set_title('{}th Order'.format(NorderValue))
plt.savefig('2th Order Imaing.eps', format='eps', dpi=100)
plt.close()
return inner_PlotSecondOrdrfun
def PlotDistrifun(self, fig):
def BoseEinstein(Nbar, n=51):
nList = np.linspace(0, n, n + 1, dtype=int)
result = 1 / (1 + Nbar) * (Nbar / (1 + Nbar))**nList
return result
def G2(FirstArray1d, SecondArray1d):
Frames = len(FirstArray1d)
#print("Frames is: " , Frames)
FirstAverage = np.sum(FirstArray1d) / Frames
SecondAverage = np.sum(SecondArray1d) / Frames
result = np.sum(FirstArray1d * SecondArray1d) / Frames / (
FirstAverage * SecondAverage)
return result
def inner_PlotDistrifun():
"""
Plot the photon number distribution ...
"""
font = {
'family': 'serif',
'color': 'darkred',
'weight': 'normal',
'size': 16
}
Nmax = 100
bins = np.linspace(0, Nmax, Nmax + 1)
nList = np.linspace(0, Nmax, Nmax + 1, dtype=int)
y_location = self.ypixel_spinbox.value()
x_location = self.xpixel_spinbox.value()
# get pixel intensity data
Array1 = self.APP_dataprocess.PixelData(y_location, x_location)
Array2 = Array1
g2 = G2(Array1, Array2)
print("g2 is:", g2)
arr = []
rv = poisson(self.firstOrdImaging[y_location, x_location])
for num in range(0, 40):
arr.append(rv.pmf(num))
ax = fig.add_subplot(111)
try:
ax.cla()
#print("clear self.cbar !")
except:
pass
#print("fail to clear self.cbar !")
ax.hist(Array1, bins, normed=True, label="Data distribution")
ax.plot(nList,
BoseEinstein(self.firstOrdImaging[y_location, x_location],
Nmax),
label="BoseEinstein distribution")
ax.plot(arr, linewidth=2.0, label="Possion distribution")
ax.set_title("Pixel Position({},{}); <$I$>:{}".format(
x_location, y_location, self.firstOrdImaging[y_location,
x_location]),
fontdict=font)
ax.text(22, .08, r"g2:{}".format(g2), fontdict=font)
ax.legend()
fig.savefig('PixelPosition({},{})PhotDist.eps'.format(
x_location, y_location),
format='eps',
dpi=300)
plt.close()
return inner_PlotDistrifun
def CoherentPlot(self):
fig = plt.figure()
ax = fig.add_subplot(111)
xCorr, yCorr = self.APP_dataprocess.SpatialCorrelation(
[self.ypixel_spinbox.value(),
self.xpixel_spinbox.value()])
ax.plot(xCorr)
ax.set_title("G2 @({}{})".format(self.ypixel_spinbox.value(),
self.xpixel_spinbox.value()))
fig.savefig("G2 @({}{}).eps".format(self.ypixel_spinbox.value(),
self.xpixel_spinbox.value()),
format="eps",
dpi=100)
plt.show()
def PostProcess(self):
app = DataProcess() # which read the data only
app.PlotFrames()
# calculated the first order which will be used at photon distribution, high order imaging, so do it firstly.
# and plot it
app.PlotFirstOdr()
app.PlotPhotonDistribution()
app.PlotSecondOdr()
from DataProcess import DataProcess
import os
import numpy as np
data = DataProcess()
try:
data.cicids_process_data_binary()
except Exception as e:
print(e)
try:
data.cicids_process_data_multiclass()
except Exception as e:
print(e)
import winsound
winsound.PlaySound("SystemHand", winsound.SND_ALIAS)
#coding=utf-8
from DataProcess import DataProcess
import Evaluate
import Geo_LFM
import datetime
import os
if __name__=="__main__":
rootdir = '../data/'
list = os.listdir(rootdir) # 列出文件夹下所有的目录与文件
for i in range(0, len(list)):
path = os.path.join(rootdir, list[i])
if os.path.exists(path):
dp = DataProcess(path)
TrainData = dp.ReadShanghai()
TestData = dp.ReadShanghaiTest()
gridFeatrue = dp.ReadShanghaiGrid()
#10,15,20,25,30,35,40,45,50,60,70,80,100,120,150,200,250,300,
for n in [20]:
print("开始训练。。。")
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
p, q = Geo_LFM.train(dp,TrainData,gridFeatrue,n)
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
print("训练完毕，持久化结果")
# dp.persistant(p, q)
#p,q = dp.load()
print("验证结果")
Evaluate.mse(dp,TestData, p, q)
