def connectIP(self): self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: self.s.bind((self.exo1.pcIP, self.exo1.ipPort)) except socket.error: with self.recDataLock: self.recData.put('IP Bind Failed\n') self.s.listen(5) conIP, addr = self.s.accept() with self.recDataLock: self.recData.put('TCP/IP Bind success\n') while self.tcpipConnect.is_set(): data = conIP.recv(1024) print(data) cmdList = [] dp.comSep(data.decode('utf-8'), cmdList) for cmd in cmdList: dataList = [] dp.dataSep(cmd, dataList) with self.plotLock: self.plotData.get() self.plotData.put(dataList) time.sleep(0.05)
def main(): global G_igraph, G_nx, nodes_vector, aca_graph G_igraph, G_nx, nodes_vector = dp.generate_data() aca_graph = dp.separate_data_by_academy(nodes_vector, G_igraph)
def main(): if not os.path.exists("train_set.npy"): # 提取数据集中的样本,并划分训练集和测试集 trainData, testData = DataProcess.getDatasetFromFile(Path) #特征提取与词典生成 vocabulary = Vocabulary.Vocabulary(numOfBag) vocabulary.generateBoW("feature_set.npy", random_state) # vocabulary.getBow("Bow.npy") # 图片表示,使用SPM方法生成图片特征向量 trainSet, trainlabels = vocabulary.Imginfo2SVMdata(trainData) testSet, testlabels = vocabulary.Imginfo2SVMdata(testData) np.save("train_set.npy", trainSet) np.save("train_label.npy", trainlabels) np.save("test_set.npy", testSet) np.save("test_label.npy", testlabels) else: trainSet = np.load("train_set.npy") trainlabels = np.load("train_label.npy") testSet = np.load("test_set.npy") testlabels = np.load("test_label.npy") # 执行分类,使用支持向量机模型对测试集图片的类别进行预测 svm = ClassifierKernel.trainSVM(trainSet, trainlabels) result = ClassifierKernel.predict(trainSet, svm) testResult = ClassifierKernel.predict(testSet, svm) # 评估预测结果,并生成分类报告和输出混淆矩阵 trainReport, _ = ClassifierKernel.evaluate(result, trainlabels) testReport, cm = ClassifierKernel.evaluate(testResult, testlabels) DataProcess.plotCM(targetName, cm, "confusion_matrix.png") print("训练集混淆矩阵:") print(trainReport) print("测试集混淆矩阵:") print(testReport)
def main(): global G_igraph, G_nx, nodes_vector, aca_graph G_igraph, G_nx, nodes_vector = dp.generate_data() # TODO 读取上传文件的图信息,获得图信息 (G_igraph, G_nx) 和 nodes_vector aca_graph = dp.separate_data_by_academy(nodes_vector, G_igraph)
def tf_model_test(path: str): model = keras.models.load_model(path) model.summary() for i in range(10): DataProcess.divideTrainAndVerify('comb_rate.csv', 'train.csv', 'verify.csv', 0.75) (verify_data, verify_labels) = DataProcess.extractFlagForRate("verify.csv") model.evaluate(verify_data, verify_labels, verbose=2)
def run_epoch(sess, model, data, batch_size, writer, saver): global global_step flag = False test_set_size = int(len(data) // 10) train_data = data[:len(data) - test_set_size] test_data = data[len(data) - test_set_size:] train_x = train_data[:, :-1] train_label = DataProcess.to_one_hot(train_data[:, -1], 4) test_x = test_data[:, :-1] test_label = DataProcess.to_one_hot(test_data[:, -1], 4) #train for i in xrange((len(data) - test_set_size) // batch_size): global_step += 1 feed_dic = {} feed_dic[model.data_in.name] = train_x[i * batch_size:(i + 1) * batch_size] feed_dic[model.true_label.name] = train_label[i * batch_size:(i + 1) * batch_size] _, accu, los, merged_summ, is_nan = sess.run([ model.train_op, model.accurate, model.loss, model.merged_summary, model.loss_is_nan ], feed_dic) if is_nan: print(train_x[i * batch_size:(i + 1) * batch_size]) print(train_label[i * batch_size:(i + 1) * batch_size]) exit() if global_step % 1000 == 0: flag = True print("training:"), print("los:%.7f, accu:%.7f" % (los, accu)) writer.add_summary(merged_summ, global_step // 1000) saver.save(sess, "checkpoint_rnn/modelRNN.ckpt") #test test_losses = [] test_accuracy = [] #***************************************** summ = np.zeros([4, 4], dtype=np.int16) for j in range(test_set_size // batch_size): feed_dic = {} feed_dic[model.data_in.name] = test_x[j * batch_size:(j + 1) * batch_size] feed_dic[model.true_label.name] = test_label[j * batch_size:(j + 1) * batch_size] actual, predict = sess.run([model.true_in, model.predict], feed_dic) for x, y in zip(actual, predict): summ[x, y] += 1 print(summ) #***************************************** if flag: print("test:"), print("los:%.2f, accu:%.2f" % (np.mean(test_losses), np.mean(test_accuracy))) flag = False
def run_epoch(sess, model, data, batch_size, writer, saver): global global_step flag = False test_set_size = int(len(data) / 10) train_data = data[:len(data) - test_set_size] test_data = data[len(data) - test_set_size:] train_x = train_data[:, :-1] train_label = DataProcess.to_one_hot(train_data[:, -1], 4) test_x = test_data[:, :-1] test_label = DataProcess.to_one_hot(test_data[:, -1], 4) # train for i in range(int((len(data) - test_set_size) / batch_size)): global_step += 1 feed_dic = {} feed_dic[model.data_in.name] = train_x[i * batch_size:(i + 1) * batch_size] feed_dic[model.true_label.name] = train_label[i * batch_size:(i + 1) * batch_size] _, accu, los, merged_summ, is_nan, fc1_, grads_ = sess.run([ model.train_op, model.accurate, model.loss, model.merged_summary, model.loss_is_nan, model.fc1, model.grads ], feed_dic) if is_nan: print(train_x[i * batch_size:(i + 1) * batch_size]) print(train_label[i * batch_size:(i + 1) * batch_size]) exit() if global_step % 10000 == 0: flag = True # print "grads", grads_[0:2] # print "fc1:",fc1_[0:4,:] # print "output_",output_[0:4,:] # print "relu_output",relu_output_[0:4,:] print("training: "), print("los:%.2f, accu:%.2f" % (los, accu)) writer.add_summary(merged_summ, global_step // 10000) saver.save(sess, "checkpoint_cnn_v1/modelCNN") # test test_losses = [] test_accuracy = [] for j in range(int(test_set_size / batch_size)): feed_dic = {} feed_dic[model.data_in.name] = test_x[j * batch_size:(j + 1) * batch_size] feed_dic[model.true_label.name] = test_label[j * batch_size:(j + 1) * batch_size] accu, los = sess.run([model.accurate, model.loss], feed_dic) test_accuracy.append(accu) test_losses.append(los) if flag: print("test: "), print("los:%.2f, accu:%.2f" % (np.mean(test_losses), np.mean(test_accuracy))) flag = False
def learning(url): DataProcess.get_csv()#首次运行时获取数据 yq=ML2.SVM() yq.testurl=url yq.Train() yq_res= yq.predict_single()[0] ML.textClassifier()#首次运行时训练分类器 t=ML2.textPredictor() t.urlpre=testurl t.crawl() t.predict() text_res=t.get_result() return res={'porn_site':text_res[0],'gaming_site':text_res[1],'phishing_site':yq_res}
def closest_group(req_id): ''' 处理学院聚集度 ---------- parameter req_id 请求编号 return 渲染的模板 ''' global G_igraph, G_nx, nodes_vector, aca_graph, data paramas = request.args # GET 请求参数 if req_id == 0: # 渲染 coefficient 页面 return render_template('new_coefficience.html') elif req_id == 1: # 选择两个学院 # 参数:学院 ID aca1_id = int(paramas['acaId1']) - 1 aca2_id = int(paramas['acaId2']) - 1 # 分别计算两个院的 Cluster Coefficient aca1_graph_json = dp.graph_json( nx.DiGraph(aca_graph[aca1_id].get_edgelist()), nodes_vector) aca1_coefficient = c.cluster_coefficient_calculation( nx.DiGraph( aca_graph[aca1_id]. get_edgelist()).to_undirected(reciprocal=False)) aca2_graph_json = dp.graph_json( nx.DiGraph(aca_graph[aca2_id].get_edgelist()), nodes_vector) aca2_coefficient = c.cluster_coefficient_calculation( nx.DiGraph( aca_graph[aca2_id]. get_edgelist()).to_undirected(reciprocal=False)) # 根据请求返回需要的对应格式的数据 data = {'AcademyGraph1': aca1_graph_json, 'AcademyGraph2': aca2_graph_json, 'AcademyCoefficient1': aca1_coefficient, 'AcademyCoefficient2': aca2_coefficient} return make_response(json.dumps(data))
def handleDataQueue(self): if self.runFlag: data_timer = Timer(0.1,self.handleDataQueue) # print "Enter HandleDataQueue____________________" itemdata = None try: itemdata = self.dataQueue.get_nowait() except Exception as e: print "handleDataQueue Error: ", e if itemdata: # print itemdata DataProcess.dataEntrance(itemdata,self.mysqlConnect_data) # self.addTask(time.time()+2, PumpCheck("1#","2#","3#")) data_timer.start()
def Initialization(self, init): contents = init.split("|") version = contents[1] BestHeight = contents[2] if contents[4]: #客户端参与挖矿 DataProcess.addIp(contents[4]) db = Conlmdb() if version == Utils().blockversion: #验证版本号是否一致 if BestHeight >= str(db.dbsize()): return "Lastest" else: return str(db.dbsize()) else: return None
def setDfMonthData(self, startDate=None, endDate=None): dpMonthObj = dp.DataProcess(self.code, self.name, period=dp.MONTH) dpMonthObj.readData() self.dpMonthObj = dpMonthObj if (os.path.exists(dpMonthObj.dataGenCsvFile)): self.dfMonthGenData = pd.read_csv(dpMonthObj.dataGenCsvFile, encoding=sd.UTF_8, dtype={'code': str}) self.dfMonthFilterData = self.filterData(self.dfMonthGenData, startDate, endDate) else: print( '[Function:%s line:%s stock:%s] Error: File %s is not exist' % (self.setDfWeekData.__name__, sys._getframe().f_lineno, self.code, dpMonthObj.dataGenCsvFile)) sys.exit() if (os.path.exists(dpMonthObj.signalReportFile)): self.dfMonthSignalData = pd.read_csv(dpMonthObj.signalReportFile, encoding=sd.UTF_8, dtype={'code': str}) else: print( '[Function:%s line:%s stock:%s] Error: File %s is not exist' % (self.setDfWeekData.__name__, sys._getframe().f_lineno, self.code, dpMonthObj.signalReportFile)) sys.exit() self.reportPath = dpMonthObj.dataPath + self.code + '_exchange_report.csv'
def getComparePlayer(): json_data = request.json # json_data = request.get_json() name = json_data["name"] team = json_data["team"] players = DataProcess.getPlayerDetails(name, team) return json.dumps(players, indent=2)
def getPositionPlayers(): json_data = request.json # json_data = request.get_json() name = json_data["name"] positionNo = json_data["positionNo"] players = DataProcess.positionSort(ws, positionNo, "Position No") return json.dumps(players, indent=2)
def app_principal(): """ This is the main method of application. It start this app and control it """ start = dp.DataProcess() t_a, h_a, h_s = start.data_in() start.define_disease_table_data(t_a, h_a, h_s)
def geolfm1(A, X, epsilon, n): (p, q, t) = Initlfm(A, n) lam1 = 0.05 alpha = 0.05 ep = 100 error = 0 T = DataProcess.Dic2Array(t) while ep > epsilon: lasterror = error error = 0 for u, i, rui in A: # update p pui = Predict.Predict(u, i, p, q) - rui #number Pu = np.array(p[u]) #get the u_th row in p Qi = np.array(q[i]) Xi = np.array(X[i]) Qp = Predict.PredictF(Pu, T) - Xi #vector QpT = Predict.PredictF(Qp, T.T) p[u] = Pu - alpha * pui * Qi - lam1 * alpha * QpT - lam1 * alpha * Pu #update q q[i] = Qi - alpha * pui * Pu - lam1 * alpha * Qi #update t QpPu = Predict.cal(Qp, Pu) T = T - alpha * lam1 * QpPu - lam1 * alpha * T error = error + np.abs(Predict.Predict(u, i, p, q) - rui) ep = np.abs(lasterror - error) # print "error"+str(error) alpha *= 0.9 return (p, q)
def student_equivalence(req_id): global G_igraph, G_nx, nodes_vector, data paramas = request.args if req_id == 0: return render_template('/new_equivalence.html') elif req_id == 1: stu_id = int(paramas['studentId']) graph_json = dp.graph_json(G_nx, nodes_vector) # 图信息 G = G_nx.to_undirected(reciprocal=False) matrix = nx.to_numpy_matrix(G) # 分别计算struct-cosine 和 regular-cosine,结果存入字典 array1 = c.select_similar(nodes_vector, matrix, stu_id, 0) array2 = c.select_similar(nodes_vector, matrix, stu_id, 1) top_ten_list = {'structral': array1, 'regular': array2} data = {'Graph': graph_json, 'Rank': top_ten_list} # 封装成 dict return make_response(json.dumps(data))
def sendBlock(): #广播区块 iplist = DataProcess.getIp() for ip in iplist: HOST = ip PORT = Utils().sport reactor.connectTCP(HOST, PORT, BTSClntFactory()) reactor.run()
def tf_predict_na(path: str, saveTo: str): model = keras.models.load_model(path) model.summary() arr = DataProcess.ripLabels("alt_na_file.csv", ['flag', 'ID']) # arr = DataProcess.ripLabels("na_rate.csv", ['flag', 'ID']) # arr = model.predict(arr) arr = model.predict_classes(arr) np.savetxt(saveTo, arr, fmt='%f', delimiter=',')
def run_epoch(sess, model, file_name, batch_size, writer, saver): global global_step flag = False data = DataProcess.data_process(file_name) test_set_size = int(len(data) // 10) train_data = data[:len(data) - test_set_size] test_data = data[len(data) - test_set_size:] train_x = train_data[:, :-1].reshape(-1, 4, 4) train_label = DataProcess.to_one_hot(train_data[:, -1], 4) test_x = test_data[:, :-1].reshape(-1, 4, 4) test_label = DataProcess.to_one_hot(test_data[:, -1], 4) #train for i in xrange((len(data) - test_set_size) // batch_size): global_step += 1 feed_dic = {} feed_dic[model.data_in.name] = train_x[i * batch_size:(i + 1) * batch_size] feed_dic[model.true_label.name] = train_label[i * batch_size:(i + 1) * batch_size] _, accu, los, merged_summ, is_nan = sess.run([ model.train_op, model.accurate, model.loss, model.merged_summary, model.loss_is_nan ], feed_dic) if is_nan: print(train_x[i * batch_size:(i + 1) * batch_size]) print(train_label[i * batch_size:(i + 1) * batch_size]) exit() if global_step % 1000 == 0: flag = True print("training:") print("los:%.7f, accu:%.7f" % (los, accu)) writer.add_summary(merged_summ, global_step) saver.save(sess, "checkpoint/model") #test feed_dic = {} feed_dic[model.data_in.name] = test_x feed_dic[model.true_label.name] = test_label accu, los = sess.run([model.accurate, model.loss], feed_dic) if flag: flag = False print("test:") print("los:%.7f, accu:%.7f" % (los, accu))
def main(): print("Start Work") print('操作系统信息:' + str(platform.architecture())) FileHelper.Init() DataProcess.Init() t = threading.Thread(target=MonitorRunGame.Monitor) t.start() UIView.Init()
def dataObtain(new_years): '''通过访问位于项目文件夹r'360doc-spider\\urlList\\'内部的相关年份的链接列表。 依次访问网页完成下载,把原始数据保存通过调用DataProcess处理 保存至r'360doc-spider\\rawFile\\xxxx\\'中的txt文件中''' global years, response, isFailed years = new_years try: # 从位于此地址的urlFolder中查找相关年份的list urlFile = open(urlFolder + "urlList" + str(years) + ".txt") # 文件命名方式‘urlList’+年份,txt格式 targetDir_text = moduleAddress[:moduleAddress.find( r'\DataObtain.py')] + '\\' + 'rawFile' + '\\{}'.format(years) # 本机地址 r'C:\Users\user\Desktop\FTC\2018个人项目\数据爬虫\360doc-spider\rawFile\{}'.format(years) targetDir_pics = moduleAddress[:moduleAddress.find( r'\DataObtain.py')] + '\\' + 'rawPics' + '\\{}'.format(years) # 本机地址 r'C:\Users\user\Desktop\FTC\2018个人项目\数据爬虫\360doc-spider\rawPics\{}'.format(years) # 如果子目录尚不存在则创建一个 if not os.path.exists(os.path.dirname(targetDir_text)): print('指定文件保存路径不存在,将创建指定路径') os.mkdir(os.path.dirname(targetDir_text)) print('成功创建路径', os.path.dirname(targetDir_text)) if not os.path.exists(os.path.dirname(targetDir_pics)): print('指定图片保存路径不存在,将创建指定路径') os.mkdir(os.path.dirname(targetDir_pics)) print('成功创建路径', os.path.dirname(targetDir_pics)) if not os.path.exists(targetDir_text): print('指定文件年份路径不存在,将创建指定路径') os.mkdir(targetDir_text) print('成功创建路径', targetDir_text) if not os.path.exists(targetDir_pics): print('指定图片年份路径不存在,将创建指定路径') os.mkdir(targetDir_pics) print('成功创建路径', targetDir_pics) except FileNotFoundError: print('无法创建路径') for urlLine in urlFile.readlines(): # 如果文件读取完毕,结束循环 new_url = urlLine # 从urlList文件中一行行读取目标网址 request_initial(new_url) isFailed = DataProcess.dataProcess(requests(), targetDir_text, targetDir_pics) if isFailed == 1: continue elif isFailed == -1: print('由于page_source获取失败,数据处理失败') break try: if isFailed == -1: print('数据获取并处理失败') return -1 elif isFailed == 1: print('数据获取并处理成功') return 1 except isFailed == 0: print('未知错误') return 0
def main(): data = DataProcess.fetchData(__path__) users = [] for dt in data: if len(dt) == 5: user = User(dt[0], dt[1], dt[2], dt[3], dt[4]) users.append(user) else: print 'Data length error: %s' % str(dt) User.calPayment(users)
def dataReceived(self, data): """ :param data:接收的区块链文本 :接收文本--->处理文本--->发送文本 发送给客户端最后的文本 """ if data: content = data.decode("utf-8") if content[-5:].strip() != "Finsh": self._data_buffer = self._data_buffer + content else: content = self._data_buffer + content[:-5] if content[0:14] == "Initialization": CurrentHeight = DealData().Initialization(content) if CurrentHeight == "Lastest": self.senData(CurrentHeight) elif CurrentHeight: self.senData("CurrentHeight" + CurrentHeight) else: self.senData("False") elif content[0:2] == "OK": recontent = DealData().getData(content) btexts = b"Text" + Jsonstr.toStr(recontent).encode("utf-8") self.senData(btexts) elif content[0:7] == "Getdata": contents = content.split(":") count = int(contents[1]) - Conlmdb().dbsize() if count == 1: if DataProcess.ConfirmB(contents[2], pids["Mine"]) == "restart": startMine() elif count >= 2: DataProcess.SyncBlock(pids["Mine"], ipaddr=self.clnt) """ 处理接收的信息 1.验证交易 2.去掉内存池交易 3.继续转发 """ else: Log.Error("Server TSS Invalid data(" + self.clnt + ")")
def main(): # using Terminal to get parameters config = Config.Config_Table(Terminal_parser()) # get standard train and test file if not os.path.exists(config.train) or not os.path.exists(config.test): DataProcess.get_standard_file(config) # get char vocab vocab, index = {}, 0 vocab, index = DataProcess.get_vocab(config.train, vocab, index) vocab, _ = DataProcess.get_vocab(config.test, vocab, index) config.n_feature = len(vocab) # get standard train and test data train_data, train_target = DataProcess.get_data(config.train, vocab, config.class_dict) test_data, test_target = DataProcess.get_data(config.test, vocab, config.class_dict) # define HMM model model = Model.HMM_Cell(config) model.fit(train_data, train_target) label = model.predict_label(test_data) result_eval = Evaluation.Result_Eval(config) result_eval.eval_model(test_target, label)
def preprocess(subjectID, redact=1000, rescale=False, scaler='minmax'): subj_filename = './PAMAP2_Dataset/Protocol/subject10' + str( subjectID) + '.dat' col_labels = DP.col_labels col_sublabels = DP.col_sublabels HR_lim = DP.HR_lim HR_rest = HR_lim[subjectID][0] HR_max = HR_lim[subjectID][1] X_std = np.empty(0) #generate dataframe from the raw data data = pd.read_csv(subj_filename, sep=' ', names=col_labels, header=None) #linear interpolate missing data data = data.interpolate(method='linear') #drop columns for orientation and acc6g data = pd.DataFrame(data, columns=col_sublabels) #convert to array data = np.array(data) #normalize heart rate data[:, 2] = DP.HR_norm(data[:, 2], HR_rest, HR_max) #Rescale: if rescale: if scaler == 'minmax': SS = MinMaxScaler(feature_range=(-1, 1), copy=True) else: SS = StandardScaler(copy=True, with_mean=True, with_std=True) data[:, 3:] = SS.fit_transform(data[:, 3:]) X_std = np.copy(data[:, 3:]) #computes timestamp indices where the activity changes, including 0 and l l = len(data) r = np.arange(l - 1) + 1 split_ind = r[data[r, 1] != data[r - 1, 1]] split_ind = np.concatenate(([0], split_ind, [l])) #chop data into chunks of continuous time blocks with the same activity, also remove activity zero chunks = [ data[split_ind[i]:split_ind[i + 1]] for i in range(len(split_ind) - 1) if data[split_ind[i], 1] != 0 ] #drop the first and last n samples. Only keep redacted samples that #are of sufficient length chunks = [x[redact:-(redact + 1)] for x in chunks if len(x) > (2 * redact)] return X_std, chunks
def tf_train(): # 自动保存 checkpoint = tf.keras.callbacks.ModelCheckpoint("checkPoint.h5", monitor='val_accuracy', verbose=1, save_best_only=True) (train_data, train_labels) = DataProcess.extractFlagForRate("train.csv") (verify_data, verify_labels) = DataProcess.extractFlagForRate("verify.csv") verify_tuple = (verify_data, verify_labels) print(train_labels) model = tf_model() model.fit(train_data, train_labels, epochs=100, validation_data=verify_tuple, callbacks=[checkpoint]) model.evaluate(verify_data, verify_labels, verbose=2) is_save = input("[INFO] Save model? [y]/n") if is_save in ['Y', 'y']: print("[INFO] Saving...") keras.models.save_model(model, "model3.h5") print("[INFO] Save Complete!")
def handleCloudMongoData(self, uri, hostPort, file_data, idCollec): client = pym.MongoClient(uri, hostPort, connectTimeoutMS=30000, socketTimeoutMS=None, socketKeepAlive=True) db = client.get_database() print" \n DB structure: ", db print" \n DB name: ", db.name print" \n Collection client: ", db.client if idCollec == 1: # dadosNumSensores2 dadosNumSensores2 = db['dadosNumSensores2'] try: dadosNumSensores2.insert(file_data) except IOError: print("\n\n Erro de insersao de dados na colecao dadosNumSensores2") self.closeCMConection() exit(0) elif idCollec == 2: # dadosVerificSensores2 dadosVerificSensores2 = db['dadosVerificSensores2'] try: dadosVerificSensores2.insert(file_data) except IOError: print("\n\n Erro de insersao de dados na colecao dadosVerificSensores2") self.closeCMConection() exit(0) elif idCollec == 3: # controle2 controle2 = db['controle2'] try: controle2.insert(file_data) except IOError: print("\n\n Erro de insersao de dados na colecao controle2") self.closeCMConection() exit(0) else: # recuperar o tempo de resposta: idCollec = 4 # docum = db.get_collection('controle').find({"tempo":{$gte:5}}) try: docum = db.get_collection('controle').find().pretty() d = dict(docum) tempoUser = d.get('tempo') # recupera o valor do campo tempo dpo = Dp.DataProcess() dpo.tempoAtual = dpo.converterTempo(tempoUser) except IOError: print("\n\n Erro na selecao de documentos da colecao controle") self.closeCMConection() exit(0)
def run_epoch(sess, model, data, batch_size,writer=None,saver=None): global global_step flag = False test_set_size = len(data) # train_data = data[:len(data) - test_set_size] # train_x = train_data[:, :-1] # train_label = DataProcess.to_one_hot(train_data[:, -1], 4) test_data = data[len(data) - test_set_size:] test_x = test_data[:, :-1] test_label = DataProcess.to_one_hot(test_data[:, -1], 4) # train # for i in range(int((len(data) - test_set_size) / batch_size)): # global_step += 1 # feed_dic = {} # feed_dic[model.data_in.name] = train_x[i * batch_size:(i + 1) * batch_size] # feed_dic[model.true_label.name] = train_label[i * batch_size:(i + 1) * batch_size] # _, accu, los, merged_summ, is_nan, fc1_, grads_= sess.run([model.train_op, model.accurate, model.loss, # model.merged_summary, model.loss_is_nan, # model.fc1,model.grads], feed_dic) # if is_nan: # print (train_x[i * batch_size:(i + 1) * batch_size]) # print (train_label[i * batch_size:(i + 1) * batch_size]) # exit() # if global_step % 10000 == 0: # flag = True # # print "grads", grads_[0:2] # # print "fc1:",fc1_[0:4,:] # # print "output_",output_[0:4,:] # # print "relu_output",relu_output_[0:4,:] # print ("training: "), # print ("los:%.2f, accu:%.2f" % (los, accu)) # writer.add_summary(merged_summ, global_step // 10000) # saver.save(sess, "checkpoint_cnn_v2/modelCNN") # test summ = np.zeros([4,4],dtype=np.int16) for j in range(int(test_set_size / batch_size)): feed_dic = {} feed_dic[model.data_in.name] = test_x[j * batch_size:(j + 1) * batch_size] feed_dic[model.true_label.name] = test_label[j * batch_size:(j + 1) * batch_size] actual, pred = sess.run([model.true_in, model.predict], feed_dic) for x,y in zip(actual,pred): summ[x,y] += 1 print(summ)
def fileCreater(dirPath): dirPath = getDirPath() i = None initialIndex = None outputFolder = r"\__outputs__" outputdirPath = dirPath + outputFolder if not os.path.isdir(outputdirPath): os.makedirs(outputdirPath) else: shutil.rmtree(outputdirPath) os.makedirs(outputdirPath) file_list = sorted(glob.glob(dirPath + '/*.txt')) # print("file_list",file_list) i = 0 for filenamePath in file_list: initialIndex = ("00" + str(i + 1))[-2:] + "_" fileNameText = filenamePath.split("\\")[-1:][0] if (fileNameText == "env.txt"): pass else: arry = None with open(filenamePath, 'r') as fr: dp.setFileRawData(fr) # dp.startUnitChangeForCalcProgram() # dp.startCalcIonSatCurrent() dp.calcFloatVolt() # dp.showGraph() with open(outputdirPath + "\\" + initialIndex + fileNameText, "w") as fw: dp.setFileName(initialIndex + fileNameText) arry = dp.getOutputData() fw.write(arry) i += 1 # dp.showAllGraph() if (i == 0): root = tk.Tk() root.withdraw() #小さなウィンドウを表示させない shutil.rmtree(outputdirPath) messagebox.showerror("選択エラー", "変換可能なテキストファイルがありません。") else: #完了 # openExplorer(outputdirPath) pass
def main(): # Input size of each steps input_size = args.num_joint * args.coord_dim # Loading data DataLoader = DataProcess.DataProcess( path=args.data_path, batch_size=args.batch_size, num_joint=args.num_joint, coord_dim=args.coord_dim, # input_size=input_size, decoder_steps=args.decoder_steps, model=args.model) # TODO # Build graph & Train/Test solver = Solver() if args.test: solver.test(args=args, DataLoader=DataLoader) else: # Biuld net if args.model == "cnn": net = Model( name="resnet", layer_n=3, in_shape=[args.in_frames, args.num_joint, args.coord_dim], out_shape=[args.out_band], num_steps=args.epochs * DataLoader.Get_num_batch( DataLoader.train_set['source'], args.in_frames), lr=args.learning_rate) elif args.model == "rnn": # Build Sequence to Sequence Model net = Model(seq_length=args.in_frames, out_length=args.out_band, rnn_size=args.rnn_size, num_layers=args.num_layers, batch_size=args.batch_size, input_size=input_size, decoder_steps=args.decoder_steps, num_steps=args.epochs * DataLoader.Get_num_batch( DataLoader.train_set['source'], args.in_frames), lr=args.learning_rate) solver.train(args=args, DataLoader=DataLoader, net=net) solver.test(args=args, DataLoader=DataLoader)
def main(): global nodes_vector G_igraph, G_nx, nodes_vector = dp.generate_data() # G = igraph.Graph.Erdos_Renyi(n=200, p=0.08) # aca_graph = dp.separate_data_by_academy(nodes_vector, G_igraph) ig_list = centrality_calculation_by_igraph(G_igraph) nx_list = centrality_calculation_by_networkx( nx.DiGraph(G_igraph.get_edgelist())) c_list = ig_list + nx_list # Centrality 计算结果 centrality_list = copy.deepcopy(c_list) print centrality_list top_ten_list, rank_list = top_rank(centrality_list, 10) print rank_list print top_ten_list
def main(self): self.port.flushInput() #start brand new #preInput = b'' #if current data is not complete, we wait until next cycle sendPCCount = 0 while self.switch.is_set(): preTime = time.time() if self.port.in_waiting > 0: rawInput = self.port.read_until( size=45) #todo make data length a variable #state,preInput,senStr = dp.dataSep(rawInput,self.senArray,self.senLock) state, senStr = dp.dataSepSimp(rawInput, self.senArray, self.senLock, self.senRecQue) if state: # tryTime1 = time.time() #self.senRecQue.put(self.senArray) #self.senArray is a multiprocess manager.Array, cannot be directly put into a list, thus we transform it into a normal list # todo check the effect on speed # tryTime2 = time.time() # print(tryTime2-tryTime1) sendPCCount = sendPCCount + 1 if sendPCCount == 4: sendPCThread = th.Thread(target=self.sendPC, args=(senStr, )) sendPCThread.start() sendPCCount = 0 else: pass else: print('!get nothing') aftTime = time.time() while (aftTime - preTime) < self.period: aftTime = time.time() time.sleep(0.00001) print('#Sensor ends')
def main(): # using Terminal to get parameters config = Config.Config_Table(Terminal_parser()) # get standard train and test file if not os.path.exists(config.train) or not os.path.exists(config.test): DataProcess.get_standard_file(config) # get char vocab vocab, index, maxLen = {}, 5, -1 vocab, index, maxLen = DataProcess.get_vocab(config.train, vocab, index, maxLen) vocab, _, maxLen = DataProcess.get_vocab(config.test, vocab, index, maxLen) config.n_feature = len(vocab) # get standard train and test data train_data, train_target = DataProcess.get_data(config.train, vocab, config.class_dict, maxLen) test_data, test_target = DataProcess.get_data(config.test, vocab, config.class_dict, maxLen) # get train loader and test loader train_loader = Data.get_dataloader(train_data, train_target, config.batch_size) test_loader = Data.get_dataloader(test_data, test_target, config.batch_size) # define model and optimizer model = Model.LSTM_Linear(config, vocab).to(config.device) optimizer = optim.Adam(model.parameters(), lr=0.01) print("Train and Test Model") res_eval = Evaluation.Result_Eval(config.num_classes) # train and test model for epoch in range(config.epochs): TrainModel(model, optimizer, train_loader, epoch, config) TestModel(model, test_loader, config, res_eval) res_eval.best_model_result()
from tensorflow.python.keras.models import load_model import DataProcess from CallBack import LossHistory model = load_model('my_model_0.h5') model.summary() x_train, y_train, x_test, y_test, class_names = DataProcess.readData( path='D:\\迅雷下载\\npy', start=10001, each_count=12000) history = LossHistory() model.fit(x_train, y_train, batch_size=40, epochs=10, callbacks=[history]) model.save('my_model_1.h5') score = model.evaluate(x_test, y_test, verbose=1) print('Test loss:', score[0]) print('Test accuracy:', score[1]) history.loss_plot('epoch')
loss_function = nn.NLLLoss() optimizer = optim.SGD(model.parameters(), lr=0.1) for epoch in range(10): running_loss = 0.0 print("epoch : (%d)" % (epoch)) for i, sentence in enumerate(D.training_data): # clear gradients for each instance model.zero_grad() # init model's hidden model.EMB_hidden = model.init_hidden('emb') model.POS_hidden = model.init_hidden('pos') # make our target ready sentence_in = D.make_index_list(sentence, 'word') targets = D.make_index_list(sentence, 'tag') char_in = D.make_index_list(sentence, 'char') # feed the model tag_scores = model(sentence_in, char_in) # Compute the Loss, Gradients, and update the parameters loss = loss_function(tag_scores, targets) loss.backward() optimizer.step() # Print the result running_loss += loss.data[0] if i % 200 == 0: print('[%d] epoch, [%5d] Steps - loss : %.3f' %
def upload_file(): ''' 解析上传文件,构造图 ''' global G_igraph, G_nx, nodes_vector, aca_graph if request.method == 'POST': uploaded_file = request.files['file'] if uploaded_file and allowed_file(uploaded_file.filename): filename = secure_filename(uploaded_file.filename) # 保存上传文件到服务器 uploaded_file.save( os.path.join(app.config['UPLOAD_FOLDER'], filename)) file_path = app.config['UPLOAD_FOLDER'] + uploaded_file.filename # 第一次遍历文件,找出编号最大的节点 with open(file_path, 'r') as f: max_num = 0 graph_nodes = [] for line in f: if line and not line.strip().startswith('#'): from_node_str, to_node_str = line.rsplit(None, 1) from_node = map(int, from_node_str.split(',')) to_node = map(int, to_node_str.split(',')) from_node_id = from_node[0] to_node_id = to_node[0] # 判断节点是否在节点集合里 if from_node_id not in graph_nodes: graph_nodes.append(from_node_id) if to_node_id not in graph_nodes: graph_nodes.append(to_node_id) if from_node_id > max_num or to_node_id > max_num: max_num = max(from_node_id, to_node_id) nodes_vector = [0] * (max_num + 1) G_igraph.add_vertices(graph_nodes) # 添加节点集合到图里 f.seek(0) # 返回文件头 # 再次遍历文件,填充 nodes_vector,构造 G_igraph for line in f: if line and not line.strip().startswith('#'): from_node_str, to_node_str = line.rsplit(None, 1) from_node = map(int, from_node_str.split(',')) to_node = map(int, to_node_str.split(',')) from_node_id = from_node[0] from_node_vector = from_node[1:] to_node_id = to_node[0] to_node_vector = to_node[1:] # 添加边 G_igraph.add_edge(from_node_id, to_node_id) nodes_vector[from_node_id] = from_node_vector nodes_vector[to_node_id] = to_node_vector G_nx = nx.DiGraph(G_igraph.get_edgelist()) aca_graph = dp.separate_data_by_academy(nodes_vector, G_igraph) return render_template('main.html')
def top_students(req_id): ''' 处理名人推荐、学院搜索 ---------- parameter req_id 请求编号 return 前端需要的数据 ''' global G_igraph, G_nx, nodes_vector, aca_graph, data paramas = request.args # GET 请求参数 if req_id == 0: # 渲染 Centrality 页面 return render_template('new_centrality.html') elif req_id == 1: # 请求全校学生节点中心度值 ig_list = c.centrality_calculation_by_igraph(G_igraph) nx_list = c.centrality_calculation_by_networkx(G_nx) graph_json = dp.graph_json(G_nx, nodes_vector) # 图信息 c_list = ig_list + nx_list # Centrality 计算结果 # c_list = c.test(G_nx) # 测试:只使用 networkx 进行计算 # 对 Centrality 计算结果进行深度拷贝,用于计算 Top-N centrality_list = copy.deepcopy(c_list) top_n_list, rank_list = top_rank( centrality_list, min(10, len(G_nx.nodes()))) data = {'Centrality': c_list, 'Graph': graph_json, 'Rank': rank_list} # 封装成 dict return make_response(json.dumps(data)) elif req_id == 2: # 请求某个院学生节点中心度值 aca_id = int(paramas['search']) - 1 # 参数:学院 ID ig_list = c.centrality_calculation_by_igraph(aca_graph[aca_id]) nx_list = c.centrality_calculation_by_networkx( nx.DiGraph(aca_graph[aca_id].get_edgelist())) graph_json = dp.graph_json( nx.DiGraph(aca_graph[aca_id].get_edgelist()), nodes_vector) # 图信息 c_list = ig_list + nx_list # Centrality 计算结果 # c_list = c.test(G_nx) centrality_list = copy.deepcopy(c_list) top_ten_list, rank_list = top_rank(centrality_list, 10) print rank_list print top_ten_list data = {'Centrality': c_list, 'Graph': graph_json, 'Rank': rank_list} # 封装成 dict return make_response(json.dumps(data)) elif req_id == 3: # 请求某个学生节点中心度值 stu_id = int(paramas['search']) # 参数:学生 ID ig_list = c.centrality_calculation_by_igraph(G_igraph) nx_list = c.centrality_calculation_by_networkx(G_nx) graph_json = dp.graph_json(G_nx, nodes_vector) # 图信息 c_list = ig_list + nx_list # Centrality 计算结果 # c_list = c.test(G_nx) centrality_list = copy.deepcopy(c_list) rank_list = [] for c_dict in centrality_list: # 构造 Rank c_dict[c_dict.keys()[0]] = [] rank_list.append(c_dict) stu_centrality_list = [] # 取出所有计算该节点 Centrality 算法中对应的值,并以 key-value 存储 for centrality in c_list: stu_centrality = {} centrality_key = centrality.keys()[0] centrality_value = centrality.values()[0][stu_id] stu_centrality[centrality_key] = centrality_value stu_centrality_list.append(stu_centrality) data = {'StudentCentrality': stu_centrality_list, 'Centrality': c_list, 'Graph': graph_json, 'Rank': rank_list} # 封装成 dict return make_response(json.dumps(data))
for i in total_step: target_st_time = datetime.datetime(year, 1, 1, hh, mm) + datetime.timedelta(days=st_doy-1) target_time = target_st_time + datetime.timedelta(minutes = run_step*i) - datetime.timedelta(minutes = time_delay) gim_time = target_time - datetime.timedelta(days = gim_time_set) target_year = target_time.year target_doy = (datetime.datetime(target_year, target_time.month, target_time.day) - datetime.datetime(target_year, 1, 1)).days + 1 target_hh = target_time.hour target_mm = target_time.minute gim_time_doy = (datetime.datetime(gim_time.year, gim_time.month, gim_time.day) - datetime.datetime(gim_time.year, 1, 1)).days + 1 os.chdir(main_pwd) # create directory and copy needed file () to save_pwd DataProcess.preprocess(target_year, target_doy, save_pwd, download_list_fn) ##### get GIM data if stnbias_method=='GIM': gim = Getdata.GimData(gim_time.year, gim_time_doy) gim.get_gimdata() gim.create_mapfile() if download_type=='IGSRT': gim.read_station_bias() shutil.copy('bias{0}{1:03}.dat'.format(gim_time.year, gim_time_doy) ,'bias{0}{1:03}.dat'.format(target_year, target_doy)) #### download GPS file gpsfile = Getdata.GPSourceFile(target_year, target_doy, target_hh, target_mm) gpsfile.download_data(download_type, download_list_fn) gpsfile.decompress_file() gpsfile.d2o()
31: 'tree g r', 32: 'tree e r', 33: 'tree g b', 34: 'tree e b', 41: 'NB Gaussian', 42: 'NB Multi', 51: 'KNN uniform', 52: 'KNN distance', } # 6 define a black list for model result recordings ### import packages ## 1 import data Tdata import numpy as np import DataProcess as dp Tdata = dp.trainingData() ## 3 import packagees for models from sklearn import svm from sklearn import linear_model from sklearn import tree from sklearn import naive_bayes from sklearn import neighbors # 4 for cross validation from sklearn import cross_validation from sklearn.metrics import accuracy_score import time # For Reference # LabelDic = {'ang': 1, 'fru' : 2, 'sad' : 3, 'hap' : 4, 'neu' : 5, 'xxx' : 6 , 'sur' : 7, 'exc' : 8, 'dis' : 9, 'fea' : 10, 'oth': 11} # mapping of label to scalar category