def monitorHealth(self): while True: while self.gameState.mode[0] == "Realm" or self.gameState.mode[ 0] == "Loot": health = GetData.getPlayerData(self.gameState.frame[0]) outOfPotions = GetData.outOfPotions(self.gameState.frame[0]) AgentTest.monitorHealth(health, outOfPotions)
def click(self): win=self.win config_name=filedialog.askopenfilename(title='选取配置文件',filetypes=[('Excel文档','xlsx'),('Excel文档','xls')]) if config_name=='': return code_name='config/code.txt' code='' try: with open(code_name,'r',encoding='utf-8') as f: code=f.read() except: messagebox.showerror(title='一个很确定的错误框',message="虽然报错了,但是代码不可能错的,必然是配置文件有问题") return data=GetData(config_name) data_dic=data.get_data() for data in data_dic: code=code.replace(data,str(data_dic[data])) win.clipboard_clear() win.clipboard_append(code) if messagebox.askokcancel(title='一个很不确定的消息框', message='不出意外的话,估摸着应该是成功了\n且相关代码已复制至剪贴板,是否需要另存为文件?'): fname = filedialog.asksaveasfilename(title=u'保存文件', filetypes=[("文本文档", ".txt")],initialfile='new_code') try: with open(fname+'.txt','w',encoding='utf-8') as f: f.write(code) except: messagebox.showerror(title='一个很确定的错误框',message="虽然不知道为啥,但就是报错了") return messagebox.showinfo(title="又一个很不确定的消息框",message="不出意外的话,估摸着应该是写入到当前目录了")
def fn(indexes): for index in indexes: if assessment: scores = gd.assessmentScores(index) else: scores = gd.trainingScores(index) pp.plot(scores, linestyle=':', marker='x', label=legendLabel)
def getDistanceGain(self): try: distance_mapping = {} ordered_trip_list = [] for id in self.trip_list: trip = GetData.GetTripDetails(id) distance = trip_merging.find_distance_from_source( trip.destination.latitude, trip.destination.longitude)[0] distance_mapping[distance] = id for distance in sorted(list(distance_mapping)): ordered_trip_list.append(distance_mapping[distance]) trip = GetData.GetTripDetails(ordered_trip_list[0]) combined_trip_distance = trip.distance i = 1 source = 0 while (i < len(self.trip_list)): trip1 = GetData.GetTripDetails(ordered_trip_list[source]) trip2 = GetData.GetTripDetails(ordered_trip_list[i]) difference = find_distance(trip1.destination.latitude, trip1.destination.longitude, trip2.destination.latitude, trip2.destination.longitude)[0] if difference > WALKING_TOLRENCE: source = i combined_trip_distance += difference #else: #print "nope: %s" %difference i += 1 normal_trip_distance = sum(list(distance_mapping)) return (normal_trip_distance, combined_trip_distance) except Exception: return (0, 0)
def get_moc_data(self, term_id): """ 根据输入的学期id查询所有的练习 :param term_id: 第几学期的id :type term_id: int :return: req.text if success else 1 """ host = 'http://www.icourse163.org/dwr/call/plaincall/MocScoreManagerBean.getMocTermDataStatisticDto.dwr' data = GetData.get_moc_data(term_id) req = requests.post(host, headers=self.headers, data=data, cookies=self.cookies) self.oj_set, self.test_set = GetData.data_manage_ex_term(req.text) print('编程题', self.oj_set) print('客观题', self.test_set) req.close() try: ModuleIcourse.upload_moc_data({ # 上传数据库 'oj_set': self.oj_set[1:len(self.oj_set)], 'test_set': self.test_set[1:len(self.test_set)] }) except: print("数据库未连接")
def fn(indexes): scores = [] for index in indexes: if assessment: scores += [gd.assessmentScores(index)] else: scores += [gd.trainingScores(index)] scores = np.array(scores) averages = np.mean(scores, 0) pp.plot(averages, linestyle=':', marker='x', label=legendLabel)
def fn(indexes): scores = [] for index in indexes: if assessment: scores += [gd.assessmentScores(index)] else: scores += [gd.trainingScores(index)] scores = np.array(scores) meanScores = np.mean(scores, 0) highErrors = np.max(scores, 0) - meanScores lowErrors = meanScores - np.min(scores, 0) pp.errorbar(x=[i for i in range(0, len(meanScores))], y=meanScores, yerr=[lowErrors, highErrors], label=legendLabel)
def run(): order = ['SG', 'CMR', 'LMR', 'THUC'] acc_list = [] use_time = [] # SG SG_train_labels, SG_train_data = GetData.get_SG_train_dataset() SG_vail_labels, SG_vail_data = GetData.get_SG_vail_dataset() SG_test_labels, SG_test_data = GetData.get_SG_test_dataset() acc, use_time_ms = ExTfIdfVotingNB.tf_idf_voting_naive_bayes( [SG_train_labels, SG_vail_labels, SG_test_labels], [SG_train_data, SG_vail_data, SG_test_data]) acc_list.append(acc) use_time.append(use_time_ms) # # CMR # CMR_train_labels, CMR_train_data = GetData.get_CMR_train_dataset() # CMR_vail_labels, CMR_vail_data = GetData.get_CMR_vail_dataset() # CMR_test_labels, CMR_test_data = GetData.get_CMR_test_dataset() # acc, use_time_ms = tf_idf_naive_bayes([CMR_train_labels, CMR_vail_labels, CMR_test_labels], # [CMR_train_data, CMR_vail_data, CMR_test_data]) # acc_list.append(acc) # use_time.append(use_time_ms) # # # LMR # LMR_train_labels, LMR_train_data = GetData.get_LMR_train_dataset() # LMR_vail_labels, LMR_vail_data = GetData.get_LMR_vail_dataset() # LMR_test_labels, LMR_test_data = GetData.get_LMR_test_dataset() # acc, use_time_ms = tf_idf_naive_bayes([LMR_train_labels, LMR_vail_labels, LMR_test_labels], # [LMR_train_data, LMR_vail_data, LMR_test_data]) # acc_list.append(acc) # use_time.append(use_time_ms) # # # THUC # # open('a.txt', 'w', encoding='utf-8').write('') # THUC_train_labels, THUC_train_data = GetData.get_THUC_train_dataset() # # open('a.txt', 'a', encoding='utf-8').write(str(THUC_train_data)) # # THUC_vail_labels, THUC_vail_data = GetData.get_THUC_vail_dataset() # # open('a.txt', 'a', encoding='utf-8').write(str(THUC_vail_data)) # THUC_test_labels, THUC_test_data = GetData.get_THUC_test_dataset() # # open('a.txt', 'a', encoding='utf-8').write(str(THUC_test_data)) # # train_data = [THUC_train_data, THUC_vail_data, THUC_test_data] # # acc, use_time_ms = tf_idf_naive_bayes([THUC_train_labels, THUC_vail_labels, THUC_test_labels], train_data ) # acc_list.append(acc) # use_time.append(use_time_ms) for i in range(1): print("For TF-IDF NB, dataset {} , acc: {}, use time: {} .".format( order[i], acc_list[i], use_time[i]))
def realm(self): count = 0 while (self.mode == "Realm"): #print("Running Realm") self.screenEnemies = GetData.getEnemiesScreen(self.frame) self.mapEnemies = GetData.getEnemiesMap(self.frame) self.closestEnemy = AgentTest.findClosestEnemy( self.mapEnemies, self.playerPos) AgentTest.Aim(self.screenEnemies, self.gameWindow) #im wondering if this should be a thread since we dont want to die if were checking enemies on screen AgentTest.monitorHealth(self.playerHealth) self.playerPos = GetData.getPlayerPos(self.frame) print("realm return")
def test_bar_model(symbol="AAPL", order_size=10, split_ratio=0.8, epochs=1, end_date=(datetime.date.today()).strftime('%Y%m%d'), n_samples=1000, isTrain=False): _symbol = symbol _split_ratio = split_ratio _epochs = epochs data = GetData.get_A_data(ts_code=_symbol, end_date=end_date, n_samples=n_samples) # yesterday one minutes data bar_data = GetData.get_latest_bar(symbol=symbol, trade_date=end_date, freq="1M") print("test_latest_model:bar_data.shape:", bar_data.shape) print(data.columns.values) # 数据预处理 根据用户输入的split_ratio 返回划分好的训练集和测试集 # train, test, date_train, date_test = DataPreprocess.data_preprocess(data, _split_ratio) train, test, date_train, date_test = DataPreprocess.data_A_preprocess(data, _split_ratio) bar_test, time_test = DataPreprocess.data_A_bar(bar_data) print("bar_test.shape:", bar_test.shape, "time_test.shape:", time_test.shape) DataPreprocess.data_A_bar(bar_data) # 生成训练环境和测试环境 # env_test = StockEnvironment.StockEnv(test, order_size) env_test = StockEnvironment.StockEnv(bar_test, order_size) env_train = StockEnvironment.StockEnv(train, order_size) # 初始化runner runner = Runner.Runner() trained_model = None if isTrain == False: for new_dir in os.listdir(os.curdir): # 列表出该目录下的所有文件(返回当前目录'.') # 如果有success的模型就使用,否则使用train模型 if new_dir.startswith('success-model-{}'.format(_symbol)): trained_model = new_dir # 如果没有success模型,使用训练过的train模型 if trained_model == None: for dir_name in os.listdir(os.curdir): # 列表出该目录下的所有文件(返回当前目录'.') if dir_name.startswith('train-model-{}'.format(_symbol)): trained_model = dir_name if trained_model == None: print("No model for predict,now train a model") trained_model = runner.trainer(_symbol, env_train, _epochs, order_size) else: # 训练dqn网络,返回训练完毕的模型,以及训练最终结果; 显示训练情况图 trained_model = runner.trainer(_symbol, env_train, _epochs) print('Model Name: {}'.format(trained_model)) # 用训练后的trained_Q对test数据进行分析,给出预测出的最终交易行为;显示测试情况图 fortune, act, reward, cash = runner.tester(env_test, trained_model, order_size) print("profitRatio:{},fortune:{},act:{},reward:{},cash:{}".format(fortune[-1] / 100000.0, fortune[-1], act[-1], reward[-1], cash[-1])) print("fortune len:", len(fortune)) return bar_test, act, fortune, cash, time_test
def __main__(): # get data d_t = GetData.readIn( "C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\train.csv" ) d_v = GetData.readIn( "C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\test.csv" ) d_v.Fare[pd.isna(d_v.Fare)] = 400 d = pd.concat([d_v, d_t], axis=0, join="outer") d = GetData.fillMissingAgeValues(d) #d = GetData.addAgeDummies(d) #d = GetData.addAgeGroup( d ) numOfSplines = 7 d = addSplines(d, ["Fare", "Age"], numOfSplines) # break up data d_t = d[pd.isna(d.Survived) == False] d_t = shuffle(d_t) d_v = d[pd.isna(d.Survived)].sort_values(["PassengerId"]) predictors = ["Fare", "Pclass1", "Pclass2", "Pclass3", "Sex", "Age"] for p in ["Fare", "Age"]: for i in range(numOfSplines): predictors.append("bs " + str(p) + str(i)) # predict model to get prediciton of errors n = d_t.shape[0] d_tt = d_t[0:(n - 400)] d_vv = d_t[(n - 400):] reg = linear_model.LogisticRegression() reg.fit(X=d_tt.loc[:, predictors], y=d_tt["Survived"]) p = reg.predict(X=d_vv.loc[:, predictors]) print(sum(d_vv["Survived"] == p) / d_vv.shape[0]) # fit full model reg = linear_model.LogisticRegression() reg.fit(X=d_t.loc[:, predictors], y=d_t["Survived"]) p = reg.predict(X=d_v.loc[:, predictors]) pd.DataFrame({ "PassengerId": pd.to_numeric(d_v.PassengerId, downcast='integer'), "Survived": p }).to_csv( "C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\prediction.csv", index=False)
def GetData(Fields,StartTime,EndTime,CodeList,DataPar=None): ''' :param Fields: :param StartTime: :param EndTime: :param CodeList: :param DataPar:包括{'TimeInterval':'1Day','PriceAdj':'Backward','DataSource':'Local'} :return: - Temp_Data:一个列表,内部每个元素为一个Data类 - ItemList:一个列表,每个元素为对应的字段名称 - TimeList:提取数据的时间列表 ''' TimeList=[StartTime,EndTime] # a.识别参数 if DataPar is None: DataPar={} # a.1 数据类型 if 'DataType' not in DataPar: DataPar['DataType']=DataDef.DATA_TYPE.K_DAY.value # a.2 组织方式 if 'GroupByType' not in DataPar: DataPar['GroupByType']=DataDef.GROUP_BY_TYPE.FIELDS.value # a.3 数据来源 # 留坑待补,这是为了整合市面上不同数据接口,也支持本地,理想中可以用一个配置文件控制提取的先后顺序,保证获取到数据 if 'DataSource' not in DataPar: DataPar['DataSource']=DataDef.DATA_SOURCE.GTA_QTAPI.value # 目前这里以采用QTAPI的接口为主 # b.取数 Ret,RetValue=GD.GetData(CodeList,TimeList,Fields,DataPar['DataType'],DataPar['GroupByType'],DataPar['DataSource'],SpecialConfig={}) return RetValue.Value,RetValue.ValueIndex,list(RetValue.Value[RetValue.ValueIndex[0]].index)
def getZoneStats(gameFrame): centerPoint = [297,280] size = 180 edgeBuffer = 10 half = size//2 threeHalf = int(1.5 *size) zoneFrame = gameFrame[(centerPoint[1] - threeHalf - edgeBuffer):(centerPoint[1] + threeHalf + edgeBuffer), (centerPoint[0] - threeHalf - edgeBuffer):(centerPoint[0] + threeHalf + edgeBuffer)] obsFrame = gameFrame[(centerPoint[1] - 60):(centerPoint[1] + 60), (centerPoint[0] - 60):(centerPoint[0] + 60)] bulletCenters = Bullets.getBulletCenters(zoneFrame) enemyCenters = GetData.getEnemiesScreen1(zoneFrame) obsMask = Trees.getObstacleMask(obsFrame,10) # 10 - Color range +/- 10 BGR values obstacle_array = getObstacles(obsMask,50) # 50 - Mean for mask value. 0 black, 255 white bulletZoneCount = getZoneCounts(zoneFrame.shape,bulletCenters,size) enemyZoneCount = getZoneCounts(zoneFrame.shape,enemyCenters,size) zoneStats = [] for i in range(0,8): zoneStats.append((obstacle_array[i],bulletZoneCount[i],enemyZoneCount[i])) return zoneStats
def HarmonicTrend(Consult,k): ## to get data from database Data = GetData(Consult) # k is number of harmonics init_p = numpy.zeros(3*k) x_values = numpy.array(range(len(Data))) #Function to fit def ArFun (p): y = [] for i in xrange(len(p)/3): y = p(3*i)*(numpy.sin(x_values+p(3*i+1)) + numpy.cos(x_values+p(3*i+2))) return y def ArFit(p): HarmSerie = HarFun(p) Erro = numpy.square(HarmSerie-Data) RMSE = numpy.sqrt(1.*numpy.sum(Erro)/(len(HarmSerie))) return RMSE ## Make exponential regression Opt_p = scipy.optimize.fmin(HarFit,init_p) ## detrended series DetSer = Data - HarFun(Opt_p) return DetSer
def display_data(s): jsonObj = ast.literal_eval(s) project_id = jsonObj.get('project_id', "") if project_id: objGD = GetData.GetData() res = objGD.Process_interface(int(project_id)) return str(res)
def Portfolio_Risk(Data, _vec_derivative): Net_Worth = Net(Data, _vec_derivative) weights = [] Stock_Beta = [] log_returns = pd.DataFrame() Index_Return = pd.DataFrame() #Beta Calculation Index_Data = GetData.Index_Data() Index_Return = np.log(Index_Data / Index_Data.shift(1)) Index_Variance = Index_Return.var()[0] # Perform Bootstrap to get the variance of the risk??? for ticker in _vec_derivative["Tickers"]: log_returns[ticker] = np.log(Data['Adj Close: ' + ticker] / Data['Adj Close: ' + ticker].shift(1)) weights.append(Data['Worth: ' + ticker].iloc[-1] / Net_Worth) Index_Return = Index_Return.fillna(0) log_returns = log_returns.fillna(0) X = np.stack((Index_Return['Adj Close: ^OMX'], log_returns[ticker]), axis=0) Stock_Beta.append(np.cov(X)[0][1] / Index_Variance) Cov_Data = log_returns.cov() Risk = np.dot(weights, np.dot(Cov_Data, weights)) Beta = np.dot(Stock_Beta, weights) return (Risk, Beta)
def update(self, currentDay=None,\ position=dict(),stock=gd.stockData(np.array([""])),\ marketInfo=pd.DataFrame(), cash=-1): if isTime(currentDay): self.currentDay = currentDay if position != dict(): #Delete empty entries for iSymbol in position.keys(): if position[iSymbol] == None: position.pop([iSymbol]) self.position = position if stock != gd.stockData(np.array([""])): self.stock = stock if marketInfo.empty != True: self.marketInfo = marketInfo if cash != -1: self.cash = cash
def get_all_ex_info(self, term, start=0): """ 根据单元的id找到对应单元的所有的练习 :param: term :type: int :param: start: 开始的单元 :type: int :return: oj_set :rtype: list """ term_id = self.terms[term - 1]['term_id'] if int( self.terms[term - 1]['term']) == int(term) else -1 if term_id == -1: print('terms没有按照顺序!') return 1 host = 'http://www.icourse163.org/dwr/call/plaincall/MocScoreManagerBean.getStudentScoresByTestId.dwr' page = 1 # 每页显示的学生数量 stu_per_page = 1000 self.download_path = '/'.join([self.origin_path, 'term' + str(term)]) # 更新下载目录 new_folder(self.download_path) new_folder(self.download_path + '/info') self.get_moc_data(term_id) # 更新对应学期的课程列表 oj_set = [] for i in range(start + 1, len(self.oj_set)): oj = self.oj_set[i] data = GetData.get_oj_by_oj_id(oj['id'], page, stu_per_page) req = requests.post(host, headers=self.headers, data=data, cookies=self.cookies) # 数据处理 oj_stu_set = GetData.data_manage_oj_stu(req.text, oj_id=oj['id'], oj_name=oj['name']) print(oj_stu_set) print(len(oj_stu_set)) oj_set.append(oj_stu_set) req.close() time.sleep(random.randint(1, 3) / 5) # break return oj_set
def copyReplaceParams(figureIndex, paramsList): plotDir = os.path.join(plotsDir, 'plot%d' % figureIndex) for f in os.listdir(plotDir): if f.endswith('.txt'): os.remove(os.path.join(plotDir, f)) for param in paramsList: index = paramsToIndexes[param] shutil.copyfile(gd.paramsFileDir(index[0]), os.path.join(plotDir, 'params%d.txt' % index[0]))
def run(): order = ['SG', 'CMR', 'LMR', 'THUC'] acc_list = [] use_time = [] # SG SG_train_labels, SG_train_data = GetData.get_SG_train_dataset() SG_vail_labels, SG_vail_data = GetData.get_SG_vail_dataset() SG_test_labels, SG_test_data = GetData.get_SG_test_dataset() acc, use_time_ms = ExVotingNB.voting_naive_bayes(11, [SG_train_labels, SG_vail_labels, SG_test_labels], [SG_train_data, SG_vail_data, SG_test_data]) acc_list.append(acc) use_time.append(use_time_ms) # # CMR # CMR_train_labels, CMR_train_data = GetData.get_CMR_train_dataset() # CMR_vail_labels, CMR_vail_data = GetData.get_CMR_vail_dataset() # CMR_test_labels, CMR_test_data = GetData.get_CMR_test_dataset() # acc, use_time_ms = ExVotingNB.voting_naive_bayes(3, [CMR_train_labels, CMR_vail_labels, CMR_test_labels], # [CMR_train_data, CMR_vail_data, CMR_test_data]) # acc_list.append(acc) # use_time.append(use_time_ms) # # # LMR # LMR_train_labels, LMR_train_data = GetData.get_LMR_train_dataset() # LMR_vail_labels, LMR_vail_data = GetData.get_LMR_vail_dataset() # LMR_test_labels, LMR_test_data = GetData.get_LMR_test_dataset() # acc, use_time_ms = ExVotingNB.voting_naive_bayes(2, [LMR_train_labels, LMR_vail_labels, LMR_test_labels], # [LMR_train_data, LMR_vail_data, LMR_test_data]) # acc_list.append(acc) # use_time.append(use_time_ms) # # # THUC # THUC_train_labels, THUC_train_data = GetData.get_THUC_train_dataset() # THUC_vail_labels, THUC_vail_data = GetData.get_THUC_vail_dataset() # THUC_test_labels, THUC_test_data = GetData.get_THUC_test_dataset() # # train_data = [THUC_train_data, THUC_vail_data, THUC_test_data] # # acc, use_time_ms = ExVotingNB.voting_naive_bayes(14, [THUC_train_labels, THUC_vail_labels, THUC_test_labels], train_data ) # acc_list.append(acc) # use_time.append(use_time_ms) for i in range(1): print("For Our NB, dataset {} , acc: {}, use time: {} ." .format(order[i], acc_list[i], use_time[i]))
def sendElectric(): data = GetData.requestElec() jsonObj = {'used': data, 'notused': 100 - data} print(jsonObj) response = app.response_class(response=json.dumps(jsonObj), status=200, mimetype='application/json') return response
def get_term_info(self): """ 得到所有的学期及其对应的term_id :return: 0 """ if not self.is_login: print('请先登陆') exit(1) data = GetData.get_term_info() host = 'https://www.icourse163.org/dwr/call/plaincall/PublishCourseBean.getTermsByTeacher.dwr' req = requests.post(host, headers=self.headers, data=data, cookies=self.cookies) self.terms = GetData.data_manage_term_info(req.text) print(self.terms) req.close() return 0
def __main__(): d_t = GetData.readIn( "C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\train.csv" ) d_v = GetData.readIn( "C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\test.csv" ) d_v.Fare[pd.isna(d_v.Fare)] = 400 d = pd.concat([d_v, d_t], axis=0, join="outer") d = GetData.fillMissingAgeValues(d) d = GetData.addAgeDummies(d) # break up data d_t = d[pd.isna(d.Survived) == False] d_t = shuffle(d_t) d_v = d[pd.isna(d.Survived)].sort_values(["PassengerId"]) predictors = [ "Fare", "Pclass1", "Pclass2", "Pclass2", "Sex", "Age", "child", "middle", "old", 'childAge', 'middleAge', 'oldAge' ] # predict model to get prediciton of error n = d_t.shape[0] d_tt = d_t[0:(n - 100)] reg = linear_model.LogisticRegression() reg.fit(X=d_tt.loc[:, predictors], y=d_tt["Survived"]) d_vv = d_t[(n - 100):] p = reg.predict(X=d_vv.loc[:, predictors]) print(sum(d_vv["Survived"] == p) / d_vv.shape[0]) # fit full model reg = linear_model.LogisticRegression() reg.fit(X=d_t.loc[:, predictors], y=d_t["Survived"]) p = reg.predict(X=d_v.loc[:, predictors]) pd.DataFrame({ "PassengerId": pd.to_numeric(d_v.PassengerId, downcast='integer'), "Survived": p }).to_csv( "C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\prediction.csv", index=False)
def preprocess(data, content): # word tokenize X_wt = [word_tokenize(x) for x in GetData.get_content(data, content)] # 去停用词及文末句号 X_st = [[del_tail_dot(word.lower()) for word in document if not (word.lower() in stop_words or word == "")] for document in X_wt] X_ls = [[lancaster_stemmer.stem(word) for word in document] for document in X_st] # 字符串列表连接 X_ls_merge = [MergeWord(document) for document in X_ls] return X_ls_merge
def train(filepath, batch_size, turn): Notes = GetData.GetNotes_Melody(filepath) Notes_reshape = [] #因为set不能对多层嵌套列表作用,所以先把它压平 for note in Notes: Notes_reshape += note Notes_Num = len(set(Notes_reshape)) Note_name = sorted(set(Notes_reshape)) #获得排序的不重复的音符名字 sequence_length = 15 #序列长度 note2int_dict = dict( (j, i) for i, j in enumerate(Note_name)) #设计一个字典,把音符转换成数字,方便训练 int2note_dict = dict( (i, j) for i, j in enumerate(Note_name)) #一个字典,把数字转换回音符 with open('int2note_dict.file', 'wb') as f: pickle.dump(int2note_dict, f) network_input = [] #输入序列 network_output = [] #输出序列 for Note in Notes: for i in range(0, len(Note) - sequence_length): #每输入sequence_length个音符,输出一个音符 sequence_in = Note[i:i + sequence_length] sequence_out = Note[i + sequence_length] network_input.append([note2int_dict[k] for k in sequence_in]) network_output.append(note2int_dict[sequence_out]) network_input = np.reshape(network_input, (len(network_input), sequence_length, 1)) with open('networkinput.file', 'wb') as f: pickle.dump(network_input, f) model = GetModel.MelodyLSTM(Notes_Num).to(device) #网络inputsize=1,输出维数为1 Loss_Func = torch.nn.CrossEntropyLoss() optimizer = torch.optim.RMSprop(model.parameters(), lr=1e-3, alpha=0.9) for j in range(turn): for i in range(int(len(network_input) / batch_size)): model.zero_grad() #清空梯度缓存 data = np.array(network_input[i:batch_size + i]) data = np.reshape(data.transpose(), (sequence_length, batch_size, 1)) #(seq_len,batch_size,input_size) #print(data.shape) input1 = torch.autograd.Variable(torch.Tensor(data)) #targets=torch.autograd.Variable(torch.Tensor([int(network_output[i])])) #targets =torch.LongTensor(targets) targets_data = np.array(network_output[i:i + batch_size]) targets = torch.tensor(targets_data.transpose(), dtype=torch.long) targets = targets.to(device) input1 = input1.to(device) output = model(input1) loss = Loss_Func(output, targets) loss.backward() optimizer.step() if (j * int(len(network_input) / batch_size) + i + 1) % 10 == 0: print('Epoch: {}, Loss:{:.5f}'.format( j * int(len(network_input) / batch_size) + i + 1, loss.item())) torch.save(model, 'Melody_LSTM/model_1857.pkl') return 0
def addOrder(): if getOrder() != []: for i in getOrder(): for g in GetData.read(): if int(i.ticket) == int(g.ticket): i.stato = "Open" else: i.stato = "Closed" modifyOrder(i) for t in GetData.read(): found = "no" t.stato = "Open" if getOrder() !=[]: for o in getOrder(): if int(t.ticket) == int(o.ticket): found = "yes" if found == "no": ticket = t.ticket typeop = "BUY" if (int(t.typeop) == 1): typeop = "SELL" symbol=t.symbol Orderop = t.Orderop SL = t.SL TP= t.TP close= t.close stato = t.stato profit = t.profit quantity = t.quantity #BinanceApi.Object(t) name = request.cookies.get('userID') print(name) sql = ("INSERT INTO position VALUES (%s, %s,%s, %s,%s, %s,%s,%s,%s,%s,%s)") values = (ticket, typeop,symbol,SL,TP,Orderop,close,stato,quantity,profit,"FauxL") conn = mysql.connect() cursor = conn.cursor() cursor.execute(sql,values) conn.commit() cursor.close() if found == "yes": modifyOrder(t) return render_template("Home.html")
def main(): # Retrieve the Cleaned Data: df = Data.get_clean_dataframe() # train_test_split and balance data if desired by setting imbalanced to true: train_x, test_x, train_y, test_y = GetData.get_train_test_split(df, imbalanced=True) # Call both Models: lr_model, lr_preds = logistic_regression_model(train_x, test_x, train_y, test_y) rf_model, random_forest_preds = random_forest(train_x, test_x, train_y, test_y) # # Find out where models differ in predictions: Unmute this code to run this: # different_predictions = GetData.cross_validation(lr_model=lr_model, rf_model=rf_model, x=test_x, y=test_y) # different_predictions.to_csv('data/Compare_Models.csv') # Print Classification Reports: print("Logistic Regression Metrics:") print(GetData.get_metrics(test_y, lr_preds) ) print("Random Foreset Metrics:") print(GetData.get_metrics(test_y, random_forest_preds) )
def register(): if request.method == 'POST': asd = request.json sql = ("INSERT INTO users VALUES (%s, %s,%s,%s, %s,%s,%s, %s)") Account = GetData.readAccountInfo() values = (asd["username"], asd["password"],asd["email"],"","", Account.id, Account.balance, Account.name) conn = mysql.connect() cursor = conn.cursor() cursor.execute(sql, values) conn.commit() return render_template("login.html")
def download(self, term, oj_stu): """ :param term: :type term: int :param oj_stu: :type oj_stu: dict """ host = 'http://www.icourse163.org/dwr/call/plaincall/YocOJQuizBean.getOJPaperDto.dwr' data = GetData.get_download_data(term, oj_stu['id']) info = { 'term': term, 'stuId': oj_stu['id'], 'stuName': oj_stu['realName'], 'stuNickname': oj_stu['nickname'], 'ojName': oj_stu['ojName'], 'ojId': oj_stu['ojId'], } req = requests.post(host, headers=self.headers, data=data, cookies=self.cookies) if not self.ex_id_set: self.ex_id_set = GetData.write_ex_info(req.text, info) code_dict = GetData.data_manage_code(req.text, info, self.ex_id_set) if not code_dict: return 1 req.close() file_name = '_'.join([info['stuNickname'], info['stuName']]) download_path = '/'.join( [self.download_path, info['ojName'], file_name]) new_folder(download_path) with open('/'.join([download_path, 'information.json']), 'w') as js_file: js = json.dumps(code_dict) js_file.write(js) self.write_in(code_dict, download_path) return 0
def test_check_metaFile_exists(self, os_mock): #------------------------ Arrange path = '/temp/poo' os_mock.return_value = True # Act inst = GetData.GetData() result = inst._check_metaFile_exists(filePath=path) # Assert self.assertEqual(result, True) self.assertTrue(call(path) in os_mock.mock_calls)
def lone_trips(list_of_merged_trips): lone_trips_list = [] all_trips = GetData.GetData(WINDOW_SIZE, DATA_LIMIT) for trip in all_trips: flag = True for merged_trip in list_of_merged_trips: if (merged_trip.contains(trip.id)): flag = False break if flag: lone_trips_list.append(trip.id) return lone_trips_list
def data(): name = request.cookies.get('userID') balance = GetData.readAccountInfo() sql = ("UPDATE users SET AccountID=%s, AccountBalance=%s, AccountName=%s where user=%s") print(balance.PrintAccount()) values = (balance.id, balance.balance, balance.name, name) conn = mysql.connect() cursor = conn.cursor() cursor.execute(sql, values) conn.commit() cursor.close() return render_template("Data.html")
def main(): print('Run: ',__name__, '.main') np_x = gd.GenerateX(50, TIME) np_x_degree = gd.RadianToDegree(np_x) np_y = gd.GetSinus(np_x, AMPLITUDE, 0, 0, 0) np_y_delay = dul.Delay_Uline(np_y) np_atan = atan.atangens2(np_y, np_y_delay) np_x_51 = gd.GenerateX(55, TIME) np_x_51_degree = gd.RadianToDegree(np_x_51) np_y_51 = gd.GetSinus(np_x_51, AMPLITUDE, 0, 0, 0) np_y_51_delay = dul.Delay_Uline(np_y_51) np_51_atan = atan.atangens2(np_y, np_y_51_delay) plt.plot(np_x_degree, np_y/np.amax(np_y)) plt.plot(np_x_degree, np_y_delay/np.amax(np_y_delay)) plt.plot(np_x_degree, np_atan/(np.amax(np_atan))) #plt.plot(np_x_degree, np_y_51/np.amax(np_y_51)) PhaseShift = np_atan - np_51_atan print( 'np.amin(PhaseShift):', np.amin(PhaseShift) ) print( 'np.amax(PhaseShift):', np.amax(PhaseShift) ) #plt.plot( np_x_degree, PhaseShift/np.amin(PhaseShift) ) plt.grid(True) plt.show()
def insert(): print(GetData.readInfo()) if request.method == 'POST': symbol = request.form.get("symbol") tp = request.form.get("tp") sl = request.form.get("sl") volume = request.form.get("volume") tipo = request.form.get("tipo") if WriteOrder(symbol, tp, sl,volume,tipo): flask.flash('Your order has been sent to Metatrader successfully, check if it is open on your account!') return redirect("/insert") else: flask.flash('There is a problem with your order!') return render_template("NewOrder.html")
# encoding=utf-8 import twitter, GetData import time # 初始化Twitter的用户信息 api = twitter.Api(consumer_key='EKEMZjnkpUu7p8CbICyFKnUfD', consumer_secret='lKnVGbM4V8fTwwBjy7uVwo1IyQBpgUBzB0r9R6wAvUmGaNdBwe', access_token_key='720077099688706048-KYyr0iV9aF5vsBl0yQGrjLB8sXBizmr', access_token_secret='D7kbKR9N1rHdYmtnUa6CdPs9qt1gNy8rEsdAIFBoC4Rhu') # 获取需要爬取的用户数据,元组或列表 data = GetData.get_id_from_xls("F:\\0722\\twitter-52-following-id.xls") # 定义存储路径 folder_path = "" def follow_by_id(): i = 0 for a in data: try: api.CreateFriendship(user_id=a, follow=True) print str(i) + " " + str(a) + "已经关注" time.sleep(5) except Exception, e: print e print str(i) + " " + str(a) + "关注失败" # for ti in xrange(5): # try: # api.CreateFriendship(user_id=a, follow=True)
#!/usr/bin/env python #coding: utf-8 from matplotlib.pylab import * rcParams['legend.numpoints'] = 1 import GetData import perceptron option = [1,2,3,4] trainset = GetData.generateData(1) # train set generation testset = GetData.generateData(2) # test set generation axis([-2, 15, -4, 10]) flag1=0 flag2=0 for x in trainset: if x[3] > 0: if (flag1==0): plot(x[0], x[1], 'ob', label='class:1') flag1=1 else: plot(x[0], x[1], 'ob') else: if(flag2==0): plot(-x[0], -x[1], 'or',label='class:-1') flag2=1 else: plot(-x[0], -x[1], 'or') legend()
# encoding=utf-8 import GetData import twitter import SaveData import time # 初始化Twitter的用户信息 api = twitter.Api(consumer_key='EKEMZjnkpUu7p8CbICyFKnUfD', consumer_secret='lKnVGbM4V8fTwwBjy7uVwo1IyQBpgUBzB0r9R6wAvUmGaNdBwe', access_token_key='720077099688706048-KYyr0iV9aF5vsBl0yQGrjLB8sXBizmr', access_token_secret='D7kbKR9N1rHdYmtnUa6CdPs9qt1gNy8rEsdAIFBoC4Rhu') # 获取需要爬取的用户数据,元组或列表 data = GetData.get_data_from_xls("H:\\TWITTER\\IOT.xls") # 用户数据存储文件夹 folder_path = "H:\\TWITTER\\friendship\\" def get_following(): for screen_name in data: print screen_name followers = api.GetFriends(screen_name=screen_name) print len(followers) SaveData.sava_user_to_xml(users=followers, folder_path=folder_path + "followering/" + screen_name + "/") time.sleep(60) def get_following_id_by_name(): id_ = 1 for screen_name in data:
__author__ = 'Pierzchalski' import GetData as gd import matplotlib.pyplot as pp import numpy as np import os import shutil maxIndex = 720 paramsToIndexes = gd.paramsToIndexesDict(maxIndex) paramsToDicts = gd.paramsToDicts(maxIndex) plotsDir = os.path.join('C:\\Users', 'Pierzchalski', 'IdeaProjects', 'trafficLightRL', 'out', 'plots') def plotFn(params, figureIndex, fn, save=True, saveName="plot", show=False): indexes = paramsToIndexes[params] pp.figure(figureIndex) fn(indexes) if save: plotDir = os.path.join(plotsDir, 'plot%d' % figureIndex) if not os.path.exists(plotDir): os.makedirs(plotDir) pp.savefig(os.path.join(plotDir, saveName)) if show: pp.show() def copyReplaceParams(figureIndex, paramsList): plotDir = os.path.join(plotsDir, 'plot%d' % figureIndex) for f in os.listdir(plotDir): if f.endswith('.txt'): os.remove(os.path.join(plotDir, f)) for param in paramsList:
# ========================================================================= # Dependances / Librairies # ========================================================================= import multiprocessing import GetData import TreatWeft # thread de traitement et insertion en base treatWeftThread = multiprocessing.Process(target=TreatWeft.doIt) treatWeftThread.start() # recuperation de la tram GetData.doIt()
def getFormatData(self,tmp): res = dict() res['timestamp'] = tmp['timestamp'] res['srcMac'] = tmp['smac'] res['dstMac'] = tmp['dmac'] res['origin'] = tmp['origin'] if tmp['IP'] or tmp['IP6']: if(tmp['IP6']): ipData = tmp['IP6'] res['srcIP'] = GetData.ipv6AddrFormat(ipData.src) res['dstIP'] = GetData.ipv6AddrFormat(ipData.dst) else: ipData = tmp['IP'] res['srcIP'] = GetData.ipAddrFormat(ipData.src) res['dstIP'] = GetData.ipAddrFormat(ipData.dst) if isinstance(ipData.data,dpkt.tcp.TCP): res['type'] = 'TCP' tcpData = ipData.data res['srcPort'] = str(tcpData.sport) res['dstPort'] = str(tcpData.dport) if(str(tcpData.sport) == '80' or str(tcpData.dport) == '80'): res['type'] = 'HTTP' if(str(tcpData.sport) == '443' or str(tcpData.dport) == '443'): res['type'] = 'HTTPS' if(str(tcpData.sport) == '53' or str(tcpData.dport) == '53'): res['type'] = 'DNS' elif isinstance(ipData.data,dpkt.udp.UDP): res['type'] = 'UDP' udpData = ipData.data res['srcPort'] = str(udpData.sport) res['dstPort'] = str(udpData.dport) if(str(udpData.sport) == '53' or str(udpData.dport) == '53'): res['type'] = 'DNS' elif isinstance(ipData.data,dpkt.icmp.ICMP): res['type'] = 'ICMP' elif isinstance(ipData.data,dpkt.igmp.IGMP): res['type'] = 'IGMP' else: if tmp['IP6']: res['type'] = 'IPV6' else: res['type'] = 'IPV4' if res['type'] in ['TCP','UDP','DNS','HTTP','HTTPS']: resStr = ' '+res['type']+' '+res['srcMac']+' |->| '+res['dstMac']+' '+res['srcIP']+' (:'+res['srcPort']+') ===>>> '+res['dstIP']+' (:'+res['dstPort']+')' else: resStr = ' '+res['type']+' '+res['srcMac']+' |->| '+res['dstMac']+' '+res['srcIP']+' ===>>> '+res['dstIP'] + ' ' elif tmp['ARP'] : res['type'] = 'ARP' resStr = ' '+res['type']+' '+res['srcMac']+' |->| '+res['dstMac'] else: res['type'] = '暂未支持分析的网络层数据包种类' resStr = ' '+res['type']+' '+res['srcMac']+' |->| '+res['dstMac'] if self.filterInfo : if res['type'] not in self.filterInfo['protocols']: #print self.filterInfo['protocols'] #print res['type'] print '不符合的数据包~~~!' return None #print self.filterInfo for name,weath in (self.filterInfo['detal']).items(): if weath : #print name #print weath try: if not (res[name] == weath): #print '不符合的数据包~~~!' return None except Exception,e: #print '不符合的数据包~~~!' return None
#encoding=utf-8 import GetData import twitter import SaveData import time # 初始化Twitter的用户信息 api = twitter.Api(consumer_key='EKEMZjnkpUu7p8CbICyFKnUfD', consumer_secret='lKnVGbM4V8fTwwBjy7uVwo1IyQBpgUBzB0r9R6wAvUmGaNdBwe', access_token_key='720077099688706048-KYyr0iV9aF5vsBl0yQGrjLB8sXBizmr', access_token_secret='D7kbKR9N1rHdYmtnUa6CdPs9qt1gNy8rEsdAIFBoC4Rhu') # 获取需要爬取的用户数据,元组或列表 data = GetData.get_data_from_xls("F:\\0901\\400.xls") # 定义存储路径 folder_path = "F:\\0901\\400\\" """ err_log 记载出错的账户名称 right_log记载正确账户名称 id_log 记载截至程序停止为止当前最大ID,启动程序时需要更新 """ err_log = open("F:/err_log.txt", 'a') right_log = open("F:/right_log.txt", 'a') id_log = open("F:/id_log.txt", 'w') def main(): for per in data:
# encoding=utf-8 import GetData import twitter import SaveData import time # 初始化Twitter的用户信息 api = twitter.Api(consumer_key='EKEMZjnkpUu7p8CbICyFKnUfD', consumer_secret='lKnVGbM4V8fTwwBjy7uVwo1IyQBpgUBzB0r9R6wAvUmGaNdBwe', access_token_key='720077099688706048-KYyr0iV9aF5vsBl0yQGrjLB8sXBizmr', access_token_secret='D7kbKR9N1rHdYmtnUa6CdPs9qt1gNy8rEsdAIFBoC4Rhu') # 获取需要爬取的用户数据,元组或列表 data = GetData.get_data_from_xls("F:\\6000\\6000\\6000.xls") # 定义存储路径 folder_path = "F:\\6000\\6000\\6000-0901\\" def main(): for per in data: try: get_status_by_id(per) except Exception, e: print e def get_status(screen_name): print screen_name + " start crawler" statuses = api.GetUserTimeline(screen_name=screen_name, count=200)