def monitorHealth(self):
while True:
while self.gameState.mode[0] == "Realm" or self.gameState.mode[
0] == "Loot":
health = GetData.getPlayerData(self.gameState.frame[0])
outOfPotions = GetData.outOfPotions(self.gameState.frame[0])
AgentTest.monitorHealth(health, outOfPotions)
def click(self):
win=self.win
config_name=filedialog.askopenfilename(title='选取配置文件',filetypes=[('Excel文档','xlsx'),('Excel文档','xls')])
if config_name=='':
return
code_name='config/code.txt'
code=''
try:
with open(code_name,'r',encoding='utf-8') as f:
code=f.read()
except:
messagebox.showerror(title='一个很确定的错误框',message="虽然报错了，但是代码不可能错的，必然是配置文件有问题")
return
data=GetData(config_name)
data_dic=data.get_data()
for data in data_dic:
code=code.replace(data,str(data_dic[data]))
win.clipboard_clear()
win.clipboard_append(code)
if messagebox.askokcancel(title='一个很不确定的消息框', message='不出意外的话,估摸着应该是成功了\n且相关代码已复制至剪贴板，是否需要另存为文件？'):
fname = filedialog.asksaveasfilename(title=u'保存文件', filetypes=[("文本文档", ".txt")],initialfile='new_code')
try:
with open(fname+'.txt','w',encoding='utf-8') as f:
f.write(code)
except:
messagebox.showerror(title='一个很确定的错误框',message="虽然不知道为啥，但就是报错了")
return
messagebox.showinfo(title="又一个很不确定的消息框",message="不出意外的话，估摸着应该是写入到当前目录了")
def fn(indexes):
for index in indexes:
if assessment:
scores = gd.assessmentScores(index)
else:
scores = gd.trainingScores(index)
pp.plot(scores, linestyle=':', marker='x', label=legendLabel)
def getDistanceGain(self):
try:
distance_mapping = {}
ordered_trip_list = []
for id in self.trip_list:
trip = GetData.GetTripDetails(id)
distance = trip_merging.find_distance_from_source(
trip.destination.latitude, trip.destination.longitude)[0]
distance_mapping[distance] = id
for distance in sorted(list(distance_mapping)):
ordered_trip_list.append(distance_mapping[distance])
trip = GetData.GetTripDetails(ordered_trip_list[0])
combined_trip_distance = trip.distance
i = 1
source = 0
while (i < len(self.trip_list)):
trip1 = GetData.GetTripDetails(ordered_trip_list[source])
trip2 = GetData.GetTripDetails(ordered_trip_list[i])
difference = find_distance(trip1.destination.latitude,
trip1.destination.longitude,
trip2.destination.latitude,
trip2.destination.longitude)[0]
if difference > WALKING_TOLRENCE:
source = i
combined_trip_distance += difference
#else:
#print "nope: %s" %difference
i += 1
normal_trip_distance = sum(list(distance_mapping))
return (normal_trip_distance, combined_trip_distance)
except Exception:
return (0, 0)
def get_moc_data(self, term_id):
"""
根据输入的学期id查询所有的练习
:param term_id: 第几学期的id
:type term_id: int
:return: req.text if success else 1
"""
host = 'http://www.icourse163.org/dwr/call/plaincall/MocScoreManagerBean.getMocTermDataStatisticDto.dwr'
data = GetData.get_moc_data(term_id)
req = requests.post(host,
headers=self.headers,
data=data,
cookies=self.cookies)
self.oj_set, self.test_set = GetData.data_manage_ex_term(req.text)
print('编程题', self.oj_set)
print('客观题', self.test_set)
req.close()
try:
ModuleIcourse.upload_moc_data({ # 上传数据库
'oj_set':
self.oj_set[1:len(self.oj_set)],
'test_set':
self.test_set[1:len(self.test_set)]
})
except:
print("数据库未连接")
def fn(indexes):
scores = []
for index in indexes:
if assessment:
scores += [gd.assessmentScores(index)]
else:
scores += [gd.trainingScores(index)]
scores = np.array(scores)
averages = np.mean(scores, 0)
pp.plot(averages, linestyle=':', marker='x', label=legendLabel)
def fn(indexes):
scores = []
for index in indexes:
if assessment:
scores += [gd.assessmentScores(index)]
else:
scores += [gd.trainingScores(index)]
scores = np.array(scores)
meanScores = np.mean(scores, 0)
highErrors = np.max(scores, 0) - meanScores
lowErrors = meanScores - np.min(scores, 0)
pp.errorbar(x=[i for i in range(0, len(meanScores))], y=meanScores, yerr=[lowErrors, highErrors], label=legendLabel)
def run():
order = ['SG', 'CMR', 'LMR', 'THUC']
acc_list = []
use_time = []
# SG
SG_train_labels, SG_train_data = GetData.get_SG_train_dataset()
SG_vail_labels, SG_vail_data = GetData.get_SG_vail_dataset()
SG_test_labels, SG_test_data = GetData.get_SG_test_dataset()
acc, use_time_ms = ExTfIdfVotingNB.tf_idf_voting_naive_bayes(
[SG_train_labels, SG_vail_labels, SG_test_labels],
[SG_train_data, SG_vail_data, SG_test_data])
acc_list.append(acc)
use_time.append(use_time_ms)
# # CMR
# CMR_train_labels, CMR_train_data = GetData.get_CMR_train_dataset()
# CMR_vail_labels, CMR_vail_data = GetData.get_CMR_vail_dataset()
# CMR_test_labels, CMR_test_data = GetData.get_CMR_test_dataset()
# acc, use_time_ms = tf_idf_naive_bayes([CMR_train_labels, CMR_vail_labels, CMR_test_labels],
# [CMR_train_data, CMR_vail_data, CMR_test_data])
# acc_list.append(acc)
# use_time.append(use_time_ms)
#
# # LMR
# LMR_train_labels, LMR_train_data = GetData.get_LMR_train_dataset()
# LMR_vail_labels, LMR_vail_data = GetData.get_LMR_vail_dataset()
# LMR_test_labels, LMR_test_data = GetData.get_LMR_test_dataset()
# acc, use_time_ms = tf_idf_naive_bayes([LMR_train_labels, LMR_vail_labels, LMR_test_labels],
# [LMR_train_data, LMR_vail_data, LMR_test_data])
# acc_list.append(acc)
# use_time.append(use_time_ms)
#
# # THUC
# # open('a.txt', 'w', encoding='utf-8').write('')
# THUC_train_labels, THUC_train_data = GetData.get_THUC_train_dataset()
# # open('a.txt', 'a', encoding='utf-8').write(str(THUC_train_data))
#
# THUC_vail_labels, THUC_vail_data = GetData.get_THUC_vail_dataset()
# # open('a.txt', 'a', encoding='utf-8').write(str(THUC_vail_data))
# THUC_test_labels, THUC_test_data = GetData.get_THUC_test_dataset()
# # open('a.txt', 'a', encoding='utf-8').write(str(THUC_test_data))
#
# train_data = [THUC_train_data, THUC_vail_data, THUC_test_data]
#
# acc, use_time_ms = tf_idf_naive_bayes([THUC_train_labels, THUC_vail_labels, THUC_test_labels], train_data )
# acc_list.append(acc)
# use_time.append(use_time_ms)
for i in range(1):
print("For TF-IDF NB, dataset {} , acc: {}, use time: {} .".format(
order[i], acc_list[i], use_time[i]))
def realm(self):
count = 0
while (self.mode == "Realm"):
#print("Running Realm")
self.screenEnemies = GetData.getEnemiesScreen(self.frame)
self.mapEnemies = GetData.getEnemiesMap(self.frame)
self.closestEnemy = AgentTest.findClosestEnemy(
self.mapEnemies, self.playerPos)
AgentTest.Aim(self.screenEnemies, self.gameWindow)
#im wondering if this should be a thread since we dont want to die if were checking enemies on screen
AgentTest.monitorHealth(self.playerHealth)
self.playerPos = GetData.getPlayerPos(self.frame)
print("realm return")
def test_bar_model(symbol="AAPL", order_size=10, split_ratio=0.8, epochs=1,
end_date=(datetime.date.today()).strftime('%Y%m%d'),
n_samples=1000, isTrain=False):
_symbol = symbol
_split_ratio = split_ratio
_epochs = epochs
data = GetData.get_A_data(ts_code=_symbol, end_date=end_date, n_samples=n_samples)
# yesterday one minutes data
bar_data = GetData.get_latest_bar(symbol=symbol, trade_date=end_date, freq="1M")
print("test_latest_model:bar_data.shape:", bar_data.shape)
print(data.columns.values)
# 数据预处理 根据用户输入的split_ratio 返回划分好的训练集和测试集
# train, test, date_train, date_test = DataPreprocess.data_preprocess(data, _split_ratio)
train, test, date_train, date_test = DataPreprocess.data_A_preprocess(data, _split_ratio)
bar_test, time_test = DataPreprocess.data_A_bar(bar_data)
print("bar_test.shape:", bar_test.shape, "time_test.shape:", time_test.shape)
DataPreprocess.data_A_bar(bar_data)
# 生成训练环境和测试环境
# env_test = StockEnvironment.StockEnv(test, order_size)
env_test = StockEnvironment.StockEnv(bar_test, order_size)
env_train = StockEnvironment.StockEnv(train, order_size)
# 初始化runner
runner = Runner.Runner()
trained_model = None
if isTrain == False:
for new_dir in os.listdir(os.curdir): # 列表出该目录下的所有文件(返回当前目录'.')
# 如果有success的模型就使用，否则使用train模型
if new_dir.startswith('success-model-{}'.format(_symbol)):
trained_model = new_dir
# 如果没有success模型，使用训练过的train模型
if trained_model == None:
for dir_name in os.listdir(os.curdir): # 列表出该目录下的所有文件(返回当前目录'.')
if dir_name.startswith('train-model-{}'.format(_symbol)):
trained_model = dir_name
if trained_model == None:
print("No model for predict,now train a model")
trained_model = runner.trainer(_symbol, env_train, _epochs, order_size)
else:
# 训练dqn网络，返回训练完毕的模型，以及训练最终结果; 显示训练情况图
trained_model = runner.trainer(_symbol, env_train, _epochs)
print('Model Name: {}'.format(trained_model))
# 用训练后的trained_Q对test数据进行分析，给出预测出的最终交易行为；显示测试情况图
fortune, act, reward, cash = runner.tester(env_test, trained_model, order_size)
print("profitRatio:{},fortune:{},act:{},reward:{},cash:{}".format(fortune[-1] / 100000.0, fortune[-1], act[-1],
reward[-1], cash[-1]))
print("fortune len:", len(fortune))
return bar_test, act, fortune, cash, time_test
def __main__():
# get data
d_t = GetData.readIn(
"C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\train.csv"
)
d_v = GetData.readIn(
"C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\test.csv"
)
d_v.Fare[pd.isna(d_v.Fare)] = 400
d = pd.concat([d_v, d_t], axis=0, join="outer")
d = GetData.fillMissingAgeValues(d)
#d = GetData.addAgeDummies(d)
#d = GetData.addAgeGroup( d )
numOfSplines = 7
d = addSplines(d, ["Fare", "Age"], numOfSplines)
# break up data
d_t = d[pd.isna(d.Survived) == False]
d_t = shuffle(d_t)
d_v = d[pd.isna(d.Survived)].sort_values(["PassengerId"])
predictors = ["Fare", "Pclass1", "Pclass2", "Pclass3", "Sex", "Age"]
for p in ["Fare", "Age"]:
for i in range(numOfSplines):
predictors.append("bs " + str(p) + str(i))
# predict model to get prediciton of errors
n = d_t.shape[0]
d_tt = d_t[0:(n - 400)]
d_vv = d_t[(n - 400):]
reg = linear_model.LogisticRegression()
reg.fit(X=d_tt.loc[:, predictors], y=d_tt["Survived"])
p = reg.predict(X=d_vv.loc[:, predictors])
print(sum(d_vv["Survived"] == p) / d_vv.shape[0])
# fit full model
reg = linear_model.LogisticRegression()
reg.fit(X=d_t.loc[:, predictors], y=d_t["Survived"])
p = reg.predict(X=d_v.loc[:, predictors])
pd.DataFrame({
"PassengerId":
pd.to_numeric(d_v.PassengerId, downcast='integer'),
"Survived":
p
}).to_csv(
"C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\prediction.csv",
index=False)
def GetData(Fields,StartTime,EndTime,CodeList,DataPar=None):
'''
:param Fields:
:param StartTime:
:param EndTime:
:param CodeList:
:param DataPar：包括｛'TimeInterval':'1Day','PriceAdj':'Backward','DataSource':'Local'｝
:return:
- Temp_Data：一个列表，内部每个元素为一个Data类
- ItemList：一个列表，每个元素为对应的字段名称
- TimeList：提取数据的时间列表
'''
TimeList=[StartTime,EndTime]
# a.识别参数
if DataPar is None:
DataPar={}
# a.1 数据类型
if 'DataType' not in DataPar:
DataPar['DataType']=DataDef.DATA_TYPE.K_DAY.value
# a.2 组织方式
if 'GroupByType' not in DataPar:
DataPar['GroupByType']=DataDef.GROUP_BY_TYPE.FIELDS.value
# a.3 数据来源
# 留坑待补，这是为了整合市面上不同数据接口，也支持本地，理想中可以用一个配置文件控制提取的先后顺序，保证获取到数据
if 'DataSource' not in DataPar:
DataPar['DataSource']=DataDef.DATA_SOURCE.GTA_QTAPI.value # 目前这里以采用QTAPI的接口为主
# b.取数
Ret,RetValue=GD.GetData(CodeList,TimeList,Fields,DataPar['DataType'],DataPar['GroupByType'],DataPar['DataSource'],SpecialConfig={})
return RetValue.Value,RetValue.ValueIndex,list(RetValue.Value[RetValue.ValueIndex[0]].index)
def getZoneStats(gameFrame):
centerPoint = [297,280]
size = 180
edgeBuffer = 10
half = size//2
threeHalf = int(1.5 *size)
zoneFrame = gameFrame[(centerPoint[1] - threeHalf - edgeBuffer):(centerPoint[1] + threeHalf + edgeBuffer),
(centerPoint[0] - threeHalf - edgeBuffer):(centerPoint[0] + threeHalf + edgeBuffer)]
obsFrame = gameFrame[(centerPoint[1] - 60):(centerPoint[1] + 60),
(centerPoint[0] - 60):(centerPoint[0] + 60)]
bulletCenters = Bullets.getBulletCenters(zoneFrame)
enemyCenters = GetData.getEnemiesScreen1(zoneFrame)
obsMask = Trees.getObstacleMask(obsFrame,10) # 10 - Color range +/- 10 BGR values
obstacle_array = getObstacles(obsMask,50) # 50 - Mean for mask value. 0 black, 255 white
bulletZoneCount = getZoneCounts(zoneFrame.shape,bulletCenters,size)
enemyZoneCount = getZoneCounts(zoneFrame.shape,enemyCenters,size)
zoneStats = []
for i in range(0,8):
zoneStats.append((obstacle_array[i],bulletZoneCount[i],enemyZoneCount[i]))
return zoneStats
def HarmonicTrend(Consult,k):
## to get data from database
Data = GetData(Consult)
# k is number of harmonics
init_p = numpy.zeros(3*k)
x_values = numpy.array(range(len(Data)))
#Function to fit
def ArFun (p):
y = []
for i in xrange(len(p)/3):
y = p(3*i)*(numpy.sin(x_values+p(3*i+1)) + numpy.cos(x_values+p(3*i+2)))
return y
def ArFit(p):
HarmSerie = HarFun(p)
Erro = numpy.square(HarmSerie-Data)
RMSE = numpy.sqrt(1.*numpy.sum(Erro)/(len(HarmSerie)))
return RMSE
## Make exponential regression
Opt_p = scipy.optimize.fmin(HarFit,init_p)
## detrended series
DetSer = Data - HarFun(Opt_p)
return DetSer
def display_data(s):
jsonObj = ast.literal_eval(s)
project_id = jsonObj.get('project_id', "")
if project_id:
objGD = GetData.GetData()
res = objGD.Process_interface(int(project_id))
return str(res)
def Portfolio_Risk(Data, _vec_derivative):
Net_Worth = Net(Data, _vec_derivative)
weights = []
Stock_Beta = []
log_returns = pd.DataFrame()
Index_Return = pd.DataFrame()
#Beta Calculation
Index_Data = GetData.Index_Data()
Index_Return = np.log(Index_Data / Index_Data.shift(1))
Index_Variance = Index_Return.var()[0]
# Perform Bootstrap to get the variance of the risk???
for ticker in _vec_derivative["Tickers"]:
log_returns[ticker] = np.log(Data['Adj Close: ' + ticker] /
Data['Adj Close: ' + ticker].shift(1))
weights.append(Data['Worth: ' + ticker].iloc[-1] / Net_Worth)
Index_Return = Index_Return.fillna(0)
log_returns = log_returns.fillna(0)
X = np.stack((Index_Return['Adj Close: ^OMX'], log_returns[ticker]),
axis=0)
Stock_Beta.append(np.cov(X)[0][1] / Index_Variance)
Cov_Data = log_returns.cov()
Risk = np.dot(weights, np.dot(Cov_Data, weights))
Beta = np.dot(Stock_Beta, weights)
return (Risk, Beta)
def update(self, currentDay=None,\
position=dict(),stock=gd.stockData(np.array([""])),\
marketInfo=pd.DataFrame(), cash=-1):
if isTime(currentDay):
self.currentDay = currentDay
if position != dict():
#Delete empty entries
for iSymbol in position.keys():
if position[iSymbol] == None:
position.pop([iSymbol])
self.position = position
if stock != gd.stockData(np.array([""])):
self.stock = stock
if marketInfo.empty != True:
self.marketInfo = marketInfo
if cash != -1:
self.cash = cash
def get_all_ex_info(self, term, start=0):
"""
根据单元的id找到对应单元的所有的练习
:param: term
:type: int
:param: start: 开始的单元
:type: int
:return: oj_set
:rtype: list
"""
term_id = self.terms[term - 1]['term_id'] if int(
self.terms[term - 1]['term']) == int(term) else -1
if term_id == -1:
print('terms没有按照顺序！')
return 1
host = 'http://www.icourse163.org/dwr/call/plaincall/MocScoreManagerBean.getStudentScoresByTestId.dwr'
page = 1
# 每页显示的学生数量
stu_per_page = 1000
self.download_path = '/'.join([self.origin_path,
'term' + str(term)]) # 更新下载目录
new_folder(self.download_path)
new_folder(self.download_path + '/info')
self.get_moc_data(term_id) # 更新对应学期的课程列表
oj_set = []
for i in range(start + 1, len(self.oj_set)):
oj = self.oj_set[i]
data = GetData.get_oj_by_oj_id(oj['id'], page, stu_per_page)
req = requests.post(host,
headers=self.headers,
data=data,
cookies=self.cookies)
# 数据处理
oj_stu_set = GetData.data_manage_oj_stu(req.text,
oj_id=oj['id'],
oj_name=oj['name'])
print(oj_stu_set)
print(len(oj_stu_set))
oj_set.append(oj_stu_set)
req.close()
time.sleep(random.randint(1, 3) / 5)
# break
return oj_set
def copyReplaceParams(figureIndex, paramsList):
plotDir = os.path.join(plotsDir, 'plot%d' % figureIndex)
for f in os.listdir(plotDir):
if f.endswith('.txt'):
os.remove(os.path.join(plotDir, f))
for param in paramsList:
index = paramsToIndexes[param]
shutil.copyfile(gd.paramsFileDir(index[0]), os.path.join(plotDir, 'params%d.txt' % index[0]))
def run():
order = ['SG', 'CMR', 'LMR', 'THUC']
acc_list = []
use_time = []
# SG
SG_train_labels, SG_train_data = GetData.get_SG_train_dataset()
SG_vail_labels, SG_vail_data = GetData.get_SG_vail_dataset()
SG_test_labels, SG_test_data = GetData.get_SG_test_dataset()
acc, use_time_ms = ExVotingNB.voting_naive_bayes(11, [SG_train_labels, SG_vail_labels, SG_test_labels],
[SG_train_data, SG_vail_data, SG_test_data])
acc_list.append(acc)
use_time.append(use_time_ms)
# # CMR
# CMR_train_labels, CMR_train_data = GetData.get_CMR_train_dataset()
# CMR_vail_labels, CMR_vail_data = GetData.get_CMR_vail_dataset()
# CMR_test_labels, CMR_test_data = GetData.get_CMR_test_dataset()
# acc, use_time_ms = ExVotingNB.voting_naive_bayes(3, [CMR_train_labels, CMR_vail_labels, CMR_test_labels],
# [CMR_train_data, CMR_vail_data, CMR_test_data])
# acc_list.append(acc)
# use_time.append(use_time_ms)
#
# # LMR
# LMR_train_labels, LMR_train_data = GetData.get_LMR_train_dataset()
# LMR_vail_labels, LMR_vail_data = GetData.get_LMR_vail_dataset()
# LMR_test_labels, LMR_test_data = GetData.get_LMR_test_dataset()
# acc, use_time_ms = ExVotingNB.voting_naive_bayes(2, [LMR_train_labels, LMR_vail_labels, LMR_test_labels],
# [LMR_train_data, LMR_vail_data, LMR_test_data])
# acc_list.append(acc)
# use_time.append(use_time_ms)
#
# # THUC
# THUC_train_labels, THUC_train_data = GetData.get_THUC_train_dataset()
# THUC_vail_labels, THUC_vail_data = GetData.get_THUC_vail_dataset()
# THUC_test_labels, THUC_test_data = GetData.get_THUC_test_dataset()
#
# train_data = [THUC_train_data, THUC_vail_data, THUC_test_data]
#
# acc, use_time_ms = ExVotingNB.voting_naive_bayes(14, [THUC_train_labels, THUC_vail_labels, THUC_test_labels], train_data )
# acc_list.append(acc)
# use_time.append(use_time_ms)
for i in range(1):
print("For Our NB, dataset {} , acc: {}, use time: {} ."
.format(order[i], acc_list[i], use_time[i]))
def sendElectric():
data = GetData.requestElec()
jsonObj = {'used': data, 'notused': 100 - data}
print(jsonObj)
response = app.response_class(response=json.dumps(jsonObj),
status=200,
mimetype='application/json')
return response
def get_term_info(self):
"""
得到所有的学期及其对应的term_id
:return: 0
"""
if not self.is_login:
print('请先登陆')
exit(1)
data = GetData.get_term_info()
host = 'https://www.icourse163.org/dwr/call/plaincall/PublishCourseBean.getTermsByTeacher.dwr'
req = requests.post(host,
headers=self.headers,
data=data,
cookies=self.cookies)
self.terms = GetData.data_manage_term_info(req.text)
print(self.terms)
req.close()
return 0
def __main__():
d_t = GetData.readIn(
"C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\train.csv"
)
d_v = GetData.readIn(
"C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\test.csv"
)
d_v.Fare[pd.isna(d_v.Fare)] = 400
d = pd.concat([d_v, d_t], axis=0, join="outer")
d = GetData.fillMissingAgeValues(d)
d = GetData.addAgeDummies(d)
# break up data
d_t = d[pd.isna(d.Survived) == False]
d_t = shuffle(d_t)
d_v = d[pd.isna(d.Survived)].sort_values(["PassengerId"])
predictors = [
"Fare", "Pclass1", "Pclass2", "Pclass2", "Sex", "Age", "child",
"middle", "old", 'childAge', 'middleAge', 'oldAge'
]
# predict model to get prediciton of error
n = d_t.shape[0]
d_tt = d_t[0:(n - 100)]
reg = linear_model.LogisticRegression()
reg.fit(X=d_tt.loc[:, predictors], y=d_tt["Survived"])
d_vv = d_t[(n - 100):]
p = reg.predict(X=d_vv.loc[:, predictors])
print(sum(d_vv["Survived"] == p) / d_vv.shape[0])
# fit full model
reg = linear_model.LogisticRegression()
reg.fit(X=d_t.loc[:, predictors], y=d_t["Survived"])
p = reg.predict(X=d_v.loc[:, predictors])
pd.DataFrame({
"PassengerId":
pd.to_numeric(d_v.PassengerId, downcast='integer'),
"Survived":
p
}).to_csv(
"C:\\Users\\Ruedi\\OneDrive\\Learning\\Learning\\Machine Learning(python)\\Kaggle Data\\Titanic\\prediction.csv",
index=False)
def preprocess(data, content):
# word tokenize
X_wt = [word_tokenize(x) for x in GetData.get_content(data, content)]
# 去停用词及文末句号
X_st = [[del_tail_dot(word.lower()) for word in document if not (word.lower() in stop_words or word == "")]
for document in X_wt]
X_ls = [[lancaster_stemmer.stem(word) for word in document] for document in X_st]
# 字符串列表连接
X_ls_merge = [MergeWord(document) for document in X_ls]
return X_ls_merge
def train(filepath, batch_size, turn):
Notes = GetData.GetNotes_Melody(filepath)
Notes_reshape = [] #因为set不能对多层嵌套列表作用，所以先把它压平
for note in Notes:
Notes_reshape += note
Notes_Num = len(set(Notes_reshape))
Note_name = sorted(set(Notes_reshape)) #获得排序的不重复的音符名字
sequence_length = 15 #序列长度
note2int_dict = dict(
(j, i) for i, j in enumerate(Note_name)) #设计一个字典，把音符转换成数字，方便训练
int2note_dict = dict(
(i, j) for i, j in enumerate(Note_name)) #一个字典，把数字转换回音符
with open('int2note_dict.file', 'wb') as f:
pickle.dump(int2note_dict, f)
network_input = [] #输入序列
network_output = [] #输出序列
for Note in Notes:
for i in range(0, len(Note) - sequence_length):
#每输入sequence_length个音符，输出一个音符
sequence_in = Note[i:i + sequence_length]
sequence_out = Note[i + sequence_length]
network_input.append([note2int_dict[k] for k in sequence_in])
network_output.append(note2int_dict[sequence_out])
network_input = np.reshape(network_input,
(len(network_input), sequence_length, 1))
with open('networkinput.file', 'wb') as f:
pickle.dump(network_input, f)
model = GetModel.MelodyLSTM(Notes_Num).to(device) #网络inputsize=1，输出维数为1
Loss_Func = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.RMSprop(model.parameters(), lr=1e-3, alpha=0.9)
for j in range(turn):
for i in range(int(len(network_input) / batch_size)):
model.zero_grad() #清空梯度缓存
data = np.array(network_input[i:batch_size + i])
data = np.reshape(data.transpose(),
(sequence_length, batch_size,
1)) #(seq_len,batch_size,input_size)
#print(data.shape)
input1 = torch.autograd.Variable(torch.Tensor(data))
#targets=torch.autograd.Variable(torch.Tensor([int(network_output[i])]))
#targets =torch.LongTensor(targets)
targets_data = np.array(network_output[i:i + batch_size])
targets = torch.tensor(targets_data.transpose(), dtype=torch.long)
targets = targets.to(device)
input1 = input1.to(device)
output = model(input1)
loss = Loss_Func(output, targets)
loss.backward()
optimizer.step()
if (j * int(len(network_input) / batch_size) + i + 1) % 10 == 0:
print('Epoch: {}, Loss:{:.5f}'.format(
j * int(len(network_input) / batch_size) + i + 1,
loss.item()))
torch.save(model, 'Melody_LSTM/model_1857.pkl')
return 0
def addOrder():
if getOrder() != []:
for i in getOrder():
for g in GetData.read():
if int(i.ticket) == int(g.ticket):
i.stato = "Open"
else:
i.stato = "Closed"
modifyOrder(i)
for t in GetData.read():
found = "no"
t.stato = "Open"
if getOrder() !=[]:
for o in getOrder():
if int(t.ticket) == int(o.ticket):
found = "yes"
if found == "no":
ticket = t.ticket
typeop = "BUY"
if (int(t.typeop) == 1):
typeop = "SELL"
symbol=t.symbol
Orderop = t.Orderop
SL = t.SL
TP= t.TP
close= t.close
stato = t.stato
profit = t.profit
quantity = t.quantity
#BinanceApi.Object(t)
name = request.cookies.get('userID')
print(name)
sql = ("INSERT INTO position VALUES (%s, %s,%s, %s,%s, %s,%s,%s,%s,%s,%s)")
values = (ticket, typeop,symbol,SL,TP,Orderop,close,stato,quantity,profit,"FauxL")
conn = mysql.connect()
cursor = conn.cursor()
cursor.execute(sql,values)
conn.commit()
cursor.close()
if found == "yes":
modifyOrder(t)
return render_template("Home.html")
def main():
# Retrieve the Cleaned Data:
df = Data.get_clean_dataframe()
# train_test_split and balance data if desired by setting imbalanced to true:
train_x, test_x, train_y, test_y = GetData.get_train_test_split(df, imbalanced=True)
# Call both Models:
lr_model, lr_preds = logistic_regression_model(train_x, test_x, train_y, test_y)
rf_model, random_forest_preds = random_forest(train_x, test_x, train_y, test_y)
# # Find out where models differ in predictions: Unmute this code to run this:
# different_predictions = GetData.cross_validation(lr_model=lr_model, rf_model=rf_model, x=test_x, y=test_y)
# different_predictions.to_csv('data/Compare_Models.csv')
# Print Classification Reports:
print("Logistic Regression Metrics:")
print(GetData.get_metrics(test_y, lr_preds) )
print("Random Foreset Metrics:")
print(GetData.get_metrics(test_y, random_forest_preds) )
def register():
if request.method == 'POST':
asd = request.json
sql = ("INSERT INTO users VALUES (%s, %s,%s,%s, %s,%s,%s, %s)")
Account = GetData.readAccountInfo()
values = (asd["username"], asd["password"],asd["email"],"","", Account.id, Account.balance, Account.name)
conn = mysql.connect()
cursor = conn.cursor()
cursor.execute(sql, values)
conn.commit()
return render_template("login.html")
def download(self, term, oj_stu):
"""
:param term:
:type term: int
:param oj_stu:
:type oj_stu: dict
"""
host = 'http://www.icourse163.org/dwr/call/plaincall/YocOJQuizBean.getOJPaperDto.dwr'
data = GetData.get_download_data(term, oj_stu['id'])
info = {
'term': term,
'stuId': oj_stu['id'],
'stuName': oj_stu['realName'],
'stuNickname': oj_stu['nickname'],
'ojName': oj_stu['ojName'],
'ojId': oj_stu['ojId'],
}
req = requests.post(host,
headers=self.headers,
data=data,
cookies=self.cookies)
if not self.ex_id_set:
self.ex_id_set = GetData.write_ex_info(req.text, info)
code_dict = GetData.data_manage_code(req.text, info, self.ex_id_set)
if not code_dict:
return 1
req.close()
file_name = '_'.join([info['stuNickname'], info['stuName']])
download_path = '/'.join(
[self.download_path, info['ojName'], file_name])
new_folder(download_path)
with open('/'.join([download_path, 'information.json']),
'w') as js_file:
js = json.dumps(code_dict)
js_file.write(js)
self.write_in(code_dict, download_path)
return 0
def test_check_metaFile_exists(self, os_mock):
#------------------------ Arrange
path = '/temp/poo'
os_mock.return_value = True
# Act
inst = GetData.GetData()
result = inst._check_metaFile_exists(filePath=path)
# Assert
self.assertEqual(result, True)
self.assertTrue(call(path) in os_mock.mock_calls)
def lone_trips(list_of_merged_trips):
lone_trips_list = []
all_trips = GetData.GetData(WINDOW_SIZE, DATA_LIMIT)
for trip in all_trips:
flag = True
for merged_trip in list_of_merged_trips:
if (merged_trip.contains(trip.id)):
flag = False
break
if flag:
lone_trips_list.append(trip.id)
return lone_trips_list
def data():
name = request.cookies.get('userID')
balance = GetData.readAccountInfo()
sql = ("UPDATE users SET AccountID=%s, AccountBalance=%s, AccountName=%s where user=%s")
print(balance.PrintAccount())
values = (balance.id, balance.balance, balance.name, name)
conn = mysql.connect()
cursor = conn.cursor()
cursor.execute(sql, values)
conn.commit()
cursor.close()
return render_template("Data.html")
def main():
print('Run: ',__name__, '.main')
np_x = gd.GenerateX(50, TIME)
np_x_degree = gd.RadianToDegree(np_x)
np_y = gd.GetSinus(np_x, AMPLITUDE, 0, 0, 0)
np_y_delay = dul.Delay_Uline(np_y)
np_atan = atan.atangens2(np_y, np_y_delay)
np_x_51 = gd.GenerateX(55, TIME)
np_x_51_degree = gd.RadianToDegree(np_x_51)
np_y_51 = gd.GetSinus(np_x_51, AMPLITUDE, 0, 0, 0)
np_y_51_delay = dul.Delay_Uline(np_y_51)
np_51_atan = atan.atangens2(np_y, np_y_51_delay)
plt.plot(np_x_degree, np_y/np.amax(np_y))
plt.plot(np_x_degree, np_y_delay/np.amax(np_y_delay))
plt.plot(np_x_degree, np_atan/(np.amax(np_atan)))
#plt.plot(np_x_degree, np_y_51/np.amax(np_y_51))
PhaseShift = np_atan - np_51_atan
print( 'np.amin(PhaseShift):', np.amin(PhaseShift) )
print( 'np.amax(PhaseShift):', np.amax(PhaseShift) )
#plt.plot( np_x_degree, PhaseShift/np.amin(PhaseShift) )
plt.grid(True)
plt.show()
def insert():
print(GetData.readInfo())
if request.method == 'POST':
symbol = request.form.get("symbol")
tp = request.form.get("tp")
sl = request.form.get("sl")
volume = request.form.get("volume")
tipo = request.form.get("tipo")
if WriteOrder(symbol, tp, sl,volume,tipo):
flask.flash('Your order has been sent to Metatrader successfully, check if it is open on your account!')
return redirect("/insert")
else:
flask.flash('There is a problem with your order!')
return render_template("NewOrder.html")
# encoding=utf-8
import twitter, GetData
import time
# 初始化Twitter的用户信息
api = twitter.Api(consumer_key='EKEMZjnkpUu7p8CbICyFKnUfD',
consumer_secret='lKnVGbM4V8fTwwBjy7uVwo1IyQBpgUBzB0r9R6wAvUmGaNdBwe',
access_token_key='720077099688706048-KYyr0iV9aF5vsBl0yQGrjLB8sXBizmr',
access_token_secret='D7kbKR9N1rHdYmtnUa6CdPs9qt1gNy8rEsdAIFBoC4Rhu')
# 获取需要爬取的用户数据，元组或列表
data = GetData.get_id_from_xls("F:\\0722\\twitter-52-following-id.xls")
# 定义存储路径
folder_path = ""
def follow_by_id():
i = 0
for a in data:
try:
api.CreateFriendship(user_id=a, follow=True)
print str(i) + " " + str(a) + "已经关注"
time.sleep(5)
except Exception, e:
print e
print str(i) + " " + str(a) + "关注失败"
# for ti in xrange(5):
# try:
# api.CreateFriendship(user_id=a, follow=True)
#!/usr/bin/env python #coding: utf-8 from matplotlib.pylab import * rcParams['legend.numpoints'] = 1 import GetData import perceptron option = [1,2,3,4] trainset = GetData.generateData(1) # train set generation testset = GetData.generateData(2) # test set generation axis([-2, 15, -4, 10]) flag1=0 flag2=0 for x in trainset: if x[3] > 0: if (flag1==0): plot(x[0], x[1], 'ob', label='class:1') flag1=1 else: plot(x[0], x[1], 'ob') else: if(flag2==0): plot(-x[0], -x[1], 'or',label='class:-1') flag2=1 else: plot(-x[0], -x[1], 'or') legend()
# encoding=utf-8
import GetData
import twitter
import SaveData
import time
# 初始化Twitter的用户信息
api = twitter.Api(consumer_key='EKEMZjnkpUu7p8CbICyFKnUfD',
consumer_secret='lKnVGbM4V8fTwwBjy7uVwo1IyQBpgUBzB0r9R6wAvUmGaNdBwe',
access_token_key='720077099688706048-KYyr0iV9aF5vsBl0yQGrjLB8sXBizmr',
access_token_secret='D7kbKR9N1rHdYmtnUa6CdPs9qt1gNy8rEsdAIFBoC4Rhu')
# 获取需要爬取的用户数据，元组或列表
data = GetData.get_data_from_xls("H:\\TWITTER\\IOT.xls")
# 用户数据存储文件夹
folder_path = "H:\\TWITTER\\friendship\\"
def get_following():
for screen_name in data:
print screen_name
followers = api.GetFriends(screen_name=screen_name)
print len(followers)
SaveData.sava_user_to_xml(users=followers, folder_path=folder_path + "followering/" + screen_name + "/")
time.sleep(60)
def get_following_id_by_name():
id_ = 1
for screen_name in data:
__author__ = 'Pierzchalski'
import GetData as gd
import matplotlib.pyplot as pp
import numpy as np
import os
import shutil
maxIndex = 720
paramsToIndexes = gd.paramsToIndexesDict(maxIndex)
paramsToDicts = gd.paramsToDicts(maxIndex)
plotsDir = os.path.join('C:\\Users', 'Pierzchalski', 'IdeaProjects', 'trafficLightRL', 'out', 'plots')
def plotFn(params, figureIndex, fn, save=True, saveName="plot", show=False):
indexes = paramsToIndexes[params]
pp.figure(figureIndex)
fn(indexes)
if save:
plotDir = os.path.join(plotsDir, 'plot%d' % figureIndex)
if not os.path.exists(plotDir): os.makedirs(plotDir)
pp.savefig(os.path.join(plotDir, saveName))
if show: pp.show()
def copyReplaceParams(figureIndex, paramsList):
plotDir = os.path.join(plotsDir, 'plot%d' % figureIndex)
for f in os.listdir(plotDir):
if f.endswith('.txt'):
os.remove(os.path.join(plotDir, f))
for param in paramsList:
# ========================================================================= # Dependances / Librairies # ========================================================================= import multiprocessing import GetData import TreatWeft # thread de traitement et insertion en base treatWeftThread = multiprocessing.Process(target=TreatWeft.doIt) treatWeftThread.start() # recuperation de la tram GetData.doIt()
def getFormatData(self,tmp):
res = dict()
res['timestamp'] = tmp['timestamp']
res['srcMac'] = tmp['smac']
res['dstMac'] = tmp['dmac']
res['origin'] = tmp['origin']
if tmp['IP'] or tmp['IP6']:
if(tmp['IP6']):
ipData = tmp['IP6']
res['srcIP'] = GetData.ipv6AddrFormat(ipData.src)
res['dstIP'] = GetData.ipv6AddrFormat(ipData.dst)
else:
ipData = tmp['IP']
res['srcIP'] = GetData.ipAddrFormat(ipData.src)
res['dstIP'] = GetData.ipAddrFormat(ipData.dst)
if isinstance(ipData.data,dpkt.tcp.TCP):
res['type'] = 'TCP'
tcpData = ipData.data
res['srcPort'] = str(tcpData.sport)
res['dstPort'] = str(tcpData.dport)
if(str(tcpData.sport) == '80' or str(tcpData.dport) == '80'):
res['type'] = 'HTTP'
if(str(tcpData.sport) == '443' or str(tcpData.dport) == '443'):
res['type'] = 'HTTPS'
if(str(tcpData.sport) == '53' or str(tcpData.dport) == '53'):
res['type'] = 'DNS'
elif isinstance(ipData.data,dpkt.udp.UDP):
res['type'] = 'UDP'
udpData = ipData.data
res['srcPort'] = str(udpData.sport)
res['dstPort'] = str(udpData.dport)
if(str(udpData.sport) == '53' or str(udpData.dport) == '53'):
res['type'] = 'DNS'
elif isinstance(ipData.data,dpkt.icmp.ICMP):
res['type'] = 'ICMP'
elif isinstance(ipData.data,dpkt.igmp.IGMP):
res['type'] = 'IGMP'
else:
if tmp['IP6']:
res['type'] = 'IPV6'
else:
res['type'] = 'IPV4'
if res['type'] in ['TCP','UDP','DNS','HTTP','HTTPS']:
resStr = ' '+res['type']+' '+res['srcMac']+' |->| '+res['dstMac']+' '+res['srcIP']+' (:'+res['srcPort']+') ===>>> '+res['dstIP']+' (:'+res['dstPort']+')'
else:
resStr = ' '+res['type']+' '+res['srcMac']+' |->| '+res['dstMac']+' '+res['srcIP']+' ===>>> '+res['dstIP'] + ' '
elif tmp['ARP'] :
res['type'] = 'ARP'
resStr = ' '+res['type']+' '+res['srcMac']+' |->| '+res['dstMac']
else:
res['type'] = '暂未支持分析的网络层数据包种类'
resStr = ' '+res['type']+' '+res['srcMac']+' |->| '+res['dstMac']
if self.filterInfo :
if res['type'] not in self.filterInfo['protocols']:
#print self.filterInfo['protocols']
#print res['type']
print '不符合的数据包～～～！'
return None
#print self.filterInfo
for name,weath in (self.filterInfo['detal']).items():
if weath :
#print name
#print weath
try:
if not (res[name] == weath):
#print '不符合的数据包～～～！'
return None
except Exception,e:
#print '不符合的数据包～～～！'
return None
#encoding=utf-8
import GetData
import twitter
import SaveData
import time
# 初始化Twitter的用户信息
api = twitter.Api(consumer_key='EKEMZjnkpUu7p8CbICyFKnUfD',
consumer_secret='lKnVGbM4V8fTwwBjy7uVwo1IyQBpgUBzB0r9R6wAvUmGaNdBwe',
access_token_key='720077099688706048-KYyr0iV9aF5vsBl0yQGrjLB8sXBizmr',
access_token_secret='D7kbKR9N1rHdYmtnUa6CdPs9qt1gNy8rEsdAIFBoC4Rhu')
# 获取需要爬取的用户数据，元组或列表
data = GetData.get_data_from_xls("F:\\0901\\400.xls")
# 定义存储路径
folder_path = "F:\\0901\\400\\"
"""
err_log 记载出错的账户名称
right_log记载正确账户名称
id_log 记载截至程序停止为止当前最大ID，启动程序时需要更新
"""
err_log = open("F:/err_log.txt", 'a')
right_log = open("F:/right_log.txt", 'a')
id_log = open("F:/id_log.txt", 'w')
def main():
for per in data:
# encoding=utf-8
import GetData
import twitter
import SaveData
import time
# 初始化Twitter的用户信息
api = twitter.Api(consumer_key='EKEMZjnkpUu7p8CbICyFKnUfD',
consumer_secret='lKnVGbM4V8fTwwBjy7uVwo1IyQBpgUBzB0r9R6wAvUmGaNdBwe',
access_token_key='720077099688706048-KYyr0iV9aF5vsBl0yQGrjLB8sXBizmr',
access_token_secret='D7kbKR9N1rHdYmtnUa6CdPs9qt1gNy8rEsdAIFBoC4Rhu')
# 获取需要爬取的用户数据，元组或列表
data = GetData.get_data_from_xls("F:\\6000\\6000\\6000.xls")
# 定义存储路径
folder_path = "F:\\6000\\6000\\6000-0901\\"
def main():
for per in data:
try:
get_status_by_id(per)
except Exception, e:
print e
def get_status(screen_name):
print screen_name + " start crawler"
statuses = api.GetUserTimeline(screen_name=screen_name, count=200)
