class HtmlParser(object):
def __init__(self):
self.sa = SaveData()
def parser(self, response, filename):
html = etree.HTML(response, etree.HTMLParser())
try:
data = html.xpath("//tbody")
for i in data:
title = i.xpath(".//a[@class='s xst']/text()")
l = i.xpath(".//a[@class='s xst']/@href") # 链接的部分内容
time = i.xpath(".//td[@class='by']/em/span/text()")
power = i.xpath(".//span[@class='xw1']/text()")
if len(title) != 0:
# 构建完整的链接
link = 'https://www.52pojie.cn/' + l[0]
print(title[0])
print(link)
print(time[0])
# 如果需要阅读权限则不显示该链接,不保存数据
if len(power) == 0:
with open(filename + '.csv',
'a+',
encoding='utf-8-sig',
newline='') as f:
w = csv.writer(f)
# 使用savedata方法保存数据到csv文件中
self.sa.savedata(w, title[0], link, time[0])
print('------------------------------')
except Exception as e:
print(e)
def __init__(self):
# 初始化程序
self.download = Downloader()
self.parser = HtmlParser()
self.save = SaveData()
self.workbook = Workbook()
self.ch = Choice()
print('初始化完成...')
class SpiderMain(object):
def __init__(self):
# 初始化程序
self.download = Downloader()
self.parser = HtmlParser()
self.save = SaveData()
self.workbook = Workbook()
self.ch = Choice()
print('初始化完成...')
def run(self):
while True:
try:
p = int(input('想要爬多少页的数据？' + '\n'))
break
except ValueError:
print('输入错误！请输入数字')
page = p + 1
print("================================")
print(' A.原创发布区 B.精品软件区 ')
print(' C.脱壳破解区 D.移动安全区 ')
print(' E.病毒分析区 F.编程语言区 ')
print(' G.软件调试区 H.动画发布区 ')
print(' I.逆向资源区 J.安全工具区 ')
print(' K.招聘求职区 ')
print("================================")
while True:
choice = input("选择爬取的专区，输入 Q 退出程序(输入的字母必须大写)：")
half_url, name = self.ch.make_the_arrg(choice)
if name != 'Error':
break
print(half_url + '\n' + name)
self.save.createfile(name)
for i in range(1, page):
url = half_url + str(i) + '.html'
response = self.download.download(url)
self.parser.parser(response, name)
sleep = random.randint(2, 10)
print('爬取第' + str(i) + '页完成，程序休息' + str(sleep) + '秒')
time.sleep(sleep) # 程序睡眠
if i != page - 1:
print('-----------------------------')
print(' 下一页 ')
print('-----------------------------')
print('数据写入完成，正在进行数据去重...')
self.save.delete_same_data()
try:
self.workbook.save('将csv的数据导入此表.xlsx')
except:
print('创建xlsx文件失败，请手动创建')
print('程序运行完毕')
def read_data_event(self):
scr_text.delete(0.0, END) # 清空text
#禁用按钮
b1['state'] = DISABLED
b3['state'] = DISABLED
# 初始化
save_data = SaveData()
scr_text.insert(END, "读取数据:\n")
scr_text.see(END)
data = save_data.data_read()
scr_text.insert(END, "读取完成:\n")
scr_text.see(END)
#开启按钮
b1['state'] = NORMAL
b3['state'] = NORMAL
return 1
def courseSave(self,db='../db/course.txt'):
dict={}
dict['courseName'] = self.courseName
dict['cycle'] = self.cycle
dict['price'] = self.price
#gradeName,schoolName通过查询获取
dict['gradeName'] = self.gradeName
dict['schoolName'] = self.schoolName
SaveData(dict,db)
def save_data_event(self):
self.path = ""
self.path = input("请输入数据所在路径:")
# 路径为空直接返回
if self.path == "":
print("请输入路径!")
return -1
self.path = self.path + "/"
# 初始化
save_data = SaveData()
print("开始存储.")
# 获取指定文件夹列表
dir_names = [name for name in os.listdir(self.path) if 'WIFI+' in name]
for dir_name in dir_names:
phone_model = (dir_name.split('+'))[1] # 获取手机型号
dir_path = self.path + dir_name + "/"
# 显示进度
print("目录 " + dir_path + " 下的文件正在存储.")
# 获取指定目录下所有数据的文件名
self.file_name = []
tt = os.walk(dir_path)
for i in tt:
for j in i[2]:
if ".txt" in j:
self.file_name.append(j)
# 遍历文件夹,存储所有数据
file_len = len(self.file_name)
for x in range(file_len):
file = self.file_name[x]
begin_time = time.time()
print(file + " saving...[" + str(x + 1) + "/" + str(file_len) +
"]")
# 存储单文件数据
flag = save_data.data_save(dir_path, file, phone_model)
if flag != 1: # 存储出错
print("数据库插入数据出错.")
print("出错位置:" + file + " " + flag)
save_data.close_connect()
return -1
end_time = time.time()
print(file + " saved,time:" + str(int(end_time - begin_time)) +
"s")
print("该文件夹存储完成.")
# 全部存储结束断开连接
save_data.close_connect()
print("结束存储.")
return 1
def __init__(self, sleep_time, debug=False):
super(Thread_COM, self).__init__()
print "COM Init"
self.arduino = COMwithArduino(0x12, 10, debug=False)
self.transmission_time = 0.005
self.sleep_time = sleep_time - self.transmission_time
self.debug = debug
#Logging init
self.log = SaveData("./images/test/log")
COM_header = ["Battery", "Luminosity", "Remote Speed", "Remote Steering", "Remote Switch", "Motor Speed", "Steering", "Motor State"]
Motor_header = ["Motor speed", "Steering"]
#IMU_header = ["IMU Roll", "IMU Pitch", "IMU Yaw", "Gyro Roll", "Gyro Pitch", "Gyro Yaw", "Acc X", "Acc Y", "Acc Z"]
#GPS_header = ["GPS Longitude", "GPS Lattitude", "GPS altitude"]
Camera_header = ["Camera X", "Camera Y"]
header = ["Time"]
header.extend(COM_header)
header.extend(Motor_header)
#header.extend(IMU_header)
#header.extend(GPS_header)
header.extend(Camera_header)
self.log.write(header)
def save_data_event():
wifi_path = "Wi-Fi_Data/"
bt_path = "BT_Data/"
save_data = SaveData()
print("开始存储")
# 获取文件夹列表
dir_names = [name for name in os.listdir(wifi_path)]
for dir_name in dir_names:
room_name = dir_name.split("_")[0]
room_device = room_devices[room_name] #设备匹配
addr = "shilintong" + room_name
# 显示进度
print(room_name + " 下的文件正在存储")
file_names = [name for name in os.listdir(wifi_path + dir_name)]
for file_name in file_names:
file_num = file_name.split("_")[0]
# 说明：按照遥感测绘坐标系规定应该是，N为x正方向，E为y正方向
# 但此处在处理之前n和e弄混了，因此x，y和n，e的对应刚好是反的
# 警示后人
coo_x = file_name.split("_")[1]
coo_y = (file_name.split("_")[2])[:-4]
begin_time = time.time()
# 存储wifi文件数据
flag = save_data.data_save(wifi_path + dir_name + "/" + file_name,
1, addr, coo_x, coo_y, room_device)
if flag != 1: # 存储出错
print("出错位置:" + wifi_path + dir_name + "/" + file_num + " " +
flag)
save_data.close_connect()
return -1
# 存储bt单文件数据
bt_file_names = [name for name in os.listdir(bt_path + dir_name)]
if file_name in bt_file_names:
flag = save_data.data_save(
bt_path + dir_name + "/" + file_name, 2, addr, coo_x,
coo_y, room_device)
else:
flag = save_data.data_save(
bt_path + dir_name + "/" + file_num + ".csv", 2, addr,
coo_x, coo_y, room_device)
if flag != 1: # 存储出错
print("出错位置:" + bt_path + dir_name + "/" + file_num + " " +
flag)
save_data.close_connect()
return -1
end_time = time.time()
# print(dir_name + "/"+ file_num + " saved,time:"+str(int(end_time-begin_time))+"s")
# print("该文件夹存储完成.")
# 全部存储结束断开连接
save_data.close_connect()
return 1
def gradeSave(self,db='../db/grade.txt'):
dict={}
dict['gradeName'] = self.gradeName
dict['schoolName'] = self.schoolName
SaveData(dict,db)
def __init__(self):
self.sa = SaveData()
def save_data_event(self):
scr_text.delete(0.0, END) # 清空text
self.path = r_path.get() # 获取路径
# 路径为空直接返回
if self.path == "":
scr_text.insert(END, "请选择路径!\n")
scr_text.see(END)
return -1
self.path = self.path + "/"
#禁用按钮
b1['state'] = DISABLED
b3['state'] = DISABLED
# 初始化
save_data = SaveData()
scr_text.insert(END, "开始存储\n")
scr_text.see(END)
phone_model = "none" # 获取手机型号
dir_path = self.path
# 显示进度
scr_text.insert(END, "目录" + dir_path + " 下的文件正在存储\n")
scr_text.see(END)
# 获取指定目录下所有数据的文件名
self.file_name = []
tt = os.walk(dir_path)
for i in tt:
for j in i[2]:
if ".txt" in j:
self.file_name.append(j)
# 遍历文件夹,存储所有数据
file_len = len(self.file_name)
for x in range(file_len):
file = self.file_name[x]
begin_time = time.time()
scr_text.insert(
END, file + " saving...[" + str(x + 1) + "/" + str(file_len) +
"]\n")
scr_text.see(END)
# 存储单文件数据
flag = save_data.data_save(dir_path, file, phone_model)
if flag != 1: # 存储出错
scr_text.insert(END, "数据库插入数据出错\n")
scr_text.see(END)
scr_text.insert(END, "出错位置:" + file + " " + flag)
scr_text.see(END)
save_data.close_connect()
#开启按钮
b1['state'] = NORMAL
b3['state'] = NORMAL
return -1
end_time = time.time()
scr_text.insert(
END, file + " saved,time:" + str(int(end_time - begin_time)) +
"s\n")
scr_text.see(END)
scr_text.insert(END, "该文件夹存储完成.\n")
scr_text.see(END)
# 全部存储结束断开连接
save_data.close_connect()
scr_text.insert(END, "结束存储\n")
scr_text.see(END)
#开启按钮
b1['state'] = NORMAL
b3['state'] = NORMAL
return 1
def save_data_event(self, set_button):
scr_text.delete(0.0, END) # 清空text
self.path = r_path.get() # 获取路径
# 路径为空直接返回
if self.path == "":
scr_text.insert(END, "请选择路径!\n")
scr_text.see(END)
return -1
self.path = self.path + "/"
#禁用按钮
b1['state'] = DISABLED
b2['state'] = DISABLED
b3['state'] = DISABLED
# 初始化
save_data = SaveData()
# 显示进度
scr_text.insert(END, "目录" + self.path + " 下的文件正在存储\n")
scr_text.see(END)
# 获取指定目录下所有数据的文件名
self.file_name = []
tt = os.walk(self.path)
for i in tt:
for j in i[2]:
if ".txt" in j:
self.file_name.append(j)
# 遍历文件夹,存储所有数据
file_len = len(self.file_name)
for x in range(file_len):
file = self.file_name[x]
begin_time = time.time()
scr_text.insert(
END, file + " saving...[" + str(x + 1) + "/" + str(file_len) +
"]\n")
scr_text.see(END)
# 存储单文件数据
if set_button == 0: # 0=完整数据存储
flag = save_data.complete_data_save(self.path, file)
else: # 1=原始数据存储
flag = save_data.initial_data_save(self.path, file)
if flag != 1: # 存储出错
scr_text.insert(END, "数据库插入数据出错\n")
scr_text.see(END)
scr_text.insert(END, "出错位置:" + file + " " + flag)
scr_text.see(END)
break
end_time = time.time()
scr_text.insert(
END, file + " saved,time:" + str(int(end_time - begin_time)) +
"s\n")
scr_text.see(END)
scr_text.insert(END, "操作结束.")
scr_text.see(END)
# 操作结束断开连接
save_data.close_connect()
#开启按钮
b1['state'] = NORMAL
b2['state'] = NORMAL
b3['state'] = NORMAL
return 1
class Thread_COM(Thread):
def __init__(self, sleep_time, debug=False):
super(Thread_COM, self).__init__()
print "COM Init"
self.arduino = COMwithArduino(0x12, 10, debug=False)
self.transmission_time = 0.005
self.sleep_time = sleep_time - self.transmission_time
self.debug = debug
#Logging init
self.log = SaveData("./images/test/log")
COM_header = ["Battery", "Luminosity", "Remote Speed", "Remote Steering", "Remote Switch", "Motor Speed", "Steering", "Motor State"]
Motor_header = ["Motor speed", "Steering"]
#IMU_header = ["IMU Roll", "IMU Pitch", "IMU Yaw", "Gyro Roll", "Gyro Pitch", "Gyro Yaw", "Acc X", "Acc Y", "Acc Z"]
#GPS_header = ["GPS Longitude", "GPS Lattitude", "GPS altitude"]
Camera_header = ["Camera X", "Camera Y"]
header = ["Time"]
header.extend(COM_header)
header.extend(Motor_header)
#header.extend(IMU_header)
#header.extend(GPS_header)
header.extend(Camera_header)
self.log.write(header)
def run(self):
while not Done:
global COM_message, Mode
COM_message = self.arduino.Read() #Receive data from Arduino
if COM_message == False:
continue #skip the rest of this while loop and goes back to testing the expression
#if self.debug:
#print "COM: ", COM_message
battery_voltage = COM_message[0]
remote_speed = COM_message[2]-100
remote_steering = COM_message[3]-100
Mode = COM_message[4] #Mode = switch value
if battery_voltage > 70:
if Mode == MODE_REMOTE: #if Mode = 0, no processing, data are only logged
motor_speed = remote_speed
motor_steering = remote_steering
elif Mode == MODE_CAMERA: #Autonomous mode with camera
if BestBlob:
motor_steering = int( (BestBlob[0]-Camera_target[0]) ) #image width
motor_speed = int( (Camera_target[1] - BestBlob[1])/2 ) #image height
else:
motor_speed = 0
motor_steering = 0
else: #if Mode = 2, stop the car
motor_speed = 0
motor_steering = 0
else: # Low battery : stop the car
motor_speed = 0
motor_steering = 0
message2Arduino = [motor_speed+100, motor_steering+100, 0]
if self.debug:
print' BestBlob: ',BestBlob[0],' \t', motor_steering
self.arduino.Send(message2Arduino) #Send data to Arduino
#Log data
data = []
data.extend([time.time()])
data.extend(COM_message)
data.extend(message2Arduino)
data.extend(BestBlob)
#data.extend(IMU_position)
#data.extend(IMU_gyro)
#data.extend(IMU_accel)
#data.extend(GPS_position)
#data.extend(im_proc)
# if self.debug:
# print "Logging: ", data
self.log.write(data)
time.sleep(self.sleep_time)
self.log.close()
if self.debug:
print "Thread COM exiting"
def schoolSave(self, db='../db/school.txt'):
dict = {}
dict['schoolName'] = self.schoolName
SaveData(dict, db)
def save_data_event(self):
scr_text.delete(0.0, END) # 清空text
self.path = r_path.get() # 获取路径
# 路径为空直接返回
if self.path == "":
scr_text.insert(END, "请选择路径!\n")
scr_text.see(END)
return -1
self.path = self.path + "/"
tt_time = []
with open('time.csv', 'r') as file_read:
while True:
lines = file_read.readline() # 读取整行数据
if not lines:
break
tt_time.append(lines.split("\n")[0].split(","))
tt_time = tt_time[0:-1]
print(tt_time)
#禁用按钮
b1['state'] = DISABLED
b3['state'] = DISABLED
# 初始化
save_data = SaveData()
scr_text.insert(END, "开始存储\n")
scr_text.see(END)
# 获取指定文件夹列表
dir_names = [name for name in os.listdir(self.path) if 'WIFI+' in name]
for dir_name in dir_names:
phone_model = (dir_name.split('+'))[1] # 获取手机型号
dir_path = self.path + dir_name + "/"
# 显示进度
scr_text.insert(END, "目录" + dir_path + " 下的文件正在存储\n")
scr_text.see(END)
# 获取指定目录下所有数据的文件名
self.file_name = []
tt = os.walk(dir_path)
for i in tt:
for j in i[2]:
if ".txt" in j:
self.file_name.append(j)
# 遍历文件夹,存储所有数据
file_len = len(self.file_name)
for x in range(file_len):
file = self.file_name[x]
begin_time = time.time()
scr_text.insert(
END, file + " saving...[" + str(x + 1) + "/" +
str(file_len) + "]\n")
scr_text.see(END)
# 存储单文件数据
flag = save_data.data_save(dir_path, file, phone_model,
tt_time)
if isinstance(flag, str): # 存储出错
scr_text.insert(END, "数据库插入数据出错\n")
scr_text.see(END)
scr_text.insert(END, "出错位置:" + file + " " + flag)
scr_text.see(END)
save_data.close_connect()
#开启按钮
b1['state'] = NORMAL
b3['state'] = NORMAL
return -1
end_time = time.time()
scr_text.insert(
END, file + " saved,time:" +
str(int(end_time - begin_time)) + "s\n")
scr_text.see(END)
scr_text.insert(END, "该文件夹存储完成.\n")
scr_text.see(END)
# 全部存储结束断开连接
save_data.close_connect()
scr_text.insert(END, "结束存储\n")
scr_text.see(END)
#开启按钮
b1['state'] = NORMAL
b3['state'] = NORMAL
return 1
@author: Raivo Koot
"""
from regressionbasics import RegressionBasics
from dataStorage import DataStorage
from saveData import SaveData
# initialize objects
regressionObject = RegressionBasics()
dataStorage = DataStorage()
# import data from pc and set up different matrices
regressionObject.loadAndPrepareData("mlr03.xls", dataStorage)
#compute the cost of our hypothesis
#cost = regressionObject.computeCost(dataStorage)
#print(cost)
regressionObject.gradientDescent(dataStorage, 0.000003)
print("error")
print(regressionObject.computeCost(dataStorage))
print(dataStorage.thetaVector)
dataSave = SaveData("theta.txt", "w")
dataSave.writeThetaToFile(dataStorage)
dataSave.closeFile()