def get_ships():
page = 1
ship_json = get_ship_json(page)
html = ship_json.get('data').get('html')
totalrows = ship_json.get('data').get('totalrows')
rowcount = ship_json.get('data').get('rowcount')
init_ship(html)
need_req_time = float(totalrows) / rowcount
if need_req_time != 0:
need_req_time += 1
for page in range(2, 2 + int(need_req_time)):
print "正在获取第 " + str(page) + "页数据"
ship_json = get_ship_json(page)
html = ship_json.get('data').get('html')
init_ship(html)
mysql_helper = MysqlHelper()
# 插入公司
Log.d('开始将公司插入数据库')
mysql_helper.insert_manufacturer(manufacturer_list)
from CitizenWikiRobot.ship_robot import get_ships
from CitizenWikiRobot.roadmap import init_road_map
from CitizenWikiRobot.MysqlHelper import MysqlHelper
from CitizenWikiRobot.comm_link_spectrum_dispatch import get_galactic_guide
if __name__ == '__main__':
my = MysqlHelper()
my.insert_constant_translate()
def get_galactic_guide():
mysql_helper = MysqlHelper()
s = requests.Session()
url = 'https://robertsspaceindustries.com/api/hub/getCommlinkItems'
road_header = {
'Connection': 'close',
'x-rsi-token': '6ec0661bc4216ad2b6017727e59349e7',
'accept': 'application/json',
'accept-encoding': 'gzip, deflate, br',
'user-agent': 'Mozilla/5.0 Chrome/66.0.3359.139 Mobile Safari/537.36',
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8'
}
for page in range(0, 6):
galactic_guide_req = ''
pamas = {"channel": "spectrum-dispatch", "page": str(page), "series": "galactic-guide", "sort": "publish_new"}
while galactic_guide_req == '':
try:
galactic_guide_req = s.post(url=url, data=pamas, headers=road_header)
except:
print("Connection refused by the server..")
print("Let me sleep for 5 seconds")
print("ZZzzzz...")
time.sleep(5)
print("Was a nice sleep, now let me continue...")
continue
galactic_guide_json = json.loads(galactic_guide_req.content)
data = galactic_guide_json['data']
guide_soup = bs4.BeautifulSoup(data, "lxml")
for link in guide_soup.findAll('a'):
guide_url = link.get('href')
if guide_url is None:
continue
icon_origin_data = link.find('div', class_='background').get('style')
start_index = icon_origin_data.find('(')
end_index = icon_origin_data.find(')')
background = icon_origin_data[start_index + +2:end_index - 1]
title_tag = link.find('div', class_='title trans-opacity trans-03s')
title = title_tag.string
comm_link = CommLink()
comm_link.type = 'galactic_guide'
comm_link.url = guide_url
comm_link.title = title
comm_link.background = background
link_page = ''
while link_page == '':
try:
link_page = requests.get(StaticField.BASE_URL + guide_url)
except:
print("Connection refused by the server..")
print("Let me sleep for 5 seconds")
print("ZZzzzz...")
time.sleep(5)
print("Was a nice sleep, now let me continue...")
continue
link_soup = bs4.BeautifulSoup(link_page.content, "lxml")
content_block1_tag = link_soup.find('div', class_='content-block1 rsi-markup')
content_tag = content_block1_tag.find('div', class_='content')
first_image_tag = link_soup.find('div', class_='content clearfix')
if first_image_tag is not None:
image_tag = first_image_tag.find('img')
if image_tag is not None:
first_image_comm_content = CommLinkContent()
first_image_comm_content.content_type = "image"
first_image_comm_content.content_data = image_tag.get('src')
first_image_comm_content.data_index = 0
# Log.d(first_image_comm_content.content_data)
comm_link.content.append(first_image_comm_content)
content = content_tag.find_all('div', class_='variant-block')
if len(content) > 0:
for div in content:
get_comm_link_content(comm_link, div.children)
else:
get_comm_link_content(comm_link, content_tag.children)
mysql_helper.insert_comm_link(comm_link)
Log.d(title)
from CitizenWikiRobot.MysqlHelper import MysqlHelper
import os
from CitizenWikiRobot.Log import Log
import json
if __name__ == '__main__':
my = MysqlHelper()
jsonFile = open(os.curdir + "/priceJson", 'r')
jsonStr = json.loads(jsonFile.read())
for ship_price in jsonStr:
name = ship_price.get('name')
price = ship_price.get('msrp')
price_final = price[1:]
Log.d(price_final)
my.update_ship_price(name, price_final)
def init_vehicle(url):
mysql_helper = MysqlHelper()
vehicle = Vehicle(url)
page = ''
while page == '':
try:
page = requests.get(StaticField.BASE_URL + vehicle.url)
except:
print("Connection refused by the server..")
print("Let me sleep for 5 seconds")
print("ZZzzzz...")
time.sleep(5)
print("Was a nice sleep, now let me continue...")
continue
ship_soup = bs4.BeautifulSoup(page.content, "lxml")
# all_script_tag = ship_soup.find_all('script')
# chassis_id_tag = all_script_tag[1].text
#
# start_index = chassis_id_tag.find('chassis_id:')
# chassis_id_content = chassis_id_tag[start_index:start_index + 30]
# start_index = chassis_id_content.find(':')
# end_index = chassis_id_content.find('}')
# chassis_id = chassis_id_content[start_index + 1: end_index]
chassis_id = __get_chassis_id(ship_soup)
if chassis_id == '':
return
id_url = 'https://robertsspaceindustries.com/ship-matrix/index?chassis_id=' + chassis_id
ships_req = ''
while ships_req == '':
try:
ships_req = requests.get(url=id_url)
except:
print("Connection refused by the server..")
print("Let me sleep for 5 seconds")
print("ZZzzzz...")
time.sleep(5)
print("Was a nice sleep, now let me continue...")
continue
content_json = json.loads(ships_req.content)
data = content_json.get('data')
for ship_item in data:
ship_url = ship_item.get('url')
if url == ship_url:
vehicle.data_json = ship_item
vehicle.init_with_data_json()
break
# icon store_large
thumbnails = ship_soup.find("span", attrs={'class': 'thumbnails clearfix'})
if thumbnails is not None:
img_urls = []
for img_tag in thumbnails.contents:
if type(img_tag) is bs4.Tag and img_tag.attrs['href'] != 'vimeo':
img_url = img_tag.attrs['href']
url_array = img_url.split(".")
if len(url_array) > 1 and (url_array[1] == 'jpg'
or url_array[1] == 'png'):
img_urls.append(img_url)
vehicle.img_urls = img_urls
mysql_helper.insert_vehicle(vehicle)
# -*- coding: UTF-8 -*-
from CitizenWikiRobot.ship_robot import get_ships
from CitizenWikiRobot.roadmap import init_road_map
from CitizenWikiRobot.MysqlHelper import MysqlHelper
from CitizenWikiRobot.comm_link_spectrum_dispatch import get_galactic_guide
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
if __name__ == '__main__':
my = MysqlHelper()
# my.update_apk(2, '1.0.1', "新增功能: 飞船浏览筛选,银河指南系列文章", 'sc_date_view_1.0.1-release.apk')
# my.update_apk(3, '1.0.2', "1.飞船搜索 \n2.飞船列表优化 \n3.修复bug \n下载地址 : https://pan.baidu.com/s/18W7x95GsZ1ME8nyi2AlmiA", 'sc_data_view_1.0.2-release.apk',"test","https://pan.baidu.com/s/18W7x95GsZ1ME8nyi2AlmiA")
# my.update_apk(4, '1.0.3', "1.修复bug \n下载地址 : https://pan.baidu.com/s/1jve5pn1sup-q6GiautlagA", 'sc_data_view_1.0.3-release.apk',"test","https://pan.baidu.com/s/1jve5pn1sup-q6GiautlagA")
my.update_apk(5, '1.0.4', "1.添加飞船价格 \n 2.由于签名由jar签名升级为全Apk签名,所以此次更新不支持直接安装,需要将app卸载重装! 建议使用网盘下载安装\n网盘下载地址 : https://pan.baidu.com/s/1uA89oxDXeRhRl2Itnxtx_Q", 'sc_data_view_1.0.4-release.apk',"test","https://pan.baidu.com/s/1uA89oxDXeRhRl2Itnxtx_Q")
from CitizenWikiRobot.ship_robot import get_ships
from CitizenWikiRobot.roadmap import init_road_map
from CitizenWikiRobot.MysqlHelper import MysqlHelper
from CitizenWikiRobot.comm_link_spectrum_dispatch import get_galactic_guide
if __name__ == '__main__':
mysqlHelper = MysqlHelper()
mysqlHelper.insert_all_ship_type()