def get_stock_smt_cfg_en_name():
# en_list = []
cfg_exist = os.path.isfile("c:/scrapy/stockSmtCfg.xml")
print('c:/scrapy/stockSmtCfg.xml is', cfg_exist)
if cfg_exist:
print('c:/scrapy/stockSmtCfg.xml is already exist')
else:
create_stock_smt_cfg()
tree = ET.parse(globalPars.getToolBaseWorkDirectory() + "stockSmtCfg.xml") #打开xml文档
root = tree.getroot() #获得root节点 '
sNode = root.find('yidian')
if( sNode.find('enable').text == 'Enable' ) :
# en_list.append('yidian')
return 'yidian'
sNode = root.find('tonghuashun')
if( sNode.find('enable').text == 'Enable' ) :
# en_list.append('tonghuashun')
return 'tonghuashun'
sNode = root.find('dazhihui')
if( sNode.find('enable').text == 'Enable' ) :
# en_list.append('dazhihui')
return 'dazhihui'
sNode = root.find('xinlang')
if( sNode.find('enable').text == 'Enable' ) :
# en_list.append('xinlang')
return 'xinlang'
def generateThsUrls():
urls = []
dict = generate_stock_dic()
for key in dict.keys():
urls.append('http://stockpage.10jqka.com.cn/' + key + '/holder/')
#print(urls)
with open(globalPars.getToolBaseWorkDirectory() + 'ths_urls.txt',
'wb') as f:
for url in urls:
f.write(url.encode('utf-8'))
f.write('\n'.encode('utf-8'))
def generateXlUrls():
urls = []
dict = generate_stock_dic()
for key in dict.keys():
# http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_CirculateStockHolder/stockid/603009.phtml#2010-09-30
urls.append(
'http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_CirculateStockHolder/stockid/'
+ key + '.phtml#2010-09-30')
#print(urls)
with open(globalPars.getToolBaseWorkDirectory() + 'xl_urls.txt',
'wb') as f:
for url in urls:
f.write(url.encode('utf-8'))
f.write('\n'.encode('utf-8'))
def generateDzhUrls():
urls = []
print('begin generateDzhUrls')
dict = generate_stock_dic()
for key in dict.keys():
if key.isdigit():
if int(key) < 600000:
urls.append('http://webf10.gw.com.cn/SZ/B10/SZ' + key +
'_B10.html')
else:
urls.append('http://webf10.gw.com.cn/SH/B10/SH' + key +
'_B10.html')
#print(urls)
with open(globalPars.getToolBaseWorkDirectory() + 'dzh_urls.txt',
'wb') as f:
for url in urls:
f.write(url.encode('utf-8'))
f.write('\n'.encode('utf-8'))
def create_stock_smt_cfg():
print('create_stock_smt_cfg')
doc = Dom.Document()
root_node = doc.createElement("stock_crawl_cfg")
root_node.setAttribute("developer", "zhonghua")
doc.appendChild(root_node)
yiDianNode = create_yidian_cfg(doc)
thsNode = create_ths_cfg(doc)
dzhNode = create_dzh_cfg(doc)
xinlangNode = create_xinlang_cfg(doc)
root_node.appendChild(yiDianNode)
root_node.appendChild(thsNode)
root_node.appendChild(dzhNode)
root_node.appendChild(xinlangNode)
with open(globalPars.getToolBaseWorkDirectory() + "stockSmtCfg.xml", "w") as f:
str = doc.toprettyxml(indent = "\t", newl = "\n", encoding = "utf-8")
f.write(bytes.decode(str))
def generateYiDianGDUrls(stock_number_list):
urls = []
with open('date_cfg.cfg', 'r') as f:
dates = f.readlines()
print(dates)
for date in dates:
for number in stock_number_list:
a_date = date.strip()
new_url = 'http://www.yidiancangwei.com/gudong/sdlt_' + number + '_' + a_date + '.html'
new_url = str(new_url)
urls.append(new_url)
#print(urls)
with open(globalPars.getToolBaseWorkDirectory() + 'yidian_urls.txt',
'wb') as f:
for url in urls:
f.write(url.encode('utf-8'))
f.write('\n'.encode('utf-8'))
return urls
class DzhgudongSpider(scrapy.Spider):
name = "dzh"
allowed_domains = ['webf10.gw.com.cn']
start_urls = [
# 'http://webf10.gw.com.cn/SH/B10/SH600767_B10.html#'
]
with open(globalPars.getToolBaseWorkDirectory() + 'dzh_urls.txt', 'r') as f:
for line in f.readlines():
start_urls.append(line.strip())
self_dict = generate_stock_dic()
#print(start_urls)
#流通股东数据
def checkLTGD(self, response):
sel = scrapy.selector.Selector(response)
# <section class="sdgdBox dzh_index" id="十大流通股东">
LTGD_body = sel.xpath('//section[@id="十大流通股东"]')
body = sel.xpath('//nav[@class="sdgd_nav clearfix"]')
dateList = body.xpath('.//a/text()').extract()
# print('dateList = ', dateList)
h2 = LTGD_body.xpath('.//h2/text()').extract()
# print('h2 = ', h2)
all_info = []
#date_length = len(date_time)
cur_seq = 0
one_jidu_info_divs = LTGD_body.xpath('.//div[@class="sdgdCon sdltgdTb"]')
jd_parts = len(one_jidu_info_divs)
# print("gudong info parts %d" % len(one_jidu_info_divs))
for one_jidu_info in one_jidu_info_divs:
#<table class="f10tabel sdgd_table">
one_gudong_info_divs = one_jidu_info.xpath('.//table[@class="f10tabel sdgd_table"]')
#print("gudong info member %d" % len(one_gudong_info_divs))
#for one_gudong_info in one_gudong_info_divs:
name = one_gudong_info_divs.xpath('.//td/a/text()').extract()
info = one_gudong_info_divs.xpath('.//td/text()').extract()
# print('---------------------------------------------------------')
name_len = len(name) + 1
if (name and info):
for i in range(1,name_len):
# print('cur = ', i)
"""
'1.', '5350.83', '未变', '15.70', '流通A股',
"""
gdname = name[i-1]
hold_num = info[5*(i-1) + 1]
change = info[5*(i-1) + 2]
percent = info[5*(i-1) + 3]
stock_type = info[5*(i-1) + 4]
date = dateList[-jd_parts + cur_seq]
stock_number = self.stock_num
Cname = self.self_dict[stock_number]
#gdinfo = (stock_number, stock_name, gdname, hold_num, percent, date, stock_type)
gdinfo = (stock_number, Cname, gdname, hold_num, percent, change, date, stock_type)
#print('gdinfo = ', gdinfo)
all_info.append(gdinfo)
cur_seq += 1
# print('all_info = ', all_info)
store_lt_gudong_db(all_info)
def parse(self, response):
pattern = re.compile(r'60[0-3]\d{3}|00[0,2]\d{3}|300\d{3}')
match = pattern.search(str(response))
if match:
self.stock_num = match.group()
print("stock_num =", str(self.stock_num))
self.checkLTGD(response)
class XlgudongSpider(scrapy.Spider):
name = "xl"
allowed_domains = ['finance.sina.com.cn']
start_urls = [
# 'http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_CirculateStockHolder/stockid/600079.phtml#2010-09-30'
# 'http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_CirculateStockHolder/stockid/300521.phtml#2010-09-30'
]
self_dict = update_stock.generate_stock_dic()
print("dic len = ", len(self_dict))
with open(globalPars.getToolBaseWorkDirectory() + 'xl_urls.txt', 'r') as f:
for line in f.readlines():
start_urls.append(line.strip())
def checkLTNumber(self, response):
pass
def reCalcGDData(self, times, all_info):
for j in range(times):
jidu_new_div = all_info[j]
if (j+1) == times:
return
jidu_old_div = all_info[j+1]
# 从一个季度里拿出每个股东数据和上衣季度对比
for i in range(len(jidu_new_div)):
find = 0
new_gd_info = jidu_new_div[i]
# 当前股东信息和上一个季度每个股东对比
new_val = int(new_gd_info[3]) / 10000.0
for k in range(len(jidu_old_div)):
old_gd_info = jidu_old_div[k]
# 如果股东在上个季度存在， 则计算差值
find = 0
old_val = int(old_gd_info[3]) / 10000.0
if new_gd_info[2] == old_gd_info[2] :
find = 1
if new_gd_info[3] == old_gd_info[3]:
new_gd_info[-3] = u'不变'
else:
val = '%.2f' % (new_val - old_val)
new_gd_info[-3] = str(val)
# new_gd_info[3] = '%.2f' % new_val
# 找到了就停止本次循环
break
if find == 0:
new_gd_info[-3] = u'新进'
new_gd_info[3] = '%.2f' % new_val
def checkLTGD(self, response):
sel = scrapy.selector.Selector(response)
# <td colspan="5" align="left" class="tdr">2011-09-30</td>
date_list_sel = sel.xpath('//td[@class="tdr"]')
date_list_txt = date_list_sel.xpath('.//text()').extract()
# print(date_list_txt)
# print('date_list_txt len = ', len(date_list_txt))
date_cur = 0
#for data_list_s in date_list_sel:
# <div align="center">上海机场(集团)有限公司</div>
tbody = sel.xpath('//div[@align="center"]')
tbody_td_list = tbody.xpath('.//text()').extract()
# print(tbody_td_list)
length = len(tbody_td_list)
all_info = []
jidu_info = []
ret_info_list = []
name = self.self_dict[self.stock_num]
# print('length = ', length)
for i in range(length):
# print('i = %d date_cur=%d' % (i, date_cur))
if tbody_td_list[i].isdigit() and int(tbody_td_list[i]) < 13 and date_cur < len(date_list_txt) and i+3 < length and '.' in tbody_td_list[i+3] :
# (stock_number varchar(20), cname varchar(20),gudong_name varchar(100), hold_num varchar(20), percent varchar(20), ichange varchar(20), date varchar(20), stock_type varchar(20))
gd_info = [self.stock_num, name, tbody_td_list[i+1], tbody_td_list[i+2], tbody_td_list[i+3], '-', date_list_txt[date_cur], '流通A股']
jidu_info.append(gd_info)
# print(gd_info)
if (i+5 < length) and tbody_td_list[i+5].isdigit() == False :
all_info.append(jidu_info)
jidu_info = []
date_cur += 1
i += 3
i += 1
# 信息保存到文件中
# with open('temp.txt', 'w') as f:
# i = 0
# for info in all_info:
# cmd = '\n %d %s ---------------------------\n' % (i, date_list_txt[i])
# f.writelines(cmd)
# number = 0
# for l in info:
# f.writelines(l)
# f.writelines('\n')
# number += 1
# cmd2 = '******************%d***********' % number
# f.writelines(cmd2)
# i += 1
# print('all_info len = ', len(all_info))
# jidu_ptr = 0
cacl_times = len(all_info) - 1
self.reCalcGDData(cacl_times, all_info)
# print('cacl_times =', cacl_times)
# for j in range(cacl_times):
# jidu_new_div = all_info[j]
# if (j+1) == cacl_times:
# return
# jidu_old_div = all_info[j+1]
# # 从一个季度里拿出每个股东数据和上衣季度对比
# for i in range(len(jidu_new_div)):
# find = 0
# new_gd_info = jidu_new_div[i]
# # 当前股东信息和上一个季度每个股东对比
# new_val = int(new_gd_info[3]) / 10000.0
# for k in range(len(jidu_old_div)):
# old_gd_info = jidu_old_div[k]
# # 如果股东在上个季度存在， 则计算差值
# find = 0
# old_val = int(old_gd_info[3]) / 10000.0
# if new_gd_info[2] == old_gd_info[2] :
# find = 1
# if new_gd_info[3] == old_gd_info[3]:
# new_gd_info[-3] = u'不变'
# else:
# val = '%.2f' % (new_val - old_val)
# new_gd_info[-3] = str(val)
# # new_gd_info[3] = '%.2f' % new_val
# # 找到了就停止本次循环
# break
# if find == 0:
# new_gd_info[-3] = u'新进'
# new_gd_info[3] = '%.2f' % new_val
# ret_info_list.append(new_gd_info)
# print('---------------------------------------------------')
# print('all_info = ', all_info)
# print(all_info)
# print('all_info len = ', len(all_info))
# 信息保存到文件中
# with open('over.txt', 'w') as f:
# i = 0
# for info in all_info:
# cmd = '\n %d %s ---------------------------\n' % (i, date_list_txt[i])
# f.writelines(cmd)
# number = 0
# for l in info:
# f.writelines(l)
# f.writelines('\n')
# number += 1
# cmd2 = '******************%d***********' % number
# f.writelines(cmd2)
# i += 1
# print(all_info)
for info in all_info:
stockSql.store_old_lt_gudong_db(info)
def parse(self, response):
pattern = re.compile(r'60[0-3]\d{3}|00[0,2]\d{3}|300\d{3}')
match = pattern.search(str(response))
if match:
self.stock_num = match.group()
self.checkLTGD(response)
class ThsgudongSpider(scrapy.Spider):
name = "ths"
allowed_domains = ['10jqka.com.cn']
start_urls = [
# 'http://stockpage.10jqka.com.cn/300575/holder/'
#'http://stockpage.10jqka.com.cn/300371/holder/'
]
with open(globalPars.getToolBaseWorkDirectory() + 'ths_urls.txt',
'r') as f:
for line in f.readlines():
start_urls.append(line.strip())
#print(start_urls)
#股东人数披露
def checkLTNumber(self, response):
all_info = []
sel = scrapy.selector.Selector(response)
# <div class="td_w">4.73</div>
data_list = sel.xpath('//div[@class="td_w"]/text()').extract()
# print('len = ', len(data_list))
length = len(data_list)
half = (length // 2) - 1
date_list = data_list[0:half]
num_list = data_list[half + 1:length - 1]
body = sel.xpath('//table[@class="tbody"]')
info_list_text = body.xpath('.//td').extract()
for i in range(half):
# ret1 = info_list_text[i].replace('</td>', '').replace('</div>', '').split('>')[-1]
number = num_list[i]
ichange_percent = info_list_text[i + half + 1].replace(
'</td>', '').replace('</span>', '').split('>')[-1]
date = date_list[i]
gd_num_info = (self.stock_num, self.cName, number, ichange_percent,
date)
# print(gd_num_info)
all_info.append(gd_num_info)
# print(all_info)
stockSql.store_lt_gudong_num_db(all_info)
#和上期比较持股数量变化
def checkIchange(self, str):
ret = str.replace('\r\n', '').replace('\t', '').replace(
'</s>', '').replace('<s>', '').replace('<td>', '').replace(
'</td>', '').strip().replace(' ',
'').replace('spanclass=',
'').split('</span>')
if (len(ret) == 1):
str = ret[0]
else:
str_temp1 = ret[0].split('>')
if '.' in str_temp1[-1]:
# print(str_temp1[-1])
str_tmp2 = str_temp1[-2].replace('<', '').replace('"', '')
# print(str_tmp2)
if (str_tmp2 == 'upcolor'):
str = '+' + str_temp1[-1]
else:
str = str_temp1[-1]
else:
str = str_temp1[-1]
return str
#流通股东数据
def checkLTGD(self, response):
sel = scrapy.selector.Selector(response)
LTGD_body = sel.xpath('//div[@id="flowholder"]')
date_time = LTGD_body.xpath(
'.//li/a[@class="fdates"]/text()').extract()
# print(date_time)
all_info = []
date_length = len(date_time)
if date_time:
ret = stockSql.query_lt_gudong_info_exist(self.stock_num,
date_time[0])
# print('query stock(%s) date(%s) ret %s' % (self.stock_num, date_time[0], ret))
if ret:
return 0
cur_seq = 0
one_jidu_info_divs = LTGD_body.xpath(
'.//table[@class="m_table m_hl ggintro"]')
# print("gudong info parts %d" % len(one_jidu_info_divs))
for one_jidu_info in one_jidu_info_divs:
one_gudong_info_divs = one_jidu_info.xpath('.//tbody/tr')
# print("gudong info member %d" % len(one_gudong_info_divs))
# print(one_gudong_info_divs)
for one_gudong_info in one_gudong_info_divs:
name = one_gudong_info.xpath(
'.//th[@class="tl holder_th"]/a/text()').extract()
info = one_gudong_info.xpath('.//td/text()').extract()
# print(info)
info_list = one_gudong_info.xpath('.//td')
# print('info_list = ', info_list)
info_list_text = one_gudong_info.xpath('.//td').extract()
# print('name = ', name)
# print('info_list_text = ', info_list_text)
if name and info_list_text and info_list:
stock_number = self.stock_num
Cname = self.cName
gdname = name[0]
hold_num = info_list[0].xpath('.//text()').extract()[0]
# percent = info_list[2].xpath('.//text()').extract()
percent = info[2]
if percent.strip() == '':
percent = '-'
ichange = self.checkIchange(info_list_text[1])
date = date_time[cur_seq]
stock_type = info_list[4].xpath('.//text()').extract()[0]
gdinfo = (stock_number, Cname, gdname, hold_num, percent,
ichange, date, stock_type)
all_info.append(gdinfo)
# print(gdinfo)
cur_seq += 1
stockSql.store_lt_gudong_db(all_info)
def checkCName(self, response):
sel = scrapy.selector.Selector(response)
self.cName = sel.xpath('//strong/text()').extract()[0]
# print('cName = ', self.cName)
def parse(self, response):
pattern = re.compile(r'60[0-3]\d{3}|00[0,2]\d{3}|300\d{3}')
match = pattern.search(str(response))
if match:
self.stock_num = match.group()
# print("stock_num =", str(self.stock_num))
self.checkCName(response)
self.checkLTNumber(response)
self.checkLTGD(response)