def start_requests(self):
start_url = 'http://69.push2his.eastmoney.com/api/qt/stock/kline/get'
headers = {
'Connection': 'close',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36',
'Content-Type': 'application/javascript; charset=UTF-8'
}
# 查询所有港股股票列表
connection = pymysql.connect(host=config.get_db_host(),
db=config.get_db_dbname(),
port=config.get_db_port(),
user=config.get_db_username(),
password=config.get_db_passwd(),
charset=config.get_db_charset())
cursor = connection.cursor()
cursor.execute(
'select secid,market,stock_code,stock_name from hk_hs_stock_list where market = "116" '
)
hk_stocks = cursor.fetchall()
# k线频度 日 周 月
freqs = ['101', '102', '103']
end_date = datetime.datetime.now().date().strftime('%Y%m%d')
begin_date = (datetime.datetime.now().date() -
relativedelta(months=1)).strftime('%Y%m%d')
for freq in freqs:
for hk_stock in hk_stocks:
log.info('正在爬取港股[' + hk_stock[0] + hk_stock[3] + ']频度' + freq +
'k线数据...')
meta = {
'secid': hk_stock[0],
'market': hk_stock[1],
'stock_code': hk_stock[2],
'stock_name': hk_stock[3],
'klt': freq
}
params = {
'secid': hk_stock[0],
'fields1': 'f1,f2,f3,f4,f5',
'fields2': 'f51,f52,f53,f54,f55,f56,f57',
'klt': freq,
'fqt': '0',
'beg': begin_date,
'end': end_date
}
yield scrapy.FormRequest(callback=self.parse,
dont_filter=True,
url=start_url,
method='GET',
headers=headers,
meta=meta,
formdata=params)
def start_requests(self):
start_url = 'https://stock.xueqiu.com/v5/stock/chart/kline.json'
headers = {
'Connection': 'keep-alive',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36',
'Content-Type': 'application/json;charset=UTF-8'
}
# 查询所有美股股票列表
connection = pymysql.connect(host=config.get_db_host(),
db=config.get_db_dbname(),
port=config.get_db_port(),
user=config.get_db_username(),
password=config.get_db_passwd(),
charset=config.get_db_charset())
cursor = connection.cursor()
cursor.execute(
'select symbol,market,stock_name from us_stock_list limit 2')
hs_stocks = cursor.fetchall()
# k线频度及对应接口参数
# 1分钟'1m',5分钟'5m', 15分钟'15m', 30分钟'30m', 60分钟'60m', 日'day', 周'week', 月'month'
freqs = ['1m', '5m', '15m', '30m', '60m', 'day', 'week', 'month']
for freq in freqs:
for hs_stock in hs_stocks:
log.info('正在爬取美股[' + hs_stock[0] + hs_stock[2] + ']频度' + freq +
'数据...')
meta = {
'symbol': hs_stock[0],
'market': hs_stock[1],
'stock_name': hs_stock[2],
'klt': freq
}
params = {
'symbol':
'XNET',
'begin':
str(int(round(time.time() * 1000))),
'period':
freq,
'type':
'normal',
'count':
'-432',
'indicator':
'kline,pe,pb,ps,pcf,market_capital,agt,ggt,balance'
}
yield scrapy.Request(callback=self.parse,
dont_filter=True,
url=start_url,
method='GET',
headers=headers,
meta=meta,
formdata=params)
def __init__(self):
dbparms = dict(
host=config.get_db_host(),
db=config.get_db_dbname(),
port=config.get_db_port(),
user=config.get_db_username(),
password=config.get_db_passwd(),
charset=config.get_db_charset(),
cursorclass=pymysql.cursors.DictCursor,
use_unicode=True
)
self.dbpool = adbapi.ConnectionPool('pymysql', **dbparms)
def start_requests(self):
start_url = 'http://pdfm.eastmoney.com/EM_UBG_PDTI_Fast/api/js'
headers = {
'Connection': 'keep-alive',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36',
'Content-Type': 'text/json; charset=utf-8'
}
# 查询所有沪深A股股票列表
connection = pymysql.connect(host=config.get_db_host(),
db=config.get_db_dbname(),
port=config.get_db_port(),
user=config.get_db_username(),
password=config.get_db_passwd(),
charset=config.get_db_charset())
cursor = connection.cursor()
cursor.execute(
'select secid,market,stock_code,stock_name from hk_hs_stock_list where (market = "0" or market = "1") '
)
hs_stocks = cursor.fetchall()
# k线频度及对应接口参数
# m5k(5MIN) m15k(15MIN) m30k(30MIN) m60k(60MIN) k(日) wk(周) mk(月)
freqs = ['m5k', 'm15k', 'm30k', 'm60k', 'k', 'wk', 'mk']
for freq in freqs:
for hs_stock in hs_stocks:
log.info('正在爬取沪深A股[' + hs_stock[0] + hs_stock[3] + ']频度' +
freq + '数据...')
meta = {
'secid': hs_stock[0],
'market': hs_stock[1],
'stock_code': hs_stock[2],
'stock_name': hs_stock[3],
'klt': freq
}
params = {
'id': hs_stock[0],
'rtntype': '6',
'type': freq,
'authorityType': 'undefined'
}
yield scrapy.FormRequest(callback=self.parse,
dont_filter=True,
url=start_url,
method='GET',
headers=headers,
meta=meta,
formdata=params)