# -*- coding: utf-8 -*-
import sys
import os
# 临时修改环境变量 为当前目录上两级目录
root_dir = os.path.dirname(
os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
sys.path.append(root_dir)
import scrapy
import pymysql
import logging as log
import scrapy_gtn.conf.config as config
import json
from ..items import QuotItem
log.basicConfig(level=log.WARNING,
format=config.get_log_format(),
datefmt=config.get_log_datefmt(),
handlers={
log.FileHandler(filename=root_dir +
'/hs_quotation_all.log',
mode='a',
encoding='utf-8'),
log.StreamHandler(sys.stderr)
})
# 从东财爬取沪深A股行情-全量爬取，初始化时执行一次即可，后续使用增量爬虫
class HsQuotationAllSpider(scrapy.Spider):
name = 'hs_quotation_all'
def start_requests(self):
# scrapy crawl hk_quotation -o hk_quotation.csv
import sys
import os
# 临时修改环境变量 为当前目录上两级目录
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
sys.path.append(root_dir)
import scrapy
import json
from ..items import QuotItem
import scrapy_gtn.conf.config as config
import logging as log
import pymysql
import datetime
from dateutil.relativedelta import relativedelta
log.basicConfig(level=log.WARNING,format=config.get_log_format(),datefmt=config.get_log_datefmt(),
handlers={log.FileHandler(filename=root_dir + '/hk_quotation_all.log', mode='a', encoding='utf-8'),
log.StreamHandler(sys.stderr)})
# 从东财爬取港股行情
class HkQuotationAllSpider(scrapy.Spider):
name = 'hk_quotation_all'
def start_requests(self):
start_url = 'http://69.push2his.eastmoney.com/api/qt/stock/kline/get'
headers = {
'Connection': 'close',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36',
'Content-Type': 'application/javascript; charset=UTF-8'
}
# 查询所有港股股票列表
import sys
import os
# 临时修改环境变量 为当前目录上两级目录
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
sys.path.append(root_dir)
import pandas as pd
import tushare as ts
import scrapy_gtn.history.stock_basic as stock_basic
import scrapy_gtn.conf.config as config
import logging as log
import time
import datetime
from dateutil.parser import parse
log.basicConfig(level=log.INFO,format=config.get_log_format(),datefmt=config.get_log_datefmt(),
handlers={log.FileHandler(filename=root_dir + '/stock_quotation.log', mode='a', encoding='utf-8'),
log.StreamHandler(sys.stderr)})
# 获取所有沪深股票的日、周、月以及分时数据
# 沪深股票信息
stocks = stock_basic.get_stock_basic()
if(len(stocks) == 0):
print('获取股票数据失败')
sys.exit(0)
# 从参数本次采集开始日期
if(len(sys.argv) == 1):#未填写开始采集日期
log.error('执行出错，请输入数据采集开始日期')
sys.exit(1)
else: