def update_jj(table):
# table='2020-02-25' # 用于获取code列
conn = DB.get_mysql_conn('db_fund', 'qq')
# today=datetime.datetime.now().strftime('%Y-%m-%d')
logger = llogger(f'log/{table}_realtime.log')
query = 'select `基金代码`,`基金简称`,`实时价格` from `{}`'.format(table)
cursor = conn.cursor()
cursor.execute(query)
session = requests.Session()
ret = cursor.fetchall()
url = 'http://web.ifzq.gtimg.cn/fund/newfund/fundSsgz/getSsgz?app=web&symbol=jj{}&_var=LOAD_1582735233556_37'
add_column1 = 'alter table `{}` add column `实时净值` float'.format(table)
add_column2 = 'alter table `{}` add column `溢价率` float'.format(table)
update_sql = 'update `{}` set `实时净值`= %s,`溢价率`=%s where `基金代码`=%s'.format(
table)
try:
cursor.execute(add_column1)
except Exception as e:
conn.rollback()
else:
conn.commit()
try:
cursor.execute(add_column2)
except Exception as e:
conn.rollback()
else:
conn.commit()
for item in ret:
code = item[0]
realtime_price = item[2]
s_resp = session.get(url.format(code), headers=headers)
content = re.search('LOAD_\d+_\d+=(.*)', s_resp.text).group(1)
js = json.loads(content)
try:
data_list = js.get('data').get('data')
except Exception as e:
logger.error(e)
continue
last_one = data_list[-1]
time_ = last_one[0]
jj_ = last_one[1]
yjl = -1 * round((jj_ - realtime_price) / realtime_price * 100, 2)
print(f'溢价率-{yjl}')
cursor.execute(update_sql, (jj_, yjl, code))
conn.commit()
logger.info('更新成功')
conn.close()
def __init__(self,check_holiday=False,remote='qq'):
self.logger = llogger('log/' + 'jisilu.log')
if check_holiday:
self.check_holiday()
self.date = datetime.datetime.now().strftime('%Y-%m-%d')
# self.date = '2020-02-07' # 用于调整时间
self.timestamp = int(time.time() * 1000)
self.headers = {
'User-Agent': 'User-Agent:Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest'}
self.url = 'https://www.jisilu.cn/data/cbnew/cb_list/?___jsl=LST___t={}'.format(self.timestamp)
self.pre_release_url = 'https://www.jisilu.cn/data/cbnew/pre_list/?___jsl=LST___t={}'.format(self.timestamp)
self.remote = remote
self.engine = DB.get_engine('db_jisilu', self.remote)
def __init__(self):
self.logger = llogger('log/jsl_monitor')
self.session = requests.Session()
self.cookies = config.jsl_cookies
self.headers = {
'Sec-Fetch-Mode': 'cors',
'Origin': 'https://www.jisilu.cn',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh,en;q=0.9,en-US;q=0.8',
'X-Requested-With': 'XMLHttpRequest',
'Connection': 'keep-alive',
'Pragma': 'no-cache',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Cache-Control': 'no-cache',
'Referer': 'https://www.jisilu.cn/data/cbnew/',
'Sec-Fetch-Site': 'same-origin',
}
self.params = (
('___jsl', 'LST___t=1579488785838'),
)
self.data = {
'fprice': '',
'tprice': '',
'volume': '',
'svolume': '',
'premium_rt': '',
'ytm_rt': '',
'rating_cd': '',
'is_search': 'Y',
'btype': 'C',
'listed': 'Y',
'sw_cd': '',
'bond_ids': '',
'rp': '50'
}
self.history = HistorySet(expire=config.EXPIRE_TIME)
# @Time : 2020/2/26 9:38
# @File : fund_raise_speed.py
# 查看基金最后暴力拉伸的
import datetime
import random
import time
from settings import get_mysql_conn,llogger
import tushare as ts
table='2020-02-25' # 用于获取code列
conn = get_mysql_conn('db_fund', local='local')
today=datetime.datetime.now().strftime('%Y-%m-%d')
print(today)
logger = llogger(f'{today}_fund_raise_monitor.log')
query='select `基金代码`,`基金简称` from `2020-02-25`'
# print(query)
cursor = conn.cursor()
cursor.execute(query)
ret = cursor.fetchall()
code_list=[]
for item in ret:
code = item[0]
df = ts.get_realtime_quotes(code)
close_p=float(df['pre_close'].values[0])
b1=float(df['b1_p'].values[0])
a1=float(df['a1_p'].values[0])
import datetime
import os
import matplotlib
'''
昨日涨停的今日的实时情况
'''
matplotlib.use("Pdf")
from settings import get_engine, is_holiday, DATA_PATH
import pandas as pd
import tushare as ts
import numpy as np
from plot_line import plot_stock_line
from settings import llogger
filename = os.path.basename(__file__)
logger = llogger('log/' + filename)
def monitor():
engine = get_engine('db_zdt')
table = '20180409zdt'
api = ts.get_apis()
df = pd.read_sql(table, engine, index_col='index')
price_list = []
percent_list = []
amplitude_list = []
start = datetime.datetime.now()
for i in df['代码'].values:
try:
curr = ts.quotes(i, conn=api)
last_close = curr['last_close'].values[0]
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
'Accept': '*/*',
# 'Referer': 'http://stockapp.finance.qq.com/mstats/?id=fund_close',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh,en;q=0.9,en-US;q=0.8',
}
DB = DBSelector()
conn = DB.get_mysql_conn('db_fund', 'qq')
cursor = conn.cursor()
base_dir = os.path.dirname(os.path.abspath(__file__))
log_path = os.path.join(base_dir, '..', 'log')
logger = llogger(os.path.join(log_path, 'fund_info.log'))
class FundSpider(object):
def __init__(self):
self.create_tb()
def create_tb(self):
create_table = 'create table if not EXISTS `{}` (`基金代码` varchar(20) PRIMARY KEY,`基金简称` varchar(100),`最新规模-万` float,' \
'`实时价格` float,`涨跌幅` float,`成交额-万` float,`净值日期` VARCHAR(10),`单位净值` float,`累计净值` float,`折溢价率` float ,`申购状态` VARCHAR(20),`申赎状态` varchar(20),`基金经理` VARCHAR(200),' \
'`成立日期` VARCHAR(20), `管理人名称` VARCHAR(200),`实时估值` INT,`QDII` INT ,`更新时间` VARCHAR(20));'.format(
today)
try:
cursor.execute(create_table)
except Exception as e:
conn.rollback()
import os
import random
import time
# from send_mail import sender_139
import pymongo
import tushare as ts
import pandas as pd
from settings import get_engine, send_aliyun, QQ_MAIL, llogger
import config
# 大单定义 手数, 需要在大单数据获取完成后
BIG_DEAL = 1000
# 获取的历史天数的数据
DELTA_DAY = 35
logger = llogger('log/big_deal.log')
current_day = datetime.date.today().strftime('%Y-%m-%d')
# current_day='2019-12-11'
class BigDeal(object):
def __init__(self):
# self.df = self.get_tick()
today = datetime.datetime.now().strftime('%Y-%m-%d')
# today = '2019-10-18'
if ts.is_holiday(today):
logger.info('{}假期 >>>>>'.format(today))
exit()
# -*-coding=utf-8-*-
# 股市黑名单
from settings import llogger, DATA_PATH, DBSelector
import os
import codecs
logger = llogger('log/blacklist.log')
def create_tb(conn):
cmd = '''CREATE TABLE IF NOT EXISTS `tb_blacklist` (DATE TIMESTAMP DEFAULT CURRENT_TIMESTAMP,CODE VARCHAR(6) PRIMARY KEY,NAME VARCHAR(60),REASON TEXT);'''
cur = conn.cursor()
try:
cur.execute(cmd)
conn.commit()
except Exception as e:
logger.info(e)
conn.rollback()
def update_data(filename, conn):
cur = conn.cursor()
with codecs.open(filename, 'r', encoding='utf8') as f:
content = f.readlines()
if not content:
return
for line in content:
# -*-coding=utf-8-*-
# @Time : 2020/1/1 0:08
# @File : trend.py
# 统计发帖趋势
import datetime
import numpy as np
import pymongo
import pandas as pd
from settings import send_aliyun, llogger
from config import QQ_MAIL
logger = llogger('log/trend_.log')
db = pymongo.MongoClient('192.168.10.48', 17001)
doc = db['db_parker']['jsl']
total_list = []
date = datetime.datetime.now() + datetime.timedelta(days=-365) # 一年内的数据
for item in doc.find({'last_resp_date': {
'$gt': date
}}, {
'html': 0,
'resp': 0,
'content': 0
}):
del item['_id']
total_list.append(item)
df = pd.DataFrame(total_list)
df['createTime'] = pd.to_datetime(df['createTime'])
df = df.set_index('createTime', drop=True)
# -*- coding: utf-8 -*-
# website: http://30daydo.com
# @Time : 2020/8/3 19:01
# @File : closed_end_fund.py
# 抓取封闭式基金数据
import datetime
import requests
from settings import llogger, DBSelector, _json_data
import pymongo
logger = llogger('log/fund.log')
RETRY = 0
db = DBSelector()
class CloseEndFundCls():
def __init__(self):
self.url = 'https://www.jisilu.cn/data/cf/cf_list/'
self.headers = {
'User-Agent':
'User-Agent:Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest'
}
host = _json_data['mongo']['qq']['host']
port = _json_data['mongo']['qq']['port']
user = _json_data['mongo']['qq']['user']
password = _json_data['mongo']['qq']['password']
connect_uri = f'mongodb://{user}:{password}@{host}:{port}'
''' http://30daydo.com Contact: [email protected] ''' import re import time import datetime import requests import pandas as pd from settings import get_engine, llogger, is_holiday, get_mysql_conn import six # from send_mail import sender_139 from sqlalchemy import VARCHAR import os engine = get_engine('db_jisilu') logger = llogger('log/' + 'jisilu.log') # 爬取集思录 可转债的数据 class Jisilu(object): def __init__(self): self.check_holiday() self.date = datetime.datetime.now().strftime('%Y-%m-%d') # self.date = '2020-02-07' # 用于调整时间 # py2 if six.PY2: self.timestamp = long(time.time() * 1000) else:
#-*-coding=utf-8-*- # 每天收盘收运行 import datetime import time __author__ = 'Rocky' ''' http://30daydo.com Contact: [email protected] ''' import tushare as ts import os from settings import get_engine, llogger, is_holiday, DATA_PATH import pandas as pd logger = llogger('log/collect_data') # 获取市场全貌 class SaveData(): today = datetime.datetime.now().strftime("%Y-%m-%d") daily_engine = get_engine('db_daily') def __init__(self): work_space = DATA_PATH if os.path.exists(work_space) == False: os.mkdir(work_space) os.chdir(work_space) @staticmethod def daily_market():
# import tkinter
# import tkinter.messagebox # 弹窗库
import threading
from jsl_monitor import ReachTargetJSL
import config
def show_box(msg):
pass
# tkinter.messagebox.showinfo('注意', msg)
dirname = os.path.dirname(__file__)
full_name = os.path.join(dirname,
'log/alert_me_{}.log'.format(datetime.date.today()))
logger = llogger(full_name)
# 循环检测时间
LOOP_TIME = 60
EXECEPTION_TIME = 20
MARKET_OPENING = 0
# ALERT_PERCENTAGE = 3
DELTA_TIME = 30
ZG_ALERT_PERCENT = 8
ZZ_ALERT_PERCENT = 8
CW_ALERT_PERCENT = -5
DIFF_DELTA_TIME = 30
# ALERT_PERCENT_POOL = 3
DIFF_V = 40 # quote 接口以千为单位
file = 'D:\OneDrive\Stock\gj_hold.xls'
# -*-coding=utf-8-*-
# @Time : 2019/7/1 16:50
# @File : bond_daily.py
# 收集可转债的市场全景图
from settings import get_engine,sendmail,llogger,get_mysql_conn
import pandas as pd
import datetime
from config import token
import tushare as ts
today = datetime.datetime.now().strftime('%Y-%m-%d')
today_fmt = datetime.datetime.now().strftime('%Y%m%d')
# today='2020-02-06'
# today_fmt='20200206'
cons=ts.get_apis()
logger=llogger('log/bond_daily.log')
# ts.set_token(token)
# pro = ts.pro_api()
conn=get_mysql_conn('db_bond_daily','local')
def creat_table(day):
tb_name = 'bond_{}'.format(day)
create_cmd = 'create table if not exists `{tb_name}` (`date` varchar(20),`code` varchar(10) primary key,`name` varchar(16),`open` float ,' \
'`close` float,`high` float,`low` float,`vol` float,`amount` float) '.format(tb_name=tb_name)
cursor = conn.cursor()
try:
cursor.execute(create_cmd)
conn.commit()
except Exception as e:
logger.error(e)
''' http://30daydo.com Contact: [email protected] ''' import re import time import datetime import requests import pandas as pd from settings import get_engine, llogger, is_holiday, get_mysql_conn import six from send_mail import sender_139 from sqlalchemy import VARCHAR import os engine = get_engine('db_jisilu') logger = llogger('log/' + 'jisilu') # 爬取集思录 可转债的数据 class Jisilu(object): def __init__(self): # self.check_holiday() # py2 if six.PY2: self.timestamp = long(time.time() * 1000) else: self.timestamp = int(time.time() * 1000) self.headers = { 'User-Agent':
# -*-coding=utf-8-*-
# @Time : 2019/9/2 9:35
# @File : send_bond_info.py
# 发送邮件的
from settings import get_engine
import pandas as pd
import datetime
from settings import llogger, send_aliyun, QQ_MAIL
import tushare as ts
logger = llogger('log/task_jsler_double_low.log')
today = datetime.datetime.now().strftime('%Y-%m-%d')
def map_rate(x):
map_dict = {'A+': 1, 'AA-': 1.2, 'AA': 1.4, 'AA+': 1.6, 'AAA': 1.8}
x = x.replace(' ', '')
return map_dict.get(x, 1)
def cb_info():
con = get_engine('db_stock', 'local')
df = pd.read_sql('tb_bond_jisilu', con=con)
df['grade'] = df['评级'].map(lambda x: map_rate(x))
df['可转债综合价格'] = df['可转债价格'] + df['溢价率'] * df['grade']
df = df.sort_values(by='可转债综合价格')
df = df[df['强赎标志'] != 'Y']
df1 = df[['可转债代码', '可转债名称', '可转债综合价格', '可转债价格', '溢价率', '评级']].head(20)
df1 = df1.reset_index(drop=True)
# -*-coding=utf-8-*-
# @Time : 2018/8/7 13:45
# @File : foreignexchange.py
# 实时获取外汇
import os
import re
import datetime
import requests
from settings import DBSelector,llogger
logger = llogger('log/usd.log')
class ForeighExchange(object):
def __init__(self):
self.url = 'http://data.bank.hexun.com/other/cms/foreignexchangejson.ashx?callback=ShowDatalist'
self.update_req = 10
self.retry = 5
self.headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
def run(self):
content = self.fetch_web()
if content:
pattern = re.compile('\{bank:\'工商银行\',currency:\'美元\',code:\'USD\',currencyUnit:\'\',cenPrice:\'\',(buyPrice1:\'[.0-9]+\',sellPrice1:\'[.0-9]+\'),.*?\'\}')
ret_str = pattern.search(content).group(1)
buy=re.search('buyPrice1:\'([0-9.]+)\'',ret_str).group(1)
sell=re.search('sellPrice1:\'([0-9.]+)\'',ret_str).group(1)
return (buy,sell)
def notice(self):
# -*-coding=utf-8-*- __author__ = 'Rocky' ''' http://30daydo.com Contact: [email protected] ''' # 模拟买入 纯粹纪录。 import os, sys, chardet import pandas as pd import datetime import numpy as np import tushare as ts from send_mail import sender_139 from settings import get_engine, get_mysql_conn, is_holiday, llogger, DATA_PATH logger = llogger('log/simulation.log') class Simulation(): def __init__(self): # path=os.path.join(os.getcwd(),'data') path = DATA_PATH if os.path.exists(path) == False: os.mkdir(path) os.chdir(path) self.name = 'simulation.xls' self.df = pd.read_excel(self.name) self.df['代码'] = self.df['代码'].map(lambda x: str(x).zfill(6)) self.engine = get_engine('db_stock')
# -*-coding=utf-8-*-
# 股市黑名单
from settings import get_mysql_conn, llogger, DATA_PATH
import os
import codecs
logger = llogger(__file__)
def create_tb(conn):
cmd = '''CREATE TABLE IF NOT EXISTS `tb_blacklist` (DATE TIMESTAMP DEFAULT CURRENT_TIMESTAMP,CODE VARCHAR(6) PRIMARY KEY,NAME VARCHAR(60),REASON TEXT);'''
cur = conn.cursor()
try:
cur.execute(cmd)
conn.commit()
except Exception as e:
logger.info(e)
conn.rollback()
def update_data(filename, conn):
cur = conn.cursor()
with codecs.open(filename, 'r', encoding='utf8') as f:
content = f.readlines()
if not content:
return
for line in content:
(code, name, reason) = line.strip().split(';')
import re
import time
import xlrd
import xlwt
import sys
import os
import settings
from settings import is_holiday, DATA_PATH
import pandas as pd
from settings import llogger, DBSelector
import requests
# from send_mail import sender_139
import datetime
import tushare as ts
logger = llogger('log/zdt.log')
class GetZDT(object):
def __init__(self, today):
'''
today 格式 20200701
:param today:
'''
self.user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/64.0.3282.167 Chrome/64.0.3282.167 Safari/537.36"
if today is None:
self.today = time.strftime("%Y%m%d")
else:
self.today = today
# -*-coding=utf-8-*-
# @Time : 2019/2/11 14:28
# @File : cb_index.py
# 可转债指数
import datetime
import re
import pandas as pd
import requests
from settings import get_mysql_conn, llogger
import tushare as ts
logger = llogger('log/' + 'cb_index.log')
# 获取当天的记录
def get_today_index():
url = 'https://www.jisilu.cn/data/cbnew/cb_index_quote/'
headers = {
'Host':
'www.jisilu.cn',
'X-Requested-With':
'XMLHttpRequest',
'Referer':
'https://www.jisilu.cn/data/cbnew/',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
r = requests.get(url=url, headers=headers)
# print(r.text)
data_ = r.json()
import matplotlib as mpl
from mpl_finance import candlestick2_ochl, volume_overlay
import matplotlib.pyplot as plt
from settings import DBSelector
from settings import llogger
import sys
if sys.platform == 'linux':
# centos的配置, 根据自定义拷贝的字体
mpl.rcParams['font.sans-serif'] = ['Microsoft YaHei']
else:
mpl.rcParams['font.sans-serif'] = ['simhei']
mpl.rcParams['axes.unicode_minus'] = False
logger = llogger('log/plot_line.log')
DB = DBSelector()
engine = DB.get_engine('db_stock', 'qq')
base_info = pd.read_sql('tb_basic_info', engine, index_col='index')
def plot_stock_line(api,
code,
name,
table_name,
current,
start='2019-10-01',
save=False):
title = '{}_{}_{}_{}'.format(current, code, name, table_name)
title = title.replace('*', '_')
# -*-coding=utf-8-*-
import datetime
import os
import matplotlib
'''
昨日涨停的今日的实时情况
'''
matplotlib.use("Pdf")
from settings import get_engine, is_holiday, DATA_PATH
import pandas as pd
import tushare as ts
import numpy as np
from plot_line import plot_stock_line
from settings import llogger
logger = llogger('log/yester_zdt.log')
def monitor():
engine = get_engine('db_zdt', 'local')
table = '20180409zdt'
api = ts.get_apis()
df = pd.read_sql(table, engine, index_col='index')
price_list = []
percent_list = []
amplitude_list = []
start = datetime.datetime.now()
for i in df['代码'].values:
try:
curr = ts.quotes(i, conn=api)
last_close = curr['last_close'].values[0]
import pandas as pd
import datetime
import numpy as np
# pd.set_option('expand_frame_repr', False)
pd.set_option('display.max_rows', None)
# 条件
OUTSTANDING_MAX = 70 # 流通 x亿
TOTAL_MAX = 500 # 总市值
YIJIALU = 10 # 溢价率
ZZ_PRICE = 120 # 转债价格
REMAIN_SHARE = 5 # 转股剩余比例
logger = llogger('log/filter_bond.log')
today = datetime.datetime.now().strftime('%Y-%m-%d')
# today='2020-03-27'
DB = DBSelector()
# host=_json_data['mongo']['qq']['host']
# port=_json_data['mongo']['qq']['port']
try:
engine_daily = DB.get_engine('db_daily', 'qq')
engine = DB.get_engine('db_stock', 'qq')
jsl_df = pd.read_sql('tb_bond_jisilu', con=engine)
basic_df = pd.read_sql('tb_basic_info', con=engine, index_col='index')
price_df = pd.read_sql(today, con=engine_daily)
INFO = _json_data['mongo']['arm']
host = INFO['host']
port = INFO['port']
from settings import DBSelector, llogger, send_from_aliyun
import pandas as pd
import numpy as np
# from filter_stock import Filter_Stock
# 筛选出 新股 的可转债
# current = datetime.datetime.now()
# last_week = current + datetime.timedelta(days=7 * -2)
# conn = get_engine('db_stock')
# df = pd.read_sql('tb_bond_jisilu', con=conn)
# code_list = df['正股代码'].values
# kzz_code_list = df['可转债代码'].values
# kzz_name_list = df['可转债名称'].values
# db = pymongo.MongoClient('10.18.6.46', port=27001)
logger = llogger('log/' + 'bond_statistic.log')
DB = DBSelector()
# 获取次新股的可转债数据
def get_zhenggu():
obj = Filter_Stock()
ns_df = obj.get_new_stock('2016', '2018')
zg_code = ns_df[ns_df['code'].isin(code_list)]['code'].values
ret_df = df[df['正股代码'].isin(zg_code)]
ret_df = ret_df.reset_index(drop=True)
ret_df.to_sql('tb_new_stock_bond', con=conn, if_exists='replace')
# 筛选出每周 （月）涨幅排名倒数的10个
__author__ = 'Rocky' ''' http://30daydo.com Contact: [email protected] ''' import datetime import time import tushare as ts import os from settings import DBSelector, llogger, is_holiday, DATA_PATH import pandas as pd logger = llogger('log/collect_data.log') # 获取市场全貌 class SaveData(object): def __init__(self): work_space = DATA_PATH if os.path.exists(work_space) == False: os.mkdir(work_space) os.chdir(work_space) self.today = datetime.datetime.now().strftime("%Y-%m-%d") self.daily_engine = DBSelector().get_engine('db_daily', 'qq') def daily_market(self): df = ts.get_today_all() try: df.to_sql(self.today, self.daily_engine, if_exists='replace')
# @File : SPSIOP_PRICE.py
# 获取SPSIOP的价格，每天早上美股收盘
import datetime
import requests
import pymongo
from settings import llogger,_json_data,send_from_aliyun
host = _json_data['mongo']['qq']['host']
port = _json_data['mongo']['qq']['port']
user = _json_data['mongo']['qq']['user']
to_mail = _json_data['mail']['qq']['user']
password = _json_data['mongo']['qq']['password']
# host='127.0.0.1'
client = pymongo.MongoClient(host=host, port=port)
doc = client['db_stock']['SPSIOP']
# 先访问一下雪球首页得到cookies
logger = llogger('log/huabaoyouqi.log')
home_headers = {'User-Agent': 'Xueqiu App'}
headers = {'User-Agent': 'Xueqiu App',
'Access-Control-Allow-Origin': 'https://xueqiu.com',
'Content-Type': 'application/json;charset=UTF-8',
'P3P': 'CP="IDC DSP COR ADM DEVi TAIi PSA PSD IVAi IVDi CONi HIS OUR IND CNT"'}
url = 'https://stock.xueqiu.com/v5/stock/quote.json?symbol=.SPSIOP&extend=detail'
home_page = 'https://xueqiu.com'
today = datetime.datetime.now().strftime('%Y-%m-%d')
def get_price():
session = requests.Session()
# -*-coding=utf-8-*- __author__ = 'Rocky' ''' http://30daydo.com Contact: [email protected] ''' # 获取每天的行情信息 import tushare as ts import pandas as pd import time import datetime import os from settings import DBSelector, llogger, is_holiday, DATA_PATH logger = llogger('log/fetch_each_day.log') class FetchDaily(object): def __init__(self): self.today = datetime.datetime.now().strftime('%Y-%m-%d') # self.TODAY = '2020-02-07' self.path = DATA_PATH if not os.path.exists(self.path): try: os.mkdir(self.path) except Exception as e: print(e) self.df_today_all = pd.DataFrame() self.TIMEOUT = 10
''' http://30daydo.com Contact: [email protected] ''' import json import datetime import time import codecs import os, sys import requests import re from scrapy.selector import Selector from elasticsearch import Elasticsearch from settings import llogger, get_mysql_conn, DATA_PATH logger = llogger('log/stockinfo.log') my_useragent = [ 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', 'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', 'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11', 'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',