def __init__(self):
self.m = MongoDB_io()
self.nothing = ''
pass
def __init__(self):
self.m = MongoDB_io()
self.nothing = ''
self.start_trade_date = '1990-01-01'
pass
from jqdatasdk import *
import pandas as pd
from data_base.mongodb import MongoDB_io
auth('15915765128','87662638qjf')
m=MongoDB_io()
m.set_db('stock_daily_data')
m.set_collection('stock_sw_industry_code')
sw_indus=m.read_data_to_get_dataframe()
start_date='2010-01-01'
trade_date_list=get_trade_days(start_date=start_date, end_date=None, count=None)
group_day_num=1000
group_num=1
trade_date_list=get_trade_days(start_date=start_date, end_date=None, count=None)
weight_df=pd.DataFrame()
m=MongoDB_io()
m.set_db('stock_daily_data')
m.set_collection('stock_ipo_date')
ipo_df=m.read_data_to_get_dataframe()
stock_list=ipo_df.stock.tolist()
for date in trade_date_list:
print(date)
panel = get_price(stock_list, start_date=date, end_date=date, frequency='daily', fields=None, skip_paused=False, fq='none', count=None)
df=panel.iloc[:,0,:]
df.reset_index(inplace=True)
df.rename(columns={'index':'stock'},inplace=True)
from data_base.mongodb import MongoDB_io
import pandas as pd
from decorate_func.decorate_function import typing_func_name
start_date = '2016-01-01'
end_date = '2019-10-05'
m = MongoDB_io()
@typing_func_name
def get_capital_data():
"""
获得流通市值
:return:
"""
m.set_db('stock_daily_data')
m.set_collection('stock_capital_data')
df = m.read_data_to_get_dataframe_include_condition(start_date=start_date,
end_date=end_date)
return df[['circulating_market_cap', 'code',
'date']].set_index(['date',
'code']).circulating_market_cap.unstack()
pass
@typing_func_name
def get_stock_industry():
"""
获得股票行业分类
:return:
from jqdatasdk import *
import pandas as pd
from data_base.mongodb import MongoDB_io
## 查看数据插入到哪一天
m = MongoDB_io()
m.set_db('stock_daily_data')
m.set_collection('stock_sw_industry_category')
start_date, end_date = m.get_start_end_date()
pass
from data_base.mongodb import MongoDB_io
auth('15915765128','87662638qjf')
start_date='2005-01-01'
trade_date_list=get_trade_days(start_date=start_date, end_date=None, count=None)
trade_date_info_df=pd.DataFrame()
trade_date_info_df['trade_date']=trade_date_list
trade_date_info_df['weekday']=trade_date_info_df['trade_date'].apply(lambda x:x.weekday())+1.0
trade_date_info_df['trade_month']=trade_date_info_df['trade_date'].apply(lambda x:str(x)[:7])
def get_ordinal_of_date(x):
x['ordinal_in_month']=range(x.shape[0])
x['ordinal_in_month']=x['ordinal_in_month']+1.0
return x
pass
trade_date_info_df=trade_date_info_df.groupby('trade_month').apply(get_ordinal_of_date)
trade_date_info_df['trade_date']=pd.to_datetime(trade_date_info_df['trade_date'])
# 插入数据库
m=MongoDB_io()
m.set_db('stock_daily_data')
m.set_collection('stock_trade_date')
m.insert_huge_dataframe_by_block_to_mongodb(trade_date_info_df)
## 后面加上更新验证模块。
pass
from data_base.mongodb import MongoDB_io
collection_list = []
m = MongoDB_io()
m.set_db('stock_daily_data')
m.set_collection()
from data_base.mongodb import MongoDB_io
import pandas as pd
m = MongoDB_io()
m.set_db('stock_min_data')
collection_list = m.list_collection_names()
m.close_MongoDB_connection()
collection_list.sort()
insert_date_df = pd.DataFrame()
for stock in collection_list:
print(stock, collection_list.index(stock))
m.set_db('stock_min_data')
m.set_collection(stock)
df = m.read_data_to_get_field(field={'DATETIME': 1})
series_index = df.DATETIME.astype(str).apply(
lambda x: x[:10]).drop_duplicates().tolist()
insert_date_series = pd.Series(True, index=series_index)
insert_date_series.name = stock
insert_date_df = insert_date_df.append(insert_date_series)
m.close_MongoDB_connection()
pass
path = r'D:\code\factor_module\download_stock_min_data\stock_insert_situation.csv'
insert_date_df.to_csv(path)
from jqdatasdk import *
from data_base.mongodb import MongoDB_io
auth('15915765128', '87662638qjf')
start_date = '2005-01-01'
df = get_industries(name='sw_l1')
df = df.append(get_industries(name='sw_l2'))
df = df.append(get_industries(name='sw_l3'))
df.index.name = 'industry_code'
df.reset_index(inplace=True)
pass
# 插入数据库
m = MongoDB_io()
m.set_db('stock_daily_data')
m.set_collection('stock_sw_industry_code')
m.insert_huge_dataframe_by_block_to_mongodb(df)
## 后面加上更新验证模块。
pass
from jqdatasdk import *
import pandas as pd
from data_base.mongodb import MongoDB_io
## 查看数据插入到哪一天
m=MongoDB_io()
m.set_db('stock_daily_data')
m.set_collection('stock_pre_price')
start_date,end_date=m.get_start_end_date()
pass
def __init__(self):
self.m = MongoDB_io()
self.m.set_db('stock_min_data')
self.nothing = ''
self.collection_list = self.m.list_collection_names()
pass
class get_min_price_class(object):
def __init__(self):
self.m = MongoDB_io()
self.m.set_db('stock_min_data')
self.nothing = ''
self.collection_list = self.m.list_collection_names()
pass
def download_and_insert(self, stock, start_date, end_date):
m = self.m
print(stock, start_date, end_date)
min_data: pd.DataFrame = get_price(stock,
start_date=start_date,
end_date=end_date,
frequency='minute',
fields=None,
skip_paused=False,
fq=None,
count=None)
if min_data.shape[0] == 0:
print(stock, ' is empty')
return
min_data.index.name = 'datetime'
min_data.reset_index(inplace=True)
min_data.columns = min_data.columns.map(lambda x: x.upper())
min_data.rename({'MONEY': 'AMOUNT'}, axis=1, inplace=True)
min_data.DATETIME = min_data.DATETIME.astype(str)
m.set_collection(stock[:6])
# m.insert_huge_dataframe_by_block_to_mongodb(min_data)
m.insert_dataframe_to_mongodb(min_data)
pass
def check_stock_is_in_collection(self, stock):
collection_list = self.collection_list
if stock[:6] in collection_list:
return True
else:
return False
pass
def get_collection_insert_date(self, collection):
m = self.m
m.set_collection(collection)
df = m.read_data_to_get_field(field=['DATETIME'])
date_list = df.DATETIME.astype(str).apply(
lambda x: x[:10]).drop_duplicates().tolist()
date_list.sort()
return date_list
pass
# def multi_process_insert_min_data(stock_code_list,start_date,end_date,trade_date_list):
# process_list=[]
# sem=Semaphore(4)
# for stock in stock_code_list:
# print(stock)
# p=MyProcess(target=inserting_one_stock, args=(stock, start_date, end_date, trade_date_list), kwargs={'sem':sem})
# p.daemon=True
# p.start()
# process_list.append(p)
# pass
# for proc in process_list:
# proc.join()
# pass
# pass
def single_process_insert_min_data(self, stock_code_list, trade_date_list):
for stock in stock_code_list:
print(stock, stock_code_list.index(stock))
self.inserting_one_stock(stock, trade_date_list)
pass
pass
def inserting_one_stock(self, stock, trade_date_list):
start_date = trade_date_list[0]
end_date = trade_date_list[-1]
flag = self.check_stock_is_in_collection(stock)
if not flag:
self.download_and_insert(stock, start_date, end_date)
else:
date_list = self.get_collection_insert_date(stock[:6])
last_insert_date = date_list[-1]
if last_insert_date not in trade_date_list:
return
index = trade_date_list.index(last_insert_date)
next_trade_date = trade_date_list[index + 1]
self.download_and_insert(stock, next_trade_date,
trade_date_list[-1])
pass
def insert_stock_min_data(self):
logging_joinquant()
stock_list = get_stock_code_list()
stock_list.sort()
stock_list = stock_list[200:500]
dic = get_setting_start_end_date()
start_date = dic['start_date']
end_date = dic['end_date']
trade_date_series: pd.Series = pd.Series(
get_trade_date_list(start_date, end_date)).astype(str)
trade_date_list = trade_date_series.tolist()
self.single_process_insert_min_data(stock_list, trade_date_list)
# multi_process_insert_min_data(stock_list,start_date,end_date,trade_date_list)
pass
count=None)
weight_df = pd.DataFrame()
for date in trade_date_list[(group_num - 1) * group_day_num:group_num *
group_day_num]:
print(date)
q = query(valuation).filter(valuation.market_cap > 0)
df = get_fundamentals(q, date)
df = df.loc[:, [
'code', 'day', 'market_cap', 'circulating_cap',
'circulating_market_cap'
]]
df.rename(columns={'day': 'date'}, inplace=True)
# 打印出总市值
weight_df = weight_df.append(df)
pass
## df 格式修改
weight_df.date = pd.to_datetime(weight_df.date)
if 'index' in weight_df.columns:
weight_df.drop('index', axis=1, inplace=True)
pass
print('transfer done')
# 插入数据库
m = MongoDB_io()
m.set_db('stock_daily_data')
m.set_collection('stock_capital_data')
m.insert_huge_dataframe_by_block_to_mongodb(weight_df)
pass
from jqdatasdk import *
import pandas as pd
from data_base.mongodb import MongoDB_io
auth('15915765128', '87662638qjf')
m = MongoDB_io()
m.set_db('stock_daily_data')
m.set_collection('stock_sw_industry_code')
sw_indus = m.read_data_to_get_dataframe()
m.set_collection('stock_trade_date')
trade_day_df = m.read_data_to_get_dataframe()
trade_day_df = trade_day_df[
trade_day_df.trade_date > pd.to_datetime('2010-01-01')]
trade_day_list = trade_day_df.trade_date.astype(str)
industry_code_list = sw_indus.industry_code.iloc[:34].tolist()
industry_stock_grouping = pd.DataFrame()
for date in trade_day_list:
print(date)
daily_industry_series = pd.Series()
for industry_code in industry_code_list[:]:
stock_list = get_industry_stocks(industry_code, date=date)
daily_industry_series = daily_industry_series.append(
pd.Series(industry_code, index=stock_list))
pass
daily_industry_series.name = pd.to_datetime(date)
daily_industry_df = daily_industry_series.to_frame()
daily_industry_df_stack_up = daily_industry_df.stack().reset_index()
daily_industry_df_stack_up.columns = ['stock', 'date', 'industry_category']
# 'stock_sw_industry_code',
# 'stock_trade_date',
# 'zz500_weight']
# db_list=['stock_capital_data',
# 'stock_post_price',
# 'stock_real_price',
# 'stock_price_adj_factor',
# 'stock_sw_industry_code',
# 'zz500_weight']
db_list = [
# 'stock_capital_data',
# 'stock_sw_industry_code',
'stock_real_price_',
# 'zz500_weight'
]
m = MongoDB_io()
m.set_db('stock_daily_data')
condition_df = pd.DataFrame()
for db in db_list:
print(db)
m.set_collection(db)
document_per_date = m.get_document_in_date()
document_per_date.name = db
condition_df = condition_df.append(document_per_date)
pass
# condition_df=condition_df.loc[:,'2010-01-01':]
condition_df.to_csv('update_condition.csv')