def __init__(self): self.m = MongoDB_io() self.nothing = '' pass
def __init__(self): self.m = MongoDB_io() self.nothing = '' self.start_trade_date = '1990-01-01' pass
from jqdatasdk import * import pandas as pd from data_base.mongodb import MongoDB_io auth('15915765128','87662638qjf') m=MongoDB_io() m.set_db('stock_daily_data') m.set_collection('stock_sw_industry_code') sw_indus=m.read_data_to_get_dataframe() start_date='2010-01-01' trade_date_list=get_trade_days(start_date=start_date, end_date=None, count=None) group_day_num=1000 group_num=1 trade_date_list=get_trade_days(start_date=start_date, end_date=None, count=None) weight_df=pd.DataFrame() m=MongoDB_io() m.set_db('stock_daily_data') m.set_collection('stock_ipo_date') ipo_df=m.read_data_to_get_dataframe() stock_list=ipo_df.stock.tolist() for date in trade_date_list: print(date) panel = get_price(stock_list, start_date=date, end_date=date, frequency='daily', fields=None, skip_paused=False, fq='none', count=None) df=panel.iloc[:,0,:] df.reset_index(inplace=True) df.rename(columns={'index':'stock'},inplace=True)
from data_base.mongodb import MongoDB_io import pandas as pd from decorate_func.decorate_function import typing_func_name start_date = '2016-01-01' end_date = '2019-10-05' m = MongoDB_io() @typing_func_name def get_capital_data(): """ 获得流通市值 :return: """ m.set_db('stock_daily_data') m.set_collection('stock_capital_data') df = m.read_data_to_get_dataframe_include_condition(start_date=start_date, end_date=end_date) return df[['circulating_market_cap', 'code', 'date']].set_index(['date', 'code']).circulating_market_cap.unstack() pass @typing_func_name def get_stock_industry(): """ 获得股票行业分类 :return:
from jqdatasdk import * import pandas as pd from data_base.mongodb import MongoDB_io ## 查看数据插入到哪一天 m = MongoDB_io() m.set_db('stock_daily_data') m.set_collection('stock_sw_industry_category') start_date, end_date = m.get_start_end_date() pass
from data_base.mongodb import MongoDB_io auth('15915765128','87662638qjf') start_date='2005-01-01' trade_date_list=get_trade_days(start_date=start_date, end_date=None, count=None) trade_date_info_df=pd.DataFrame() trade_date_info_df['trade_date']=trade_date_list trade_date_info_df['weekday']=trade_date_info_df['trade_date'].apply(lambda x:x.weekday())+1.0 trade_date_info_df['trade_month']=trade_date_info_df['trade_date'].apply(lambda x:str(x)[:7]) def get_ordinal_of_date(x): x['ordinal_in_month']=range(x.shape[0]) x['ordinal_in_month']=x['ordinal_in_month']+1.0 return x pass trade_date_info_df=trade_date_info_df.groupby('trade_month').apply(get_ordinal_of_date) trade_date_info_df['trade_date']=pd.to_datetime(trade_date_info_df['trade_date']) # 插入数据库 m=MongoDB_io() m.set_db('stock_daily_data') m.set_collection('stock_trade_date') m.insert_huge_dataframe_by_block_to_mongodb(trade_date_info_df) ## 后面加上更新验证模块。 pass
from data_base.mongodb import MongoDB_io collection_list = [] m = MongoDB_io() m.set_db('stock_daily_data') m.set_collection()
from data_base.mongodb import MongoDB_io import pandas as pd m = MongoDB_io() m.set_db('stock_min_data') collection_list = m.list_collection_names() m.close_MongoDB_connection() collection_list.sort() insert_date_df = pd.DataFrame() for stock in collection_list: print(stock, collection_list.index(stock)) m.set_db('stock_min_data') m.set_collection(stock) df = m.read_data_to_get_field(field={'DATETIME': 1}) series_index = df.DATETIME.astype(str).apply( lambda x: x[:10]).drop_duplicates().tolist() insert_date_series = pd.Series(True, index=series_index) insert_date_series.name = stock insert_date_df = insert_date_df.append(insert_date_series) m.close_MongoDB_connection() pass path = r'D:\code\factor_module\download_stock_min_data\stock_insert_situation.csv' insert_date_df.to_csv(path)
from jqdatasdk import * from data_base.mongodb import MongoDB_io auth('15915765128', '87662638qjf') start_date = '2005-01-01' df = get_industries(name='sw_l1') df = df.append(get_industries(name='sw_l2')) df = df.append(get_industries(name='sw_l3')) df.index.name = 'industry_code' df.reset_index(inplace=True) pass # 插入数据库 m = MongoDB_io() m.set_db('stock_daily_data') m.set_collection('stock_sw_industry_code') m.insert_huge_dataframe_by_block_to_mongodb(df) ## 后面加上更新验证模块。 pass
from jqdatasdk import * import pandas as pd from data_base.mongodb import MongoDB_io ## 查看数据插入到哪一天 m=MongoDB_io() m.set_db('stock_daily_data') m.set_collection('stock_pre_price') start_date,end_date=m.get_start_end_date() pass
def __init__(self): self.m = MongoDB_io() self.m.set_db('stock_min_data') self.nothing = '' self.collection_list = self.m.list_collection_names() pass
class get_min_price_class(object): def __init__(self): self.m = MongoDB_io() self.m.set_db('stock_min_data') self.nothing = '' self.collection_list = self.m.list_collection_names() pass def download_and_insert(self, stock, start_date, end_date): m = self.m print(stock, start_date, end_date) min_data: pd.DataFrame = get_price(stock, start_date=start_date, end_date=end_date, frequency='minute', fields=None, skip_paused=False, fq=None, count=None) if min_data.shape[0] == 0: print(stock, ' is empty') return min_data.index.name = 'datetime' min_data.reset_index(inplace=True) min_data.columns = min_data.columns.map(lambda x: x.upper()) min_data.rename({'MONEY': 'AMOUNT'}, axis=1, inplace=True) min_data.DATETIME = min_data.DATETIME.astype(str) m.set_collection(stock[:6]) # m.insert_huge_dataframe_by_block_to_mongodb(min_data) m.insert_dataframe_to_mongodb(min_data) pass def check_stock_is_in_collection(self, stock): collection_list = self.collection_list if stock[:6] in collection_list: return True else: return False pass def get_collection_insert_date(self, collection): m = self.m m.set_collection(collection) df = m.read_data_to_get_field(field=['DATETIME']) date_list = df.DATETIME.astype(str).apply( lambda x: x[:10]).drop_duplicates().tolist() date_list.sort() return date_list pass # def multi_process_insert_min_data(stock_code_list,start_date,end_date,trade_date_list): # process_list=[] # sem=Semaphore(4) # for stock in stock_code_list: # print(stock) # p=MyProcess(target=inserting_one_stock, args=(stock, start_date, end_date, trade_date_list), kwargs={'sem':sem}) # p.daemon=True # p.start() # process_list.append(p) # pass # for proc in process_list: # proc.join() # pass # pass def single_process_insert_min_data(self, stock_code_list, trade_date_list): for stock in stock_code_list: print(stock, stock_code_list.index(stock)) self.inserting_one_stock(stock, trade_date_list) pass pass def inserting_one_stock(self, stock, trade_date_list): start_date = trade_date_list[0] end_date = trade_date_list[-1] flag = self.check_stock_is_in_collection(stock) if not flag: self.download_and_insert(stock, start_date, end_date) else: date_list = self.get_collection_insert_date(stock[:6]) last_insert_date = date_list[-1] if last_insert_date not in trade_date_list: return index = trade_date_list.index(last_insert_date) next_trade_date = trade_date_list[index + 1] self.download_and_insert(stock, next_trade_date, trade_date_list[-1]) pass def insert_stock_min_data(self): logging_joinquant() stock_list = get_stock_code_list() stock_list.sort() stock_list = stock_list[200:500] dic = get_setting_start_end_date() start_date = dic['start_date'] end_date = dic['end_date'] trade_date_series: pd.Series = pd.Series( get_trade_date_list(start_date, end_date)).astype(str) trade_date_list = trade_date_series.tolist() self.single_process_insert_min_data(stock_list, trade_date_list) # multi_process_insert_min_data(stock_list,start_date,end_date,trade_date_list) pass
count=None) weight_df = pd.DataFrame() for date in trade_date_list[(group_num - 1) * group_day_num:group_num * group_day_num]: print(date) q = query(valuation).filter(valuation.market_cap > 0) df = get_fundamentals(q, date) df = df.loc[:, [ 'code', 'day', 'market_cap', 'circulating_cap', 'circulating_market_cap' ]] df.rename(columns={'day': 'date'}, inplace=True) # 打印出总市值 weight_df = weight_df.append(df) pass ## df 格式修改 weight_df.date = pd.to_datetime(weight_df.date) if 'index' in weight_df.columns: weight_df.drop('index', axis=1, inplace=True) pass print('transfer done') # 插入数据库 m = MongoDB_io() m.set_db('stock_daily_data') m.set_collection('stock_capital_data') m.insert_huge_dataframe_by_block_to_mongodb(weight_df) pass
from jqdatasdk import * import pandas as pd from data_base.mongodb import MongoDB_io auth('15915765128', '87662638qjf') m = MongoDB_io() m.set_db('stock_daily_data') m.set_collection('stock_sw_industry_code') sw_indus = m.read_data_to_get_dataframe() m.set_collection('stock_trade_date') trade_day_df = m.read_data_to_get_dataframe() trade_day_df = trade_day_df[ trade_day_df.trade_date > pd.to_datetime('2010-01-01')] trade_day_list = trade_day_df.trade_date.astype(str) industry_code_list = sw_indus.industry_code.iloc[:34].tolist() industry_stock_grouping = pd.DataFrame() for date in trade_day_list: print(date) daily_industry_series = pd.Series() for industry_code in industry_code_list[:]: stock_list = get_industry_stocks(industry_code, date=date) daily_industry_series = daily_industry_series.append( pd.Series(industry_code, index=stock_list)) pass daily_industry_series.name = pd.to_datetime(date) daily_industry_df = daily_industry_series.to_frame() daily_industry_df_stack_up = daily_industry_df.stack().reset_index() daily_industry_df_stack_up.columns = ['stock', 'date', 'industry_category']
# 'stock_sw_industry_code', # 'stock_trade_date', # 'zz500_weight'] # db_list=['stock_capital_data', # 'stock_post_price', # 'stock_real_price', # 'stock_price_adj_factor', # 'stock_sw_industry_code', # 'zz500_weight'] db_list = [ # 'stock_capital_data', # 'stock_sw_industry_code', 'stock_real_price_', # 'zz500_weight' ] m = MongoDB_io() m.set_db('stock_daily_data') condition_df = pd.DataFrame() for db in db_list: print(db) m.set_collection(db) document_per_date = m.get_document_in_date() document_per_date.name = db condition_df = condition_df.append(document_per_date) pass # condition_df=condition_df.loc[:,'2010-01-01':] condition_df.to_csv('update_condition.csv')