def 下载板块数据(self):
# 获取不同分类数据
ts.get_industry_classified().to_csv('./股票数据/基本信息/行业分类.csv')
ts.get_concept_classified().to_csv('./股票数据/基本信息/概念分类.csv')
ts.get_area_classified().to_csv('./股票数据/基本信息/地域分类.csv')
ts.get_sme_classified().to_csv('./股票数据/基本信息/中小板.csv')
ts.get_gem_classified().to_csv('./股票数据/基本信息/创业板.csv')
ts.get_st_classified().to_csv('./股票数据/基本信息/风险警示板.csv')
ts.get_hs300s().to_csv('./股票数据/基本信息/沪深300.csv')
ts.get_sz50s().to_csv('./股票数据/基本信息/上证50.csv')
ts.get_zz500s().to_csv('./股票数据/基本信息/中证500.csv')
def main():
logger = logging.getLogger("mylog")
formatter = logging.Formatter(
'%(name)-12s %(asctime)s %(levelname)-8s %(message)s',
'%a, %d %b %Y %H:%M:%S',
)
file_handler = logging.FileHandler("./log/test_log.txt", encoding='utf-8')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG)
zxb_tickers = ts.get_sme_classified()
one_dateee = date(2015, month=7, day=30)
for i in range(zxb_tickers.index.size):
one_ticker = zxb_tickers.iloc[i, 0]
for j in range(3650):
one_data = input_data_label(one_ticker, one_dateee, 30, 0.1)
x = one_data.get_label()
#检查数据合理性
if (not one_data.data_valid):
logger.debug('invalid: ' + str(one_ticker) + '_' +
date_to_str(one_dateee) + "_" + str(x))
#发现买入点
if (x == 1):
logger.debug('valid: 买入' + str(one_ticker) + '_' +
date_to_str(one_dateee) + "_" + str(x))
one_dateee = one_dateee - timedelta(1)
def load_data():
try:
rs=ts.get_sme_classified()
pd.DataFrame.to_sql(rs, table_name, con=conn , flavor='mysql', if_exists='replace',index=True)
except Exception as e:
print(e.message)
print("公司地域分类信息数据出错")
def load_tushare_df(df_type):
file = 'ts.' + df_type + '.dat'
try:
obj = pickle.load(open(file, "rb"))
except:
#print("---load in the fly",df_type)
if df_type == "basic":
obj = ts.get_stock_basics()
elif df_type == "sme":
obj = ts.get_sme_classified()
elif df_type == "gem":
obj = ts.get_gem_classified()
elif df_type == "industry":
#print(ts, pickle)
obj = ts.get_industry_classified(
) #该函数不全,只有2800多个,实际股票有3326,所以换个方法取 get_stock_basics
elif df_type == "st":
obj = ts.get_st_classified()
else:
raise Exception("Error TSshare Type!!!")
pickle.dump(obj, open(file, "wb", 0))
else:
#print("***Read from file %s" % df_type)
pass
return obj
def core_function(self, func):
self.set_data()
mongo = MongoClient("127.0.0.1", 27017)
if (func == "industry_classified"):
df = ts.get_industry_classified()
elif (func == "concept_classified"):
df = ts.get_concept_classified()
elif (func == "area_classified"):
df = ts.get_area_classified()
elif (func == "gem_classified"):
df = ts.get_gem_classified()
elif (func == "sme_classified"):
df = ts.get_sme_classified()
elif (func == "st_classified"):
df = ts.get_st_classified()
elif (func == "hs300s"):
df = ts.get_hs300s()
elif (func == "sz50s"):
df = ts.get_sz50s()
elif (func == "zz500s"):
df = ts.get_zz500s()
print(df)
elif (func == "terminated"):
df = ts.get_terminated()
else:
df = {}
insert_string = df.to_json(orient='records')
items = json.loads(insert_string)
coll = mongo.classify[func]
coll.insert(items)
def get_sme_info():
# 中小板分类
df = ts.get_sme_classified()
res = df.to_sql(all_sme_table, engine, if_exists='replace')
msg = 'ok' if res is None else res
print('获取中小板分类: ' + msg + '\n')
print(msg)
def fetch_classification(self):
# 数据来源自新浪财经的行业分类/概念分类/地域分类
print("Trying: get_today_all")
today_all = ts.get_today_all() #一次性获取今日全部股价
set_today_all = set(today_all.T.values[0])
print("Trying: get_industry_classified")
industry_classified = ts.get_industry_classified()
set_industry_classified = set(industry_classified.T.values[0])
print("Trying: get_area_classified")
area_classified = ts.get_area_classified()
set_area_classified = set(area_classified.T.values[0])
print("Trying: get_concept_classified")
concept_classified = ts.get_concept_classified()
set_concept_classified = set(concept_classified.T.values[0])
print("Trying: get_sme_classified")
sme_classified = ts.get_sme_classified()
set_sme_classified = set(sme_classified.T.values[0])
return [
today_all, set_today_all, industry_classified,
set_industry_classified, area_classified, set_area_classified,
concept_classified, set_concept_classified, sme_classified,
set_sme_classified
]
def test_ts_stock():
logger = logging.getLogger("mylog")
formatter = logging.Formatter(
'%(name)-12s %(asctime)s %(levelname)-8s %(message)s',
'%a, %d %b %Y %H:%M:%S',
)
file_handler = logging.FileHandler("./log/test_log.txt", encoding='utf-8')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG)
zxb_tickers = ts.get_sme_classified()
one_dateee = date(2015, month=7, day=30)
for i in range(zxb_tickers.index.size):
one_ticker = zxb_tickers.iloc[i, 0]
for j in range(3650):
one_data = price_and_volume_ts(one_ticker, one_dateee)
x = one_data.get_normalized_price_and_volume()
# 检查数据合理性
if (one_data.data_valid):
logger.debug('valid: ' + str(one_ticker) + '_' +
date_to_str(one_dateee) + "_" + str(x)) # 数据有效
elif (one_data.no_week_ma):
logger.debug('invalid: no week MA20 ' + str(one_ticker) + '_' +
date_to_str(one_dateee) + "_" + str(x))
break # 数据无效，而且再往前的日子的数据都不会有效了
else: # 数据无效
logger.debug('invalid: ' + str(one_ticker) + '_' +
date_to_str(one_dateee) + "_" + str(x))
one_dateee = one_dateee - timedelta(1)
def classification(class_types):
if class_types == 'industry':
industry_classified = ts.get_industry_classified()
industry_classified.to_csv('D:\\ts\\classification\\industry_classified.csv', encoding='gbk')
elif class_types == 'concept':
concept_classified = ts.get_concept_classified()
concept_classified.to_csv('D:\\ts\\classification\\concept_classified.csv', encoding='gbk')
elif class_types == 'area':
area_classified = ts.get_area_classified()
area_classified.to_csv('D:\\ts\\classification\\area_classified.csv', encoding='gbk')
elif class_types == 'sme':
sme_classified = ts.get_sme_classified()
sme_classified.to_csv('D:\\ts\\classification\\sme_classified.csv', encoding='gbk')
elif class_types == 'gem':
gem_classified = ts.get_gem_classified()
gem_classified.to_csv('D:\\ts\\classification\\gem_classified.csv', encoding='gbk')
elif class_types == 'st':
st_classified = ts.get_st_classified()
st_classified.to_csv('D:\\ts\\classification\\st_classified.csv', encoding='gbk')
elif class_types == 'hs300':
hs300s = ts.get_hs300s()
hs300s.to_csv('D:\\ts\\classification\\hs300s.csv', encoding='gbk')
elif class_types == 'sz50':
sz50s = ts.get_sz50s()
sz50s.to_csv('D:\\ts\\classification\\sz50s.csv', encoding='gbk')
elif class_types == 'zz500':
zz500s = ts.get_zz500s()
zz500s.to_csv('D:\\ts\\classification\\zz500s.csv', encoding='gbk')
elif class_types == 'terminated':
terminated = ts.get_terminated()
terminated.to_csv('D:\\ts\\classification\\terminated.csv', encoding='gbk')
elif class_types == 'suspended':
suspended = ts.get_suspended()
suspended.to_csv('D:\\ts\\classification\\suspended.csv', encoding='gbk')
def fetch_classification(self): # 数据来源自新浪财经的行业分类/概念分类/地域分类 print( "Trying: get_today_all" ) today_all = ts.get_today_all() #一次性获取今日全部股价 set_today_all = set(today_all.T.values[0]) print( "Trying: get_industry_classified" ) industry_classified = ts.get_industry_classified() set_industry_classified = set(industry_classified.T.values[0]) print( "Trying: get_area_classified" ) area_classified = ts.get_area_classified() set_area_classified = set(area_classified.T.values[0]) print( "Trying: get_concept_classified" ) concept_classified = ts.get_concept_classified() set_concept_classified = set(concept_classified.T.values[0]) print( "Trying: get_sme_classified" ) sme_classified = ts.get_sme_classified() set_sme_classified = set(sme_classified.T.values[0]) return [ today_all , set_today_all , industry_classified , set_industry_classified , area_classified , set_area_classified , concept_classified , set_concept_classified , sme_classified , set_sme_classified ]
def get_sme_classified():
try:
logger.info('get sme data starting...')
df = ts.get_sme_classified()
logger.info('get sme data end...')
except:
logger.exception('some errors between get sme data end...')
return df
def get_002_300(self):
g002 = ts.get_sme_classified()
g300 = ts.get_gem_classified()
engine = create_engine(
'mysql://*****:*****@127.0.0.1/tushareapp?charset=utf8')
g002.to_sql('g002', engine, if_exists='append')
time.sleep(60)
g300.to_sql('g300', engine, if_exists='append')
def get_sme_classified(self):
"""中小板分类"""
sme = ts.get_sme_classified()
sme['smeName'] = 1
sme = sme[['code','smeName']]
sme.columns = ['code','smeName']
self.data = pd.merge(self.data,sme,on=['code'],how='outer')
def get_stockcodes(stocktype): #stocktype 股票类型：sh 上证 sz 深证 cyb 创业板
if(stocktype=='cyb'): #创业板
codes=ts.get_gem_classified().code.tolist()
elif (stocktype=='sz'): #深证
codes=ts.get_sme_classified().code.tolist()
elif(stocktype=='sh'): #上证
codes=ts.get_sz50s().code.tolist() #目前只调用了上证50
return codes #返回的股票代码列表
def getSmeClassified():
'''
获取中小板分类。获取中小板股票数据，即查找所有002开头的股票
参数说明：
file_path:文件路径，默认为None即由TuShare提供，可以设定自己的股票文件路径。
返回值说明：
code：股票代码
name：股票名称
'''
return ts.get_sme_classified()
def main():
#按照日期和code取数据
hs300s = ts.get_sme_classified()
date_list = _datelist((2014, 1, 1), (2017, 1, 1))
for one_code in hs300s['code']:
for one_date in date_list:
prices, volumes = _get_price_and_volume_ts(one_code, one_date)
if (prices is not None):
prices = prices > np.mean(prices)
volumes = volumes > np.mean(volumes)
def stk_get_sme_classified():
try:
conn = connMysql().createconn()
cur = conn.cursor()
cur.execute('truncate table stk_sme_classified;')
cur.close()
conn.close()
df = ts.get_sme_classified()
df.to_sql('stk_sme_classified', engine, if_exists='append')
except:
print("获取中小板数据失败")
def load_data():
try:
rs = ts.get_sme_classified()
pd.DataFrame.to_sql(rs,
name=table_name,
con=con,
schema=db,
if_exists='replace',
index=True)
except Exception as e:
print(e)
print("公司地域分类信息数据出错")
def getstockclassifieddata():
mylogger = getmylogger()
print("start")
scdict = {
'industry': '行业分类',
'concept': '概念分类',
'area': '地域分类',
'sme': '中小板分类',
'gem': '创业板分类',
'st': '风险警示板分类',
'hs300s': '沪深300成份及权重',
'sz50s': '上证50成份股',
'zz500s': '中证500成份股',
'terminated': '终止上市股票列表',
'suspended': '暂停上市股票列表'
}
for sc in scdict:
sctbname = sc
scinfo = scdict[sctbname]
try:
if sc == "industry":
df = ts.get_industry_classified()
elif sc == "concept":
df = ts.get_concept_classified()
elif sc == "area":
df = ts.get_area_classified()
elif sc == "sme":
df = ts.get_sme_classified()
elif sc == "gem":
df = ts.get_gem_classified()
elif sc == "st":
df = ts.get_st_classified()
elif sc == "hs300s":
df = ts.get_hs300s()
elif sc == "zz500s":
df = ts.get_zz500s()
elif sc == "sz50s":
df = ts.get_sz50s()
elif sc == "terminated":
df = ts.get_terminated()
elif sc == "suspended":
df = ts.get_suspended()
else:
mylogger.info("没有执行命令。")
if df is not None:
tosql(df, sctbname, "replace", scinfo, mylogger)
else:
mylogger.info("没有%s数据。" % scinfo)
except Exception:
tracelog = traceback.format_exc()
mylogger.info("获取数据异常。")
mylogger.info(tracelog)
def updatesmall():
smalldatalist=ts.get_sme_classified()
smalldata=pd.DataFrame(smalldatalist)
conn= ms.connect(host='localhost',port = 3306,user='******', passwd='123456',db ='investment',charset="utf8")
cur = conn.cursor()
values=[]
for index,row in smalldata.iterrows():
values.append((row['code'],row['name']))
cur.executemany('insert into small (code,name) values(%s,%s)',values)
conn.commit()
cur.close()
conn.close()
def getZXCodeList():
df_zx = None
try:
df_zx = pd.read_sql_table('zx', con=engine)
except:
pass
if df_zx is None or df_zx.empty:
df_zx = ts.get_sme_classified()
df_zx.to_sql('zx',
con=engine,
if_exists='replace',
index=False,
index_label='code')
return df_zx['code'].tolist()
def update_sme(request):
'''更新中小板数据'''
stock_sme.objects.all().delete()
rltobj = ts.get_sme_classified()
objlist = []
for i in range(len(rltobj)):
tmpobj = stock_sme(code=rltobj.loc[i]['code'],
name=rltobj.loc[i]['name'])
objlist.append(tmpobj)
stock_sme.objects.bulk_create(objlist)
return HttpResponse('succ.{0}条!'.format(len(objlist)))
def list(self, stock_block_type):
stock_block = None
if stock_block_type == self.industry:
stock_block = db.get(STOCK_BLOCK_INDUSTRY)
if stock_block is None:
stock_block = ts.get_industry_classified()
db.save(STOCK_BLOCK_INDUSTRY, stock_block)
elif stock_block_type == self.concept:
stock_block = db.get(STOCK_BLOCK_CONCEPT)
if stock_block is None:
stock_block = ts.get_concept_classified()
db.save(STOCK_BLOCK_CONCEPT, stock_block)
elif stock_block_type == self.area:
stock_block = db.get(STOCK_BLOCK_AREA)
if stock_block is None:
stock_block = ts.get_area_classified()
db.save(STOCK_BLOCK_AREA, stock_block)
elif stock_block_type == self.sme:
stock_block = db.get(STOCK_BLOCK_SME)
if stock_block is None:
stock_block = ts.get_sme_classified()
db.save(STOCK_BLOCK_SME, stock_block)
elif stock_block_type == self.gem:
stock_block = db.get(STOCK_BLOCK_GEM)
if stock_block is None:
stock_block = ts.get_gem_classified()
db.save(STOCK_BLOCK_GEM, stock_block)
elif stock_block_type == self.st:
stock_block = db.get(STOCK_BLOCK_ST)
if stock_block is None:
stock_block = ts.get_st_classified()
db.save(STOCK_BLOCK_ST, stock_block)
elif stock_block_type == self.hs300s:
stock_block = db.get(STOCK_BLOCK_HS300S)
if stock_block is None:
stock_block = ts.get_hs300s()
db.save(STOCK_BLOCK_HS300S, stock_block)
elif stock_block_type == self.sz50s:
stock_block = db.get(STOCK_BLOCK_SZ50S)
if stock_block is None:
stock_block = ts.get_sz50s()
db.save(STOCK_BLOCK_SZ50S, stock_block)
elif stock_block_type == self.zz500s:
stock_block = db.get(STOCK_BLOCK_ZZ500S)
if stock_block is None:
stock_block = ts.get_zz500s()
db.save(STOCK_BLOCK_ZZ500S, stock_block)
else:
return None
return stock_block
def save_sme_classified():
"""获取中小板股票数据，即查找所有002开头的股票"""
logger.info('Begin get and save sme中小板 clssified.')
try:
data_df = ts.get_sme_classified()
if data_df is not None and not data_df.empty:
data = data_df.values
data_dicts = [{'code': row[0], 'name': row[1]} for row in data]
SmeClassified.insert_many(data_dicts).execute()
logger.info('Success get and save sme中小板 classified.')
else:
logger.warn('Empty get and save sme中小板 classified.')
except Exception as e:
logger.exception('Error get and save sme中小板 classified.')
def sme_classified(request):
try:
data = ts.get_sme_classified()
res = {
'columns': [
'股票代码',
'股票名称',
],
'data': json.loads(json.dumps(data.values, cls=DateEncoder))
}
except (Exception):
return HttpResponse('error', Exception)
else:
return HttpResponse(json.dumps(res), content_type="application/json")
def get_stock_codes(markets=['zz500s'], ipo_date=None):
'''
markets: list
e.g.['sme','gem','st','hs300s','sz50s','zz500s','general'];
ipo_date: str
e.g.'2015-03-30'
'''
code_list = []
if 'sme' in markets: #中小板
df = ts.get_sme_classified()
codes = list(df.code)
code_list.extend(codes)
if 'gem' in markets: #创业板
df = ts.get_gem_classified()
codes = list(df.code)
code_list.extend(codes)
if 'st' in markets: #风险板
df = ts.get_st_classified()
codes = list(df.code)
code_list.extend(codes)
if 'hs300s' in markets: #沪深300
df = ts.get_hs300s()
codes = list(df.code)
code_list.extend(codes)
if 'sz50s' in markets: #上证50
df = ts.get_sz50s()
codes = list(df.code)
code_list.extend(codes)
if 'zz500s' in markets: #中证500
df = ts.get_zz500s()
codes = list(df.code)
code_list.extend(codes)
if ipo_date:
new_stock_df = ts.new_stocks()
new_stock_df = new_stock_df[new_stock_df['ipo_date'] > ipo_date]
new_stock_codes = list(new_stock_df.code)
#得到输入时间之后发行的股票
code_list = list(set(code_list))
desired_codes = list(set(code_list) - set(new_stock_codes))
#剔除新股
desired_codes = list(set(code_list))
return desired_codes
def updatesmall():
smalldatalist = ts.get_sme_classified()
smalldata = pd.DataFrame(smalldatalist)
conn = ms.connect(host='localhost',
port=3306,
user='******',
passwd='123456',
db='investment',
charset="utf8")
cur = conn.cursor()
values = []
for index, row in smalldata.iterrows():
values.append((row['code'], row['name']))
cur.executemany('insert into small (code,name) values(%s,%s)', values)
conn.commit()
cur.close()
conn.close()
def preload():
stock_block = ts.get_industry_classified()
db.save(STOCK_BLOCK_INDUSTRY, stock_block)
stock_block = ts.get_concept_classified()
db.save(STOCK_BLOCK_CONCEPT, stock_block)
stock_block = ts.get_area_classified()
db.save(STOCK_BLOCK_AREA, stock_block)
stock_block = ts.get_sme_classified()
db.save(STOCK_BLOCK_SME, stock_block)
stock_block = ts.get_gem_classified()
db.save(STOCK_BLOCK_GEM, stock_block)
stock_block = ts.get_st_classified()
db.save(STOCK_BLOCK_ST, stock_block)
stock_block = ts.get_hs300s()
db.save(STOCK_BLOCK_HS300S, stock_block)
stock_block = ts.get_sz50s()
db.save(STOCK_BLOCK_SZ50S, stock_block)
stock_block = ts.get_zz500s()
db.save(STOCK_BLOCK_ZZ500S, stock_block)
def initial_pool():
concept_df = ts.get_concept_classified()
for idx in concept_df.index:
if(concept_df['c_name'][idx] == "汽车电子"):
a_concept_list.append(concept_df['code'][idx])
print("汽车电子 concept is ")
#concept_df.to_csv('concept.csv')
industry_df = ts.get_industry_classified()
for idx in industry_df.index:
if(concept_df['c_name'][idx] == "电子器件"):
a_concept_list.append(concept_df['code'][idx])
if(concept_df['c_name'][idx] == "电子信息"):
a_concept_list.append(concept_df['code'][idx])
print(a_concept_list)
# print(industry_df)
#industry_df.to_csv('industry.csv')
# small and chuangye
try:
startup_df = ts.get_gem_classified()
sm_df = ts.get_sme_classified()
basic_df = ts.get_stock_basics()
except:
traceback.print_exc()
pass
print(startup_df)
print("start to parse startup stocks\n") ;
for idx in startup_df.index:
pe = basic_df['pe'][startup_df['code'][idx]]
if(pe < 80 and pe != 0):
startup_pool.append(startup_df['code'][idx])
# print(str(startup_df['code'][idx])+" pe is "+str(pe))
print(startup_pool)
print("start to parse small and middle stocks\n") ;
for idx in sm_df.index:
pe = basic_df['pe'][sm_df['code'][idx]]
if(pe < 50 and pe != 0):
sm_pool.append(sm_df['code'][idx])
def getStockClassfied():
filepath = 'f:\\stockdata\\股票分类数据\\'
index = 0
# 行业分类
df = ts.get_industry_classified()
saveCsv(df, filepath, stockClassList, index)
index += 1
# 概念分类
df = ts.get_concept_classified()
saveCsv(df, filepath, stockClassList, index)
index += 1
# 地域分类
df = ts.get_area_classified()
saveCsv(df, filepath, stockClassList, index)
index += 1
# 中小板分类
df = ts.get_sme_classified()
saveCsv(df, filepath, stockClassList, index)
index += 1
# 创业板分类
df = ts.get_gem_classified()
saveCsv(df, filepath, stockClassList, index)
index += 1
# 风险警示板分类
df = ts.get_st_classified()
saveCsv(df, filepath, stockClassList, index)
index += 1
# 沪深300成份及权重
df = ts.get_hs300s()
saveCsv(df, filepath, stockClassList, index)
index += 1
# 上证50成份股
df = ts.get_sz50s()
saveCsv(df, filepath, stockClassList, index)
index += 1
# 中证500成份股
df = ts.get_zz500s()
saveCsv(df, filepath, stockClassList, index)
index += 1
def get_stockcode_list(dataset=None, update=False):
filepath = dataset + '.csv'
codefilepath = os.path.join(rootDir, 'codelist', filepath)
if (os.path.exists(codefilepath) and update == False):
codelist = pd.read_csv(codefilepath, encoding='gbk')
return codelist
if (dataset == 'zxb'):
codelist = ts.get_sme_classified()
codelist = codelist[['code', 'name']]
codelist.to_csv(codefilepath)
return codelist
elif (dataset == 'cyb'):
codelist = ts.get_gem_classified()
codelist = codelist[['code', 'name']]
codelist.to_csv(codefilepath)
return codelist
elif (dataset == 'hs300'):
codelist = ts.get_hs300s()
codelist = codelist[['code', 'name']]
codelist.to_csv(codefilepath)
return codelist
elif (dataset == 'sz50'):
codelist = ts.get_sz50s()
codelist = codelist[['code', 'name']]
codelist.to_csv(codefilepath)
return codelist
elif (dataset == 'zz500'):
codelist = ts.get_zz500s()
codelist = codelist[['code', 'name']]
codelist.to_csv(codefilepath)
return codelist
elif (dataset == 'whole'):
codelist = ts.get_today_all()
codelist = codelist[['code', 'name']]
codelist.to_csv(codefilepath)
return codelist
def get_sme():
'''
函数功能：
--------
获取所有中小板上市的公司。
输入参数：
--------
无
输出参数：
--------
DataFrame
code,代码
name,名称
'''
# Download data
sme = ts.get_sme_classified()
# Return Dataframe
return sme
def generate_according_sme(filename):
category = ts.get_sme_classified()
reload(sys)
sys.setdefaultencoding('utf-8')
category.to_csv("tmp")
with open("tmp") as f:
lines = f.readlines()
code = []
name = []
for index in range(len(lines)):
if index > 0:
code.append(lines[index].split(",")[1])
name.append(lines[index].split(",")[2])
code = [str(i) for i in code]
name = [str(i) for i in name]
stock_pair = zip(code, name)
data = {"stock":stock_pair}
with open(filename,'w') as f:
yaml.dump(data, f)
def test():
logger = logging.getLogger("mylog")
formatter = logging.Formatter('%(name)-12s %(asctime)s %(levelname)-8s %(message)s', '%a, %d %b %Y %H:%M:%S',)
file_handler = logging.FileHandler("test_log.txt",encoding='utf-8')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG)
zxb_tickers = ts.get_sme_classified()
one_dateee = date(2017,month=4,day=1)
for i in range(zxb_tickers.index.size):
one_ticker = zxb_tickers.iloc[i,0]
for j in range(3650):
one_data = input_data_label(one_ticker,one_dateee)
x = one_data.get_input_data()
y = one_data.get_label_data()
#检查数据合理性
if(one_data.data_valid):
logger.debug('valid: '+str(one_ticker)+'_'+date_to_str(one_dateee)+"_"+str(x)+'_'+str(y)) #数据有效
elif(one_data.no_week_ma):
logger.debug('invalid: '+str(one_ticker)+'_'+date_to_str(one_dateee)+"_"+str(x)+'_'+str(y))
break #数据无效，而且再往前的日子的数据都不会有效了
one_dateee = one_dateee-timedelta(1)
Created on 2015年6月4日 @author: Administrator ''' import tushare as ts # 行业分类 ts.get_industry_classified() # 概念分类 ts.get_concept_classified() # 地域分类 ts.get_area_classified() # 获取沪深300当前成份股及所占权重 ts.get_hs300s() # 获取中小板股票数据，即查找所有002开头的股票 ts.get_sme_classified() # 获取创业板股票数据，即查找所有300开头的股票 ts.get_gem_classified() # 获取风险警示板股票数据，即查找所有st股票 ts.get_st_classified() # 获取上证50成份股 ts.get_sz50s() # 获取中证500成份股 ts.get_zz500s()
def zxb():
return ts.get_sme_classified()
def get_sme_info(file_path):
sme_info = ts.get_sme_classified()
sme_info.to_csv(file_path, encoding='utf-8')
print '\ndownload sme info finished\n'
#Concept = ts.get_concept_classified()
#Concept = Concept.set_index('code')
#Concept = Concept.rename(columns = {'c_name': 'concept'})
#Concept = Concept.drop('name', axis = 1)
# Merge
#StockInfo1 = StockInfo.join(Concept, how = 'outer')
# get all stocks
StockName = StockInfo['name']
StockName.to_csv('./ASHR/DATA/StockName.csv', index = True, header = True)
# Small & Medium Enterprise
# Note SME is a pd.series data type
SME = ts.get_sme_classified()
SME.to_csv('./ASHR/DATA/SME.csv', index = False)
# Growth Enterprise Market
GEM = ts.get_gem_classified()
GEM.to_csv('./ASHR/DATA/GEM.csv', index = False)
# ST Enterprise
ST = ts.get_st_classified()
ST.to_csv('./ASHR/DATA/ST.csv', index = False)
ts.get_h_data()
# HS 300
HS300S = ts.get_hs300s()
HS300S.to_csv('./ASHR/DATA/HS300S.csv', index = False)
import ilhb
# fs = ts.top_list()
# fs['dif'] = fs.apply(lambda x: float(x.buy) - float(x.sell), axis=1)
# fs.mask(lambda x: float(x.dif) > 0)
# print(fs)
# cf = fs[fs['dif'] > 0]
# grouped.filter(lambda x: x['A'].sum() + x['B'].sum() > 0)
# [x for x in fs if x['dif'] > 0]
# print (cf)
# fs.to_sql('lhb_daily', engine, if_exists='append')
# df = pd.read_sql_query("select code,reason,date from lhb_daily", engine)
# for x, y, z in df.values:
# print x, y, z
zxb = ts.get_sme_classified()
engine = create_engine("mysql://*****:*****@55e4043822731.gz.cdb.myqcloud.com:5073/stock?charset=utf8")
sql = "INSERT INTO five_minute_data(code,date,t_3_00,t_2_55,t_2_50,t_2_45,t_2_40,t_2_35) VALUES('%s','%s','%s','%s','%s','%s','%s','%s')"
for code in zxb["code"]:
connection = engine.connect()
df = ts.get_hist_data(code, start="2016-01-29", end="2016-01-30", ktype="5")
# df['code'] = '600637'
df = df.drop(
[
"ma5",
"ma10",
"ma20",
"v_ma5",
"v_ma10",
"v_ma20",
"open",
def load_company_industry_info():
#下载加载行业分类数据
try:
rs=ts.get_industry_classified()
sql.write_frame(rs, "company_industry_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载公司行业分类信息ok")
except:
print("下载公司行业分类信息出错")
#下载加载概念分类数据
try:
rs=ts.get_concept_classified()
sql.write_frame(rs, "company_concept_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("载公司概念分类信息ok")
except:
print("下载公司概念分类信息出错")
#下载加载地域分类数据
try:
rs=ts.get_area_classified()
sql.write_frame(rs, "company_area_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载公司区域分类信息ok")
except:
print("下载公司区域分类信息出错")
#下载加载中小板分类数据
try:
rs=ts.get_sme_classified()
sql.write_frame(rs, "company_sme_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载中小板分类数据ok")
except:
print("下载中小板分类数据出错")
#下载加载创业板分类数据
try:
rs=ts.get_gem_classified()
sql.write_frame(rs, "company_gem_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载创业板分类数据ok")
except:
print("下载创业板分类数据出错")
#下载加载st板分类数据
try:
rs=ts.get_st_classified()
sql.write_frame(rs, "company_st_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载st板分类数据ok")
except:
print("下载st板分类数据出错")
#下载加载沪深300板分类数据
try:
rs=ts.get_hs300s()
sql.write_frame(rs, "company_hs300_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载沪深300板分类数据ok")
except:
print("下载加载沪深300板分类数据出错")
#下载加载上证50板分类数据
try:
rs=ts.get_sz50s()
sql.write_frame(rs, "company_sz50_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载上证50板分类数据ok")
except:
print("下载加载上证50板分类数据出错")
#下载加载中证500板分类数据
try:
rs=ts.get_zz500s()
sql.write_frame(rs, "company_zz500_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载中证500板分类数据ok")
except:
print("下载加载中证500板分类数据出错")
#下载加载终止上市分类数据
try:
rs=ts.get_terminated()
sql.write_frame(rs, "company_terminated_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载终止上市分类数据ok")
except:
print("下载加载终止上市分类数据出错")
#下载加载暂停上市分类数据
try:
rs=ts.get_suspended()
sql.write_frame(rs, "company_suspended_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载暂停上市分类数据ok")
except:
print("下载加载暂停上市分类数据出错")
def get_sme():
"""获取中小板股票数据，即查找所有002开头的股票"""
df = ts.get_sme_classified()
print(df)
# 行业分类
inds = ts.get_industry_classified()
inds.columns = ('code', 'name', 'industry')
time.sleep(1)
# 概念分类
concepts = ts.get_concept_classified()
concepts.columns = ('code', 'name', 'concept')
time.sleep(1)
# 地域分类
areas = ts.get_area_classified()
areas.columns = ('code', 'name', 'area')
time.sleep(1)
# 中小板块
smalls = ts.get_sme_classified()
time.sleep(1)
# 创业版
news = ts.get_gem_classified()
time.sleep(1)
# st版块
sts = ts.get_st_classified()
time.sleep(1)
# 沪深300
hss = ts.get_hs300s()
time.sleep(1)
# 上证50