def update_static_info(force=False):
now = datetime.datetime.now()
t = None
rf = open(StaticInfo.storage_update_date, "r")
line = rf.read()
rf.close()
wf = open(StaticInfo.storage_update_date, "w")
if line:
t = parser.parse(line)
else:
wf.write(str(now))
delta_day = (now - t).days
if delta_day > 0 or force:
wf.write(str(now))
print "static info need to update"
ts.get_industry_classified().to_csv(
StaticInfo.data_type["industry"], index=False)
ts.get_concept_classified().to_csv(StaticInfo.data_type["concept"],
index=False)
ts.get_area_classified().to_csv(StaticInfo.data_type["area"],
index=False)
InitStaticInfo._get_stock_hist_data(
StaticInfo.get_stock_pool().keys())
wf.close()
def 下载板块数据(self):
# 获取不同分类数据
ts.get_industry_classified().to_csv('./股票数据/基本信息/行业分类.csv')
ts.get_concept_classified().to_csv('./股票数据/基本信息/概念分类.csv')
ts.get_area_classified().to_csv('./股票数据/基本信息/地域分类.csv')
ts.get_sme_classified().to_csv('./股票数据/基本信息/中小板.csv')
ts.get_gem_classified().to_csv('./股票数据/基本信息/创业板.csv')
ts.get_st_classified().to_csv('./股票数据/基本信息/风险警示板.csv')
ts.get_hs300s().to_csv('./股票数据/基本信息/沪深300.csv')
ts.get_sz50s().to_csv('./股票数据/基本信息/上证50.csv')
ts.get_zz500s().to_csv('./股票数据/基本信息/中证500.csv')
def gen_data():
ret=ts.get_concept_classified()
arr=[]
for row in ret.iterrows():
arr.append((row[1]['code'],row[1]['name'],row[1]['c_name']))
output = open('data.json', 'wb')
json.dump(arr,output)
def craw_stock():
concept = ts.get_concept_classified()
# print concept
data = concept['name']
# print len(data)
data = data.drop_duplicates()
print len(data)
def classification(class_types):
if class_types == 'industry':
industry_classified = ts.get_industry_classified()
industry_classified.to_csv('D:\\ts\\classification\\industry_classified.csv', encoding='gbk')
elif class_types == 'concept':
concept_classified = ts.get_concept_classified()
concept_classified.to_csv('D:\\ts\\classification\\concept_classified.csv', encoding='gbk')
elif class_types == 'area':
area_classified = ts.get_area_classified()
area_classified.to_csv('D:\\ts\\classification\\area_classified.csv', encoding='gbk')
elif class_types == 'sme':
sme_classified = ts.get_sme_classified()
sme_classified.to_csv('D:\\ts\\classification\\sme_classified.csv', encoding='gbk')
elif class_types == 'gem':
gem_classified = ts.get_gem_classified()
gem_classified.to_csv('D:\\ts\\classification\\gem_classified.csv', encoding='gbk')
elif class_types == 'st':
st_classified = ts.get_st_classified()
st_classified.to_csv('D:\\ts\\classification\\st_classified.csv', encoding='gbk')
elif class_types == 'hs300':
hs300s = ts.get_hs300s()
hs300s.to_csv('D:\\ts\\classification\\hs300s.csv', encoding='gbk')
elif class_types == 'sz50':
sz50s = ts.get_sz50s()
sz50s.to_csv('D:\\ts\\classification\\sz50s.csv', encoding='gbk')
elif class_types == 'zz500':
zz500s = ts.get_zz500s()
zz500s.to_csv('D:\\ts\\classification\\zz500s.csv', encoding='gbk')
elif class_types == 'terminated':
terminated = ts.get_terminated()
terminated.to_csv('D:\\ts\\classification\\terminated.csv', encoding='gbk')
elif class_types == 'suspended':
suspended = ts.get_suspended()
suspended.to_csv('D:\\ts\\classification\\suspended.csv', encoding='gbk')
def print_interested_concept_stocks():
interested_concepts = ['5G概念', '雄安新区', '物联网']
concept_classified_stocks = ts.get_concept_classified()
for concept_name, group in concept_classified_stocks.groupby(['c_name']):
for interested_concept in interested_concepts:
if concept_name == interested_concept:
print(interested_concept)
def get_pre_of_industry_concept():
# get info
df = ts.get_industry_classified()
df.to_csv("stock_industry_prep.csv")
df = ts.get_concept_classified()
df.to_csv("stock_concept_prep.csv")
def exportConcept_classified(fromDate, toDate, code):
df = ts.get_concept_classified()
engine = create_engine(sql_str)
# 存入数据库
df.to_sql('concept_classified', engine, if_exists='append')
Log.logger.info("concept_classified数据导入完成。")
return True
def update_stock_basic() -> None:
df = ts.get_stock_basics()
df.reset_index(inplace=True)
data = df.to_dict(orient="records")
# update industry
industry_df = ts.get_industry_classified()
industry_df = industry_df.groupby(["code"])["c_name"].apply(lambda x: x.tolist()).reset_index()
industry_mapper = dict([(d["code"], d["c_name"]) for
d in industry_df.to_dict(orient="records")])
# update concept
concept_df = ts.get_concept_classified()
concept_df = concept_df.groupby(["code"])["c_name"].apply(lambda x: x.tolist()).reset_index()
concept_mapper = dict([(d["code"], d["c_name"]) for
d in concept_df.to_dict(orient="records")])
# update area
area_df = ts.get_area_classified()
area_df = area_df.groupby(["code"])["area"].apply(lambda x: x.tolist()).reset_index()
area_mapper = dict([(d["code"], d["area"]) for
d in area_df.to_dict(orient="records")])
bar = ProgressBar(total=len(data))
for d in data:
d["industry"] = list(set(industry_mapper.get(d["code"], []) + [d["industry"]]))
d["concept"] = concept_mapper.get(d["code"], [])
d["area"] = list(set(area_mapper.get(d["code"], []) + [d["area"]]))
bar.move()
db.stock_basics.update({"code": d["code"]}, {"$set": d}, True)
bar.log("code: {} name: {} industry {} concept {} area {}"
.format(d["code"], d["name"], d["industry"], d["concept"], d["area"]))
def core_function(self, func):
self.set_data()
mongo = MongoClient("127.0.0.1", 27017)
if (func == "industry_classified"):
df = ts.get_industry_classified()
elif (func == "concept_classified"):
df = ts.get_concept_classified()
elif (func == "area_classified"):
df = ts.get_area_classified()
elif (func == "gem_classified"):
df = ts.get_gem_classified()
elif (func == "sme_classified"):
df = ts.get_sme_classified()
elif (func == "st_classified"):
df = ts.get_st_classified()
elif (func == "hs300s"):
df = ts.get_hs300s()
elif (func == "sz50s"):
df = ts.get_sz50s()
elif (func == "zz500s"):
df = ts.get_zz500s()
print(df)
elif (func == "terminated"):
df = ts.get_terminated()
else:
df = {}
insert_string = df.to_json(orient='records')
items = json.loads(insert_string)
coll = mongo.classify[func]
coll.insert(items)
def gen_data():
ret = ts.get_concept_classified()
arr = []
for row in ret.iterrows():
arr.append((row[1]['code'], row[1]['name'], row[1]['c_name']))
output = open('data.json', 'wb')
json.dump(arr, output)
def init_concept_from_ts(self):
table = 'concept'
sql = 'create table if not exists %s(c_name varchar(50),code varchar(5000))' % table
if table not in self.tables:
if not create_table(DB_USER, DB_PASSWD, DB_NAME, DB_HOSTNAME, sql):
raise Exception("create table %s failed" % table)
df_concept = ts.get_concept_classified()
df_concept = df_concept[['code', 'c_name']]
df_concept_dict = {}
for index, code_id in df_concept['code'].iteritems():
concept_name = df_concept['c_name'][index]
if concept_name in df_concept_dict:
if not code_id in df_concept_dict:
df_concept_dict[concept_name].append(code_id)
else:
df_concept_dict[concept_name] = []
df_concept_dict[concept_name].append(code_id)
concept_obj = {}
with open("concepts/concepts.json") as f:
concept_obj = json.load(f)
for key, value in concept_obj.items():
if key in df_concept_dict:
if value not in df_concept_dict[key]:
df_concept_dict[key].extend(value)
else:
df_concept_dict[key] = value
for key in df_concept_dict:
df_concept_dict[key] = json.dumps(df_concept_dict[key],
ensure_ascii=False)
df_concept = DataFrame({
'c_name': df_concept_dict.keys(),
'code': df_concept_dict.values()
})
set(self.engine, df_concept, table)
return get(self.engine, SQL % table)
def get_concept_classified():
'''
Return the concept classified DataFrame
:return: the DataFrame
'''
content = ts.get_concept_classified()
return content
def get_stock_basic() -> list:
nodes, links = [], []
Node = namedtuple("Node", "name type source")
Link = namedtuple("Link", "head link tail source")
df = ts.get_stock_basics()
df.reset_index(inplace=True)
data = df.to_dict(orient="records")
industry_mapper = ts.get_industry_classified().groupby(
["code"])["c_name"].apply(lambda x: x.tolist()).to_dict()
concept_mapper = ts.get_concept_classified().groupby(
["code"])["c_name"].apply(lambda x: x.tolist()).to_dict()
area_mapper = ts.get_area_classified().groupby(
["code"])["area"].apply(lambda x: x.tolist()).to_dict()
for d in data:
nodes.append(Node(d["code"], "公司", "tu"))
nodes.append(Node(d["name"], "公司", "tu"))
links.append(Link(d["code"], "等于", d["name"], "tu"))
for k in set(industry_mapper.get(d["code"], []) + [d["industry"]]):
nodes.append(Node(k, "行业", "tu"))
links.append(Link(d["code"], "属于", k, "tu"))
for k in concept_mapper.get(d["code"], []):
nodes.append(Node(k, "概念", "tu"))
links.append(Link(d["code"], "属于", k, "tu"))
for k in set(area_mapper.get(d["code"], []) + [d["area"]]):
nodes.append(Node(k, "区域", "tu"))
links.append(Link(d["code"], "属于", k, "tu"))
del d["industry"], d["area"]
nodes = list(set(nodes))
return nodes, links, data
def test_get_concept_classified():
df = ts.get_concept_classified()
df.to_sql("concept_classify",
engine,
if_exists='replace',
index=False,
dtype={"code": String(6)})
def save_concept_classified():
'''
更新概念分类信息
:return:
'''
cur = mysql_connect.cursor()
try:
sta = cur.execute("delete from py_db.gp_concept_classified")
print(sta)
print("删除概念分类gp_concept_classified:", sta, "条数据完成")
except Exception as e:
print(e)
mysql_connect.commit()
cur.close()
df = tushare.get_concept_classified()
try:
pd.io.sql.to_sql(df,
'gp_concept_classified',
yconnect,
schema='py_db',
if_exists='append',
index=df['code'])
print("概念分类", len(df), "更新完成")
return len(df)
except ValueError as e:
print(e)
return False
return -1
def fetch_classification(self): # 数据来源自新浪财经的行业分类/概念分类/地域分类 print( "Trying: get_today_all" ) today_all = ts.get_today_all() #一次性获取今日全部股价 set_today_all = set(today_all.T.values[0]) print( "Trying: get_industry_classified" ) industry_classified = ts.get_industry_classified() set_industry_classified = set(industry_classified.T.values[0]) print( "Trying: get_area_classified" ) area_classified = ts.get_area_classified() set_area_classified = set(area_classified.T.values[0]) print( "Trying: get_concept_classified" ) concept_classified = ts.get_concept_classified() set_concept_classified = set(concept_classified.T.values[0]) print( "Trying: get_sme_classified" ) sme_classified = ts.get_sme_classified() set_sme_classified = set(sme_classified.T.values[0]) return [ today_all , set_today_all , industry_classified , set_industry_classified , area_classified , set_area_classified , concept_classified , set_concept_classified , sme_classified , set_sme_classified ]
def load_data():
try:
rs=ts.get_concept_classified()
pd.DataFrame.to_sql(rs, table_name, con=conn , flavor='mysql', if_exists='replace',index=True)
except Exception as e:
print(e.message)
print("公司行业分类信息数据出错")
def download_concept_classified(self,
start='2017-01-01',
end='2018-01-01'):
"""
返回值说明：
code：股票代码
name：股票名称
c_name：概念名称
"""
#print('downloading the concept_classified info...')
total = 0
concept_classified = ts.get_concept_classified()
if concept_classified is not None:
concept_classified['date_download'] = end
filename = 'concept_classified'
concept_classified.to_excel(self.working_folder + filename +
'.xlsx',
encoding='GBK')
else:
warning_code = pd.DataFrame({
'return code': ['for all stocks'],
'description': ['concept_classified download failed']
})
self.warning_list = self.warning_list.append(warning_code)
return
def GetStockConcept( stockId ):
global g_mapStockConcept
global g_pReprots
try:
if g_pReprots == None and g_mapStockConcept == None:
g_mapStockConcept = {}
allConcept = set([])
try:
g_pReprots = ts.get_concept_classified()
for index, row in g_pReprots.iterrows():
try:
StockConceptDataCollection.update( {'StockId':row['code']}, {"$set":{'name':row['name'], 'c_name':row['c_name']}}, upsert = True)
except Exception as ex:
print(ex)
if row['code'] not in g_mapStockConcept:
g_mapStockConcept[row['code']] = {'name':row['name'], 'c_name':row['c_name']}
allConcept.add(row['c_name'])
except:
g_pReprots = None
except Exception as ex:
print(ex)
if len(g_mapStockConcept) <= 0:
cursor = StockConceptDataCollection.find()
for cursorItem in cursor:
g_mapStockConcept[cursorItem["StockId"]] = {"name":cursorItem["name"], "c_name":cursorItem["c_name"]}
if stockId not in g_mapStockConcept:
return None
return g_mapStockConcept[stockId]
def get_concept_info():
# 概念分类
df = ts.get_concept_classified()
res = df.to_sql(all_concept_table, engine, if_exists='replace')
msg = 'ok' if res is None else res
print('获取概念分类: ' + msg + '\n')
print(msg)
def conceptData(self):
table = 'stock_etl.stock_concept'
column = 'ticker,tickerName,tickerType'
result = ts.get_concept_classified()
sql = SqlBuildUtil.SqlBuildUtil.insertBuildts(table, column, result.values)
EtlDao.EtlDao().delAllDate(table)
EtlDao.EtlDao().save(sql)
def get_concept_classified_dict():
ccdf = ts.get_concept_classified()
concept_classified_dict = ccdf.to_dict(orient='records')
ret = {}
for data in concept_classified_dict:
ret[data['code']] = data['c_name']
return ret
def fetch_classification(self):
# 数据来源自新浪财经的行业分类/概念分类/地域分类
print("Trying: get_today_all")
today_all = ts.get_today_all() #一次性获取今日全部股价
set_today_all = set(today_all.T.values[0])
print("Trying: get_industry_classified")
industry_classified = ts.get_industry_classified()
set_industry_classified = set(industry_classified.T.values[0])
print("Trying: get_area_classified")
area_classified = ts.get_area_classified()
set_area_classified = set(area_classified.T.values[0])
print("Trying: get_concept_classified")
concept_classified = ts.get_concept_classified()
set_concept_classified = set(concept_classified.T.values[0])
print("Trying: get_sme_classified")
sme_classified = ts.get_sme_classified()
set_sme_classified = set(sme_classified.T.values[0])
return [
today_all, set_today_all, industry_classified,
set_industry_classified, area_classified, set_area_classified,
concept_classified, set_concept_classified, sme_classified,
set_sme_classified
]
def load_concept():
file_name = 'concept_classified.csv'
if os.path.exists(file_name):
df = pd.read_csv(file_name)
else:
df = ts.get_concept_classified()
df.to_csv(file_name)
return df
def creaw_concept():
concept = ts.get_concept_classified()
data = concept['c_name']
data = data.drop_duplicates()
print len(data)
for i in data:
sql = "insert into plate(name, type) VALUE (%s, %s)"
execute_sql(sql, [i, 1])
def refresh_stock_concept():
engine = get_pg_engine()
code_df = ts.get_concept_classified()
code_df.to_sql(constants.STOCK_CONCEPT, engine, if_exists='replace')
engine.dispose()
runner = SqlRunner()
runner.execute("CREATE INDEX on %s(code)" % constants.STOCK_CONCEPT)
runner.dispose()
def conceptData(self):
table = 'stock_etl.stock_concept'
column = 'ticker,tickerName,tickerType'
result = ts.get_concept_classified()
sql = SqlBuildUtil.SqlBuildUtil.insertBuildts(table, column,
result.values)
EtlDao.EtlDao().delAllDate(table)
EtlDao.EtlDao().save(sql)
def get_concept_classified():
try:
df = ts.get_concept_classified();
engine = create_engine('mysql://*****:*****@127.0.0.1/stock?charset=utf8')
# df.insert(0,'code','600848')
df.to_sql('concept_classified', engine, if_exists='append')
except Exception, e:
e.message
def get_concept_classification():
filename = 'concept_classification'
filepath = os.path.join(rootDir, 'StockBasis', filename + '.csv')
if (os.path.exists(filepath)):
data = pd.read_csv(filepath, encoding='gbk')
else:
data = ts.get_concept_classified()
data.to_csv(filepath)
return data
def get_concept_classified():
try:
logger.info('get concept data starting...')
df = ts.get_concept_classified()
logger.info('get concept data end...')
except:
logger.exception('some errors between get concept data end...')
return df
def getGainian():
'''
获取概念
:return:
'''
gainian = ts.get_concept_classified()
#gainian.to_csv("../../data/gainianData.csv")
#print gainian
return gainian
def __concept_classified(hs):
"""增加概念分类"""
clazz = ts.get_concept_classified()
clazz = clazz.groupby('name').agg({'code': 'first', 'c_name': ','.join})
clazz.rename({'c_name': 'concept'}, axis=1, inplace=True)
clazz = clazz.reset_index(drop=True).reindex(['code', 'concept'], axis=1)
hs = hs.merge(clazz, on='code', how='left')
hs.fillna({'concept': ''}, axis=0, inplace=True)
return hs
def ini_classfication():
Industry = ts.get_industry_classified()
Concept = ts.get_concept_classified()
Area = ts.get_area_classified()
_Codelist = Area[['code']]
Codelist = []
for i in range(len(_Codelist)):
Codelist.append(_Codelist.ix[i, 0])
return [Codelist, Area, Concept, Industry]
def init_stock_basic_info(self):
table = 'info'
text = 'create table if not exists %s(code int,\
name varchar(20),\
industry varchar(20),\
c_name varchar(100),\
area varchar(20),\
pe float,\
outstanding float,\
totals float,\
totalAssets float,\
fixedAssets float,\
reserved float,\
reservedPerShare float,\
esp float,\
bvps float,\
pb float,\
timeToMarket int,\
undp float,\
perundp float,\
rev float,\
profit float,\
gpr float,\
npr float,\
limit-up-num int,\
continuous-limit-up-num int,\
holders int)' % table
if table not in self.tables:
if not create_table(DB_USER, DB_PASSWD, DB_NAME, DB_HOSTNAME,
text):
raise Exception("create table %s failed" % table)
df = ts.get_stock_basics()
df = df.reset_index().rename_axis({'index': 'code'}, axis="columns")
df_concept = ts.get_concept_classified()
df_concept = df_concept[['code', 'c_name']]
df_concept_dict = {}
for index, code_id in df_concept['code'].iteritems():
concept_name = df_concept['c_name'][index]
if code_id in df_concept_dict:
if not concept_name in df_concept_dict:
df_concept_dict[code_id].append(concept_name)
else:
df_concept_dict[code_id] = []
df_concept_dict[code_id].append(concept_name)
for key in df_concept_dict:
df_concept_dict[key] = json.dumps(df_concept_dict[key],
ensure_ascii=False)
df_concept = DataFrame({
'code': df_concept_dict.keys(),
'c_name': df_concept_dict.values(),
'limit-up-num': 0,
'continuous-limit-up-num': 0
})
df_all = pd.merge(df, df_concept, how='left', on=['code'])
df_all.reset_index(drop=True)
set(self.engine, df_all, table)
return get(self.engine, SQL % table)
def getConceptClassified():
'''
获取概念分类
返回值说明：
code：股票代码
name：股票名称
c_name：概念名称
'''
return ts.get_concept_classified()
def get_concept(path='stock_concept.csv'):
try:
dataframe = None
if os.path.exists(path):
dataframe = pd.read_csv(path)
else:
dataframe = ts.get_concept_classified()
dataframe.to_csv(path,encoding="utf8")
return dataframe
except Exception,e:
print e
traceback.print_exc()
def list(self, stock_block_type):
stock_block = None
if stock_block_type == self.industry:
stock_block = db.get(STOCK_BLOCK_INDUSTRY)
if stock_block is None:
stock_block = ts.get_industry_classified()
db.save(STOCK_BLOCK_INDUSTRY, stock_block)
elif stock_block_type == self.concept:
stock_block = db.get(STOCK_BLOCK_CONCEPT)
if stock_block is None:
stock_block = ts.get_concept_classified()
db.save(STOCK_BLOCK_CONCEPT, stock_block)
elif stock_block_type == self.area:
stock_block = db.get(STOCK_BLOCK_AREA)
if stock_block is None:
stock_block = ts.get_area_classified()
db.save(STOCK_BLOCK_AREA, stock_block)
elif stock_block_type == self.sme:
stock_block = db.get(STOCK_BLOCK_SME)
if stock_block is None:
stock_block = ts.get_sme_classified()
db.save(STOCK_BLOCK_SME, stock_block)
elif stock_block_type == self.gem:
stock_block = db.get(STOCK_BLOCK_GEM)
if stock_block is None:
stock_block = ts.get_gem_classified()
db.save(STOCK_BLOCK_GEM, stock_block)
elif stock_block_type == self.st:
stock_block = db.get(STOCK_BLOCK_ST)
if stock_block is None:
stock_block = ts.get_st_classified()
db.save(STOCK_BLOCK_ST, stock_block)
elif stock_block_type == self.hs300s:
stock_block = db.get(STOCK_BLOCK_HS300S)
if stock_block is None:
stock_block = ts.get_hs300s()
db.save(STOCK_BLOCK_HS300S, stock_block)
elif stock_block_type == self.sz50s:
stock_block = db.get(STOCK_BLOCK_SZ50S)
if stock_block is None:
stock_block = ts.get_sz50s()
db.save(STOCK_BLOCK_SZ50S, stock_block)
elif stock_block_type == self.zz500s:
stock_block = db.get(STOCK_BLOCK_ZZ500S)
if stock_block is None:
stock_block = ts.get_zz500s()
db.save(STOCK_BLOCK_ZZ500S, stock_block)
else:
return None
return stock_block
def preload():
stock_block = ts.get_industry_classified()
db.save(STOCK_BLOCK_INDUSTRY, stock_block)
stock_block = ts.get_concept_classified()
db.save(STOCK_BLOCK_CONCEPT, stock_block)
stock_block = ts.get_area_classified()
db.save(STOCK_BLOCK_AREA, stock_block)
stock_block = ts.get_sme_classified()
db.save(STOCK_BLOCK_SME, stock_block)
stock_block = ts.get_gem_classified()
db.save(STOCK_BLOCK_GEM, stock_block)
stock_block = ts.get_st_classified()
db.save(STOCK_BLOCK_ST, stock_block)
stock_block = ts.get_hs300s()
db.save(STOCK_BLOCK_HS300S, stock_block)
stock_block = ts.get_sz50s()
db.save(STOCK_BLOCK_SZ50S, stock_block)
stock_block = ts.get_zz500s()
db.save(STOCK_BLOCK_ZZ500S, stock_block)
def read_classify():
#读取行业分类
db_classify=ts.get_industry_classified()
#读取概念分类
db_concept=ts.get_concept_classified()
#修改列名
db_classify.columns=['code','name','class']
db_concept.columns=['code','name','concept']
#去除name、c_name列里的空格
db_classify['name']=db_classify['name'].map(lambda x : x.replace(" ",''))
db_classify['class']=db_classify['class'].map(lambda x : x.replace(" ",''))
db_concept['name']=db_concept['name'].map(lambda x : x.replace(" ",''))
db_concept['concept']=db_concept['concept'].map(lambda x : x.replace(" ",''))
#写入数据库
db=DB_ENGINE.connect()
db_classify.to_sql('stock_classify',db,if_exists='append',index=False)
db_concept.to_sql('stock_concept',db,if_exists='append',index=False)
db.close()
def generate_according_concept(filename, filter_list):
category = ts.get_concept_classified()
selection = category[category["c_name"].isin(filter_list)]
reload(sys)
sys.setdefaultencoding('utf-8')
selection.to_csv("tmp")
with open("tmp") as f:
lines = f.readlines()
code = []
name = []
for index in range(len(lines)):
if index > 0:
code.append(lines[index].split(",")[1])
name.append(lines[index].split(",")[2])
code = [str(i) for i in code]
name = [str(i) for i in name]
stock_pair = zip(code, name)
data = {"stock":stock_pair}
with open(filename,'w') as f:
yaml.dump(data, f)
#coding = 'UTF-8'
#get_data
import tushare as ts
import sqlalchemy as sqlalc
import pandas as pd
import datetime as dt
route = 'sqlite:////var/www/Stock/Stock/DataBase/stock.db'
engine = sqlalc.create_engine(route)
today = str(dt.date.today())
stock_basics = pd.read_sql('SELECT * from stock_basics', engine)
if stock_basics.empty == True:
stock_basics = ts.get_stock_basics()
stock_concept = ts.get_concept_classified()
stock_basics.to_sql('stock_basics', engine, if_exists = 'replace')
stock_concept.to_sql('stock_concept', engine, if_exists = 'replace')
for stock in stock_basics['code']:
try:
stock_price = ts.get_hist_data(stock, start=today, end=today)
stock_price.loc[:, 'code'] = stock
index = stock_price.index
stock_price = stock_price.set_index(['code', index])
print(stock_price.index)
stock_price.to_sql('stock_price', engine, if_exists = 'append')
except AttributeError:
print("AttributeError", stock)
continue
except ValueError:
def convert_timetomarket(str_time2m):
try:
time2m = time.strptime(str(str_time2m), "%Y%m%d")
except Exception, e:
print e
time2m = time.strptime("19700101", "%Y%m%d")
# 转换成pandas的datetime类型
rtn_date = pd.datetime(time2m.tm_year, time2m.tm_mon, time2m.tm_mday)
return rtn_date
#module test code
if __name__ == "__main__":
#cls_orig = ts.get_industry_classified()
cls_orig = ts.get_concept_classified()
cls_trans = convert_cls_stockdata(cls_orig)
print cls_trans
cls_trans.to_csv('a.csv', encoding = 'utf-8')
df_stock_list = ts.get_stock_basics()
# random pick 50 stocks
rand_stocks = random.sample(df_stock_list.index,50)
rand_stock_ex = rand_stocks[:1]
st_date = '2014-07-01'
#stock_data_list = []
stock_data_list = dld.get_stock_data(rand_stock_ex, st_date)
idx_all = all_date_list(stock_data_list)
def get_area_info(file_path):
area_info = ts.get_concept_classified()
area_info.to_csv(file_path, encoding='utf-8')
print '\ndownload area info finished\n'
#coding=utf-8 ''' Created on 2015年6月4日 @author: Administrator ''' import tushare as ts # 行业分类 ts.get_industry_classified() # 概念分类 ts.get_concept_classified() # 地域分类 ts.get_area_classified() # 获取沪深300当前成份股及所占权重 ts.get_hs300s() # 获取中小板股票数据，即查找所有002开头的股票 ts.get_sme_classified() # 获取创业板股票数据，即查找所有300开头的股票 ts.get_gem_classified() # 获取风险警示板股票数据，即查找所有st股票 ts.get_st_classified() # 获取上证50成份股 ts.get_sz50s()
def get_concept_df():
'''
概念股分类
'''
df = ts.get_concept_classified()
# =================================================industry data=================================================
# import industry data
#code：股票代码
#name：股票名称
#c_name：行业名称
#df = ts.get_industry_classified()
#df.to_sql('industry_data',engine, if_exists='replace')
#==================================================concept data==============================================
# import concept data
# code
# name
# c_name
df = ts.get_concept_classified()
df.to_sql('concept_data',engine, if_exists='replace')
# =================================================bonus data=================================================
# import bonus data
#code:股票代码
#name:股票名称
#year:分配年份
#report_date:公布日期
#divi:分红金额（每10股）
#shares:转增和送股数（每10股）
df = ts.profit_data(year=lastYear,top=1000)
df.to_sql('bonus_data',engine, if_exists='replace')
def load_company_industry_info():
#下载加载行业分类数据
try:
rs=ts.get_industry_classified()
sql.write_frame(rs, "company_industry_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载公司行业分类信息ok")
except:
print("下载公司行业分类信息出错")
#下载加载概念分类数据
try:
rs=ts.get_concept_classified()
sql.write_frame(rs, "company_concept_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("载公司概念分类信息ok")
except:
print("下载公司概念分类信息出错")
#下载加载地域分类数据
try:
rs=ts.get_area_classified()
sql.write_frame(rs, "company_area_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载公司区域分类信息ok")
except:
print("下载公司区域分类信息出错")
#下载加载中小板分类数据
try:
rs=ts.get_sme_classified()
sql.write_frame(rs, "company_sme_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载中小板分类数据ok")
except:
print("下载中小板分类数据出错")
#下载加载创业板分类数据
try:
rs=ts.get_gem_classified()
sql.write_frame(rs, "company_gem_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载创业板分类数据ok")
except:
print("下载创业板分类数据出错")
#下载加载st板分类数据
try:
rs=ts.get_st_classified()
sql.write_frame(rs, "company_st_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载st板分类数据ok")
except:
print("下载st板分类数据出错")
#下载加载沪深300板分类数据
try:
rs=ts.get_hs300s()
sql.write_frame(rs, "company_hs300_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载沪深300板分类数据ok")
except:
print("下载加载沪深300板分类数据出错")
#下载加载上证50板分类数据
try:
rs=ts.get_sz50s()
sql.write_frame(rs, "company_sz50_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载上证50板分类数据ok")
except:
print("下载加载上证50板分类数据出错")
#下载加载中证500板分类数据
try:
rs=ts.get_zz500s()
sql.write_frame(rs, "company_zz500_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载中证500板分类数据ok")
except:
print("下载加载中证500板分类数据出错")
#下载加载终止上市分类数据
try:
rs=ts.get_terminated()
sql.write_frame(rs, "company_terminated_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载终止上市分类数据ok")
except:
print("下载加载终止上市分类数据出错")
#下载加载暂停上市分类数据
try:
rs=ts.get_suspended()
sql.write_frame(rs, "company_suspended_classified", con=conn_company_classified , flavor='mysql', if_exists='replace',index=True)
print("下载加载暂停上市分类数据ok")
except:
print("下载加载暂停上市分类数据出错")
def typeconcept(): global dftypeconcept dftypeconcept = ts.get_concept_classified() dftypeconcept.insert(0,'uploadtime',nowtime) typeconcept_tosql()
