def getRZRQDetailsSH():
# Download RZRQ Stock Data of SH Market
rzrq_sh_details = pd.DataFrame()
rzrq_sh = u.read_csv(c.fullpath_dict['rzrq'] % 'Market_SH')
date_number = len(rzrq_sh)
for i in range(date_number):
date = rzrq_sh.ix[i,'date']
rzrq = get_rzrq_sh_details(date, date)
print(rzrq.head(10))
if i == 0:
rzrq_sh_details = pd.DataFrame.copy(rzrq)
else:
rzrq_sh_details = pd.concat([rzrq_sh_details, rzrq])
print(rzrq_sh_details.head(10))
rzrq.set_index('date', inplace=True)
if not u.isNoneOrEmpty(rzrq):
u.to_csv(rzrq, c.path_dict['rzrq'], c.file_dict['rzrq'] % ('Details_SH_%s'%date))
# Process RZRQ Stock Data of SH Market
rzrq_sh_details.set_index('date', inplace=True)
rzrq_sh_details.sort_index(ascending=True,inplace=True)
if gs.is_debug:
print(rzrq_sh_details.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(rzrq_sh_details):
u.to_csv(rzrq_sh_details, c.path_dict['rzrq'], c.file_dict['rzrq'] % 'Details_SH')
def extractCommodityPrice(code, column):
# Check Pre-requisite
fullpath = c.fullpath_dict['commodity'] % code
if not u.hasFile(fullpath):
print('Require File Exists:', fullpath)
return
# Load Commodity Data
data = loadCommodityPrice(code)
data.set_index(u'发布时间', inplace=True)
print(data.head(10))
# Extract Price Data based on Given Column
market = data[column].drop_duplicates()
market_number = len(market)
print('Market Number:', market_number)
print('Markets:', market)
i = 0
for m in market:
print('Market %s: %s' % (i + 1, m))
m_name = 'Market_%s' % (i + 1)
m_data = data[data[column].isin([m])]
if not u.isNoneOrEmpty(m_data):
u.to_csv(m_data, c.path_dict['commodity'],
c.file_dict['commodity_m'] % (code, m_name))
i = i + 1
'''
def extractConceptSina():
# Load Sina Concept Data
concept_sina = loadConceptSina()
if gs.is_debug:
print(concept_sina.head(10))
# Extract Concept List
concept_list = concept_sina.drop(['code', 'name'], axis=1)
concept_list.drop_duplicates(['concept'], inplace=True)
concept_list.set_index('concept', inplace=True)
if gs.is_debug:
print(concept_list.index)
# Save to CSV File
if not u.isNoneOrEmpty(concept_list):
u.to_csv(concept_list, c.path_dict['conc_sina'], c.file_dict['conc_list'])
# Extract Stocks for Each Concept
concept_number = len(concept_list)
print('#Concept =', concept_number)
for i in range(concept_number):
concept_name = concept_list.index[i]
concept_stock = concept_sina[concept_sina.concept.isin([concept_name])]
concept_stock['code'] = concept_stock['code'].map(lambda x:str(x).zfill(6))
concept_stock.set_index('code', inplace=True)
if gs.is_debug:
print(concept_stock.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(concept_stock):
u.to_csv(concept_stock, c.path_dict['conc_sina'], c.file_dict['conc_stock'] % concept_name)
def extractIndustrySina():
# Load Sina Industry Data
industry_sina = loadIndustrySina()
if gs.is_debug:
print(industry_sina.head(10))
# Extract Industry List
industry_list = industry_sina.drop(['code', 'name'], axis=1)
industry_list.drop_duplicates(['industry'], inplace=True)
industry_list.set_index('industry', inplace=True)
if gs.is_debug:
print(industry_list.index)
# Save to CSV File
if not u.isNoneOrEmpty(industry_list):
u.to_csv(industry_list, c.path_dict['indu_sina'], c.file_dict['indu_list'])
# Extract Stocks for Each Industry
industry_number = len(industry_list)
print('#Industry =', industry_number)
for i in range(industry_number):
industry_name = industry_list.index[i]
industry_stock = industry_sina[industry_sina.industry.isin([industry_name])]
industry_stock['code'] = industry_stock['code'].map(lambda x:str(x).zfill(6))
industry_stock.set_index('code', inplace=True)
if gs.is_debug:
print(industry_stock.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(industry_stock):
u.to_csv(industry_stock, c.path_dict['indu_sina'], c.file_dict['indu_stock'] % industry_name)
def updateSamplePriceAllIndex(benchmark_id, period):
'''
函数功能：
--------
根据基准指数的时间范围和采样周期，对所有指数的收盘价格进行全采样。
输入参数：
--------
benchmark_id : string, 指数代码 e.g. '000300'，隐含起止时间。
period : string, 采样周期 e.g. 'M'，支持'D', 'W', and 'M'。
输出参数：
--------
True/False : boolean，是否采样成功。
数据文件
Strategy_Common_AllPrice_Benchmark_Period_AllStock.csv : 所有指数收盘价格的采样结果数据文件。
'''
# Sample Price for All Index
allindex = loadAllIndex()
allprice = updateSamplePrice(benchmark_id, allindex, True, period)
# Save to CSV File
if not u.isNoneOrEmpty(allindex):
file_postfix = '_'.join(
['Common', 'AllPrice', benchmark_id, period, 'AllIndex'])
u.to_csv(allprice, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
return False
def updateAllStocks():
'''
函数功能：
--------
更新所有股票列表。
输入参数：
--------
无
输出参数：
--------
True/False : boolean，是否更新成功。
数据文件
Strategy_Common_AllStock.csv : 参与策略计算的所有股票列表
'''
# Load from Fundamental Stock Basics
allstock = loadStockBasics()
if not u.isNoneOrEmpty(allstock):
# Save to CSV File
allstock.set_index('code', inplace=True)
file_postfix = '_'.join(['Common', 'AllStock'])
u.to_csv(allstock, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
return False
def extractRollingBeta(postfix):
# Load Rolling Coefficient
fullpath = c.path_dict['strategy'] + c.file_dict['strategy'] % postfix
coef = u.read_csv(fullpath)
if u.isNoneOrEmpty(coef):
print('Require Coefficient File: %s!' % fullpath)
return False
# Extract Rolling Beta
row_number = len(coef)
beta = u.createDataFrame(row_number, ['date', 'beta'])
beta['date'] = coef['date']
for column in coef.columns:
if len(column) >= 4 and column[0:4] == 'beta':
beta[column] = coef[column]
# Calculate Rolling Beta Average
beta_number = len(beta.columns) - 2
for i in range(row_number):
beta_avg = 0.0
beta_count = 0
for j in range(beta_number):
b = beta.ix[i, beta.columns[j + 2]]
if not np.isnan(b):
beta_avg = beta_avg + b
beta_count = beta_count + 1
if beta_count > 0:
beta.ix[i, 'beta'] = beta_avg / float(beta_count)
beta.set_index('date', inplace=True)
postfix = '_'.join([postfix, 'Beta'])
u.to_csv(beta, c.path_dict['strategy'], c.file_dict['strategy'] % postfix)
def extractArea():
# Load Area Data
area = loadArea()
if gs.is_debug:
print(area.head(10))
# Extract Area List
area_list = area.drop(['code', 'name'], axis=1)
area_list.drop_duplicates(['area'], inplace=True)
area_list.set_index('area', inplace=True)
if gs.is_debug:
print(area_list.index)
# Save to CSV File
if not u.isNoneOrEmpty(area_list):
u.to_csv(area_list, c.path_dict['area'], c.file_dict['area_list'])
# Extract Stocks for Each Area
area_number = len(area_list)
print('#Area =', area_number)
for i in range(area_number):
area_name = area_list.index[i]
area_stock = area[area.area.isin([area_name])]
area_stock['code'] = area_stock['code'].map(lambda x:str(x).zfill(6))
area_stock.set_index('code', inplace=True)
if gs.is_debug:
print(area_stock.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(area_stock):
u.to_csv(area_stock, c.path_dict['area'], c.file_dict['area_stock'] % area_name)
def updateAllIndex():
'''
函数功能：
--------
更新所有指数列表。
输入参数：
--------
无
输出参数：
--------
True/False : boolean，是否更新成功。
数据文件
Strategy_Common_AllIndex.csv : 参与策略计算的所有指数列表
'''
# Load from Constants.index_list
allindex = pd.DataFrame({'code': c.index_list})
if not u.isNoneOrEmpty(allindex):
# Save to CSV File
allindex.set_index('code', inplace=True)
file_postfix = '_'.join(['Common', 'AllIndex'])
u.to_csv(allindex, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
return False
def getFinanceSummary(stock_id):
# Download Finance Summary for Given Stock
fs = get_finance_summary(stock_id)
if gs.is_debug:
print(fs.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(fs):
u.to_csv(fs, c.path_dict['finsum'], c.file_dict['finsum'] % stock_id)
def getStockBasics():
# Download Stock Basics
basics = get_stock_basics()
if gs.is_debug:
print(basics.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(basics):
u.to_csv(basics, c.path_dict['basics'], c.file_dict['basics'])
def getCommodityPrice(code):
# Download Commodity Data
data = get_commodity_price(code)
if gs.is_debug:
print(data.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(data):
u.to_csv(data, c.path_dict['commodity'],
c.file_dict['commodity'] % code)
def calcQFQ(stock_id, period):
# Calculate QFQ DataFrame
df = calc_qfq(stock_id, period)
if gs.is_debug:
print(df.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(df):
path = c.path_dict['qfq'] % period
file = c.file_dict['qfq'] % (period, stock_id)
u.to_csv(df, path, file)
def calcHPE(stock_id, period, ratio):
# Calculate HPE DataFrame
df = calc_hpe(stock_id, period, ratio)
if gs.is_debug:
print(df.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(df):
key = 'hpe' if ratio == 'PE' else 'hep'
path = c.path_dict[key] % period
file = c.file_dict[key] % (period, stock_id)
u.to_csv(df, path, file)
def histogramAlpha(postfix, completeness_threshold):
# Load Coefficient File
allcoef = loadCoefficient(postfix, completeness_threshold)
if u.isNoneOrEmpty(allcoef):
return False
# Calculate Coefficient Histogram
columns = [
'Total', 'Very High', 'High', 'Medium', 'Low', 'Very Low',
'Negative Very Low', 'Negative Low', 'Negative Medium',
'Negative High', 'Negative Very High'
]
histogram = u.createDataFrame(1, columns, 0)
stock_number = len(allcoef)
histogram.ix[0, 'Total'] = stock_number
for i in range(stock_number):
alpha = allcoef.ix[i, 'alpha']
if alpha >= 0.4: # [0.4, +Infinity)
histogram.ix[0, 'Very High'] = histogram.ix[0, 'Very High'] + 1
elif alpha >= 0.3: # [0.3, 0.4)
histogram.ix[0, 'High'] = histogram.ix[0, 'High'] + 1
elif alpha >= 0.2: # [0.2, 0.3)
histogram.ix[0, 'Medium'] = histogram.ix[0, 'Medium'] + 1
elif alpha >= 0.1: # [0.1, 0.2)
histogram.ix[0, 'Low'] = histogram.ix[0, 'Low'] + 1
elif alpha >= 0.0: # [0.0, 0.1)
histogram.ix[0, 'Very Low'] = histogram.ix[0, 'Very Low'] + 1
elif alpha >= -0.1: # [-0.1, 0.0)
histogram.ix[
0,
'Negative Very Low'] = histogram.ix[0, 'Negative Very Low'] + 1
elif alpha >= -0.2: # [-0.2, -0.1)
histogram.ix[0,
'Negative Low'] = histogram.ix[0, 'Negative Low'] + 1
elif alpha >= -0.3: # [-0.3, -0.2)
histogram.ix[
0, 'Negative Medium'] = histogram.ix[0, 'Negative Medium'] + 1
elif alpha >= -0.4: # [-0.4, -0.3)
histogram.ix[0,
'Negative High'] = histogram.ix[0,
'Negative High'] + 1
else: # (-Infinity, -0.4)
histogram.ix[0, 'Negative Very High'] = histogram.ix[
0, 'Negative Very High'] + 1
# Save to CSV File
histogram.set_index('Total', inplace=True)
file_postfix = '_'.join(
[postfix, completeness_threshold, 'HistogramAlpha'])
u.to_csv(histogram, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
def histogramCorrelation(postfix, completeness_threshold):
# Load Coefficient File
allcoef = loadCoefficient(postfix, completeness_threshold)
if u.isNoneOrEmpty(allcoef):
return False
# Calculate Coefficient Histogram
columns = [
'Total', 'Very Strong', 'Strong', 'Medium', 'Weak', 'Very Weak',
'Negative Very Weak', 'Negative Weak', 'Negative Medium',
'Negative Strong', 'Negative Very Strong'
]
histogram = u.createDataFrame(1, columns, 0)
stock_number = len(allcoef)
histogram.ix[0, 'Total'] = stock_number
for i in range(stock_number):
correlation = allcoef.ix[i, 'correlation']
if correlation > 0.8: # (0.8, 1.0]
histogram.ix[0, 'Very Strong'] = histogram.ix[0, 'Very Strong'] + 1
elif correlation > 0.6: # (0.6, 0.8]
histogram.ix[0, 'Strong'] = histogram.ix[0, 'Strong'] + 1
elif correlation > 0.4: # (0.4, 0.6]
histogram.ix[0, 'Medium'] = histogram.ix[0, 'Medium'] + 1
elif correlation > 0.2: # (0.2, 0.4]
histogram.ix[0, 'Weak'] = histogram.ix[0, 'Weak'] + 1
elif correlation >= 0.0: # [0.0, 0.2]
histogram.ix[0, 'Very Weak'] = histogram.ix[0, 'Very Weak'] + 1
elif correlation > -0.2: # (-0.2, 0.0)
histogram.ix[0, 'Negative Very Weak'] = histogram.ix[
0, 'Negative Very Weak'] + 1
elif correlation > -0.4: # (-0.4, -0.2]
histogram.ix[0,
'Negative Weak'] = histogram.ix[0,
'Negative Weak'] + 1
elif correlation > -0.6: # (-0.6, -0.4]
histogram.ix[
0, 'Negative Medium'] = histogram.ix[0, 'Negative Medium'] + 1
elif correlation > -0.8: # (-0.8, -0.6]
histogram.ix[
0, 'Negative Strong'] = histogram.ix[0, 'Negative Strong'] + 1
else: # [-1.0, -0.8]
histogram.ix[0, 'Negative Very Strong'] = histogram.ix[
0, 'Negative Very Strong'] + 1
# Save to CSV File
histogram.set_index('Total', inplace=True)
file_postfix = '_'.join(
[postfix, completeness_threshold, 'HistogramCorrelation'])
u.to_csv(histogram, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
def getRZRQMarketSH(date_start, date_end):
# Download RZRQ Data of SH Market
rzrq_sh = get_rzrq_sh(date_start, date_end)
# Process RZRQ Market Data
rzrq_sh.set_index('date',inplace=True)
rzrq_sh.sort_index(ascending=True,inplace=True)
if gs.is_debug:
print(rzrq_sh.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(rzrq_sh):
u.to_csv(rzrq_sh, c.path_dict['rzrq'], c.file_dict['rzrq'] % 'Market_SH')
def getConceptSina():
# Download Sina Concept Data
concept = get_concept_sina()
if not u.isNoneOrEmpty(concept):
concept.set_index('code', inplace=True)
if gs.is_debug:
print(concept.head(10))
# Save to CSV File
u.to_csv(concept, c.path_dict['classify'], c.file_dict['conc_sina'])
else:
print('getConceptSina() Failed!')
raise SystemExit
def getGEM():
# Download GEM Data
gem = get_gem()
if not u.isNoneOrEmpty(gem):
gem.set_index('code', inplace=True)
if gs.is_debug:
print(gem.head(10))
# Save to CSV File
u.to_csv(gem, c.path_dict['classify'], c.file_dict['gem'])
else:
print('getGEM() Failed!')
raise SystemExit
def getSuspended():
# Download Suspended Data
suspended = get_suspended()
if not u.isNoneOrEmpty(suspended):
suspended.set_index('code', inplace=True)
if gs.is_debug:
print(suspended.head(10))
# Save to CSV File
u.to_csv(suspended, c.path_dict['classify'], c.file_dict['suspended'])
else:
print('getSuspended() Failed!')
raise SystemExit
def getTerminated():
# Download Terminated Stock Data
terminated = get_terminated()
if not u.isNoneOrEmpty(terminated):
terminated.set_index('code', inplace=True)
if gs.is_debug:
print(terminated.head(10))
# Save to CSV File
u.to_csv(terminated, c.path_dict['classify'], c.file_dict['terminated'])
else:
print('getTerminated() Failed!')
raise SystemExit
def getZZ500():
# Download ZZ500 Data
zz500 = get_zz500()
if not u.isNoneOrEmpty(zz500):
zz500.set_index('code', inplace=True)
if gs.is_debug:
print(zz500.head(10))
# Save to CSV File
u.to_csv(zz500, c.path_dict['classify'], c.file_dict['zz500'])
else:
print('getZZ500() Failed!')
raise SystemExit
def getSZ50():
# Download SZ50 Data
sz50 = get_sz50()
if not u.isNoneOrEmpty(sz50):
sz50.set_index('code', inplace=True)
if gs.is_debug:
print(sz50.head(10))
# Save to CSV File
u.to_csv(sz50, c.path_dict['classify'], c.file_dict['sz50'])
else:
print('getSZ50() Failed!')
raise SystemExit
def getArea():
# Download Area Data
area = get_area()
if not u.isNoneOrEmpty(area):
area.set_index('code', inplace=True)
if gs.is_debug:
print(area.head(10))
# Save to CSV File
u.to_csv(area, c.path_dict['classify'], c.file_dict['area'])
else:
print('getArea() Failed!')
raise SystemExit
def getST():
# Download ST Data
st = get_st()
if not u.isNoneOrEmpty(st):
st.set_index('code', inplace=True)
if gs.is_debug:
print(st.head(10))
# Save to CSV File
u.to_csv(st, c.path_dict['classify'], c.file_dict['st'])
else:
print('getST() Failed!')
raise SystemExit
def getStockList(cutoff_date):
stocks = get_stock_list(cutoff_date)
if not u.isNoneOrEmpty(stocks):
stocks['code'] = stocks['code'].map(lambda x:str(x).zfill(6))
stocks.set_index('code', inplace=True)
if gs.is_debug:
print(stocks.head(10))
# Save to CSV File
u.to_csv(stocks, c.path_dict['classify'], c.file_dict['stock_list'] % cutoff_date)
else:
print('getStockList() Failed!')
raise SystemExit
def getIndustrySina():
# Download Sina Industry Data
industry = get_industry_sina()
if not u.isNoneOrEmpty(industry):
industry.set_index('code', inplace=True)
if gs.is_debug:
print(industry.head(10))
# Save to CSV File
u.to_csv(industry, c.path_dict['classify'], c.file_dict['indu_sina'])
else:
print('getIndustrySina() Failed!')
raise SystemExit
def getSME():
# Download SME Data
sme = get_sme()
if not u.isNoneOrEmpty(sme):
sme.set_index('code', inplace=True)
if gs.is_debug:
print(sme.head(10))
# Save to CSV File
u.to_csv(sme, c.path_dict['classify'], c.file_dict['sme'])
else:
print('getSME() Failed!')
raise SystemExit
def getCXG(date):
# Get CXG Stock Data
cxg = get_cxg(date)
if not u.isNoneOrEmpty(cxg):
cxg['code'] = cxg['code'].map(lambda x:str(x).zfill(6))
cxg.set_index('code', inplace=True)
if gs.is_debug:
print(cxg.head(10))
# Save to CSV File
u.to_csv(cxg, c.path_dict['classify'], c.file_dict['cxg'])
else:
print('getCXG() Failed!')
raise SystemExit
def mergeRZRQMarketSZ(files_number):
rzrq_sz = pd.DataFrame()
for index in range(files_number):
rzrq = u.read_csv(c.fullpath_dict['rzrq'] % ('Market_SZ_%s'%index))
if index == 0:
rzrq_sz = pd.DataFrame.copy(rzrq)
else:
rzrq_sz = pd.concat([rzrq_sz, rzrq])
rzrq_sz.set_index('date',inplace=True)
rzrq_sz.sort_index(ascending=True,inplace=True)
if gs.is_debug:
print(rzrq_sz.head(10))
if not u.isNoneOrEmpty(rzrq_sz):
u.to_csv(rzrq_sz, c.path_dict['rzrq'], c.file_dict['rzrq'] % 'Market_SZ')