def updateSamplePriceAllIndex(benchmark_id, period):
'''
函数功能：
--------
根据基准指数的时间范围和采样周期，对所有指数的收盘价格进行全采样。
输入参数：
--------
benchmark_id : string, 指数代码 e.g. '000300'，隐含起止时间。
period : string, 采样周期 e.g. 'M'，支持'D', 'W', and 'M'。
输出参数：
--------
True/False : boolean，是否采样成功。
数据文件
Strategy_Common_AllPrice_Benchmark_Period_AllStock.csv : 所有指数收盘价格的采样结果数据文件。
'''
# Sample Price for All Index
allindex = loadAllIndex()
allprice = updateSamplePrice(benchmark_id, allindex, True, period)
# Save to CSV File
if not u.isNoneOrEmpty(allindex):
file_postfix = '_'.join(
['Common', 'AllPrice', benchmark_id, period, 'AllIndex'])
u.to_csv(allprice, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
return False
def loadSamplePriceAllIndex(benchmark_id, period):
'''
函数功能：
--------
根据基准指数的时间范围和采样周期，加载所有指数的收盘价格采样文件。
输入参数：
--------
benchmark_id : string, 指数代码 e.g. '000300'，隐含起止时间。
period : string, 采样周期 e.g. 'M'，支持'D', 'W', and 'M'。
输出参数：
--------
加载成功时，allprice : pandas.DataFrame, 所有指数的收盘价格采样结果。
加载失败时，None
'''
# Check if AllPrice File Already Exists
file_postfix = '_'.join(
['Common', 'AllPrice', benchmark_id, period, 'AllIndex'])
fullpath = c.path_dict['strategy'] + c.file_dict['strategy'] % file_postfix
allprice = u.read_csv(fullpath)
if not u.isNoneOrEmpty(allprice):
return allprice
print('Failed to Load File: %s!' % fullpath)
return None
def loadAllIndex():
'''
函数功能：
--------
加载所有指数列表。
输入参数：
--------
无
输出参数：
--------
加载成功时，index_ids : pandas.Series, 所有指数列表。
加载失败时，None
'''
# Load Local Cache
file_postfix = '_'.join(['Common', 'AllIndex'])
fullpath = c.path_dict['strategy'] + c.file_dict['strategy'] % file_postfix
allindex = u.read_csv(fullpath)
if not u.isNoneOrEmpty(allindex):
allindex['code'] = allindex['code'].map(lambda x: str(x).zfill(6))
return allindex['code']
print('Failed to Load File: %s!' % fullpath)
return None
def updateAllIndex():
'''
函数功能：
--------
更新所有指数列表。
输入参数：
--------
无
输出参数：
--------
True/False : boolean，是否更新成功。
数据文件
Strategy_Common_AllIndex.csv : 参与策略计算的所有指数列表
'''
# Load from Constants.index_list
allindex = pd.DataFrame({'code': c.index_list})
if not u.isNoneOrEmpty(allindex):
# Save to CSV File
allindex.set_index('code', inplace=True)
file_postfix = '_'.join(['Common', 'AllIndex'])
u.to_csv(allindex, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
return False
def updateAllStocks():
'''
函数功能：
--------
更新所有股票列表。
输入参数：
--------
无
输出参数：
--------
True/False : boolean，是否更新成功。
数据文件
Strategy_Common_AllStock.csv : 参与策略计算的所有股票列表
'''
# Load from Fundamental Stock Basics
allstock = loadStockBasics()
if not u.isNoneOrEmpty(allstock):
# Save to CSV File
allstock.set_index('code', inplace=True)
file_postfix = '_'.join(['Common', 'AllStock'])
u.to_csv(allstock, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
return False
def fileCompare(src_fullpath, tar_fullpath):
# Check File Existence
if not u.hasFile(src_fullpath):
print('Source File %s Does Not Exist' % src_fullpath)
raise SystemExit
if not u.hasFile(tar_fullpath):
print('Target File %s Does Not Exist' % tar_fullpath)
raise SystemExit
# Load Data Files
src = u.read_csv(src_fullpath)
tar = u.read_csv(tar_fullpath)
src_row_number = len(src)
tar_row_number = len(tar)
src_col_number = len(src.columns)
tar_col_number = len(tar.columns)
matched = True
print('File Compare Start: %s vs %s' % (src_fullpath, tar_fullpath))
if src_row_number != tar_row_number:
matched = False
print('Row Number Un-matched')
elif src_col_number != tar_col_number:
matched = False
print('Col Number Un-matched')
else:
for i in range(src_row_number):
for j in range(src_col_number):
if src.iloc[i,j] != tar.iloc[i,j]:
matched = False
print('Element(%s,%s) Un-matched' % (i,j))
print('File Compare End: %s' % ('Matched' if matched else 'Un-Matched'))
# Return Result
return matched
def extractRollingBeta(postfix):
# Load Rolling Coefficient
fullpath = c.path_dict['strategy'] + c.file_dict['strategy'] % postfix
coef = u.read_csv(fullpath)
if u.isNoneOrEmpty(coef):
print('Require Coefficient File: %s!' % fullpath)
return False
# Extract Rolling Beta
row_number = len(coef)
beta = u.createDataFrame(row_number, ['date', 'beta'])
beta['date'] = coef['date']
for column in coef.columns:
if len(column) >= 4 and column[0:4] == 'beta':
beta[column] = coef[column]
# Calculate Rolling Beta Average
beta_number = len(beta.columns) - 2
for i in range(row_number):
beta_avg = 0.0
beta_count = 0
for j in range(beta_number):
b = beta.ix[i, beta.columns[j + 2]]
if not np.isnan(b):
beta_avg = beta_avg + b
beta_count = beta_count + 1
if beta_count > 0:
beta.ix[i, 'beta'] = beta_avg / float(beta_count)
beta.set_index('date', inplace=True)
postfix = '_'.join([postfix, 'Beta'])
u.to_csv(beta, c.path_dict['strategy'], c.file_dict['strategy'] % postfix)
def extractCommodityPrice(code, column):
# Check Pre-requisite
fullpath = c.fullpath_dict['commodity'] % code
if not u.hasFile(fullpath):
print('Require File Exists:', fullpath)
return
# Load Commodity Data
data = loadCommodityPrice(code)
data.set_index(u'发布时间', inplace=True)
print(data.head(10))
# Extract Price Data based on Given Column
market = data[column].drop_duplicates()
market_number = len(market)
print('Market Number:', market_number)
print('Markets:', market)
i = 0
for m in market:
print('Market %s: %s' % (i + 1, m))
m_name = 'Market_%s' % (i + 1)
m_data = data[data[column].isin([m])]
if not u.isNoneOrEmpty(m_data):
u.to_csv(m_data, c.path_dict['commodity'],
c.file_dict['commodity_m'] % (code, m_name))
i = i + 1
'''
def analyzePriceFollow(target_date, stock_id, is_index, threshold):
file_postfix = 'Timing_%s_%s' % (u.stockFileName(stock_id,
is_index), threshold)
timing = u.read_csv(c.path_dict['strategy'] + file_postfix + '.csv',
encoding='gbk')
timing_number = len(timing)
# Find the matched timing date and trend
timing_index = -1
for i in range(timing_number):
date = dt.datetime.strptime(timing.ix[i, 'date'], '%Y-%m-%d').date()
if date <= target_date:
timing_index = i
else:
break
# Report results
if timing_index != -1:
date = dt.datetime.strptime(timing.ix[timing_index, 'date'],
'%Y-%m-%d').date()
trend = timing.ix[timing_index, 'trend']
if date == target_date: # Given target_date is Timing Date
print('Date', target_date, ': Trend of',
u.stockFileName(stock_id, is_index), 'Goes', trend)
else:
print('Date', target_date, ': Trend of',
u.stockFileName(stock_id, is_index),
'Does Not Change, Still', trend)
else:
print('Date', target_date, ': Trend of',
u.stockFileName(stock_id, is_index),
'Not Available, No Timing Data')
def loadFinanceSummary(stock_id):
# Ensure data file is available
fullpath = c.fullpath_dict['finsum'] % stock_id
if not u.hasFile(fullpath):
print('Require Finance Summary of %s!' % fullpath)
return None
fs = u.read_csv(fullpath)
return fs
def getFinanceSummary(stock_id):
# Download Finance Summary for Given Stock
fs = get_finance_summary(stock_id)
if gs.is_debug:
print(fs.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(fs):
u.to_csv(fs, c.path_dict['finsum'], c.file_dict['finsum'] % stock_id)
def loadStockBasics():
# Ensure data file is available
fullpath = c.fullpath_dict['basics']
if not u.hasFile(fullpath):
print('Require Stock Basics: %s!' % fullpath)
return None
basics = u.read_csv(fullpath)
return basics
def getStockBasics():
# Download Stock Basics
basics = get_stock_basics()
if gs.is_debug:
print(basics.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(basics):
u.to_csv(basics, c.path_dict['basics'], c.file_dict['basics'])
def loadCoefficient(postfix, completeness_threshold):
# Load Coefficient File
fullpath = c.path_dict['strategy'] + c.file_dict['strategy'] % '_'.join(
[postfix, completeness_threshold])
allcoef = u.read_csv(fullpath)
if u.isNoneOrEmpty(allcoef):
print('Require Coefficient File: %s!' % fullpath)
return None
return allcoef
def getCommodityPrice(code):
# Download Commodity Data
data = get_commodity_price(code)
if gs.is_debug:
print(data.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(data):
u.to_csv(data, c.path_dict['commodity'],
c.file_dict['commodity'] % code)
def loadDailyHFQ(stock_id, is_index):
fullpath = c.fullpath_dict['lshq'] % u.stockFileName(stock_id, is_index)
# Ensure data file is available
if not u.hasFile(fullpath):
print('Require LSHQ of %s!' % u.stockFileName(stock_id, is_index))
return None
# Read data file
df = u.read_csv(fullpath)
return df
def load_component(index_name):
fullpath = c.path_dict['index'] + c.file_dict['index_c'] % index_name
# Ensure data file is available
if not u.hasFile(fullpath):
print('Require Index Component of %s!' % index_name)
return None
# Read data file
df = u.read_csv(fullpath)
return df
def plot_index(index_name, benchmark_name):
# Load Index Data File
index_path = c.path_dict['index']
index_file = c.file_dict['index_r'] % index_name
df = u.read_csv(index_path + index_file)
# Plot Figure
fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True)
fig.set_size_inches(32, 18)
# Define Font
font = {
'family': 'serif',
'color': 'black',
'weight': 'normal',
'size': 18,
}
# Plot Sub-figure 1
title = '%s vs. %s' % (index_name, benchmark_name)
ax1.set_title(title, fontdict=font)
ax1.set_xlabel('', fontdict=font)
ax1.set_ylabel('Ratio', fontdict=font)
for column in ['ratio', 'b_ratio']:
df.plot(x='date', y=column, ax=ax1)
# Plot Sub-figure 2
title = 'Index %s' % index_name
ax2.set_title(title, fontdict=font)
ax2.set_xlabel('', fontdict=font)
ax2.set_ylabel('Close Price', fontdict=font)
df.plot(x='date', y='index', ax=ax2)
# Plot Sub-figure 3
title = 'Index %s' % benchmark_name
ax3.set_title(title, fontdict=font)
ax3.set_xlabel('', fontdict=font)
ax3.set_ylabel('Close Price', fontdict=font)
df.plot(x='date', y='b_index', ax=ax3)
# Common Format for Both Sub-figures
for ax in [ax1, ax2, ax3]:
ax.grid(True)
fig.autofmt_xdate()
fig.tight_layout()
plt.setp(plt.gca().get_xticklabels(), rotation=30)
plt.show()
# Save Figure
fig_key = 'fig_index'
fig_path = c.path_dict[fig_key]
fig_file = c.file_dict[fig_key] % (index_name + '_' +
u.dateToStr(u.today()))
u.saveFigure(fig, fig_path, fig_file)
def calcQFQ(stock_id, period):
# Calculate QFQ DataFrame
df = calc_qfq(stock_id, period)
if gs.is_debug:
print(df.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(df):
path = c.path_dict['qfq'] % period
file = c.file_dict['qfq'] % (period, stock_id)
u.to_csv(df, path, file)
def histogramAlpha(postfix, completeness_threshold):
# Load Coefficient File
allcoef = loadCoefficient(postfix, completeness_threshold)
if u.isNoneOrEmpty(allcoef):
return False
# Calculate Coefficient Histogram
columns = [
'Total', 'Very High', 'High', 'Medium', 'Low', 'Very Low',
'Negative Very Low', 'Negative Low', 'Negative Medium',
'Negative High', 'Negative Very High'
]
histogram = u.createDataFrame(1, columns, 0)
stock_number = len(allcoef)
histogram.ix[0, 'Total'] = stock_number
for i in range(stock_number):
alpha = allcoef.ix[i, 'alpha']
if alpha >= 0.4: # [0.4, +Infinity)
histogram.ix[0, 'Very High'] = histogram.ix[0, 'Very High'] + 1
elif alpha >= 0.3: # [0.3, 0.4)
histogram.ix[0, 'High'] = histogram.ix[0, 'High'] + 1
elif alpha >= 0.2: # [0.2, 0.3)
histogram.ix[0, 'Medium'] = histogram.ix[0, 'Medium'] + 1
elif alpha >= 0.1: # [0.1, 0.2)
histogram.ix[0, 'Low'] = histogram.ix[0, 'Low'] + 1
elif alpha >= 0.0: # [0.0, 0.1)
histogram.ix[0, 'Very Low'] = histogram.ix[0, 'Very Low'] + 1
elif alpha >= -0.1: # [-0.1, 0.0)
histogram.ix[
0,
'Negative Very Low'] = histogram.ix[0, 'Negative Very Low'] + 1
elif alpha >= -0.2: # [-0.2, -0.1)
histogram.ix[0,
'Negative Low'] = histogram.ix[0, 'Negative Low'] + 1
elif alpha >= -0.3: # [-0.3, -0.2)
histogram.ix[
0, 'Negative Medium'] = histogram.ix[0, 'Negative Medium'] + 1
elif alpha >= -0.4: # [-0.4, -0.3)
histogram.ix[0,
'Negative High'] = histogram.ix[0,
'Negative High'] + 1
else: # (-Infinity, -0.4)
histogram.ix[0, 'Negative Very High'] = histogram.ix[
0, 'Negative Very High'] + 1
# Save to CSV File
histogram.set_index('Total', inplace=True)
file_postfix = '_'.join(
[postfix, completeness_threshold, 'HistogramAlpha'])
u.to_csv(histogram, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
def calcHPE(stock_id, period, ratio):
# Calculate HPE DataFrame
df = calc_hpe(stock_id, period, ratio)
if gs.is_debug:
print(df.head(10))
# Save to CSV File
if not u.isNoneOrEmpty(df):
key = 'hpe' if ratio == 'PE' else 'hep'
path = c.path_dict[key] % period
file = c.file_dict[key] % (period, stock_id)
u.to_csv(df, path, file)
def histogramCorrelation(postfix, completeness_threshold):
# Load Coefficient File
allcoef = loadCoefficient(postfix, completeness_threshold)
if u.isNoneOrEmpty(allcoef):
return False
# Calculate Coefficient Histogram
columns = [
'Total', 'Very Strong', 'Strong', 'Medium', 'Weak', 'Very Weak',
'Negative Very Weak', 'Negative Weak', 'Negative Medium',
'Negative Strong', 'Negative Very Strong'
]
histogram = u.createDataFrame(1, columns, 0)
stock_number = len(allcoef)
histogram.ix[0, 'Total'] = stock_number
for i in range(stock_number):
correlation = allcoef.ix[i, 'correlation']
if correlation > 0.8: # (0.8, 1.0]
histogram.ix[0, 'Very Strong'] = histogram.ix[0, 'Very Strong'] + 1
elif correlation > 0.6: # (0.6, 0.8]
histogram.ix[0, 'Strong'] = histogram.ix[0, 'Strong'] + 1
elif correlation > 0.4: # (0.4, 0.6]
histogram.ix[0, 'Medium'] = histogram.ix[0, 'Medium'] + 1
elif correlation > 0.2: # (0.2, 0.4]
histogram.ix[0, 'Weak'] = histogram.ix[0, 'Weak'] + 1
elif correlation >= 0.0: # [0.0, 0.2]
histogram.ix[0, 'Very Weak'] = histogram.ix[0, 'Very Weak'] + 1
elif correlation > -0.2: # (-0.2, 0.0)
histogram.ix[0, 'Negative Very Weak'] = histogram.ix[
0, 'Negative Very Weak'] + 1
elif correlation > -0.4: # (-0.4, -0.2]
histogram.ix[0,
'Negative Weak'] = histogram.ix[0,
'Negative Weak'] + 1
elif correlation > -0.6: # (-0.6, -0.4]
histogram.ix[
0, 'Negative Medium'] = histogram.ix[0, 'Negative Medium'] + 1
elif correlation > -0.8: # (-0.8, -0.6]
histogram.ix[
0, 'Negative Strong'] = histogram.ix[0, 'Negative Strong'] + 1
else: # [-1.0, -0.8]
histogram.ix[0, 'Negative Very Strong'] = histogram.ix[
0, 'Negative Very Strong'] + 1
# Save to CSV File
histogram.set_index('Total', inplace=True)
file_postfix = '_'.join(
[postfix, completeness_threshold, 'HistogramCorrelation'])
u.to_csv(histogram, c.path_dict['strategy'],
c.file_dict['strategy'] % file_postfix)
return True
def getStockList(cutoff_date):
stocks = get_stock_list(cutoff_date)
if not u.isNoneOrEmpty(stocks):
stocks['code'] = stocks['code'].map(lambda x:str(x).zfill(6))
stocks.set_index('code', inplace=True)
if gs.is_debug:
print(stocks.head(10))
# Save to CSV File
u.to_csv(stocks, c.path_dict['classify'], c.file_dict['stock_list'] % cutoff_date)
else:
print('getStockList() Failed!')
raise SystemExit
def plotHPE(period='M', ratio='PE'):
# Check pre-requisite
basics = loadStockBasics()
if u.isNoneOrEmpty(basics):
print('Need to have stock basics!')
raise SystemExit
# Iterate over all stocks
basics_number = len(basics)
for i in range(basics_number):
stock_id = u.stockID(basics.loc[i, 'code'])
# Plot HPE Data
plot_HPE(stock_id=stock_id, period=period, ratio=ratio)
def getIndustrySina():
# Download Sina Industry Data
industry = get_industry_sina()
if not u.isNoneOrEmpty(industry):
industry.set_index('code', inplace=True)
if gs.is_debug:
print(industry.head(10))
# Save to CSV File
u.to_csv(industry, c.path_dict['classify'], c.file_dict['indu_sina'])
else:
print('getIndustrySina() Failed!')
raise SystemExit
def getSZ50():
# Download SZ50 Data
sz50 = get_sz50()
if not u.isNoneOrEmpty(sz50):
sz50.set_index('code', inplace=True)
if gs.is_debug:
print(sz50.head(10))
# Save to CSV File
u.to_csv(sz50, c.path_dict['classify'], c.file_dict['sz50'])
else:
print('getSZ50() Failed!')
raise SystemExit
def getZZ500():
# Download ZZ500 Data
zz500 = get_zz500()
if not u.isNoneOrEmpty(zz500):
zz500.set_index('code', inplace=True)
if gs.is_debug:
print(zz500.head(10))
# Save to CSV File
u.to_csv(zz500, c.path_dict['classify'], c.file_dict['zz500'])
else:
print('getZZ500() Failed!')
raise SystemExit
def getTerminated():
# Download Terminated Stock Data
terminated = get_terminated()
if not u.isNoneOrEmpty(terminated):
terminated.set_index('code', inplace=True)
if gs.is_debug:
print(terminated.head(10))
# Save to CSV File
u.to_csv(terminated, c.path_dict['classify'], c.file_dict['terminated'])
else:
print('getTerminated() Failed!')
raise SystemExit
def getConceptSina():
# Download Sina Concept Data
concept = get_concept_sina()
if not u.isNoneOrEmpty(concept):
concept.set_index('code', inplace=True)
if gs.is_debug:
print(concept.head(10))
# Save to CSV File
u.to_csv(concept, c.path_dict['classify'], c.file_dict['conc_sina'])
else:
print('getConceptSina() Failed!')
raise SystemExit
def getSuspended():
# Download Suspended Data
suspended = get_suspended()
if not u.isNoneOrEmpty(suspended):
suspended.set_index('code', inplace=True)
if gs.is_debug:
print(suspended.head(10))
# Save to CSV File
u.to_csv(suspended, c.path_dict['classify'], c.file_dict['suspended'])
else:
print('getSuspended() Failed!')
raise SystemExit