def set_start_date(self, date):
"""Sets the start date for this Simulator.
Args:
date: A start date
"""
self.dates_testing = (date_str(date), self.dates_testing[1])
def set_end_date(self, date):
"""Sets the end date for this Simulator.
Args:
date: An end date
"""
self.dates_testing = (self.dates_testing[0], date_str(date))
def current_date(self):
"""Returns the current date of this Market.
Returns:
A string representing the current date in this Market
"""
return date_str(self.date[1])
def set_default_dates(self):
"""Sets a default range for this Market's dates.
Based on existing stocks in this Market, decides an appropriate
range in which all stocks have prices.
"""
date_range = (date_str(dt.fromordinal(1)),
date_str(dt.fromordinal(999999)))
for price_lut in self.stocks.values():
dates = sorted(price_lut.keys())
date_range = (max(date_range[0], dates[0]), min(
date_range[1], dates[-1]))
date_idxs = (dates.index(
date_range[0]), dates.index(date_range[1]))
self.dates = dates[date_idxs[0]:date_idxs[1] + 1]
self.date = (0, self.dates[0])
if len(sys.argv) < 2:
print("usage: ./dump_ind_choice.py date country")
quit()
dt = sys.argv[1]
country = sys.argv[2]
data_path = root_path + 'data/industry_choice/{}/'.format(country)
os.makedirs(data_path, 0o777, exist_ok = True)
if not utils.is_bday(dt, country):
print('{} is {} holiday'.format(dt, country))
c.stop()
quit()
if country == 'cn':
import utils_jq
stocks = utils_jq.get_universe(utils.date_str(dt))
stocks = [x.replace('XSHG', 'SH').replace('XSHE', 'SZ') for x in stocks]
else:
stocks = utils_choice.get_choice_universe(dt, country)
res_gics = c.css(stocks, 'GICSCODE', 'ClassiFication=3,EndDate={},Ispandas=1'.format(dt)).reset_index().rename(columns = { 'GICSCODE': 'gics' })
res_list = [res_gics]
if country == 'cn':
res_citic1 = c.css(stocks,"CITICCODE2020", "ClassiFication=1,EndDate={},Ispandas=1".format(dt)).reset_index().rename(columns = { 'CITICCODE2020': 'citic_l1' })
res_citic2 = c.css(stocks, "CITICCODE2020", "ClassiFication=2,EndDate={},Ispandas=1".format(dt)).reset_index().rename(columns = { 'CITICCODE2020': 'citic_l2' })
res_citic3 = c.css(stocks, "CITICCODE2020", "ClassiFication=3,EndDate={},Ispandas=1".format(dt)).reset_index().rename(columns = { 'CITICCODE2020': 'citic_l3' })
res_list = res_list + [res_citic1, res_citic2, res_citic3]
res = reduce(lambda left, right: pd.merge(left, right, on = ['CODES', 'DATES'], how = 'left'), res_list)
t = res
t = t.rename(columns = { 'CODES': 'ric', 'DATES': 'date' })
t = t.drop_duplicates()
t['date'] = utils.date_str(dt)
import sys
import os
root_path = os.path.dirname(os.path.realpath(__file__)) + '/../'
sys.path.append(root_path + 'scripts/')
import utils
import utils_jq
import jqdatasdk
if len(sys.argv) < 3:
print("usage: ./dump_ind.py date cn")
quit()
d = sys.argv[1]
country = sys.argv[2]
if not utils.is_bday(d, country):
print('{} is {} holiday'.format(d, country))
quit()
file_path = root_path + 'data/industry/{}/'.format(country)
os.makedirs(file_path, 0o777, exist_ok=True)
stocks = utils_jq.get_universe(utils.date_str(d))
t = jqdatasdk.get_industry(stocks, date=utils.date_str(d))
t = pd.DataFrame([
dict(
zip(['ric', 'date'] + list(t[r].keys()),
[r, d] + [x['industry_code'] for x in t[r].values()]))
for r in t.keys()
])
if 0 == t.size:
quit()
print(file_path + '{}.txt'.format(d))
t.to_csv(file_path + '{}.txt'.format(d), sep='\t', index=False)
'inc_operation_profit_year_on_year', 'inc_net_profit_year_on_year',
'inc_net_profit_to_shareholders_year_on_year'
],
'profit': [
'pubDate', 'statDate', 'total_operating_revenue', 'operating_revenue',
'operating_profit', 'total_profit', 'net_profit',
'np_parent_company_owners', 'basic_eps', 'diluted_eps'
]
}
data_path = root_path + 'data/fund/{}/'.format(statement)
os.makedirs(data_path, 0o777, exist_ok=True)
if not utils.is_bday(d, country):
print('{} is {} holiday'.format(d, country))
quit()
import jqdatasdk
import utils_jq
stocks = utils_jq.get_universe(utils.date_str(d))
f = jqdatasdk.get_fundamentals(jqdatasdk.query(
jqdatasdk.valuation, jqdatasdk.income,
jqdatasdk.indicator).filter(jqdatasdk.valuation.code.in_(stocks)),
date=utils.date_str(d))
f['date'] = d
f.fillna(0)
f = f.rename(columns={'code': 'ric'})
f = f.drop_duplicates()
f = f[['date', 'ric'] + fund_map[statement]]
data_file = data_path + '/{}.txt'.format(d)
print(data_file)
f.to_csv(data_file, sep='\t', index=False)
import os
root_path = os.path.dirname(os.path.realpath(__file__)) + '/../'
sys.path.append(root_path + 'scripts/')
import utils
import utils_jq
import jqdatasdk
if len(sys.argv) < 2:
print("usage: ./dump_compo.py date index")
quit()
d = sys.argv[1]
idx = sys.argv[2]
country = utils.country_map[idx]
# csi300: 000030.XSHG csi500: 000905.XSHG
compo_mapping = {'csi300': '000300.XSHG', 'csi500': '000905.XSHG'}
compo_path = root_path + 'data/compo/' + idx + '/'
if not utils.is_bday(d, country):
print('{} is {} holiday'.format(d, country))
quit()
stocks = jqdatasdk.get_index_stocks(compo_mapping[idx], date = utils.date_str(d))
t = jqdatasdk.get_index_weights(compo_mapping[idx], date = utils.date_str(d))
t = t.reset_index()
t['weight'] = t['weight'] / 100
t = t.drop(columns = ['date'])
t = t.rename(columns={'index': 'ric', 'display_name': 'ric_name'})
t = t.drop_duplicates()
t = t[['ric', 'ric_name', 'weight']]
print(compo_path + '{}.txt'.format(d))
t.to_csv(compo_path + '{}.txt'.format(d), sep='\t', index = False)
def get_choice_universe(dt, country):
stocks = c.sector(utils.choice_country_map[country],
utils.date_str(dt)).Data
return stocks[::2]
#!/usr/bin/python3
#/usr/local/bin/python3
import pandas as pd
import datetime
import sys
import os
root_path = os.path.dirname(os.path.realpath(__file__)) + '/../'
sys.path.append(root_path + 'scripts/')
import utils
import utils_jq
import jqdatasdk
data_path = root_path + 'data/holder/cn/'
dt = sys.argv[1]
os.makedirs(data_path, 0o777, exist_ok = True)
#stocks = utils_jq.get_universe(utils.date_str(dt))
stocks = utils_jq.get_universe(utils.date_str(datetime.datetime.now().strftime('%Y%m%d')))
df_list = []
df = jqdatasdk.finance.run_query(jqdatasdk.query(jqdatasdk.finance.STK_HOLDER_NUM).filter(jqdatasdk.finance.STK_HOLDER_NUM.pub_date == utils.date_str(dt)))
df['date'] = utils.date_str(dt)
df.fillna(0)
df = df.rename(columns = {'code': 'ric'})
df = df[df['ric'].isin(stocks)]
df = df.drop_duplicates()
df = df[['date', 'ric', 'end_date', 'pub_date', 'share_holders']]
data_file = data_path + '{}.txt'.format(dt)
print(data_file)
df.to_csv(data_file, sep='\t', index = False)
quit()
stocks = utils_choice.get_choice_universe(dt, country)
data_dict = {
'ric': [],
'date': [],
'open': [],
'high': [],
'low': [],
'close': [],
'pre_close': [],
'volume': [],
'money': []
}
data = c.css(stocks, 'OPEN,HIGH,LOW,CLOSE,PRECLOSE,AMOUNT,VOLUME',
'TradeDate={},CurType=1,AdjustFlag=3'.format(
utils.date_str(dt))).Data # local currency, pre adjustment
for k, v in data.items():
data_dict['ric'].append(k)
data_dict['date'] = utils.date_str(dt)
data_dict['open'].append(v[0])
data_dict['high'].append(v[1])
data_dict['low'].append(v[2])
data_dict['close'].append(v[3])
data_dict['pre_close'].append(v[4])
data_dict['volume'].append(v[5])
data_dict['money'].append(v[6])
t = pd.DataFrame(data_dict)
t.replace([np.nan], 0, inplace=True)
if 0 == t.size: quit()
t = t.reindex(columns=[
'ric', 'date', 'open', 'high', 'low', 'close', 'pre_close', 'volume',
if len(sys.argv) > 1: dt = sys.argv[1]
else:
dt = datetime.datetime.now().strftime('%Y-%m-%d')
print(dt)
dt = utils.bday_offset(dt, -1, country).replace('-', '')
if not utils.is_bday(dt, country):
print('{} is {} holiday'.format(dt, country))
c.stop()
quit()
data_path = root_path + 'data/choice/est/'
os.makedirs(data_path, 0o777, exist_ok=True)
if not utils.is_bday(dt, country):
print('{} is {} holiday'.format(dt, country))
quit()
stocks = utils_jq.get_universe(utils.date_str(dt))
stocks = [x.replace('XSHG', 'SH').replace('XSHE', 'SZ') for x in stocks]
data_file = data_path + '/{}.txt'.format(dt)
res = c.csd(stocks, "SESTNIFY1,SESTNIFY2,SESTROEFY1,ESTPEFTM", dt, dt,
'Period=1,Ispandas=1')
print(res)
res = res.reset_index()
res = res.rename(
columns={
'CODES': 'ric',
'DATES': 'date',
'SESTNIFY1': 'est_pni_fy1',
'SESTNIFY2': 'est_pni_fy2',
'SESTROEFY1': 'est_roe_fy1',
'ESTPEFTM': 'pe_fw12m'
import os
import datetime
import pandas as pd
root_path = os.path.dirname(os.path.realpath(__file__)) + '/../'
sys.path.append(root_path + 'scripts/')
import utils
sd = sys.argv[1]
ed = sys.argv[2]
exch = sys.argv[3]
data_path = root_path + '/data/calendar/'
if exch != 'ftx':
exch_cal = mcal.get_calendar(exch.upper())
days = [
x.strftime('%Y-%m-%d')
for x in exch_cal.valid_days(start_date=utils.date_str(sd),
end_date=utils.date_str(ed))
]
df = pd.DataFrame({'date': days})
else:
days = []
sd = datetime.datetime.strptime(sd, '%Y%m%d')
ed = datetime.datetime.strptime(ed, '%Y%m%d')
for dt in [sd + datetime.timedelta(x) for x in range((ed - sd).days + 1)]:
dt = dt.strftime('%Y-%m-%d')
days.append(dt)
df = pd.DataFrame({'date': days})
data_file = data_path + '/{}.txt'.format(utils.exch_map[exch])
print(data_file)
df.to_csv(data_file, sep='\t', index=False)