def main(rootdir, dt_today):
reportdir = os.path.join(rootdir, "report")
filename = "{0}-{1}-{2}损益日报表(收盘价).xlsm".format(dt_today.year,
dt_today.month,
dt_today.day)
filepath = os.path.join(reportdir, filename)
datadir = os.path.join(rootdir, "dataotc")
srcpath = os.path.join(datadir, "Blank模板勿删(收盘价).xlsm")
if os.path.exists(filepath):
pass
else:
shutil.copyfile(srcpath, filepath)
#[1]
oholiday = BD.CHolidayData()
pre_workday = oholiday.preWorkDay(dt_today)
baseobj = BD.CSnapData4(dt_today, pre_workday)
baseobj.start()
#[2]
obj = historyAccount()
obj.setData(oholiday, baseobj)
today_rows, later_rows = obj.readAccount(datadir, dt_today)
obj.writeTemplate(filepath, later_rows, today_rows)
pass
def main(rootdir, dt_today):
tmap = {
"林秉玮": ('0009', 'jn'),
"谢晨星": ('0019', 'jn'),
"何剑桥": ('0029', 'jn'),
"刘剑溥": ('0039', 'jn'),
"场内CU": ('8001888888', 'ctp1', 'CU'),
"场内RU": ('8001888888', 'ctp1', 'RU'),
"场内CF": ('8001888888', 'ctp1', 'CF'),
"场内i": ('8001888888', 'ctp1', 'i'),
"911005": ('9110000005', 'ctp2'),
"911002": ('9110000002', 'ctp2')
}
#[1]基础数据
oproduct = BD.CProductData()
oholiday = BD.CHolidayData()
pre_workday = oholiday.preWorkDay(dt_today)
baseobj = BD.CSnapData4(dt_today, pre_workday)
baseobj.start()
traderlist = []
for k, v in tmap.items():
if v[1] == 'ctp1': #计算ctp场内期权，需要2日的结算单
prefilepath = scan_find(rootdir, pre_workday, v[0])
filepath = scan_find(rootdir, dt_today, v[0])
if prefilepath is None:
continue
if filepath is None:
continue
pass
else: #计算金牛，或ctp, 需要当日的结算单
filepath = scan_find(rootdir, dt_today, v[0])
if filepath is None:
continue
#every trader
trader = Trader(k, dt_today)
trader.setBaseData(baseobj, oproduct)
if v[1] == 'ctp1':
trader.parseClearData2(prefilepath, filepath, v[1], v[2])
else:
trader.parseClearData(filepath, v[1])
trader.printself()
traderlist.append(trader)
pass
#[4]汇总并输出报表
makeReport(rootdir, traderlist, dt_today)
pass
def main_batch():
#目标目录
curfile = os.path.realpath(__file__)
rootdir = os.path.dirname( os.path.dirname(curfile) )
curdir = os.path.join(rootdir, "sharedata\\MarketData")
#dt = datetime.datetime.strptime("2019-09-06 15:00:00","%Y-%m-%d %H:%M:%S")
#节假日
oholiday = BD.CHolidayData()
#指定一段时间，再获取这一段时间的工作日，然后逐日下载
start = datetime.datetime(2019, 12, 30)
end = datetime.datetime(2019, 12, 31)
dt_list = oholiday.getWorklist(start, end)
for dt in dt_list:
#下载
main_only_download(curdir, dt)
#更新
#updateEveryDay(curdir, dt)
#print("update one ======")
print("main_batch end!!")
pass
def paradata_float(self, ruler_k):
single_paradata = []
if ruler_k['value'] != None:
paraattr1 = {}
paraattr1['value'] = ruler_k['value']
paraattr1['except'] = False
single_paradata.append(paraattr1)
return single_paradata
elif ruler_k['max'] != None and ruler_k['min'] != None:
# 产生边界值和一个随机等效
down = ruler_k['min']
up = ruler_k['max']
paraattr1 = {}
paraattr1['value'] = down - 0.01
paraattr1['except'] = True
single_paradata.append(paraattr1)
paraattr2 = {}
paraattr2['value'] = up + 0.01
paraattr2['except'] = True
single_paradata.append(paraattr2)
paraattr3 = {}
paraattr3['value'] = BaseData.BASEDATA().gen_ranfloat(down, up)
paraattr3['except'] = False
single_paradata.append(paraattr3)
return single_paradata
elif ruler_k['max'] != None and ruler_k['min'] == None:
up = ruler_k['max']
paraattr1 = {}
paraattr1['value'] = up + 0.01
paraattr1['except'] = True
single_paradata.append(paraattr1)
paraattr2 = {}
paraattr2['value'] = up
paraattr2['except'] = False
single_paradata.append(paraattr2)
return single_paradata
elif ruler_k['max'] == None and ruler_k['min'] != None:
down = ruler_k['min']
paraattr1 = {}
paraattr1['value'] = down - 0.01
paraattr1['except'] = True
single_paradata.append(paraattr1)
paraattr2 = {}
paraattr2['value'] = down
paraattr2['except'] = False
single_paradata.append(paraattr2)
return single_paradata
else:
HandleException.Exception().syntaxerror('The paraset is invalid!')
def paradata_context(self, ruler_k):
single_paradata = []
paraattr = {}
if ruler_k['value'] != None:
paraattr['value'] = ruler_k['value']
paraattr['except'] = False
single_paradata.append(paraattr)
return single_paradata
elif ruler_k['long'] != None:
long = ruler_k['long']
paraattr['value'] = BaseData.BASEDATA().gen_rancontext(long)
paraattr['except'] = False
single_paradata.append(paraattr)
return single_paradata
else:
HandleException.Exception().syntaxerror('The paraset is invalid!')
def main_batch():
curfile = os.path.realpath(__file__)
curdir = os.path.dirname( os.path.dirname(curfile) )
rootdir = os.path.join( curdir, "sharedata")
#节假日
oholiday = BD.CHolidayData()
#指定一段时间，再获取这一段时间的工作日，然后逐日执行
start = datetime.datetime(2019, 4, 30)
end = datetime.datetime(2019, 6, 1)
dt_list = oholiday.getWorklist(start, end)
for dt in dt_list:
HistoryOTCClose.main(rootdir, dt )
HistoryCtpjnClose.main(rootdir, dt )
pass
pass
print("a")
np.set_printoptions(suppress=True)
print("b")
xy, y_score = load_data(config["task"])
print("c1")
#y_score, best_score = normalize(y_score)
print("c2")
model, i, modelhistory = best_feedforward_model(xy, y_score, True)
print("d")
print("We take:")
print(i)
print("Best score for normalization scale:")
#print(best_score)
WineData = BaseData('data/winequality-red.csv',
';',
11,
10,
config["nr_base_columns"],
rifs=True)
WineData.load()
_ = evaluation_wrapper(config["task"], model, 'data/winequality-red.csv', True,
WineData)
def load_data(task):
if task == Task.regression:
WineData = BaseData('data/winequality-red.csv',
';',
7,
10,
config["nr_base_columns"],
rifs=True)
GoogleData = BaseData(
'data/google-safe-browsing-transparency-report-data.csv',
',',
10,
10,
config["nr_base_columns"],
rifs=True)
CampusData = BaseData('data/placement_data_full_class.csv',
',',
14,
10,
config["nr_base_columns"],
rifs=True,
text_columns=[1, 3, 5, 6, 8, 9, 11, 13])
FootballData = BaseData('data/results_football.csv',
',',
3,
10,
config["nr_base_columns"],
rifs=True,
text_columns=[1, 2, 5, 6, 7, 8],
date=0)
KingSalesData = BaseData('data/kc_house_data.csv',
',',
2,
10,
config["nr_base_columns"],
rifs=True,
date=1)
AvocadoSalesData = BaseData('data/avocado.csv',
',',
2,
10,
config["nr_base_columns"],
rifs=True,
text_columns=[11, 13],
date=1)
Brazil_Rent = BaseData('data/houses_to_rent.csv',
',',
12,
10,
config["nr_base_columns"],
rifs=True,
text_columns=[6, 7])
TeslaStocksData = BaseData('data/TSLA.csv',
',',
6,
10,
config["nr_base_columns"],
rifs=True,
date=0)
WeatherHistoryData = BaseData('data/weatherHistory.csv',
',',
8,
10,
config["nr_base_columns"],
rifs=True,
text_columns=[1, 2, 11],
date=0)
VoiceData = BaseData('data/voice.csv',
',',
19,
10,
config["nr_base_columns"],
rifs=True,
text_columns=[20])
CountriesData = BaseData('data/countries_of_the_world.csv',
',',
14,
10,
config["nr_base_columns"],
rifs=True,
text_columns=[0, 1])
train_datasets = [
WineData, GoogleData, CampusData, FootballData, KingSalesData,
AvocadoSalesData, Brazil_Rent, TeslaStocksData, WeatherHistoryData,
VoiceData, CountriesData
]
if config["prod"] == False:
train_datasets = train_datasets[0:2]
xy, y_score = merge_data_sets(train_datasets)
config["current_dataset_names"] = list(
map(lambda d: d.dataset_path, train_datasets))
if task == Task.multivariate_time_series:
BirthDeaths3 = BaseData(
'data/multivariate_time_series/_births_and_deaths.csv',
';',
3,
1,
base_size=config["nr_base_columns"])
BirthDeaths4 = BaseData(
'data/multivariate_time_series/_births_and_deaths.csv',
';',
4,
1,
base_size=config["nr_base_columns"])
xy, y_score = merge_data_sets([BirthDeaths3, BirthDeaths4])
if task == Task.classification:
pass
return xy, y_score
def evaluation_wrapper(task, model, _path, _print, eval_dataset):
if task == Task.multivariate_time_series:
BirthDeaths3 = BaseData(_path,
';',
3,
1,
base_size=config["nr_base_columns"])
BirthDeaths3.load()
evaluations = evaluation(BirthDeaths3.base_xy, add_eval_columns,
eval_dataset.base_dataset[:, 11], model)
if task == Task.regression:
def get_wine_column():
content = open('data/wineexport.csv').read().split("\n")
values = list(map(lambda line: line.split(";")[1], content))
return np.array(values).astype(float)
wc = get_wine_column()
import pdb
pdb.set_trace()
if config["budget_join"]:
add_eval_columns = [
eval_dataset.base_dataset[:, 0],
eval_dataset.base_dataset[:, 1],
eval_dataset.base_dataset[:, 2],
eval_dataset.base_dataset[:, 3],
eval_dataset.base_dataset[:, 11],
]
for i in np.arange(config["nr_add_columns_budget"] -
len(add_eval_columns)):
add_eval_columns.append(
np.random.normal(0, 1, eval_dataset.base_dataset[:,
0].shape))
#add_eval_columns.append(np.ones(eval_dataset.base_dataset[:, 0].shape))
else:
add_eval_columns = [eval_dataset.base_dataset[:, 0]]
import pdb
pdb.set_trace()
evaluations = evaluation(
eval_dataset.base_xy, #5
add_eval_columns, #1200
eval_dataset.base_dataset[:, 11], #
model #model
)
import pdb
pdb.set_trace()
import pdb
pdb.set_trace()
import pdb
pdb.set_trace()
#plot_performance([evaluations[1]], [evaluations[3]])
regr = linear_model.LinearRegression()
regr.fit(eval_dataset.base_x, eval_dataset.base_y)
regression_score_base_data = regr.score(eval_dataset.base_x,
eval_dataset.base_y)
indices = np.where(evaluations[0] > config["nfs_output_threshold"])
regr = linear_model.LinearRegression()
add_columns = np.array(list(map(lambda c: c[0],
eval_dataset.add_columns)))[indices[0], :]
augmented_x = np.transpose(
np.concatenate((np.transpose(eval_dataset.base_x,
(1, 0)), add_columns),
axis=0), (1, 0))
regr.fit(augmented_x, eval_dataset.base_y)
regression_score_augmented = regr.score(augmented_x, eval_dataset.base_y)
output = ""
output += "Path: " + str(_path)
output += "\nBase DataSet size: " + str(eval_dataset.base_x.shape)
output += "\nScore for Base DataSet: " + str(regression_score_base_data)
output += "\nColumns presented to NFS: " + str(
config["nr_add_columns_budget"])
output += "\nColumns chosen by NFS by threshold: " + str(len(indices[0]))
output += "\nAugmented DataSet size: " + str(augmented_x.shape)
output += "\nScore for Augmented DataSet: " + str(
regression_score_augmented)
output += "\nNFS Time: " + str(evaluations[1])
output += "\nPearson Time: " + str(evaluations[3])
output += "\nConfig: " + str(config)
f = open("output/" + str(time.time()).split(".")[0] + ".txt", "a")
f.write(output)
f.close()
return _path, evaluations, regression_score_base_data, regression_score_augmented