class Application(object):
def __init__(self, options):
self.options = options
self.datasource = DataSource(options)
def run(self):
from lib.man.generator import Generator
if self.options.dump_isa or self.options.dump_arch:
if self.options.dump_isa:
self.dump_isa()
if self.options.dump_arch:
self.dump_arch()
else:
gen = Generator(self.options, self.datasource)
generated_files = gen.generate()
if self.options.deb_dir is not None:
create_deb_files(self.options, self.datasource)
if self.options.rpm_spec_dir:
create_spec_file(self.options, self.datasource,
generated_files)
def dump_isa(self):
instr_db = self.datasource.get_instructions()
text = "List of ISAs defined in %s: %s." % \
(self.options.instructions_xml, fmtset(instr_db.get_cpuids()))
print '\n'.join(textwrap.wrap(text))
def dump_arch(self):
from lib.uops import architecture_name
arch_db = self.datasource.get_architecture_details()
print "List of architectures defined in %s" % self.options.uops_xml
for symbol in sorted(arch_db.get_architectures()):
name = architecture_name(symbol)
if name != symbol:
print '* %s (%s)' % (name, symbol)
else:
print '* %s' % symbol
def preload_data(data):
symbol, start_date, end_date = data[0], data[1].start_date, data[1].end_date
dataset = ds.loadFeaturedData(symbol, start_date, end_date)
# time.sleep(0.1)
# 观察数据
with counter.get_lock():
counter.value +=1
progress = counter.value / total
time_elapsed = datetime.datetime.now(tz=None) - start_ts
time_eta = (time_elapsed/progress) * (1 - progress)
bar_width = 25
print("\rProgress: {:>5.2f}% ({:04d}/{:04d}) Symbol: {:s}\t[{}{}]\tElapsed Time: {} ETA: {}".format(
round(progress*100,2), counter.value, total, symbol,
"#"*int(progress*bar_width),"."*(bar_width-int(progress*bar_width)),
str(time_elapsed).split(".")[0], str(time_eta).split(".")[0]
), end="")
return
def __init__(self, options):
self.options = options
self.datasource = DataSource(options)
default=3,
type=int,
help='Stop learning if N batch of improving the result')
parser.add_argument(
'--random',
'-r',
default=1,
type=int,
help='By default using random data samples for learning')
args = vars(parser.parse_args())
if args['random'] != 1: np.random.seed(0)
securities = ds.loadSecuirtyList()
# 设置训练的指标模块
if args['all'] is not None:
strategies = indicators.values()
else:
strategies = [indicators[args['indicator']]]
for i in range(args['batch_size']):
#skip batch logic
if i < args['skip_batch']: continue
print("Learning batch :{}".format(i))
print("Preloading datasets: {}".format(args['training_set_size'] * 2))
processed_counter = mp.Value('i', 0)
print(
"\rProgress: {:>5.2f}% ({:04d}/{:04d}) Symbol: {:s}\t[{}{}]\tElapsed Time: {} ETA: {}"
.format(round(progress * 100, 2), counter.value, total, symbol,
"#" * int(progress * bar_width),
"." * (bar_width - int(progress * bar_width)),
str(time_elapsed).split(".")[0],
str(time_eta).split(".")[0]),
end="")
return
if __name__ == "__main__":
mp.freeze_support()
np.random.seed(5)
# set output
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
securities = ds.loadSecuirtyList()
days = ds.loadTradeDays()
print("Max processes: {}".format(MAX_PROCESSES))
processed_counter = mp.Value('i', 0)
pool = mp.Pool(min(MAX_PROCESSES, mp.cpu_count()),
initializer=init_globals,
initargs=(processed_counter, len(securities)))
res = pool.map(preload_data, securities.iterrows())
pool.close()
print("")
#!/usr/bin/env python3
'''
从最原始的数据中提取出交易日和股票代码信息
这是最初级别的数据预处理
'''
import pandas as pd
import warnings
import multiprocessing as mp
from lib.datasource import DataSource as ds
warnings.simplefilter(action='ignore', category=FutureWarning)
if __name__ == "__main__":
mp.freeze_support()
# set output
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
ds.preload()