def train(config, model, train_iter, dev_iter, test_iter):
start_time = time.time()
# EMA初始化
ema = EMA(model, 0.999)
ema.register()
model.train()
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
# optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
optimizer = BertAdam(optimizer_grouped_parameters,
lr=config.learning_rate,
warmup=0.05,
t_total=len(train_iter) * config.num_epochs)
total_batch = 0 # 记录进行到多少batch
dev_best_loss = float('inf')
last_improve = 0 # 记录上次验证集loss下降的batch数
flag = False # 记录是否很久没有效果提升
model.train()
for epoch in range(config.num_epochs):
print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs))
for i, (trains, labels) in enumerate(train_iter):
outputs = model(trains)
model.zero_grad()
loss = F.cross_entropy(outputs, labels)
loss.backward()
optimizer.step()
ema.update() # 训练过程中，更新完参数后，同步update shadow weights
if total_batch % 100 == 0:
# 每多少轮输出在训练集和验证集上的效果
true = labels.data.cpu()
predic = torch.max(outputs.data, 1)[1].cpu()
train_acc = metrics.accuracy_score(true, predic)
ema.apply_shadow() # model应用shadow weights，进行验证、保存模型
dev_acc, dev_loss = evaluate(config, model, dev_iter)
if dev_loss < dev_best_loss:
dev_best_loss = dev_loss
torch.save(model.state_dict(), config.save_path)
improve = '*'
last_improve = total_batch
else:
improve = ''
ema.restore() # 下一次训练之前，恢复模型参数
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Train Acc: {2:>6.2%}, Val Loss: {3:>5.2}, Val Acc: {4:>6.2%}, Time: {5} {6}'
print(msg.format(total_batch, loss.item(), train_acc, dev_loss, dev_acc, time_dif, improve))
model.train()
total_batch += 1
if total_batch - last_improve > config.require_improvement:
# 验证集loss超过1000batch没下降，结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
test(config, model, test_iter)
def MACD(price,fast_period=12,slow_period=26,signal_period=9):
macd=[]
fast=EMA(price,fast_period)
slow=EMA(price,slow_period)
macd=[x-y for (x,y) in zip(fast,slow)]
signal=EMA(macd,signal_period)
return macd,signal
class MACDStrategy(Strategy):
def __init__(self, symbol, fast_time, slow_time, signal_time):
Strategy.__init__(self, symbol)
self.fast_ema = EMA(0)
self.slow_ema = EMA(0)
self.signal_ema = EMA(0)
self.fast_time = fast_time
self.slow_time = slow_time
self.signal_time = signal_time
self.prev_hist = 0
def update_signal(self, symbol, signal):
self.signal = signal
self.symbol = symbol
def on_new_tick(self, tick_object):
# generate macd --> e.g. 12 sec EMA of p - 26 sec EMA of p
fast_line = self.fast_ema.next(tick_object.ltp, self.fast_time)
slow_line = self.slow_ema.next(tick_object.ltp, self.slow_time)
macd_line = fast_line - slow_line if slow_line is not None else None
# generate signal --> e.g. 9 sec EMA of MACD
signal_line = self.signal_ema.next(macd_line, self.signal_time) if macd_line is not None else None
if signal_line is not None:
histogram = macd_line - signal_line
if histogram > 0 > self.prev_hist:
self.update_signal(tick_object.symbol, 2)
if histogram < 0 < self.prev_hist:
self.update_signal(tick_object.symbol, -2)
self.prev_hist = histogram
self.last_tick_object = tick_object
def strategy(alpha1, alpha2, window1, window2, multiple):
d = SP500Backtest()
s_long = EMA(alpha1, alpha2, d.df)
s_short = EMA(alpha1, alpha2, d.df)
vol_filter = RollingStdFilter(window1, window2, multiple,
d.df.Close).filter
volume_filter = RollingMeanFilter(window1, window2, multiple,
d.df.Volume).filter
filter_all = volume_filter & vol_filter
long_signal = [False]
for val in s_long.long.index[1:]:
if long_signal[-1] and s_long.long[val]:
long_signal.append(True)
elif not long_signal[-1] and s_long.long[val] and filter_all[val]:
long_signal.append(True)
else:
long_signal.append(False)
long_signal = pd.Series(long_signal, index=s_long.long.index)
short_signal = [False]
for val in s_short.short.index[1:]:
if short_signal[-1] and s_short.short[val]:
short_signal.append(True)
elif not short_signal[-1] and s_short.short[val] and filter_all[val]:
short_signal.append(True)
else:
short_signal.append(False)
short_signal = pd.Series(short_signal, index=s_short.short.index)
d.mini_backtest(long_signal, short_signal)
return float(d.annualized_sharpe_ratio), float(d.drawdown), float(
d.annual_return)
def PVO(volume, shortperiod=12, longperiod=26):
emashort = EMA(volume, shortperiod)
emalong = EMA(volume, longperiod)
pvo = list(np.zeros(longperiod - 1))
for i in range(longperiod - 1, len(emalong)):
pvo.append(100 * (emashort[i] - emalong[i]) / emalong[i])
return pvo
def MassIndex(high, low, emaperiod=9, period=25):
hl = [i - j for (i, j) in zip(high, low)]
singleEMA = EMA(hl, emaperiod)
doubleEMA = EMA(singleEMA, emaperiod)
MI = list(np.zeros(period - 1))
EMAratio = list(np.zeros(period - 1))
EMAratio += [
i / j for (i, j) in zip(singleEMA[period - 1:], doubleEMA[period - 1:])
]
for i in range(period - 1, len(high)):
MI.append(sum(EMAratio[i - period + 1:i]))
return MI
def KVO(close, high, low, volume, period=13, shortperiod=34, longperiod=55):
keyprice = [(i + j + k) / 3 for (i, j, k) in zip(close, high, low)]
trend = [0]
for i in range(1, len(close)):
if keyprice[i] > keyprice[i - 1]:
trend.append(volume[i])
else:
trend.append(-volume[i])
emashort = EMA(trend, shortperiod)
emalong = EMA(trend, longperiod)
KVO = [i - j for (i, j) in zip(emashort, emalong)]
KVOsignal = EMA(KVO, period)
return KVO, KVOsignal
def TMF(close, high, low, volume):
HH = [high[0]]
LL = [low[0]]
for i in range(1, len(high)):
HH.append(max(high[i], close[i - 1]))
LL.append(min(low[i], close[i - 1]))
num = EMA([
i * (2 * (j - k) / (l - k) - 1)
for (i, j, k, l) in zip(volume, close, LL, HH)
], 21)
denum = EMA(volume, 21)
TMF = list(np.zeros(20))
TMF += [i / j for (i, j) in zip(num[20:], denum[20:])]
return TMF
def train_model(args, model, dataloader, loaders_len, criterion, optimizer,
scheduler, use_gpu):
'''
train the model
'''
since = time.time()
ema = None
# exponential moving average
if args.ema_decay > 0:
ema = EMA(model, decay=args.ema_decay)
ema.register()
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
correspond_top5 = 0.0
for epoch in range(args.start_epoch, args.num_epochs):
epoch_time = time.time()
train(args, model, dataloader['train'], loaders_len['train'],
criterion, optimizer, scheduler, use_gpu, epoch, ema)
top1_acc, top5_acc = validate(args, model, dataloader['val'],
loaders_len['val'], criterion, use_gpu,
epoch, ema)
epoch_time = time.time() - epoch_time
print('Time of epoch-[{:d}/{:d}] : {:.0f}h {:.0f}m {:.0f}s\n'.format(
epoch, args.num_epochs, epoch_time // 3600,
(epoch_time % 3600) // 60, epoch_time % 60))
# deep copy the model if it has higher top-1 accuracy
if top1_acc > best_acc:
best_acc = top1_acc
correspond_top5 = top5_acc
if args.ema_decay > 0:
ema.apply_shadow()
best_model_wts = copy.deepcopy(model.state_dict())
if args.ema_decay > 0:
ema.restore()
print(os.path.split(args.save_path)[-1])
print('Best val top-1 Accuracy: {:4f}'.format(best_acc))
print('Corresponding top-5 Accuracy: {:4f}'.format(correspond_top5))
time_elapsed = time.time() - since
print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(
time_elapsed // 3600, (time_elapsed % 3600) // 60, time_elapsed % 60))
# load best model weights
model.load_state_dict(best_model_wts)
# save best model weights
if args.save:
torch.save(
model.state_dict(),
os.path.join(
args.save_path,
'best_model_wts-' + '{:.2f}'.format(best_acc) + '.pth'))
return model
def CV(high, low, period=20, change_period=10):
hl = [x - y for (x, y) in zip(high, low)]
hlema = EMA(hl)
CV = 0
for i in range(change_period - 1):
CV.append(0)
for i in range(change_period, len(hl)):
j = i - change_period
CV.append((hl[i] - hl[j]) * 100 / hl[j])
return CV
def EIS(close):
eis = []
macd, signal, histogram = MACD(close)
eis.append('b')
ema13 = EMA(close, 13)
for i in range(1, len(close)):
if ema13[i] > ema13[i - 1] and histogram[i] > histogram[i - 1]:
eis.append('g')
elif ema13[i] < ema13[i - 1] and histogram[i] < histogram[i - 1]:
eis.append('r')
else:
eis.append('b')
return eis
def ADX(close, high, low, period=14):
pdm = [0]
ndm = [0]
for i in range(1, len(close)):
h = high[i]
yh = high[i - 1]
l = low[i]
yl = low[i - 1]
if h - yh > yl - l:
pdm.append(h - yh)
ndm.append(0)
else:
pdm.append(0)
ndm.append(yl - l)
expdm = EMA(pdm, period)
exndm = EMA(ndm, period)
atr = ATR(close, high, low, period)
PDI = [100 * i / (j + 1) for (i, j) in zip(expdm, atr)]
NDI = [100 * i / (j + 1) for (i, j) in zip(exndm, atr)]
DX = [100 * abs(i - j) / (i + j + 1) for (i, j) in zip(PDI, NDI)]
ADX = EMA(DX, period)
return ADX, PDI, NDI
def __init__(self, symbol, fast_time, slow_time, signal_time):
Strategy.__init__(self, symbol)
self.fast_ema = EMA(0)
self.slow_ema = EMA(0)
self.signal_ema = EMA(0)
self.fast_time = fast_time
self.slow_time = slow_time
self.signal_time = signal_time
self.prev_hist = 0
def ERI(close, high, low, period=13):
ema = EMA(close, period)
BullPower = [i - j for (i, j) in zip(high, ema)]
BearPower = [i - j for (i, j) in zip(ema, low)]
return BullPower, BearPower
def BB(price, period=20, std_factor=2):
middle_band = EMA(price, period)
std = std_period(middle_band, period)
upper_band = [x + 2 * y for (x, y) in zip(middle_band, std)]
lower_band = [x - 2 * y for (x, y) in zip(middle_band, std)]
return middle_band, upper_band, lower_band
def TRIX(close,period):
tripleema=EMA(EMA(EMA(close,period),period),period)
trix=list(np.zeros(period))
for i in range(period,len(close)):
trix.append(100*(tripleema[i]-tripleema[i-1])/tripleema[i-1])
return trix
model_vars = []
for var in learn_vars:
if var.name.find('_loss') < 0:
model_vars.append(var)
model_vars = learn_vars
# latest_checkpoint = get_latest_checkpoint(model_path)
latest_checkpoint = model_path
# latest_checkpoint = None
if latest_checkpoint is not None:
restore = slim.assign_from_checkpoint_fn(latest_checkpoint,
var_list=model_vars,
ignore_missing_vars=True)
restore(sess)
# 4 begin iteration
ema_iloss = EMA()
ema_tloss = EMA()
ema_acc = EMA()
print('\n\nTraining started ...')
for i in range(args.epoch):
for batch in range(batch_per_epoch):
try:
# get batch
images_train, labels_train = db.getBatch()
# print(images_train)
images_train = (np.float32(images_train) - 127.5) / 128
# images_train = np.float32(images_train)
# print(images_train)
feed_dict = {
images: images_train,
labels: labels_train,
def EFI(close, volume, period=14):
efi1 = [0]
for i in range(1, len(close)):
efi1.append((close[i] - close[i - 1]) * volume[i])
EFI = EMA(efi1, period)
return EFI
def Envelop(price, period=20, envelop_width=5):
avg = EMA(price, period)
upper_envelop = [x + x * envelop_width / 100 for x in avg]
lower_envelop = [x - x * envelop_width / 100 for x in avg]
return upper_envelop, lower_envelop
def Qstick(closep, openp, period=8):
diff = [i - j for (i, j) in zip(closep, openp)]
Qstick = EMA(diff, period)
return Qstick
def CoppockCurve(close, shortRocPeriod=11, longRocPeriod=14, period=10):
shortRoc = ROC(close, shortRocPeriod)
longRoc = ROC(close, longRocPeriod)
CC = EMA([i + j for (i, j) in zip(shortRoc, longRoc)], period)
return CC
def KeltnerChannel(close,high,low,ema_period=20,atr_period=10,shift=2):
middle_line=EMA(close,ema_period)
atr=ATR(close,high,low,atr_period)
upper_line=[i+shift*j for (i,j) in zip(middle_line,atr)]
lower_line=[i-shift*j for (i,j) in zip(middle_line,atr)]
return middle_line,upper_line,lower_line