def main(debug):
rmsds = []
ppts = []
wrs = []
for info in DATA:
currency = info['currency']
min_trail = info['trail']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
actions = calculateActions(min_trail)
df = loadData(currency, interval, 'test')
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
rewards = []
errors = []
ticks = []
for i, row in df.iterrows():
df_inner = df.loc[i:]
q, r, error, tick = test(df_inner, q, PERIODS, actions, pip_mul)
# logging.warn('{0} {1}'.format(i, r))
# results
rewards.append(r)
errors.append(error * pip_mul)
ticks.append(tick)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
win_ratio = np.mean(wins)
# ppt
ppt = np.mean(rewards) * pip_mul
logging.warn('{0} RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}]'.format(
currency,
None,
int(rmsd),
int(ppt),
sum(rewards),
win_ratio * 100,
np.mean(ticks),
))
rmsds.append(rmsd)
ppts.append(ppt)
wrs.append(win_ratio)
logging.error('RMSD {0:.0f} +- {1:.0f}'.format(np.mean(rmsds), np.std(rmsds)))
logging.error('PPT {0:.0f} +- {1:.0f}'.format(np.mean(ppts), np.std(ppts)))
logging.error('WR {0:.0f} +- {1:.0f}'.format(np.mean(wrs) * 100, np.std(wrs) * 100))
def main(debug):
interval = choice(INTERVALS)
for currency, min_trail in CURRENCIES.iteritems():
pip_mul = 100. if 'JPY' in currency else 10000.
actions = calculateActions(min_trail)
df = loadData(currency, interval)
df = df[-1000:]
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
df_last = df[-1:]
a = predict(df, q, PERIODS, actions)
row = df_last.iloc[-1]
a_trade, a_trail = a.split('-')
if a_trade == 'buy':
stop_loss = row['close'] - (float(a_trail) / pip_mul)
else:
stop_loss = row['close'] + (float(a_trail) / pip_mul)
logging.warn('{0} {1} a:{2} t:{3} sl:{4:.4f}'.format(
row.name,
currency,
a_trade,
a_trail,
stop_loss,
))
def main():
for item in DATA:
df = loadData(item['currency'], item['timeframe'])
# print df
# adding indicators
addEwma(df, FIBOS)
addRsi(df, FIBOS)
# print df
# break
# set labels
labels = getLabels(df)
# print labels
print df
print labels
# split and scale
X_train, X_test, y_train, y_test = splitAndScale(df, labels)
print X_train
print y_train
# fitting regressor
clf = GradientBoostingClassifier()
clf.fit(X_train, y_train)
log.info('Classifier fitted')
# saving
joblib.dump(clf, 'models/{0}.gbrt'.format(item['currency']), compress=9)
log.info('Classifier saved')
def main(debug):
interval = choice(INTERVALS)
for currency, min_trail in CURRENCIES.iteritems():
pip_mul = 100. if 'JPY' in currency else 10000.
actions = calculateActions(min_trail)
df = loadData(currency, interval)
df = df[-1000:]
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
df_last = df[-1:]
a = predict(df, q, PERIODS, actions)
row = df_last.iloc[-1]
a_trade, a_trail = a.split('-')
if a_trade == 'buy':
stop_loss = row['close'] - (float(a_trail) / pip_mul)
else:
stop_loss = row['close'] + (float(a_trail) / pip_mul)
logging.warn('{0} {1} a:{2} t:{3} sl:{4:.4f}'.format(
row.name,
currency,
a_trade,
a_trail,
stop_loss,
))
def main():
for item in DATA:
df = loadData(item['currency'], item['timeframe'])
# print df
# adding indicators
addEwma(df, FIBOS)
addRsi(df, FIBOS)
# set labels
labels = getLabels(df)
# print labels
# print df.tail()
# print labels.tail()
# split and scale
X_train, X_test, y_train, y_test = splitAndScale(df, labels)
# print X_test
# print y_test
# loading regressor
clf = joblib.load('models/{0}.gbrt'.format(item['currency']))
log.info('Classifier loading')
# predict last day
prediction = clf.predict(X_test[-1])
predict_proba = clf.predict_proba(X_test[-1])
log.warn('{0} {1} {2} {3}'.format(df.ix[-1].name, item['currency'], prediction[0], max(predict_proba[0])))
def main(debug):
pt = PrettyTable(
['Currency', 'min trail', 'date', '1', '2', '3', '4', '5'])
for info in DATA:
currency = info['currency']
min_trail = info['trail']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
actions = calculateActions(min_trail)
df = loadData(currency, interval)
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
df_last = df[-1:]
row = df_last.iloc[-1]
predictions = predict(df, q, PERIODS, actions, pip_mul, row)
# logging.warn('{0} {1} {2}'.format(currency, row.name, a))
pt.add_row([currency, min_trail, row.name] + predictions)
print pt
def main():
for item in DATA:
df = loadData(item['currency'], item['timeframe'])
# print df
# adding indicators
addEwma(df, FIBOS)
addRsi(df, FIBOS)
# print df
# break
# set labels
labels = getLabels(df)
# print labels
print df
print labels
# split and scale
X_train, X_test, y_train, y_test = splitAndScale(df, labels)
print X_train
print y_train
# fitting regressor
clf = GradientBoostingClassifier()
clf.fit(X_train, y_train)
log.info('Classifier fitted')
# saving
joblib.dump(clf,
'models/{0}.gbrt'.format(item['currency']),
compress=9)
log.info('Classifier saved')
def main(debug):
for info in DATA:
currency = info['currency']
min_trail = info['trail']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
actions = calculateActions(min_trail)
df = loadData(currency, interval)
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
rewards = []
errors = []
ticks = []
for x in xrange(2000):
index_start = randint(0, len(df) - 20)
df_inner = df.iloc[index_start:]
q, r, error, tick = test(df_inner, q, PERIODS, actions, pip_mul)
# results
rewards.append(r)
errors.append(error * pip_mul)
ticks.append(tick)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
logging.warn(
'{0} RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}]'
.format(
currency,
None,
int(rmsd),
int(np.mean(rewards) * pip_mul),
sum(rewards),
np.mean(wins) * 100,
np.mean(ticks),
))
def main():
for info in DATA:
currency = info['currency']
min_trail = info['trail']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
logging.info(currency)
df = loadData(currency, interval, 'all')
# print df
df['yesterday'] = df['close'].shift(1)
# print df
df['up'] = df.apply(lambda x: True
if x['close'] - x['yesterday'] > 0 else False,
axis=1)
df.dropna(inplace=True)
print df
results = {}
run_dir = True
run_len = 0
for idx, row in df.iterrows():
# logging.info(row['up'])
if row['up'] != run_dir:
key = '{0}'.format(run_len)
if key not in results:
results[key] = 0
results[key] += 1
run_dir = row['up']
run_len = 1
else:
run_len += 1
pprint(results)
logging.warn('probabilities:')
for n in xrange(1, 10):
p = 0.5**(n + 1)
every = 1 / p
expecting = len(df) / every
logging.info(
'{0}: {1:.2f}% every {2:.0f} expecting {3:.0f}'.format(
n, p * 100, every, expecting))
break
def main(debug):
interval = choice(INTERVALS)
for currency, min_trail in CURRENCIES.iteritems():
pip_mul = 100. if 'JPY' in currency else 10000.
actions = calculateActions(min_trail)
df = loadData(currency, interval)
df = df[-2000:]
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
rewards = []
errors = []
ticks = []
for x in xrange(1000):
index_start = randint(0, len(df) - 1)
df_inner = df.iloc[index_start:]
q, r, error, tick = test(df_inner, q, PERIODS, actions, pip_mul)
# results
rewards.append(r)
errors.append(error * pip_mul)
ticks.append(tick)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
logging.warn(
'{0} RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}]'
.format(
currency,
None,
int(rmsd),
int(np.mean(rewards) * pip_mul),
sum(rewards),
np.mean(wins) * 100,
np.mean(ticks),
))
def main(debug):
for info in DATA:
currency = info['currency']
min_trail = info['trail']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
actions = calculateActions(min_trail)
df = loadData(currency, interval)
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
rewards = []
errors = []
ticks = []
for x in xrange(2000):
index_start = randint(0, len(df)-20)
df_inner = df.iloc[index_start:]
q, r, error, tick = test(df_inner, q, PERIODS, actions, pip_mul)
# results
rewards.append(r)
errors.append(error * pip_mul)
ticks.append(tick)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
logging.warn('{0} RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}]'.format(
currency,
None,
int(rmsd),
int(np.mean(rewards) * pip_mul),
sum(rewards),
np.mean(wins) * 100,
np.mean(ticks),
))
def main(debug):
interval = choice(INTERVALS)
for currency, min_trail in CURRENCIES.iteritems():
pip_mul = 100. if 'JPY' in currency else 10000.
actions = calculateActions(min_trail)
df = loadData(currency, interval)
df = df[-2000:]
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
rewards = []
errors = []
ticks = []
for x in xrange(1000):
index_start = randint(0, len(df)-1)
df_inner = df.iloc[index_start:]
q, r, error, tick = test(df_inner, q, PERIODS, actions, pip_mul)
# results
rewards.append(r)
errors.append(error * pip_mul)
ticks.append(tick)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
logging.warn('{0} RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}]'.format(
currency,
None,
int(rmsd),
int(np.mean(rewards) * pip_mul),
sum(rewards),
np.mean(wins) * 100,
np.mean(ticks),
))
def main():
for info in DATA:
currency = info['currency']
min_trail = info['trail']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
logging.info(currency)
df = loadData(currency, interval, 'all')
# print df
df['yesterday'] = df['close'].shift(1)
# print df
df['up'] = df.apply(lambda x: True if x['close'] - x['yesterday'] > 0 else False, axis=1)
df.dropna(inplace=True)
print df
results = {}
run_dir = True
run_len = 0
for idx, row in df.iterrows():
# logging.info(row['up'])
if row['up'] != run_dir:
key = '{0}'.format(run_len)
if key not in results:
results[key] = 0
results[key] += 1
run_dir = row['up']
run_len = 1
else:
run_len += 1
pprint(results)
logging.warn('probabilities:')
for n in xrange(1, 10):
p = 0.5**(n+1)
every = 1/p
expecting = len(df) / every
logging.info('{0}: {1:.2f}% every {2:.0f} expecting {3:.0f}'.format(n, p * 100, every, expecting))
break
def main(equity, debug):
pips = []
pt = PrettyTable(
['Currency', 'min trail', 'date', '1', '2', '3', '4', '5'])
for info in DATA:
currency = info['currency']
min_trail = info['trail']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
logging.warn('{0}...'.format(currency))
actions = calculateActions(min_trail)
df = loadData(currency, interval, 'test')
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
df_last = df[-1:]
row = df_last.iloc[-1]
predictions = predict(df, q, PERIODS, actions, pip_mul, row)
# logging.warn('{0} {1} {2}'.format(currency, row.name, a))
pt.add_row([currency, min_trail,
str(row.name).split(' ')[0]] + predictions)
pips.append(int(predictions[0].split(' ')[0].split('-')[1]))
print pt
equity = float(equity)
risk = 0.10
available = equity * risk
logging.info('Risk ${0:.0f} from ${1:.0f} at {2:.0f}%'.format(
available, equity, risk * 100))
total_pips = sum(pips)
lot_size = available / total_pips
lot_size /= len(pips)
logging.warn('Lot size = {0:.2f}'.format(lot_size))
def main(equity, debug):
pips = []
pt = PrettyTable(['Currency', 'min trail', 'date', '1', '2', '3', '4', '5'])
for info in DATA:
currency = info['currency']
min_trail = info['trail']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
logging.warn('{0}...'.format(currency))
actions = calculateActions(min_trail)
df = loadData(currency, interval, 'test')
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
df_last = df[-1:]
row = df_last.iloc[-1]
predictions = predict(df, q, PERIODS, actions, pip_mul, row)
# logging.warn('{0} {1} {2}'.format(currency, row.name, a))
pt.add_row([currency, min_trail, str(row.name).split(' ')[0]] + predictions)
pips.append(int(predictions[0].split(' ')[0].split('-')[1]))
print pt
equity = float(equity)
risk = 0.10
available = equity * risk
logging.info('Risk ${0:.0f} from ${1:.0f} at {2:.0f}%'.format(available, equity, risk * 100))
total_pips = sum(pips)
lot_size = available / total_pips
lot_size /= len(pips)
logging.warn('Lot size = {0:.2f}'.format(lot_size))
def main(debug):
rmsds = []
ppts = []
wrs = []
for info in DATA:
currency = info['currency']
min_trail = info['trail']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
actions = calculateActions(min_trail)
df = loadData(currency, interval, 'test')
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
q = loadQ(currency, interval)
rewards = []
errors = []
ticks = []
for i, row in df.iterrows():
df_inner = df.loc[i:]
q, r, error, tick = test(df_inner, q, PERIODS, actions, pip_mul)
# logging.warn('{0} {1}'.format(i, r))
# results
rewards.append(r)
errors.append(error * pip_mul)
ticks.append(tick)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
win_ratio = np.mean(wins)
# ppt
ppt = np.mean(rewards) * pip_mul
logging.warn(
'{0} RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}]'
.format(
currency,
None,
int(rmsd),
int(ppt),
sum(rewards),
win_ratio * 100,
np.mean(ticks),
))
rmsds.append(rmsd)
ppts.append(ppt)
wrs.append(win_ratio)
logging.error('RMSD {0:.0f} +- {1:.0f}'.format(np.mean(rmsds),
np.std(rmsds)))
logging.error('PPT {0:.0f} +- {1:.0f}'.format(np.mean(ppts), np.std(ppts)))
logging.error('WR {0:.0f} +- {1:.0f}'.format(
np.mean(wrs) * 100,
np.std(wrs) * 100))
import pandas as pd
import numpy as np
import math
import importlib as il
import main
if __name__ == '__main__':
sp.call('cls', shell = True)
plt.close('all')
il.reload(main)
# ----------------------------------------
# Data loading and formatting
# ----------------------------------------
df, dfTest = main.loadData()
# Count number of rows with missing values.
tplTrain = main.countNan(df)
tplTest = main.countNan(dfTest)
print('Training set missing values: {0}/{1} = {2:.6f}'.format(tplTrain[0], tplTrain[1], tplTrain[2]))
print('Test set missing values: {0}/{1} = {2:.6f}'.format(tplTest[0], tplTest[1], tplTest[2]))
print('')
# Load data with feature engineering, imputation, and type conversions.
df, dfTest = main.preprocData(df, dfTest)
# ----------------------------------------
# Constants
# ----------------------------------------
np.set_printoptions(precision = 4, suppress = True)
return True
def testLayer3(self):
'''For each classifier, checks if the classifier assigns all the mass to a the prototypes label if the label is certain (alpha=1, mem_degree=delta(i,label))'''
self.alphas = np.ones(self.n_prototypes)
self.setRandomPrototypesMembersip()
l3p = self.layer3OnPrototypes()
for i in range(self.n_prototypes):
labelMasses,uncMass = l3p[i]
if uncMass != 0. or np.argmax(labelMasses) != np.argmax(self.mem_degrees[i]):
return False
return True
if __name__ == '__main__':
from main import loadData
features, labels, labelEnc = loadData('data/iris/iris.data')
testAbsCla = TestAbs(n_prototypes=10)
testAbsCla.fit(features, labels)
if testAbsCla.testLayer1():
print('Layer1 test success!')
else:
print('Layer1 test failed')
if testAbsCla.testLayer2():
print('Layer2 test success!')
else:
print('Layer2 test failed')
if testAbsCla.testLayer3():
print('Layer3 test success!')
else:
print('Layer3 test failed')
def main(debug):
minutes = 0
while True:
minutes += 1
seconds_to_run = 60 * minutes
seconds_info_intervals = seconds_to_run / 5
logging.error('Training each currency for {0} minutes'.format(minutes))
shuffle(DATA)
for info in DATA:
logging.debug('Currency info: {0}'.format(info))
currency = info['currency']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
df = loadData(currency, interval, 'train')
df = getBackgroundKnowledge(df, PERIODS)
alpha = 0.
epsilon = alpha / 2.
q = loadQ(currency, interval)
time_start = time()
time_interval = seconds_info_intervals
epoch = 0
rewards = []
errors = []
ticks = []
logging.warn('Training {0} on {1} with {2} ticks [m:{3}]'.format(
currency,
interval,
len(df),
minutes,
))
while True:
epoch += 1
logging.info(' ')
logging.info('{0}'.format('=' * 20))
logging.info('EPOCH {0}'.format(epoch))
index_start = randint(0, len(df)-20)
df_inner = df.iloc[index_start:]
logging.info('Epoch: at {0} with {1} ticks'.format(index_start, len(df_inner)))
q, r, error, tick = train(df_inner, q, alpha, epsilon, PERIODS, ACTIONS, pip_mul, info['std'])
# results
error *= pip_mul
rewards.append(r)
errors.append(error)
ticks.append(tick)
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
win_ratio = np.mean(wins)
# logging.error('wr {0}'.format(win_ratio))
# adjust values
alpha = 1.0102052281586786e+000 + (-2.0307383627607809e+000 * win_ratio) + (1.0215546892913909e+000 * win_ratio**2)
epsilon = 3.9851080604500078e-001 + (2.1874724815820201e-002 * win_ratio) + (-4.1444101741886652e-001 * win_ratio**2)
# logging.error('new alpha = {0}'.format(alpha))
# only do updates at interval
if time() - time_start > time_interval or debug:
# prune lengths
while len(rewards) > 1000 + minutes * 1000:
rewards.pop(0)
errors.pop(0)
ticks.pop(0)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
# logging.error('RMSD: {0} from new error {1}'.format(rmsd, error))
logging.warn('{0} [{1:05d}] RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}, a:{7:.2f}, e:{8:.2f}]'.format(
currency,
epoch,
int(rmsd),
int(np.mean(rewards) * pip_mul),
sum(rewards),
np.mean(wins) * 100,
np.mean(ticks),
alpha * 100,
epsilon * 100,
))
# exit
if time_interval >= seconds_to_run or debug:
break
# continue
time_interval += seconds_info_intervals
saveQ(currency, interval, q)
saveQ(currency, interval, q)
summarizeActions(q)
if debug:
break # currencies
if debug:
break # forever
def main(debug):
interval = choice(INTERVALS)
minutes = 0
while True:
minutes += 1
seconds_to_run = 60 * minutes
seconds_info_intervals = seconds_to_run / 4
logging.error('Training each currency for {0} minutes'.format(minutes))
# shuffle(CURRENCIES)
for currency, min_trail in CURRENCIES.iteritems():
pip_mul = 100. if 'JPY' in currency else 10000.
actions = calculateActions(min_trail)
df = loadData(currency, interval)
df = df[-2000:]
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
alpha = 0
epsilon = 0
q = loadQ(currency, interval)
time_start = time()
time_interval = 0
epoch = 0
rewards = []
errors = []
ticks = []
logging.warn('Training {0} on {1} with {2} ticks...'.format(
currency, interval, len(df)))
while True:
epoch += 1
logging.info(' ')
logging.info('{0}'.format('=' * 20))
logging.info('EPOCH {0}'.format(epoch))
index_start = randint(0, len(df) - 20)
df_inner = df.iloc[index_start:]
logging.info('Epoch: at {0} with {1} ticks'.format(
index_start, len(df_inner)))
q, r, error, tick = train(df_inner, q, alpha, epsilon, PERIODS,
actions, pip_mul)
# results
rewards.append(r)
errors.append(error * pip_mul)
ticks.append(tick)
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
win_ratio = np.mean(wins)
# logging.error('wr {0}'.format(win_ratio))
# adjust values
epsilon = np.sqrt((1 - win_ratio) * 100.) / 100.
alpha = epsilon / 2.
# logging.error('new alpha = {0}'.format(alpha))
if time() - time_start > time_interval or debug:
# prune lengths
while len(rewards) > 1000 + minutes * 1000:
rewards.pop(0)
errors.pop(0)
ticks.pop(0)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
logging.info('RMSD: {0}'.format(rmsd))
logging.warn(
'{0} [{1:05d}] RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}, a:{7:.2f}, e:{8:.2f}]'
.format(
currency,
epoch,
int(rmsd),
int(np.mean(rewards) * pip_mul),
sum(rewards),
np.mean(wins) * 100,
np.mean(ticks),
alpha * 100,
epsilon * 100,
))
# exit
if (time() - time_start >= seconds_to_run) or debug:
break
# saveQ(currency, interval, q)
time_interval += seconds_info_intervals
saveQ(currency, interval, q)
summarizeActions(q)
if debug:
break # currencies
if debug:
break # forever
def main(debug):
minutes = 0
while True:
minutes += 1
seconds_to_run = 60 * minutes
seconds_info_intervals = seconds_to_run / 5
logging.error('Training each currency for {0} minutes'.format(minutes))
shuffle(DATA)
for info in DATA:
logging.debug('Currency info: {0}'.format(info))
currency = info['currency']
interval = info['intervals'][0]
pip_mul = info['pip_mul']
df = loadData(currency, interval, 'train')
df = getBackgroundKnowledge(df, PERIODS)
alpha = 0.
epsilon = alpha / 2.
q = loadQ(currency, interval)
time_start = time()
time_interval = seconds_info_intervals
epoch = 0
rewards = []
errors = []
ticks = []
logging.warn('Training {0} on {1} with {2} ticks [m:{3}]'.format(
currency,
interval,
len(df),
minutes,
))
while True:
epoch += 1
logging.info(' ')
logging.info('{0}'.format('=' * 20))
logging.info('EPOCH {0}'.format(epoch))
index_start = randint(0, len(df) - 20)
df_inner = df.iloc[index_start:]
logging.info('Epoch: at {0} with {1} ticks'.format(
index_start, len(df_inner)))
q, r, error, tick = train(df_inner, q, alpha, epsilon, PERIODS,
ACTIONS, pip_mul, info['std'])
# results
error *= pip_mul
rewards.append(r)
errors.append(error)
ticks.append(tick)
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
win_ratio = np.mean(wins)
# logging.error('wr {0}'.format(win_ratio))
# adjust values
alpha = 1.0102052281586786e+000 + (
-2.0307383627607809e+000 *
win_ratio) + (1.0215546892913909e+000 * win_ratio**2)
epsilon = 3.9851080604500078e-001 + (
2.1874724815820201e-002 *
win_ratio) + (-4.1444101741886652e-001 * win_ratio**2)
# logging.error('new alpha = {0}'.format(alpha))
# only do updates at interval
if time() - time_start > time_interval or debug:
# prune lengths
while len(rewards) > 1000 + minutes * 1000:
rewards.pop(0)
errors.pop(0)
ticks.pop(0)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
# logging.error('RMSD: {0} from new error {1}'.format(rmsd, error))
logging.warn(
'{0} [{1:05d}] RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}, a:{7:.2f}, e:{8:.2f}]'
.format(
currency,
epoch,
int(rmsd),
int(np.mean(rewards) * pip_mul),
sum(rewards),
np.mean(wins) * 100,
np.mean(ticks),
alpha * 100,
epsilon * 100,
))
# exit
if time_interval >= seconds_to_run or debug:
break
# continue
time_interval += seconds_info_intervals
saveQ(currency, interval, q)
saveQ(currency, interval, q)
summarizeActions(q)
if debug:
break # currencies
if debug:
break # forever
def main(debug):
interval = choice(INTERVALS)
minutes = 0
while True:
minutes += 1
seconds_to_run = 60 * minutes
seconds_info_intervals = seconds_to_run / 4
logging.error('Training each currency for {0} minutes'.format(minutes))
# shuffle(CURRENCIES)
for currency, min_trail in CURRENCIES.iteritems():
pip_mul = 100. if 'JPY' in currency else 10000.
actions = calculateActions(min_trail)
df = loadData(currency, interval)
df = df[-2000:]
df = getBackgroundKnowledge(df, PERIODS)
# print df
# break
alpha = 0
epsilon = 0
q = loadQ(currency, interval)
time_start = time()
time_interval = 0
epoch = 0
rewards = []
errors = []
ticks = []
logging.warn('Training {0} on {1} with {2} ticks...'.format(currency, interval, len(df)))
while True:
epoch += 1
logging.info(' ')
logging.info('{0}'.format('=' * 20))
logging.info('EPOCH {0}'.format(epoch))
index_start = randint(0, len(df)-20)
df_inner = df.iloc[index_start:]
logging.info('Epoch: at {0} with {1} ticks'.format(index_start, len(df_inner)))
q, r, error, tick = train(df_inner, q, alpha, epsilon, PERIODS, actions, pip_mul)
# results
rewards.append(r)
errors.append(error * pip_mul)
ticks.append(tick)
# win ratio
wins = [1. if r > 0. else 0. for r in rewards]
win_ratio = np.mean(wins)
# logging.error('wr {0}'.format(win_ratio))
# adjust values
epsilon = np.sqrt((1 - win_ratio) * 100.) / 100.
alpha = epsilon / 2.
# logging.error('new alpha = {0}'.format(alpha))
if time() - time_start > time_interval or debug:
# prune lengths
while len(rewards) > 1000 + minutes * 1000:
rewards.pop(0)
errors.pop(0)
ticks.pop(0)
# RMSD
rmsd = np.sqrt(np.mean([e**2 for e in errors]))
logging.info('RMSD: {0}'.format(rmsd))
logging.warn('{0} [{1:05d}] RMSD {2:03d} PPT {3:03d} WR {5:.0f}% [ticks:{6:.1f} sum:{4:.1f}, a:{7:.2f}, e:{8:.2f}]'.format(
currency,
epoch,
int(rmsd),
int(np.mean(rewards) * pip_mul),
sum(rewards),
np.mean(wins) * 100,
np.mean(ticks),
alpha * 100,
epsilon * 100,
))
# exit
if (time() - time_start >= seconds_to_run) or debug:
break
# saveQ(currency, interval, q)
time_interval += seconds_info_intervals
saveQ(currency, interval, q)
summarizeActions(q)
if debug:
break # currencies
if debug:
break # forever
def activbutton():
global tablicaProbek
tablicaProbek = loadData("data.txt")
FSD_button['state'] = "normal"
SFS_button['state'] = "normal"
return tablicaProbek
from main import loadData loadData()
def makeContent(env=None,linksToDocumentCopies=False):
if env is None:
import main
env=main.Environment(main.loadData())
def content(makeFns):
w,nonFirstWrite,wtd,wtdrowspan,e,a,af,ae=makeFns('w,nonFirstWrite,wtd,wtdrowspan,e,a,af,ae')
refs=Refs()
w("<h1>Ведомственная структура расходов бюджета Санкт-Петербурга</h1>\n")
abbrXls="<abbr title='Excel Binary File Format, бинарный формат файлов Excel'>xls</abbr>"
abbrPdf="<abbr title='Portable Document Format'>pdf</abbr>"
abbrCsv="<abbr title='comma-separated values, значения, разделённые запятыми'>csv</abbr>"
abbrUtf8="<abbr title='Unicode Transformation Format, 8-bit'>utf-8</abbr>"
w("<p>Последнее обновление: 25.10.2014."+refs.makeRef(
"<dl>"
"<dt>31.05.2013</dt><dd>Добавлен закон 1-й корректировки бюджета 2013 г. и закон об исполнении бюджета 2011 г.</dd>"
"<dt>04.06.2013</dt><dd>Добавлены законы 2009—2010 гг.</dd>"
"<dt>24.08.2013</dt><dd>Добавлены законы 2008 г., кроме небольших корректировок расходов.</dd>"
"<dt>26.08.2013</dt><dd>Добавлены законы 2000—2007 г., многие из которых оказались уже доступными в формате "+abbrXls+".</dd>"
"<dt>04.10.2013</dt><dd>Добавлен проект закона о бюджете на 2014 г.</dd>"
"<dt>20.04.2014</dt><dd>Добавлен закон 2014 г. и проект его 1-й корректировки.</dd>"
"<dt>21.04.2014</dt><dd>В проекте 1-й корректировки 2014 г. добавлен код Комитета по межнациональным отношениям.</dd>"
"<dt>04.06.2014</dt><dd>Добавлен закон 1-й корректировки бюджета 2014 г.</dd>"
"<dt>16.07.2014</dt><dd>Исправлены некоторые опечатки в файлах.</dd>"
"<dt>24.07.2014</dt><dd>Обновлена ссылка на исходный код.</dd>"
"<dt>25.10.2014</dt><dd>Добавлен проект закона о бюджете на 2015 г.</dd>"
"</dl>"
).ref+"</p>\n")
yearLaws=collections.defaultdict(list)
yearExtracted=collections.defaultdict(bool)
for law in env.laws:
yearLaws[law.year].append(law)
if law.documents:
yearExtracted[law.year]=True
w("<p>Документы для годов: ")
wn=nonFirstWrite()
for year in sorted(yearLaws):
wn(", ")
w(a("#"+year,e(year)))
w(".</p>\n")
w("<p>Для 2014 г. также доступен "+a("db.html","вариант со всеми поправками, собранными в одну таблицу")+".</p>\n")
w("<h2>Данные, для которых извлечение не производилось</h2>\n")
w(
"<p>До 2007 года "+ae("http://www.fincom.spb.ru/","Комитет финансов Санкт-Петербурга")+" публиковал таблицы из приложений к бюджету в формате "+abbrXls+". "
"Таблицы можно найти на сайте комитета в разделе "+ae("http://www.fincom.spb.ru/comfin/budjet/laws.htm","«Законы о бюджете»")+". "
"Позже в этом формате в разделе "+ae("http://www.fincom.spb.ru/comfin/budjet/budget_for_people.htm","«Бюджет для граждан»")+" был опубликован проект закона, закон и 1-е изменения к закону о бюджете 2014 года (но не был опубликован проект 1-х изменений), проект закона 2015 года.</p>\n"
)
w("<table>\n")
w("<thead>\n")
w("<tr><th>год</th><th>закон</th><th>исходные документы</th><th>данные в машиночитаемом виде</th></tr>\n")
w("</thead>\n")
for year,laws in sorted(yearLaws.items()):
if yearExtracted[year]:
continue
w("<tbody id='"+e(year)+"'>\n");
w("<tr>")
wtdrowspan(sum(max(len(law.documents),1) for law in laws),e(year))
wn=nonFirstWrite()
for law in laws:
wn("<tr>")
wtd("<span title='"+e(law.title)+"'>"+e(law.description)+"</span>")
wtd(ae(law.viewUrl,"страница")+
(" "+ae(law.downloadUrl,"архив") if law.isSingleDownload else "")+
(" "+af(law.zipPath,"копия") if law.isSingleZip and linksToDocumentCopies else "")
)
if law.originalXlsUrl is not None:
wtd(ae(law.originalXlsUrl,e(law.availabilityNote)))
else:
wtd(e(law.availabilityNote))
w("</tr>\n")
w("</tbody>\n");
w("</table>\n")
w("<h2>Извлечённые данные</h2>\n")
w("<p>С 2007 года приложения к законам о бюджете публикуются в формате "+abbrPdf+". Работать с таблицами в этом формате может быть неудобно. Ниже приводятся данные, автоматически"+refs.makeRef(
ae("https://github.com/AntonKhorev/spb-budget-xls","Исходный код программы для извлечения")+"."
).ref+" извлечённые из приложений.</p>\n")
refData=refs.makeRef(
"<pre><code>"
"1. \<br />"
"1.2. 1.2. ) разные уровни раскрытия<br />"
"1.2.3. 1.2.3. /<br />"
" | \<br />"
"«столбиком» «лесенкой» — в этом варианте проще формулы для суммирования"
"</code></pre>"
)
refCsv=refs.makeRef("В кодировке "+abbrUtf8+". Числа с десятичной запятой, потому что в таком формате их читают русские версии электронных таблиц.")
w("<table>\n")
w("<thead>\n")
w("<tr><th rowspan='2'>год</th><th rowspan='2'>закон</th><th rowspan='2'>исходные документы"+refs.makeRef(
"Наличие ссылок на архивы для скачивания зависит от их доступности на сайте Комитета финансов."
).ref+"</th><th rowspan='2'>приложение</th><th colspan='3'>данные в машиночитаемом виде"+refData.ref+"</th></tr>\n")
w("<tr><th>"+abbrCsv+refCsv.ref+" без формул</th><th>"+abbrCsv+refCsv.ref+" с формулами"+refs.makeRef(
"Для правильной работы формул файл необходимо читать с первой строки."
).ref+"</th><th>"+abbrXls+"</th></tr>\n")
w("</thead>\n")
refUnknownSlack=refs.makeRef("Указанные суммы могут расходиться с реальными суммами подпунктов на 100—200 руб. по неизвестным причинам.")
refRoundoffSlack=refs.makeRef(
"Указанные суммы в графе «Исполнено» могут расходиться с реальными суммами подпунктов из-за округления:"+
" выплаты производятся с точностью до копейки, а в данном приложении значения указываются с точностью до 100 руб."
)
for year,laws in sorted(yearLaws.items()):
if not yearExtracted[year]:
continue
w("<tbody id='"+e(year)+"'>\n");
w("<tr>")
wtdrowspan(sum(max(len(law.documents),1) for law in laws),e(year))
wn=nonFirstWrite()
for law in laws:
wn("<tr>")
wtdrowspan(max(len(law.documents),1),"<span title='"+e(law.title)+"'>"+e(law.description)+"</span>"+
(refUnknownSlack.ref if any('slack' in doc.quirks for doc in law.documents) else "")+
(refRoundoffSlack.ref if law.version=='i' else "")
)
wtdrowspan(max(len(law.documents),1),ae(law.viewUrl,"страница")+
(" "+ae(law.downloadUrl,"архив") if law.isSingleDownload else "")+
(" "+af(law.zipPath,"копия") if law.isSingleZip and linksToDocumentCopies else "")
)
if not law.documents:
if law.originalXlsUrl is not None:
w("<td colspan='4'>"+ae(law.originalXlsUrl,e(law.availabilityNote))+"</td>")
else:
w("<td colspan='4'>"+e(law.availabilityNote)+"</td>")
w("</tr>\n")
wn2=nonFirstWrite()
for doc in law.documents:
wn2("<tr>")
w("<td><span title='"+"Приложение "+e(doc.appendixNumber)+". "+e(doc.title)+"'>")
if len(doc.forYears)==1:
w("расходы "+e('-'.join(doc.forYears)))
else:
w("план "+e('-'.join(doc.forYears)))
w("</span></td>")
def matrix(path):
w("<td><pre><code>"+
af(path(1,False),"1.")+"<br />"+
af(path(2,False),"1.2.")+" "+af(path(2,True),"1.2.")+"<br />"+
af(path(3,False),"1.2.3.")+" "+af(path(3,True),"1.2.3.")+
("<br /> "+af(path(4,False),"...4.")+" "+af(path(4,True),"...4.") if doc.maxDepth>=4 else "")+
"</code></pre></td>")
matrix(lambda l,s: doc.getCsvPath(l,s,False))
matrix(lambda l,s: doc.getCsvPath(l,s,True))
matrix(doc.getXlsPath)
w("</tr>\n")
w("</tbody>\n");
w("</table>\n")
w("<h2>Примечания</h2>\n<ol>\n")
for ref in refs:
w("<li>"+ref.body+"</li>\n")
w("</ol>\n")
return content
