def getBarsMulti():
"""Classifies the time difference in single bars.
But uses insted of getBars() several analysis-File and calculates a mean value"""
fileIter=iglob(path.newPath(path.main,"auswertung/reisezeit/analysisFiles/taxiAnalysisInformation*.xml"))
fcdDiffDict={}
simDiffDict={}
barsDict={}
barsDictSim={}
stdDev=[]
mw=[]
#calc diffs
for file in fileIter: #for each
path.analysisWEE=path.newPath(file)
print path.analysisWEE
taxis=reader.readAnalysisInfo(WEE)
for taxi in taxis:
if len(taxi.getSteps())<1:
continue
try:
#diff=getTimeDiff(taxi.getSteps(),False)
diffSim,fcd,sim,no=getTimeDiff(taxi.getSteps())
simDiffDict.setdefault(taxi.id,[]).append(sim)
fcdDiffDict.setdefault(taxi.id,fcd)
except TypeError, e:
tueNichts=True
def main():
print "starting"
#change path for multi
path.vtypeprobe=path.newPath(path.main,"simRawFCD/t2MitVerschiedenenLosfahrzeiten/vtypeprobe_-4.out.xml")
path.simulatedRawFCD=path.newPath(path.main,"simRawFCD/t2MitVerschiedenenLosfahrzeiten/simulatedRawFCD_-4.out.dat")
inputFile=open(path.vtypeprobe,'r')
outputFile=open(path.simulatedRawFCD,'w')
for line in inputFile:
words=line.split('"')
#get time
if line.find("<timestep")!=-1:
time=words[1]
#if line starts with "<vehicle"
if line.find("<vehicle id")!=-1:
# veh_id date (time to simDate+time) x (remove and set comma new)
outputFile.write(str(getVehId(words[1]))+'\t'+getDateFromDepart(time)+'\t'+words[13][0:2]+'.'+words[13][2:7]+words[13][8:]+
# y (remove and set comma new) status speed form m/s in km/h
'\t'+words[11][0:2]+'.'+words[11][2:7]+words[11][8:]+'\t'+"90"+'\t'+str(int(round(float(words[15])*3.6)))+'\n')
outputFile.close()
inputFile.close()
for key in vehIdDict:
print key, vehIdDict[key],"\n"
#pickles the vehIdList
dump(vehIdDict, open(path.rawFcdVehIdList,'w'))
print "end"
def main():
print "starting"
#change path for multi
path.vtypeprobe=path.newPath(path.main,"simRawFCD/t2MitVerschiedenenLosfahrzeiten/vtypeprobe_-4.out.xml")
path.simulatedRawFCD=path.newPath(path.main,"simRawFCD/t2MitVerschiedenenLosfahrzeiten/simulatedRawFCD_-4.out.dat")
inputFile=open(path.vtypeprobe,'r')
outputFile=open(path.simulatedRawFCD,'w')
for line in inputFile:
words=line.split('"')
#get time
if line.find("<timestep")!=-1:
time=words[1]
#if line starts with "<vehicle"
if line.find("<vehicle id")!=-1:
# veh_id date (time to simDate+time) x (remove and set comma new)
outputFile.write(str(getVehId(words[1]))+'\t'+getDateFromDepart(time)+'\t'+words[13][0:2]+'.'+words[13][2:7]+words[13][8:]+
# y (remove and set comma new) status speed form m/s in km/h
'\t'+words[11][0:2]+'.'+words[11][2:7]+words[11][8:]+'\t'+"90"+'\t'+str(int(round(float(words[15])*3.6)))+'\n')
outputFile.close()
inputFile.close()
for key in vehIdDict:
print key, vehIdDict[key],"\n"
#pickles the vehIdList
dump(vehIdDict, open(path.rawFcdVehIdList,'w'))
print "end"
def getBarsMulti():
"""Classifies the time difference in single bars.
But uses insted of getBars() several analysis-File and calculates a mean value"""
fileIter = iglob(
path.newPath(
path.main,
"auswertung/reisezeit/analysisFiles/taxiAnalysisInformation*.xml"))
fcdDiffDict = {}
simDiffDict = {}
barsDict = {}
barsDictSim = {}
stdDev = []
mw = []
#calc diffs
for file in fileIter: #for each
path.analysisWEE = path.newPath(file)
print path.analysisWEE
taxis = reader.readAnalysisInfo(WEE)
for taxi in taxis:
if len(taxi.getSteps()) < 1:
continue
try:
#diff=getTimeDiff(taxi.getSteps(),False)
diffSim, fcd, sim, no = getTimeDiff(taxi.getSteps())
simDiffDict.setdefault(taxi.id, []).append(sim)
fcdDiffDict.setdefault(taxi.id, fcd)
except TypeError, e:
tueNichts = True
class config:
model_dir = None
base_dataset_dir = Path(
os.path.dirname(__file__)) / '..' / 'data_process' / 'HPA_ieee'
dataset_dir = Path(os.path.dirname(__file__)) / 'dataset'
ckpt_dir = Path(os.path.dirname(__file__)) / 'ckpt'
summary_dir = Path(os.path.dirname(__file__)) / 'summary'
n_labels = 6
# dataset config
stride = 16
patch_size = 32
img_size = 3000
cut_img_threshold = 0.9
padding = 750
train_samples = [30, 230]
eval_samples = [0, 30]
# batch_size = 1024
# training config
# shuffle_size = batch_size * 3
learning_rate = 0.1
min_patches_per_sample = 1000
max_patches_per_sample = 2048
n_candidates = int(max_patches_per_sample * 0.4)
def main():
print "start program"
#multi change pathes (routen müssen immer extra neu erstellt werden wegen taxiIDpickleList)
path.simFcd=path.newPath(path.main,"simProzessiertFCD/t2MitVerschiedenenLosfahrzeiten/linkspeeds.simulatedRawFCD_-4.dat")
path.vtypeprobe=path.newPath(path.main,"simRawFCD/t2MitVerschiedenenLosfahrzeiten/vtypeprobe_-4.out.xml")
path.simRawFcd=path.newPath(path.main,"simRawFCD/t2MitVerschiedenenLosfahrzeiten/simulatedRawFCD_-4.out.dat")
path.analysisWEE=path.newPath(path.main,"auswertung/reisezeit/analysisFiles/taxiAnalysisInformationWEE_-4.xml")
arrangeData()
print "end"
def make(self, projectConfig):
makeConf = {
'target_type':
projectConfig.getItem('make.configuration.target.type',
self.defaultConf['target_type']),
'ar':
projectConfig.getItem('make.configuration.archiver.ar',
self.defaultConf['ar']),
'ld':
projectConfig.getItem('make.configuration.linker.ld',
self.defaultConf['ld']),
'ldflags':
projectConfig.getItem('make.configuration.linker.flags',
self.defaultConf['ldflags']),
'ld_library_paths':
projectConfig.getItem('make.configuration.linker.library_paths',
self.defaultConf['ld_library_paths']),
'libraries':
projectConfig.getItem('make.configuration.linker.libraries', []),
'solution_dir':
projectConfig.getItem('global.solution.dir', '..'),
'sources':
projectConfig.getItem('make.configuration.linker.include', []),
}
makefile = GMakeDocument(TARGET_MAKEFILE_PATH)
objectSourcePaths = []
self.objectFiles.clear()
solutionDirPath = Path(makeConf['solution_dir'], True)
objectFilePaths = []
for objectSourcePath in makeConf['sources']:
objectSourcePath = solutionDirPath.join(objectSourcePath)
FileUtil.searchAllFiles(objectFilePaths, objectSourcePath, ['.o'])
for objectFilePath in objectFilePaths:
self.objectFiles.append(Path(objectFilePath))
finalFilePath = self.getFinalFilePath(projectConfig, makeConf)
linkRule = self.getLinkRule(finalFilePath, makeConf)
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
makefile.writeToFile()
return True
def main():
print("start program")
# multi change pathes (routen muessen immer extra neu erstellt werden wegen
# taxiIDpickleList)
path.simFcd = path.newPath(
path.main, "simProzessiertFCD/t2MitVerschiedenenLosfahrzeiten/linkspeeds.simulatedRawFCD_-4.dat")
path.vtypeprobe = path.newPath(
path.main, "simRawFCD/t2MitVerschiedenenLosfahrzeiten/vtypeprobe_-4.out.xml")
path.simRawFcd = path.newPath(
path.main, "simRawFCD/t2MitVerschiedenenLosfahrzeiten/simulatedRawFCD_-4.out.dat")
path.analysisWEE = path.newPath(
path.main, "auswertung/reisezeit/analysisFiles/taxiAnalysisInformationWEE_-4.xml")
arrangeData()
print("end")
def getBarsMulti():
"""Classifies the time difference in single bars.
But uses insted of getBars() several analysis-File and calculates a mean value"""
fileIter = iglob(
path.newPath(
path.main,
"auswertung/reisezeit/analysisFiles/taxiAnalysisInformation*.xml"))
fcdDiffDict = {}
simDiffDict = {}
barsDict = {}
barsDictSim = {}
stdDev = []
mw = []
# calc diffs
for file in fileIter: # for each
path.analysisWEE = path.newPath(file)
print(path.analysisWEE)
taxis = reader.readAnalysisInfo(WEE)
for taxi in taxis:
if len(taxi.getSteps()) < 1:
continue
try:
# diff=getTimeDiff(taxi.getSteps(),False)
diffSim, fcd, sim, no = getTimeDiff(taxi.getSteps())
simDiffDict.setdefault(taxi.id, []).append(sim)
fcdDiffDict.setdefault(taxi.id, fcd)
except TypeError as e:
tueNichts = True
# print "Error by taxi %s : %s" %(taxi.id,e.message)
for taxi, simList in simDiffDict.iteritems():
simDiffDict[taxi] = sum(simList) / (len(simList) + 0.0)
# create barsDict
for taxi in fcdDiffDict:
fcd = fcdDiffDict[taxi]
sim = simDiffDict[taxi]
diff = sim - fcd
relDiff = int(round(((100.0 * diff) / fcd)))
barsDictSim[(relDiff / 10) * 10] = barsDictSim.setdefault(
(relDiff / 10) * 10, 0) + 1
# standard deviation
stdDev.append((relDiff - 9.53) * (relDiff - 9.53))
mw.append(relDiff)
print("mw", sum(mw) / (len(mw) + 0.0)) # 9.91 #kor 0.48
print("standard deviation ", sqrt(sum(stdDev) / (len(stdDev) + 0.0)))
return (barsDictSim, barsDict)
def getBarsMulti():
"""Classifies the time difference in single bars.
But uses insted of getBars() several analysis-File and calculates a mean value"""
fileIter = iglob(path.newPath(
path.main, "auswertung/reisezeit/analysisFiles/taxiAnalysisInformation*.xml"))
fcdDiffDict = {}
simDiffDict = {}
barsDict = {}
barsDictSim = {}
stdDev = []
mw = []
# calc diffs
for file in fileIter: # for each
path.analysisWEE = path.newPath(file)
print(path.analysisWEE)
taxis = reader.readAnalysisInfo(WEE)
for taxi in taxis:
if len(taxi.getSteps()) < 1:
continue
try:
# diff=getTimeDiff(taxi.getSteps(),False)
diffSim, fcd, sim, no = getTimeDiff(taxi.getSteps())
simDiffDict.setdefault(taxi.id, []).append(sim)
fcdDiffDict.setdefault(taxi.id, fcd)
except TypeError as e:
tueNichts = True
# print "Error by taxi %s : %s" %(taxi.id,e.message)
for taxi, simList in simDiffDict.iteritems():
simDiffDict[taxi] = sum(simList) / (len(simList) + 0.0)
# create barsDict
for taxi in fcdDiffDict:
fcd = fcdDiffDict[taxi]
sim = simDiffDict[taxi]
diff = sim - fcd
relDiff = int(round(((100.0 * diff) / fcd)))
barsDictSim[
(relDiff / 10) * 10] = barsDictSim.setdefault((relDiff / 10) * 10, 0) + 1
# standard deviation
stdDev.append((relDiff - 9.53) * (relDiff - 9.53))
mw.append(relDiff)
print("mw", sum(mw) / (len(mw) + 0.0)) # 9.91 #kor 0.48
print("standard deviation ", sqrt(sum(stdDev) / (len(stdDev) + 0.0)))
return (barsDictSim, barsDict)
def make(self, projectConfig):
makeConf = {
'target_type': projectConfig.getItem('make.configuration.target.type', self.defaultConf['target_type']),
'ar': projectConfig.getItem('make.configuration.archiver.ar', self.defaultConf['ar']),
'ld': projectConfig.getItem('make.configuration.linker.ld', self.defaultConf['ld']),
'ldflags': projectConfig.getItem('make.configuration.linker.flags', self.defaultConf['ldflags']),
'ld_library_paths': projectConfig.getItem('make.configuration.linker.library_paths', self.defaultConf['ld_library_paths']),
'libraries': projectConfig.getItem('make.configuration.linker.libraries', []),
'solution_dir': projectConfig.getItem('global.solution.dir', '..'),
'sources': projectConfig.getItem('make.configuration.linker.include', []),
}
makefile = GMakeDocument(TARGET_MAKEFILE_PATH)
objectSourcePaths = []
self.objectFiles.clear()
solutionDirPath = Path(makeConf['solution_dir'], True)
objectFilePaths = []
for objectSourcePath in makeConf['sources']:
objectSourcePath = solutionDirPath.join(objectSourcePath)
FileUtil.searchAllFiles(objectFilePaths, objectSourcePath, [ '.o' ])
for objectFilePath in objectFilePaths:
self.objectFiles.append(Path(objectFilePath))
finalFilePath = self.getFinalFilePath(projectConfig, makeConf)
linkRule = self.getLinkRule(finalFilePath, makeConf)
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
makefile.writeToFile()
return True
def split_edge(self, new_len, child):
p1 = self.path
p2 = child.path
assert len(p1) < new_len < len(p2), "split length %d->%d->%d" % (
len(p1), new_len, len(p2))
edge_start = p2.start + len(p1)
edge_end = p2.start + new_len
new_node = Internal(self, Path(p2.S, p2.start, edge_end))
self.children[p2.S[edge_start]] = new_node # substitute new node
new_node.children[p2.S[edge_end]] = child
child.parent = new_node
return new_node
def make(self, projectConfig):
makeConf = {
'target_type': projectConfig.getItem('make.configuration.target.type', self.defaultConf['target_type']),
'cc': projectConfig.getItem('make.configuration.compiler.c.cc', self.defaultConf['cc']),
'cxxc': projectConfig.getItem('make.configuration.compiler.cpp.cc', self.defaultConf['cxxc']),
'cflags': projectConfig.getItem('make.configuration.compiler.c.flags', self.defaultConf['cflags']),
'cxxflags': projectConfig.getItem('make.configuration.compiler.cpp.flags', self.defaultConf['cxxflags']),
'fpic': projectConfig.getItem('make.configuration.compiler.fpic', self.defaultConf['fpic']),
'autolink': projectConfig.getItem('make.configuration.linker.autolink', self.defaultConf['autolink']),
'ar': projectConfig.getItem('make.configuration.archiver.ar', self.defaultConf['ar']),
'ld': projectConfig.getItem('make.configuration.linker.ld', self.defaultConf['ld']),
'ldflags': projectConfig.getItem('make.configuration.linker.flags', self.defaultConf['ldflags']),
'ld_library_paths': projectConfig.getItem('make.configuration.linker.library_paths', self.defaultConf['ld_library_paths']),
'libraries': projectConfig.getItem('make.configuration.linker.libraries', []),
'c_src_exts': projectConfig.getItem('make.configuration.compiler.c.src_exts', self.defaultConf['c_src_exts']),
'cxx_src_exts': projectConfig.getItem('make.configuration.compiler.cpp.src_exts', self.defaultConf['cxx_src_exts']),
'c_include_paths': projectConfig.getItem('make.configuration.compiler.c.include_paths', self.defaultConf['c_include_paths']),
'cxx_include_paths': projectConfig.getItem('make.configuration.compiler.cpp.include_paths', self.defaultConf['cxx_include_paths']),
'cxx_using_c_include_paths': projectConfig.getItem('make.configuration.compiler.cpp.inherit_c_include_path', self.defaultConf['cxx_using_c_include_paths'])
}
makefile = GMakeDocument(TARGET_MAKEFILE_PATH)
self.mainSourceFiles.clear()
self.objectFiles.clear()
cSourceFiles = []
cppSourceFiles = []
FileUtil.searchAllFiles(cSourceFiles, SRC_MAIN_PATH, makeConf['c_src_exts'])
FileUtil.searchAllFiles(cppSourceFiles, SRC_MAIN_PATH, makeConf['cxx_src_exts'])
self.mainSourceFiles.extend(cSourceFiles)
self.mainSourceFiles.extend(cppSourceFiles)
for fileName in self.mainSourceFiles:
filePath = Path(fileName, True)
relPath = filePath.getRelevantPath(SRC_MAIN_PATH)
dirName = relPath.getDirName()
baseName = relPath.getBaseName()
subMakeDirPath = TARGET_SUBMAKE_MAIN_PATH.join(dirName)
FileUtil.createDirectory(subMakeDirPath)
subMakefilePath = subMakeDirPath.join(PathUtil.getPrefix(baseName))
subMakefilePath.appendExt('mk')
subMakefile = GMakeDocument(subMakefilePath)
compileRule = GCCCompileRule(filePath, [INCLUDE_MAIN_PATH], SRC_MAIN_PATH, TARGET_OBJECT_MAIN_PATH, makeConf)
subMakefile.addRule(compileRule)
makefile.addSubDocument(subMakefile)
objectFilePath = Path(compileRule.getTarget())
self.objectFiles.append(objectFilePath)
if makeConf['autolink']:
if makeConf['target_type'] == 'executable':
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
finalFileName = '%(name)s-%(version)s' % {
'name': projectConfig.getItem('project.name', 'noname'),
'version': projectConfig.getItem('project.version', '1.0.0')}
finalFilePath = TARGET_PATH.join(finalFileName)
linkRule = GCCLinkRule(finalFilePath, self.objectFiles, makeConf)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
elif makeConf['target_type'] == 'dynamic_library':
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
finalFileName = 'lib%(name)s.so.%(version)s' % {
'name': projectConfig.getItem('project.name', 'noname'),
'version': projectConfig.getItem('project.version', '1.0.0')}
finalFilePath = TARGET_PATH.join(finalFileName)
linkRule = GCCLinkRule(finalFilePath, self.objectFiles, makeConf)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
elif makeConf['target_type'] == 'static_library':
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
finalFileName = 'lib%(name)s.a.%(version)s' % {
'name': projectConfig.getItem('project.name', 'noname'),
'version': projectConfig.getItem('project.version', '1.0.0')}
finalFilePath = TARGET_PATH.join(finalFileName)
arRule = ArRule(finalFilePath, self.objectFiles, makeConf)
subMakefile.addRule(arRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
else:
self.logger.warn('target_type is not recognized!')
sys.exit(1)
else:
allRule = GMakeSimpleRule('all', self.objectFiles)
makefile.addRule(allRule)
makefile.writeToFile()
return True
import os
import sys
import modules.module
from util import Path, PathUtil, FileUtil, Configuration, ConsoleLogger
from backends.gmake.engine import Document as GMakeDocument
from backends.gmake.engine import SimpleRule as GMakeSimpleRule
from backends.gmake.engine import GCCCompileRule
from backends.gmake.engine import GCCLinkRule
from backends.gmake.engine import ArRule
SRC_PATH = Path('src', True)
INCLUDE_PATH = Path('include', True)
TARGET_PATH = Path('target', True)
SRC_MAIN_PATH = SRC_PATH.join('main')
SRC_TEST_PATH = SRC_PATH.join('test')
INCLUDE_MAIN_PATH = INCLUDE_PATH.join('main')
INCLUDE_TEST_PATH = INCLUDE_PATH.join('test')
TARGET_OBJECT_PATH = TARGET_PATH.join('object')
TARGET_OBJECT_MAIN_PATH = TARGET_OBJECT_PATH.join('main')
TARGET_SUBMAKE_PATH = TARGET_PATH.join('submake')
TARGET_SUBMAKE_MAIN_PATH = TARGET_SUBMAKE_PATH.join('main')
TARGET_MAKEFILE_PATH = TARGET_PATH.join('Makefile')
FINAL_TARGET_SUBMAKE_PATH = TARGET_SUBMAKE_PATH.join('final_target.mk')
class Module(modules.module.Module):
mainSourceFiles = []
objectFiles = []
defaultConf = {}
def __init__(self):
class SG3TWS(EWrapper, EClient):
def __init__(
self,
contract: IBContract,
invest: float,
s_window: int,
l_window: int,
update_interval: int = 15,
band_offset: float = 0.1,
) -> None:
assert (isinstance(contract, IBContract))
assert (0 < invest if isinstance(invest, float) else False)
assert (0 < s_window if isinstance(s_window, int) else False)
assert (0 < l_window if isinstance(l_window, int) else False)
assert (s_window < l_window)
assert (15 <= update_interval
if isinstance(update_interval, int) else False)
assert (0 < band_offset if isinstance(band_offset, float) else False)
EClient.__init__(self, self)
self.connect('127.0.0.1', 7497, 0)
self.contract = contract
self.invest = invest
self.s_window, self.l_window = s_window, l_window
self.s_alpha, self.l_alpha = 2 / (s_window + 1), 2 / (l_window + 1)
self.update_interval = update_interval
self.band_offset = band_offset
self.odir = Path('sg3tws_out')
self.rid = itertools.count()
self.oid = itertools.count(start=1)
# init market data subscription
self.data = {
'DELAYED_ASK': 0,
'DELAYED_BID': 0,
'DELAYED_ASK_SIZE': 0,
'DELAYED_BID_SIZE': 0,
'DELAYED_VOLUME': 0,
'DELAYED_LAST': 0,
'DELAYED_LAST_SIZE': 0,
'DELAYED_HIGH': 0,
'DELAYED_LOW': 0,
'DELAYED_CLOSE': 0,
'DELAYED_OPEN': 0
}
self.df_data = pd.DataFrame(columns=list(self.data.keys()))
self.df_data.index.name = 'time'
self.data_file = self.odir.join('data.csv')
self.df_data.to_csv(self.data_file)
self.reqMarketDataType(3)
self.reqMktData(next(self.rid), self.contract, '', False, False, [])
# init account data subscription
self.account = {'TotalCashValue': 0., 'GrossPositionValue': 0.}
self.df_account = pd.DataFrame(columns=list(self.account.keys()))
self.df_account.index.name = 'time'
self.account_file = self.odir.join('account.csv')
self.df_account.to_csv(self.account_file)
self.reqAccountSummary(next(self.rid), 'All',
', '.join(self.account.keys()))
# init trades log
self.n_order = itertools.count(1)
self.orders = {
'time': datetime.datetime.now(),
'oid': 0,
'shares': 0,
'avg_price': 0.,
'tick_price': 0.,
'commission': 0.,
'total': 0.,
'exchange': ''
}
self.df_orders = pd.DataFrame(columns=list(self.orders.keys()))
self.df_orders.index.name = 'n'
self.orders_file = self.odir.join('orders.csv')
self.df_orders.to_csv(self.orders_file)
# init technical analysis
self.fields = [
'price', 'mal', 'mas', 'mao', 'ub', 'lb', 'cash', 'stock', 'pos',
'total hold', 'total'
]
self.stat = dict(
zip(self.fields, np.zeros_like(self.fields, dtype=np.float)))
self.df_stat = pd.DataFrame(columns=self.fields)
self.df_stat.index.name = 'time'
self.stat_file = self.odir.join('stat.csv')
self.df_stat.to_csv(self.stat_file)
# init plotting
register_matplotlib_converters()
self.fig, self.axes = plt.subplots(nrows=4, sharex='all')
self.fig.set(figwidth=10, figheight=9)
self.axes[0].set_title('SG3TWS {}'.format(self.contract.symbol),
fontsize=18)
self.axes[0].set_ylabel('Price', fontsize=14)
self.axes[1].set_ylabel('Oscillator', fontsize=14)
self.axes[1].axhline(y=0, ls='-', c='k')
self.axes[2].set_ylabel('Position', fontsize=14)
self.axes[3].set_ylabel('Portfolio value', fontsize=14)
self.axes[3].set_xlabel('Time', fontsize=14)
self.lines = {
'price': self.axes[0].plot([], [], label='price')[0],
'mas': self.axes[0].plot([], [], label='mas')[0],
'mal': self.axes[0].plot([], [], label='mal')[0],
'mao': self.axes[1].plot([], [], label='mao')[0],
'ub': self.axes[1].plot([], [], '--k', label='ub')[0],
'lb': self.axes[1].plot([], [], '--k', label='lb')[0],
'pos': self.axes[2].plot([], [], label='position')[0],
'total hold': self.axes[3].plot([], [], label='total hold')[0],
'total': self.axes[3].plot([], [], label='total')[0],
}
for ax in self.axes:
ax.legend()
self.fig.tight_layout()
# initiate TWS-API loop and strategy loop
threading.Thread(target=self.run, args=()).start()
threading.Thread(target=self.exec, args=()).start()
def exec(self):
time.sleep(2)
time0 = time.time()
# init values of first timestamp
self.append_to_dataframe(self.df_data, list(self.data.values()),
self.data_file)
self.append_to_dataframe(self.df_account, list(self.account.values()),
self.account_file)
self.stat.update({
'price': self.df_data['DELAYED_LAST'][-1],
'mas': self.df_data['DELAYED_LAST'][-1],
'mal': self.df_data['DELAYED_LAST'][-1],
'ub': self.band_offset,
'lb': -self.band_offset,
'cash': self.invest,
'total hold': self.invest,
'total': self.invest
})
self.append_to_dataframe(self.df_stat, list(self.stat.values()),
self.stat_file)
# sleep till next update
time.sleep(self.update_interval -
(time.time() + time0) % self.update_interval)
# strategy loop
while self.isConnected():
aprint(self.data, fmt='i')
self.append_to_dataframe(self.df_data, list(self.data.values()),
self.data_file)
self.strategy()
self.append_to_dataframe(self.df_account,
list(self.account.values()),
self.account_file)
self.analyze()
self.append_to_dataframe(self.df_stat, list(self.stat.values()),
self.stat_file)
self.update_plot()
# sleep till next update
time.sleep(self.update_interval -
(time.time() + time0) % self.update_interval)
@staticmethod
def append_to_dataframe(df, arr, log_file, t=None):
if t is None:
df.loc[datetime.datetime.now()] = arr
else:
df.loc[t] = arr
df.iloc[[-1]].to_csv(log_file, mode='a', header=False)
def strategy(self):
# analyze time series
self.stat['price'] = self.df_data['DELAYED_LAST'][-1]
self.stat['mal'] = self.l_alpha * self.stat['price'] + (
1 - self.l_alpha) * self.stat['mal']
self.stat['mas'] = self.s_alpha * self.stat['price'] + (
1 - self.s_alpha) * self.stat['mas']
self.stat['mao'] = (self.stat['mas'] -
self.stat['mal']) / self.stat['mal'] * 100
# create trade signals
pos_max = (self.df_stat['cash'][-1] + self.df_stat['pos'][-1] *
self.stat['price']) // self.stat['price']
dpos = 0
if self.l_window < len(self.df_stat):
ub_cross = np.diff(
np.sign(self.df_stat['mao'][-2:] -
self.df_stat['ub'][-2:])).item()
lb_cross = np.diff(
np.sign(self.df_stat['mao'][-2:] -
self.df_stat['lb'][-2:])).item()
# buy if mao crosses ub from below
if ub_cross > 0:
dpos = pos_max - self.df_stat['pos'][-1]
# sell if mao crosses lb from above
elif lb_cross < 0:
dpos = -pos_max - self.df_stat['pos'][-1]
# close if mao crosses ub from above or lb from below
elif (ub_cross < 0) | (lb_cross > 0):
dpos = -self.df_stat['pos'][-1]
# place orders in TWS
if dpos > 0:
oid = next(self.oid)
order = IBOrder(action='BUY', totalQuantity=dpos, orderType='MKT')
self.placeOrder(oid, self.contract, order)
elif dpos < 0:
oid = next(self.oid)
order = IBOrder(action='SELL',
totalQuantity=-dpos,
orderType='MKT')
self.placeOrder(oid, self.contract, order)
def analyze(self):
time.sleep(5)
self.stat['stock'] = self.stat['pos'] * self.stat['price']
# self.stat['stock'] = self.account['GrossPositionValue'] * np.sign(self.stat['pos'])
self.stat['total'] = self.stat['cash'] + self.stat['stock']
pos_hold = self.invest // self.df_data['DELAYED_LAST'][0]
self.stat['total hold'] = \
self.df_data['DELAYED_LAST'][-1] * pos_hold + (self.invest - pos_hold * self.df_data['DELAYED_LAST'][0])
def update_plot(self):
for k in [
'price', 'mas', 'mal', 'mao', 'ub', 'lb', 'total hold',
'total', 'pos'
]:
self.lines[k].set_xdata(self.df_stat.index.time)
self.lines[k].set_ydata(self.df_stat[k].values)
for ax in self.axes:
ax.relim()
ax.autoscale_view()
self.axes[3].set(xticks=np.linspace(*self.axes[0].get_xlim(), 5),
xticklabels=pd.to_datetime(
np.linspace(self.df_data.index[0].value,
self.df_data.index[-1].value,
5)).strftime('%H:%M:%S'))
self.fig.canvas.draw_idle()
def error(self, rid, error_code, error_string):
aprint('id={} / code={} / msg={}'.format(rid, error_code,
error_string),
fmt='e')
def tickPrice(self, rid, tick_type, price, attrib):
self.data[TickTypeEnum.to_str(tick_type)] = price
def tickSize(self, rid, tick_type, size):
self.data[TickTypeEnum.to_str(tick_type)] = size
def execDetails(self, rid, contract, execution):
self.orders['time'] = datetime.datetime.strptime(
execution.time, '%Y%m%d %H:%M:%S')
self.orders['oid'] = execution.orderId
self.orders[
'shares'] = execution.shares if execution.side == 'BOT' else -execution.shares
self.orders['avg_price'] = execution.avgPrice
self.orders['tick_price'] = self.df_data['DELAYED_LAST'][-1]
self.orders['exchange'] = execution.exchange
aprint(execution.__dict__, fmt='i')
self.stat['pos'] += self.orders['shares']
def commissionReport(self, commission_report):
self.orders['commission'] = commission_report.commission
self.orders['total'] = self.orders['shares'] * self.orders[
'avg_price'] + self.orders['commission']
self.append_to_dataframe(self.df_orders,
list(self.orders.values()),
self.orders_file,
t=next(self.n_order))
aprint(commission_report.__dict__, fmt='i')
self.stat['cash'] -= self.orders['total']
# self.stat['cash'] -= self.orders['shares'] * self.orders['tick_price'] + self.orders['commission']
# self.stat['cash'] = self.invest + self.df_account['TotalCashValue'][-1] - self.df_account['TotalCashValue'][0]
def accountSummary(self, rid, account, tag, value, currency):
self.account[tag] = float(value)
def __init__(
self,
contract: IBContract,
invest: float,
s_window: int,
l_window: int,
update_interval: int = 15,
band_offset: float = 0.1,
) -> None:
assert (isinstance(contract, IBContract))
assert (0 < invest if isinstance(invest, float) else False)
assert (0 < s_window if isinstance(s_window, int) else False)
assert (0 < l_window if isinstance(l_window, int) else False)
assert (s_window < l_window)
assert (15 <= update_interval
if isinstance(update_interval, int) else False)
assert (0 < band_offset if isinstance(band_offset, float) else False)
EClient.__init__(self, self)
self.connect('127.0.0.1', 7497, 0)
self.contract = contract
self.invest = invest
self.s_window, self.l_window = s_window, l_window
self.s_alpha, self.l_alpha = 2 / (s_window + 1), 2 / (l_window + 1)
self.update_interval = update_interval
self.band_offset = band_offset
self.odir = Path('sg3tws_out')
self.rid = itertools.count()
self.oid = itertools.count(start=1)
# init market data subscription
self.data = {
'DELAYED_ASK': 0,
'DELAYED_BID': 0,
'DELAYED_ASK_SIZE': 0,
'DELAYED_BID_SIZE': 0,
'DELAYED_VOLUME': 0,
'DELAYED_LAST': 0,
'DELAYED_LAST_SIZE': 0,
'DELAYED_HIGH': 0,
'DELAYED_LOW': 0,
'DELAYED_CLOSE': 0,
'DELAYED_OPEN': 0
}
self.df_data = pd.DataFrame(columns=list(self.data.keys()))
self.df_data.index.name = 'time'
self.data_file = self.odir.join('data.csv')
self.df_data.to_csv(self.data_file)
self.reqMarketDataType(3)
self.reqMktData(next(self.rid), self.contract, '', False, False, [])
# init account data subscription
self.account = {'TotalCashValue': 0., 'GrossPositionValue': 0.}
self.df_account = pd.DataFrame(columns=list(self.account.keys()))
self.df_account.index.name = 'time'
self.account_file = self.odir.join('account.csv')
self.df_account.to_csv(self.account_file)
self.reqAccountSummary(next(self.rid), 'All',
', '.join(self.account.keys()))
# init trades log
self.n_order = itertools.count(1)
self.orders = {
'time': datetime.datetime.now(),
'oid': 0,
'shares': 0,
'avg_price': 0.,
'tick_price': 0.,
'commission': 0.,
'total': 0.,
'exchange': ''
}
self.df_orders = pd.DataFrame(columns=list(self.orders.keys()))
self.df_orders.index.name = 'n'
self.orders_file = self.odir.join('orders.csv')
self.df_orders.to_csv(self.orders_file)
# init technical analysis
self.fields = [
'price', 'mal', 'mas', 'mao', 'ub', 'lb', 'cash', 'stock', 'pos',
'total hold', 'total'
]
self.stat = dict(
zip(self.fields, np.zeros_like(self.fields, dtype=np.float)))
self.df_stat = pd.DataFrame(columns=self.fields)
self.df_stat.index.name = 'time'
self.stat_file = self.odir.join('stat.csv')
self.df_stat.to_csv(self.stat_file)
# init plotting
register_matplotlib_converters()
self.fig, self.axes = plt.subplots(nrows=4, sharex='all')
self.fig.set(figwidth=10, figheight=9)
self.axes[0].set_title('SG3TWS {}'.format(self.contract.symbol),
fontsize=18)
self.axes[0].set_ylabel('Price', fontsize=14)
self.axes[1].set_ylabel('Oscillator', fontsize=14)
self.axes[1].axhline(y=0, ls='-', c='k')
self.axes[2].set_ylabel('Position', fontsize=14)
self.axes[3].set_ylabel('Portfolio value', fontsize=14)
self.axes[3].set_xlabel('Time', fontsize=14)
self.lines = {
'price': self.axes[0].plot([], [], label='price')[0],
'mas': self.axes[0].plot([], [], label='mas')[0],
'mal': self.axes[0].plot([], [], label='mal')[0],
'mao': self.axes[1].plot([], [], label='mao')[0],
'ub': self.axes[1].plot([], [], '--k', label='ub')[0],
'lb': self.axes[1].plot([], [], '--k', label='lb')[0],
'pos': self.axes[2].plot([], [], label='position')[0],
'total hold': self.axes[3].plot([], [], label='total hold')[0],
'total': self.axes[3].plot([], [], label='total')[0],
}
for ax in self.axes:
ax.legend()
self.fig.tight_layout()
# initiate TWS-API loop and strategy loop
threading.Thread(target=self.run, args=()).start()
threading.Thread(target=self.exec, args=()).start()
import os
import modules.module
from util import Path, PathUtil, FileUtil, Configuration, ConsoleLogger
from backends.gmake.engine import Document as GMakeDocument
from backends.gmake.engine import SimpleRule as GMakeSimpleRule
from backends.gmake.engine import GCCCompileRule
from backends.gmake.engine import GCCLinkRule
from backends.gmake.engine import ArRule
TARGET_PATH = Path('target', True)
TARGET_MAKEFILE_PATH = TARGET_PATH.join('Makefile')
TARGET_SUBMAKE_PATH = TARGET_PATH.join('submake')
FINAL_TARGET_SUBMAKE_PATH = TARGET_SUBMAKE_PATH.join('final_target')
class Module(modules.module.Module):
objectFiles = []
defaultConf = {}
def __init__(self):
modules.module.Module.__init__(self)
self.defaultConf['target_type'] = 'executable'
self.defaultConf['ar'] = 'ar'
self.defaultConf['ld'] = 'g++'
self.defaultConf['ldflags'] = ''
self.defaultConf['ld_library_paths'] = []
self.logger = ConsoleLogger('ldmodule')
def initKakefile(self):
kakefile = FileUtil.openFile('Kakefile')
kakefileContent = """project:
import os
import sys
import modules.module
from util import Path, PathUtil, FileUtil, Configuration, ConsoleLogger
from backends.gmake.engine import Document as GMakeDocument
from backends.gmake.engine import SimpleRule as GMakeSimpleRule
from backends.gmake.engine import GCCCompileRule
from backends.gmake.engine import GCCLinkRule
from backends.gmake.engine import ArRule
SRC_PATH = Path('src', True)
INCLUDE_PATH = Path('include', True)
TARGET_PATH = Path('target', True)
SRC_MAIN_PATH = SRC_PATH.join('main')
SRC_TEST_PATH = SRC_PATH.join('test')
INCLUDE_MAIN_PATH = INCLUDE_PATH.join('main')
INCLUDE_TEST_PATH = INCLUDE_PATH.join('test')
TARGET_OBJECT_PATH = TARGET_PATH.join('object')
TARGET_OBJECT_MAIN_PATH = TARGET_OBJECT_PATH.join('main')
TARGET_SUBMAKE_PATH = TARGET_PATH.join('submake')
TARGET_SUBMAKE_MAIN_PATH = TARGET_SUBMAKE_PATH.join('main')
TARGET_MAKEFILE_PATH = TARGET_PATH.join('Makefile')
FINAL_TARGET_SUBMAKE_PATH = TARGET_SUBMAKE_PATH.join('final_target.mk')
class Module(modules.module.Module):
mainSourceFiles = []
objectFiles = []
defaultConf = {}
def make(self, projectConfig):
makeConf = {
'target_type':
projectConfig.getItem('make.configuration.target.type',
self.defaultConf['target_type']),
'cc':
projectConfig.getItem('make.configuration.compiler.c.cc',
self.defaultConf['cc']),
'cxxc':
projectConfig.getItem('make.configuration.compiler.cpp.cc',
self.defaultConf['cxxc']),
'cflags':
projectConfig.getItem('make.configuration.compiler.c.flags',
self.defaultConf['cflags']),
'cxxflags':
projectConfig.getItem('make.configuration.compiler.cpp.flags',
self.defaultConf['cxxflags']),
'fpic':
projectConfig.getItem('make.configuration.compiler.fpic',
self.defaultConf['fpic']),
'autolink':
projectConfig.getItem('make.configuration.linker.autolink',
self.defaultConf['autolink']),
'ar':
projectConfig.getItem('make.configuration.archiver.ar',
self.defaultConf['ar']),
'ld':
projectConfig.getItem('make.configuration.linker.ld',
self.defaultConf['ld']),
'ldflags':
projectConfig.getItem('make.configuration.linker.flags',
self.defaultConf['ldflags']),
'ld_library_paths':
projectConfig.getItem('make.configuration.linker.library_paths',
self.defaultConf['ld_library_paths']),
'libraries':
projectConfig.getItem('make.configuration.linker.libraries', []),
'c_src_exts':
projectConfig.getItem('make.configuration.compiler.c.src_exts',
self.defaultConf['c_src_exts']),
'cxx_src_exts':
projectConfig.getItem('make.configuration.compiler.cpp.src_exts',
self.defaultConf['cxx_src_exts']),
'c_include_paths':
projectConfig.getItem(
'make.configuration.compiler.c.include_paths',
self.defaultConf['c_include_paths']),
'cxx_include_paths':
projectConfig.getItem(
'make.configuration.compiler.cpp.include_paths',
self.defaultConf['cxx_include_paths']),
'cxx_using_c_include_paths':
projectConfig.getItem(
'make.configuration.compiler.cpp.inherit_c_include_path',
self.defaultConf['cxx_using_c_include_paths'])
}
makefile = GMakeDocument(TARGET_MAKEFILE_PATH)
self.mainSourceFiles.clear()
self.objectFiles.clear()
cSourceFiles = []
cppSourceFiles = []
FileUtil.searchAllFiles(cSourceFiles, SRC_MAIN_PATH,
makeConf['c_src_exts'])
FileUtil.searchAllFiles(cppSourceFiles, SRC_MAIN_PATH,
makeConf['cxx_src_exts'])
self.mainSourceFiles.extend(cSourceFiles)
self.mainSourceFiles.extend(cppSourceFiles)
for fileName in self.mainSourceFiles:
filePath = Path(fileName, True)
relPath = filePath.getRelevantPath(SRC_MAIN_PATH)
dirName = relPath.getDirName()
baseName = relPath.getBaseName()
subMakeDirPath = TARGET_SUBMAKE_MAIN_PATH.join(dirName)
FileUtil.createDirectory(subMakeDirPath)
subMakefilePath = subMakeDirPath.join(PathUtil.getPrefix(baseName))
subMakefilePath.appendExt('mk')
subMakefile = GMakeDocument(subMakefilePath)
compileRule = GCCCompileRule(filePath, [INCLUDE_MAIN_PATH],
SRC_MAIN_PATH,
TARGET_OBJECT_MAIN_PATH, makeConf)
subMakefile.addRule(compileRule)
makefile.addSubDocument(subMakefile)
objectFilePath = Path(compileRule.getTarget())
self.objectFiles.append(objectFilePath)
if makeConf['autolink']:
if makeConf['target_type'] == 'executable':
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
finalFileName = '%(name)s-%(version)s' % {
'name': projectConfig.getItem('project.name', 'noname'),
'version': projectConfig.getItem('project.version',
'1.0.0')
}
finalFilePath = TARGET_PATH.join(finalFileName)
linkRule = GCCLinkRule(finalFilePath, self.objectFiles,
makeConf)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
elif makeConf['target_type'] == 'dynamic_library':
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
finalFileName = 'lib%(name)s.so.%(version)s' % {
'name': projectConfig.getItem('project.name', 'noname'),
'version': projectConfig.getItem('project.version',
'1.0.0')
}
finalFilePath = TARGET_PATH.join(finalFileName)
linkRule = GCCLinkRule(finalFilePath, self.objectFiles,
makeConf)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
elif makeConf['target_type'] == 'static_library':
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
finalFileName = 'lib%(name)s.a.%(version)s' % {
'name': projectConfig.getItem('project.name', 'noname'),
'version': projectConfig.getItem('project.version',
'1.0.0')
}
finalFilePath = TARGET_PATH.join(finalFileName)
arRule = ArRule(finalFilePath, self.objectFiles, makeConf)
subMakefile.addRule(arRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
else:
self.logger.warn('target_type is not recognized!')
sys.exit(1)
else:
allRule = GMakeSimpleRule('all', self.objectFiles)
makefile.addRule(allRule)
makefile.writeToFile()
return True
import os
import sys
import modules.module
from util import Path, PathUtil, FileUtil, Configuration, ConsoleLogger
from backends.gmake.engine import Document as GMakeDocument
from backends.gmake.engine import SimpleRule as GMakeSimpleRule
from backends.gmake.engine import GCCCompileRule
from backends.gmake.engine import GCCLinkRule
from backends.gmake.engine import ArRule
PROJECT_PATH = Path('project', True)
TARGET_PATH = Path('target', True)
class Module(modules.module.Module):
defaultConf = {}
projects = []
def __init__(self):
#'solution_dir': projectConfig.getItem('global.solution.dir', '..'),
self.logger = ConsoleLogger('cmodule')
projectNames = os.listdir('project')
projectNames.sort()
for projectName in projectNames:
project = {
'name': projectName,
'path': PROJECT_PATH.join(projectName)
}
self.projects.append(project)
import os
import modules.module
from util import Path, PathUtil, FileUtil, Configuration, ConsoleLogger
from backends.gmake.engine import Document as GMakeDocument
from backends.gmake.engine import SimpleRule as GMakeSimpleRule
from backends.gmake.engine import GCCCompileRule
from backends.gmake.engine import GCCLinkRule
from backends.gmake.engine import ArRule
TARGET_PATH = Path('target', True)
TARGET_MAKEFILE_PATH = TARGET_PATH.join('Makefile')
TARGET_SUBMAKE_PATH = TARGET_PATH.join('submake')
FINAL_TARGET_SUBMAKE_PATH = TARGET_SUBMAKE_PATH.join('final_target')
class Module(modules.module.Module):
objectFiles = []
defaultConf = {}
def __init__(self):
modules.module.Module.__init__(self)
self.defaultConf['target_type'] = 'executable'
self.defaultConf['ar'] = 'ar'
self.defaultConf['ld'] = 'g++'
self.defaultConf['ldflags'] = ''
self.defaultConf['ld_library_paths'] = []
self.logger = ConsoleLogger('ldmodule')
def initKakefile(self):
kakefile = FileUtil.openFile('Kakefile')
