class DataEngine(AspectDB):
_channel_ = CH_DATAENGINE
def __init__(self, workspace):
self.workspace = WorkSpace(workspace)
self.config = self.workspace.workspace('engine.ini')
self._socksrv = SockService()
self._socksrv.attach(self)
def close(self):
self.stop()
for drv in set(self._paths.values()):
drv.close()
def __init__(self, workspace):
self.workspace = WorkSpace(workspace)
self.config = self.workspace.config('hqdrv.ini')
# for aspect handler
self.aspectfacs = {}
self.instances = {}
self.aspects = {}
# for network communication
self.online = False
self.reqcount = 0
self.cmdbuf = []
self.inbuf = {}
# for respons dispatch
self.resp_receivers = {}
# data file update progress
self.datafile_update_map = {}
self.init_connection()
class TdxHqDrv(BaseAspectDrv):
_channel_ = CH_HQ_DRIVER
@property
def sig_connected(self):
return self.signal('connected')
@property
def sig_fileupdated(self):
return self.signal('fileupdated')
def __init__(self, workspace):
self.workspace = WorkSpace(workspace)
self.config = self.workspace.config('hqdrv.ini')
# for aspect handler
self.aspectfacs = {}
self.instances = {}
self.aspects = {}
# for network communication
self.online = False
self.reqcount = 0
self.cmdbuf = []
self.inbuf = {}
# for respons dispatch
self.resp_receivers = {}
# data file update progress
self.datafile_update_map = {}
self.init_connection()
def init_connection(self):
drvconf = self.config.section('driver')
workernum = drvconf.jsetdefault('hq.worker.num', 1)
proxy = drvconf.jsetdefault('hq.proxy', None)
if proxy:proxy = SockProxy(*proxy)
hostsec = drvconf.setdefault('hq.hosts.section', "HQHOST")
hosts = map(tuple, self.config.jitems(hostsec).values()[:workernum])
for h in hosts:
req = ConnectRequest.Value(target=h, channel=CH_HQ_SOCK, proxy=proxy)
self.schedule(ConnectRequest(value=req))
def request(self, reqcmd, *args, **kwargs):
self.reqcount += 1
self.cmdbuf.append((self.reqcount, reqcmd, args, kwargs))
return self.reqcount
def clear_request(self):
self.cmdbuf = []
self.reqcount = 0
def listen_response(self, seqno, callback):
self.resp_receivers[seqno] = callback
def dispatch_response(self, resps):
for seqno, cmd, data in resps:
if seqno in self.resp_receivers:
val = parse_response(cmd, data)
self.resp_receivers.pop(seqno)(val)
def close(self):
for hld in set(self.aspects.values()):
hld.flush()
for sock in self.inbuf.keys():
try: sock.close()
except: pass
# for aspect handler
self.aspectfacs = {}
self.instances = {}
self.aspects = {}
# for network communication
self.reqcount = 0
self.cmdbuf = []
self.inbuf = {}
self.resp_receivers = {}
self.datafile_update_map = {}
#===========================================================================
# sock event handling
#===========================================================================
@handler('connectresponse', CH_HQ_SOCK)
def evt_conresp(self, evt):
if evt.value.success:
self.inbuf[evt.value.sock] = ''
else:
self._log_exc(*evt.value.reason)
@handler('poller_connected', CH_HQ_SOCK)
def evt_connected(self, evt):
self.online = True
self.update_datafiles()
self.sig_connected.emit(True)
@handler('poller_disconnected', CH_HQ_SOCK)
def evt_disconnected(self, evt):
self.online = False
self.sig_connected.emit(False)
self.reqcount = 0
self.cmdbuf = []
self.inbuf = {}
self.resp_receivers = {}
self.datafile_update_map = {}
@handler('poller_read', CH_HQ_SOCK)
def evt_read(self, evt):
sock = evt.value
if sock in self.inbuf:
respstream = sock.recv(1024*32)
if not respstream:
sock.close()
print 'socket closed, EOF.', sock.addrs()
else:
self.inbuf[sock] = self.inbuf.setdefault(sock, '') + respstream
resps, self.inbuf[sock] = split_response(self.inbuf[sock])
self.dispatch_response(resps)
@handler('poller_write', CH_HQ_SOCK)
def evt_write(self, evt):
if self.cmdbuf:
seqno, cmd, args, kwargs = self.cmdbuf[0]
request = build_request(seqno, cmd, *args, **kwargs)
evt.value.sendall(request)
self.cmdbuf.pop(0)
@handler('poller_exc', CH_HQ_SOCK)
def evt_exc(self, evt):
sock, err = evt.value #@UnusedVariable
if sock in self.inbuf:
self.inbuf.pop(sock)
#===========================================================================
# keep socks alive
#===========================================================================
@routine(3)
def heartbeat(self):
hbreq = build_request(0xffffffff, Cmds.HeartBeat)
for sock in self.inbuf.keys():
try:
sock.sendall(hbreq)
except:
pass
#===========================================================================
# aspect handler registration/initialisation
#===========================================================================
def register_aspectfac(self, path, aspectfac):
self.aspectfacs[path] = aspectfac
def register_aspect(self, path, aspect):
self.aspects[path] = aspect
def aspect(self, path):
if path not in self.aspects:
fac = self.aspectfacs.get(path, None) or _tdx_hq_aspects.get(path, None)
handler = self.instances.get(fac, None)
if handler==None:
handler = fac(self)
self.instances[fac] = handler
self.aspects[path] = handler
return self.aspects[path]
#===========================================================================
# dispatch actions to aspects
#===========================================================================
def init_path(self, path):
self.aspect(path).init(self.subscription(path), path.category)
def close_path(self, path):
self.aspect(path).close(path.category)
def getval(self, path, ticker):
return self.aspect(path).getval(ticker, path.category)
def on_sub(self, path, ticker):
self.aspect(path).on_sub(ticker, path.category)
def on_unsub(self, path, ticker):
self.aspect(path).on_unsub(ticker, path.category)
#===========================================================================
# update datafiles from tdx server
#===========================================================================
def datafile(self, filename):
dlloc = self.config.section('driver').setdefault('hq.datafiles.path', 'datafiles')
return self.workspace.workspace(dlloc).file(filename)
def update_datafiles(self):
if not self.online: return
filenames = map(str, self.config.section('driver').jsetdefault('hq.datafiles.name', []))
for fn in filenames:
self._update_file(fn)
def _update_file(self, filename):
def recvchunk(fn, begin, chunk):
if chunk == None: #retry
seqno = self.request(Cmds.GetFile, fn, begin, 30000)
self.listen_response(seqno, partial(recvchunk, fn, begin))
else:
chunks = self.datafile_update_map[fn]
chunks[begin] = chunk
if None not in chunks.values():
fobj = self.datafile(fn)
with fobj.open('wb') as fp:
fp.write(''.join(chunks.values()))
self.datafile_update_map.pop(fn)
self.sig_fileupdated.emit(filename)
def check(fn, srvinfo):
if fn in self.datafile_update_map:
return # downloading
fileobj = self.datafile(fn)
localinfo = () if not fileobj.exist() else (fileobj.stat().size, fileobj.md5hex())
if localinfo != srvinfo:
chunks = OrderedDict()
begin = 0
while begin < srvinfo[0]:
seqno = self.request(Cmds.GetFile, filename, begin, 30000)
self.listen_response(seqno, partial(recvchunk, fn, begin))
chunks[begin] = None
begin += 30000
self.datafile_update_map[fn] = chunks
seqno = self.request(Cmds.FileInfo, filename)
self.listen_response(seqno, partial(check, filename))
def __init__(self, workspace):
self.workspace = WorkSpace(workspace)
self.config = self.workspace.workspace('engine.ini')
self._socksrv = SockService()
self._socksrv.attach(self)
