class DataEngine(AspectDB): _channel_ = CH_DATAENGINE def __init__(self, workspace): self.workspace = WorkSpace(workspace) self.config = self.workspace.workspace('engine.ini') self._socksrv = SockService() self._socksrv.attach(self) def close(self): self.stop() for drv in set(self._paths.values()): drv.close()
class TdxHqDrv(BaseAspectDrv): _channel_ = CH_HQ_DRIVER @property def sig_connected(self): return self.signal('connected') @property def sig_fileupdated(self): return self.signal('fileupdated') def __init__(self, workspace): self.workspace = WorkSpace(workspace) self.config = self.workspace.config('hqdrv.ini') # for aspect handler self.aspectfacs = {} self.instances = {} self.aspects = {} # for network communication self.online = False self.reqcount = 0 self.cmdbuf = [] self.inbuf = {} # for respons dispatch self.resp_receivers = {} # data file update progress self.datafile_update_map = {} self.init_connection() def init_connection(self): drvconf = self.config.section('driver') workernum = drvconf.jsetdefault('hq.worker.num', 1) proxy = drvconf.jsetdefault('hq.proxy', None) if proxy:proxy = SockProxy(*proxy) hostsec = drvconf.setdefault('hq.hosts.section', "HQHOST") hosts = map(tuple, self.config.jitems(hostsec).values()[:workernum]) for h in hosts: req = ConnectRequest.Value(target=h, channel=CH_HQ_SOCK, proxy=proxy) self.schedule(ConnectRequest(value=req)) def request(self, reqcmd, *args, **kwargs): self.reqcount += 1 self.cmdbuf.append((self.reqcount, reqcmd, args, kwargs)) return self.reqcount def clear_request(self): self.cmdbuf = [] self.reqcount = 0 def listen_response(self, seqno, callback): self.resp_receivers[seqno] = callback def dispatch_response(self, resps): for seqno, cmd, data in resps: if seqno in self.resp_receivers: val = parse_response(cmd, data) self.resp_receivers.pop(seqno)(val) def close(self): for hld in set(self.aspects.values()): hld.flush() for sock in self.inbuf.keys(): try: sock.close() except: pass # for aspect handler self.aspectfacs = {} self.instances = {} self.aspects = {} # for network communication self.reqcount = 0 self.cmdbuf = [] self.inbuf = {} self.resp_receivers = {} self.datafile_update_map = {} #=========================================================================== # sock event handling #=========================================================================== @handler('connectresponse', CH_HQ_SOCK) def evt_conresp(self, evt): if evt.value.success: self.inbuf[evt.value.sock] = '' else: self._log_exc(*evt.value.reason) @handler('poller_connected', CH_HQ_SOCK) def evt_connected(self, evt): self.online = True self.update_datafiles() self.sig_connected.emit(True) @handler('poller_disconnected', CH_HQ_SOCK) def evt_disconnected(self, evt): self.online = False self.sig_connected.emit(False) self.reqcount = 0 self.cmdbuf = [] self.inbuf = {} self.resp_receivers = {} self.datafile_update_map = {} @handler('poller_read', CH_HQ_SOCK) def evt_read(self, evt): sock = evt.value if sock in self.inbuf: respstream = sock.recv(1024*32) if not respstream: sock.close() print 'socket closed, EOF.', sock.addrs() else: self.inbuf[sock] = self.inbuf.setdefault(sock, '') + respstream resps, self.inbuf[sock] = split_response(self.inbuf[sock]) self.dispatch_response(resps) @handler('poller_write', CH_HQ_SOCK) def evt_write(self, evt): if self.cmdbuf: seqno, cmd, args, kwargs = self.cmdbuf[0] request = build_request(seqno, cmd, *args, **kwargs) evt.value.sendall(request) self.cmdbuf.pop(0) @handler('poller_exc', CH_HQ_SOCK) def evt_exc(self, evt): sock, err = evt.value #@UnusedVariable if sock in self.inbuf: self.inbuf.pop(sock) #=========================================================================== # keep socks alive #=========================================================================== @routine(3) def heartbeat(self): hbreq = build_request(0xffffffff, Cmds.HeartBeat) for sock in self.inbuf.keys(): try: sock.sendall(hbreq) except: pass #=========================================================================== # aspect handler registration/initialisation #=========================================================================== def register_aspectfac(self, path, aspectfac): self.aspectfacs[path] = aspectfac def register_aspect(self, path, aspect): self.aspects[path] = aspect def aspect(self, path): if path not in self.aspects: fac = self.aspectfacs.get(path, None) or _tdx_hq_aspects.get(path, None) handler = self.instances.get(fac, None) if handler==None: handler = fac(self) self.instances[fac] = handler self.aspects[path] = handler return self.aspects[path] #=========================================================================== # dispatch actions to aspects #=========================================================================== def init_path(self, path): self.aspect(path).init(self.subscription(path), path.category) def close_path(self, path): self.aspect(path).close(path.category) def getval(self, path, ticker): return self.aspect(path).getval(ticker, path.category) def on_sub(self, path, ticker): self.aspect(path).on_sub(ticker, path.category) def on_unsub(self, path, ticker): self.aspect(path).on_unsub(ticker, path.category) #=========================================================================== # update datafiles from tdx server #=========================================================================== def datafile(self, filename): dlloc = self.config.section('driver').setdefault('hq.datafiles.path', 'datafiles') return self.workspace.workspace(dlloc).file(filename) def update_datafiles(self): if not self.online: return filenames = map(str, self.config.section('driver').jsetdefault('hq.datafiles.name', [])) for fn in filenames: self._update_file(fn) def _update_file(self, filename): def recvchunk(fn, begin, chunk): if chunk == None: #retry seqno = self.request(Cmds.GetFile, fn, begin, 30000) self.listen_response(seqno, partial(recvchunk, fn, begin)) else: chunks = self.datafile_update_map[fn] chunks[begin] = chunk if None not in chunks.values(): fobj = self.datafile(fn) with fobj.open('wb') as fp: fp.write(''.join(chunks.values())) self.datafile_update_map.pop(fn) self.sig_fileupdated.emit(filename) def check(fn, srvinfo): if fn in self.datafile_update_map: return # downloading fileobj = self.datafile(fn) localinfo = () if not fileobj.exist() else (fileobj.stat().size, fileobj.md5hex()) if localinfo != srvinfo: chunks = OrderedDict() begin = 0 while begin < srvinfo[0]: seqno = self.request(Cmds.GetFile, filename, begin, 30000) self.listen_response(seqno, partial(recvchunk, fn, begin)) chunks[begin] = None begin += 30000 self.datafile_update_map[fn] = chunks seqno = self.request(Cmds.FileInfo, filename) self.listen_response(seqno, partial(check, filename))