def ownHeartbeat(uu=None): me = 'ownHeartbeat' logrec = None try: if uu is None: uu = _dt.utcut() ul = _dt.locut(uu) uuts = '%15.4f' % uu # 15.4, unblanked fraction. uuiosfs = _dt.ut2isofs(uu) uliosfs = _dt.ut2isofs(ul) rxts = txts = uuts kvs = {'_id': SRCID, '_si': SUBID, '_el': HB_EL, '_sl': HB_SL, '_ip': None, '_ts': uuts, 'dt_loc': uliosfs, 'dt_utc': uuiosfs} kvsa = json.dumps(kvs, ensure_ascii=True, sort_keys=True) kvsab = kvsa.encode(encoding=ENCODING, errors=ERRORS) h = hashlib.sha1() h.update(kvsab) sha1x = h.hexdigest() _ = '\t' logrec = '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' %('1', _, rxts, _, txts, _, SRCID, _, SUBID, _, HB_EL, _, HB_SL, _, sha1x, _, kvsa) except Exception as E: errmsg = '%s: %s @ %s' % (me, E, _m.tblineno()) DOSQUAWK(errmsg) raise finally: return logrec
def shutdown(): # A shortcut for the the post-main section of an app shutdown. me = argsMe()#!#margsMe() msg = '' #!#msg += '%s ends @ %s' % (me, mNowISOlongCS(True)) # !!! <<< msg += '%s ends @ %s' % (me, dt.ut2isofs(dt.locut())) # !!! <<< msg += '\n' #!#msg += '## <%s> %s' % (mNowISOlongCS(True), 50*'=') # !!! <<< msg += '## <%s> %s' % (dt.ut2isofs(dt.locut()), 50*'=') # !!! <<< return msg
def startup(): # A shortcut for the the pre-maininits() section of an app startup. me = argsMe()#!#margsMe() msg = '' #!#msg += '## <%s> %s' % (mNowISOlongCS(True), 50*'-') # !!! <<< msg += '## <%s> %s' % (dt.ut2isofs(dt.locut()), 50*'-') # !!! <<< msg += '\n' #!#msg += '%s begins @ %s' % (me, mNowISOlongCS(True)) # !!! <<< msg += '%s begins @ %s' % (me, dt.ut2isofs(dt.locut())) # !!! <<< return me, msg
def watcherThread(): """A thread to watch WPATH for files to process.""" global LOADRECS, FFWDB, FWTRUNNING, FWTSTOP, FWTSTOPPED LOADRECS = [] me = 'watcher thread' try: FWTRUNNING = True assert XLOGDB, 'no XLOGDB' # Connect to FlatFileWatchDataBase. FFWDB = ffwdb.FFWDB(FFWDBPFN) assert FFWDB, 'no FFWDB' uu = 0 # Unix Utc. while not FWTSTOP: # flushHeartbeats() # Wait out INTERVAL. z = time.time() w = INTERVAL - (z - uu) if w > 0: _sw.wait(w) uu = _dt.utcut() ul = _dt.locut(uu) uuts = '%15.4f' % uu # 15.4, unblanked fraction. uuiosfs = _dt.ut2isofs(uu) uliosfs = _dt.ut2isofs(ul) # Heartbeat? if OWNHEARTBEAT: logrec = ownHeartbeat(uu) addHeartbeat(logrec) # Files? t0 = time.perf_counter(); fis = getFIs(uu) t1 = time.perf_counter(); if TIMINGS: _sl.warning(' getFIs: {:9,.1f} ms'.format((1000*(t1-t0)))) if not fis: errmsg = 'no logfiles @ ' + uliosfs raise Exception(errmsg) # Update FFWDB. # Freshen the "acquired" timestamp. # Delete entries for nonexistent files. t0 = time.perf_counter(); filenames = [fi['filename'] for fi in fis] filenames.sort() # Compare real (nf) and db (nx) counts. nf = len(filenames) nx = FFWDB.count() if nf < nx: nf, nx = nf, nx elif nf > nx: nf, nx = nf, nx else: nf, nx = nf, nx FFWDB.acquired(filenames, uu) for fi in fis: z = updateDB(fi) t1 = time.perf_counter(); if TIMINGS: _sl.warning('updateDBs: {:9,.1f} ms'.format((1000*(t1-t0)))) # Find the oldest, newest unfinished files in DB. t0 = time.perf_counter(); o_dbfi, n_dbfi = FFWDB.oldestnewest('u') t1 = time.perf_counter(); if TIMINGS: _sl.warning(' oldest: {:9,.1f} ms'.format((1000*(t1-t0)))) # Something unfinished? if o_dbfi: fn = o_dbfi['filename'] historical = (fn != n_dbfi['filename']) # Export the file. exportFile(historical, o_dbfi) # !CHANGE! # Move oldest finished file? # It must not be the only, and therefore "live", file. if DONESD: o_dbfi, n_dbfi = FFWDB.oldestnewest('f') if o_dbfi and (n_dbfi['filename'] != o_dbfi['filename']): t0 = time.perf_counter(); doneWithFile(o_dbfi['filename']) t1 = time.perf_counter(); if TIMINGS: _sl.warning(' moved: {:9,.1f} ms'.format((1000*(t1-t0)))) if ONECHECK: FWTSTOP = True except KeyboardInterrupt as E: _m.beeps(1) # watcherThread: msg = '1: {}: KeyboardInterrupt: {}'.format(me, E) _sl.warning(msg) ###---DOSQUAWK(errmsg, beeps=1) pass###raise # Let the thread exit. Avoids "Exception in thread...". except Exception as E: errmsg = '%s: E: %s @ %s' % (me, E, _m.tblineno()) DOSQUAWK(errmsg) raise finally: # Flush heartbeats and loadrecs. flushHeartbeats() loadrecs2db() if FWTSTOP: FWTSTOPPED = True FFWDB.disconnect() _sl.info('%s exits. STOPPED: %s' % (me, str(FWTSTOPPED))) FWTRUNNING = False