Exemple #1
0
def ownHeartbeat(uu=None):
    me = 'ownHeartbeat'
    logrec = None
    try:
        if uu is None:
                uu = _dt.utcut()
        ul = _dt.locut(uu)
        uuts = '%15.4f' % uu                                # 15.4, unblanked fraction.
        uuiosfs = _dt.ut2isofs(uu)
        uliosfs = _dt.ut2isofs(ul)
        rxts = txts = uuts
        kvs = {'_id': SRCID, '_si': SUBID, 
               '_el': HB_EL, '_sl': HB_SL, 
               '_ip': None, '_ts': uuts, 
               'dt_loc': uliosfs, 'dt_utc': uuiosfs}
        kvsa = json.dumps(kvs, ensure_ascii=True, sort_keys=True)
        kvsab = kvsa.encode(encoding=ENCODING, errors=ERRORS)
        h = hashlib.sha1()
        h.update(kvsab)
        sha1x = h.hexdigest()
        _ = '\t'
        logrec = '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' %('1', _, rxts, _, txts, _, SRCID, _, SUBID, _, HB_EL, _, HB_SL, _, sha1x, _, kvsa)
    except Exception as E:
        errmsg = '%s: %s @ %s' % (me, E, _m.tblineno())
        DOSQUAWK(errmsg)
        raise
    finally:
        return logrec
Exemple #2
0
def shutdown():
    # A shortcut for the the post-main section of an app shutdown.
    me = argsMe()#!#margsMe()
    msg = ''
    #!#msg += '%s   ends @ %s' % (me, mNowISOlongCS(True))     # !!! <<<
    msg += '%s   ends @ %s' % (me, dt.ut2isofs(dt.locut()))     # !!! <<<
    msg += '\n'
    #!#msg += '## <%s> %s' % (mNowISOlongCS(True), 50*'=')     # !!! <<<
    msg += '## <%s> %s' % (dt.ut2isofs(dt.locut()), 50*'=')     # !!! <<<
    return msg
Exemple #3
0
def startup():
    # A shortcut for the the pre-maininits() section of an app startup.
    me = argsMe()#!#margsMe()
    msg = ''
    #!#msg += '## <%s> %s' % (mNowISOlongCS(True), 50*'-')     # !!! <<<
    msg += '## <%s> %s' % (dt.ut2isofs(dt.locut()), 50*'-')     # !!! <<<
    msg += '\n'
    #!#msg += '%s begins @ %s' % (me, mNowISOlongCS(True))     # !!! <<<
    msg += '%s begins @ %s' % (me, dt.ut2isofs(dt.locut()))     # !!! <<<
    return me, msg
Exemple #4
0
def watcherThread():
    """A thread to watch WPATH for files to process."""
    global LOADRECS, FFWDB, FWTRUNNING, FWTSTOP, FWTSTOPPED

    LOADRECS = []

    me = 'watcher thread' 
    try:
        FWTRUNNING = True
        assert XLOGDB, 'no XLOGDB'

        # Connect to FlatFileWatchDataBase.
        FFWDB = ffwdb.FFWDB(FFWDBPFN)
        assert FFWDB, 'no FFWDB'

        uu = 0                                                  # Unix Utc.
        while not FWTSTOP:

            #
            flushHeartbeats()
           
            # Wait out INTERVAL.
            z = time.time()
            w = INTERVAL - (z - uu)
            if w > 0:
                _sw.wait(w)
            uu = _dt.utcut()
            ul = _dt.locut(uu)
            uuts = '%15.4f' % uu                                # 15.4, unblanked fraction.
            uuiosfs = _dt.ut2isofs(uu)
            uliosfs = _dt.ut2isofs(ul)

            # Heartbeat?
            if OWNHEARTBEAT:
                logrec = ownHeartbeat(uu)
                addHeartbeat(logrec)

            # Files?
            t0 = time.perf_counter();
            fis = getFIs(uu)
            t1 = time.perf_counter();
            if TIMINGS:
                _sl.warning('   getFIs: {:9,.1f} ms'.format((1000*(t1-t0))))
            if not fis:
                errmsg = 'no logfiles @ ' + uliosfs
                raise Exception(errmsg)

            # Update FFWDB. 
            # Freshen the "acquired" timestamp.
            # Delete entries for nonexistent files.
            t0 = time.perf_counter();
            filenames = [fi['filename'] for fi in fis]
            filenames.sort()
            # Compare real (nf) and db (nx) counts.
            nf = len(filenames)
            nx = FFWDB.count()
            if   nf < nx:
                nf, nx = nf, nx
            elif nf > nx:
                nf, nx = nf, nx
            else:
                nf, nx = nf, nx
            FFWDB.acquired(filenames, uu)
            for fi in fis:
                z = updateDB(fi)
            t1 = time.perf_counter();
            if TIMINGS:
                _sl.warning('updateDBs: {:9,.1f} ms'.format((1000*(t1-t0))))

            # Find the oldest, newest unfinished files in DB.
            t0 = time.perf_counter();
            o_dbfi, n_dbfi = FFWDB.oldestnewest('u')
            t1 = time.perf_counter();
            if TIMINGS:
                _sl.warning('   oldest: {:9,.1f} ms'.format((1000*(t1-t0))))

            # Something unfinished?
            if o_dbfi:  

                fn = o_dbfi['filename']
                historical = (fn != n_dbfi['filename'])

                # Export the file.
                exportFile(historical, o_dbfi)    # !CHANGE!

            # Move oldest finished file? 
            # It must not be the only, and therefore "live", file.
            if DONESD:
                o_dbfi, n_dbfi = FFWDB.oldestnewest('f')
                if o_dbfi and (n_dbfi['filename'] != o_dbfi['filename']):
                    t0 = time.perf_counter();
                    doneWithFile(o_dbfi['filename'])
                    t1 = time.perf_counter();
                    if TIMINGS:
                        _sl.warning('    moved: {:9,.1f} ms'.format((1000*(t1-t0))))

            if ONECHECK:
                FWTSTOP = True

    except KeyboardInterrupt as E:
        _m.beeps(1)
        # watcherThread:
        msg = '1: {}: KeyboardInterrupt: {}'.format(me, E)
        _sl.warning(msg)
        ###---DOSQUAWK(errmsg, beeps=1)
        pass###raise                # Let the thread exit.  Avoids "Exception in thread...".
    except Exception as E:
        errmsg = '%s: E: %s @ %s' % (me, E, _m.tblineno())
        DOSQUAWK(errmsg)
        raise      
    finally:
        # Flush heartbeats and loadrecs.
        flushHeartbeats()
        loadrecs2db()
        if FWTSTOP:
            FWTSTOPPED = True
        FFWDB.disconnect()
        _sl.info('%s exits. STOPPED: %s' % (me, str(FWTSTOPPED)))
        FWTRUNNING = False