Ejemplo n.º 1
0
def sep_(sl, local, name, width, prefix, fillchar):
    """Make separator line for sepBegin, sepEnd."""
    # sl Falsey -> separator line is returned but not logged.
    if local:   ts, sep = _dt.locut(_dt.utcut()), 'L'
    else:       ts, sep = _dt.utcut(), 'U'
    sep = '{}{} '.format(prefix, _dt.ut2iso(ts, sep=sep))
    if name:
        sep += '{} '.format(name)
    n = width - len(sep)
    if n > 0:
        sep += n * fillchar
    if sl:
        sl._log(sep)
    return sep
Ejemplo n.º 2
0
def ownHeartbeat(uu=None):
    me = 'ownHeartbeat'
    logrec = None
    try:
        if uu is None:
                uu = _dt.utcut()
        ul = _dt.locut(uu)
        uuts = '%15.4f' % uu                                # 15.4, unblanked fraction.
        uuiosfs = _dt.ut2isofs(uu)
        uliosfs = _dt.ut2isofs(ul)
        rxts = txts = uuts
        kvs = {'_id': SRCID, '_si': SUBID, 
               '_el': HB_EL, '_sl': HB_SL, 
               '_ip': None, '_ts': uuts, 
               'dt_loc': uliosfs, 'dt_utc': uuiosfs}
        kvsa = json.dumps(kvs, ensure_ascii=True, sort_keys=True)
        kvsab = kvsa.encode(encoding=ENCODING, errors=ERRORS)
        h = hashlib.sha1()
        h.update(kvsab)
        sha1x = h.hexdigest()
        _ = '\t'
        logrec = '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' %('1', _, rxts, _, txts, _, SRCID, _, SUBID, _, HB_EL, _, HB_SL, _, sha1x, _, kvsa)
    except Exception as E:
        errmsg = '%s: %s @ %s' % (me, E, _m.tblineno())
        DOSQUAWK(errmsg)
        raise
    finally:
        return logrec
Ejemplo n.º 3
0
def update_ts(utcut=None):
    global UTC_TS, UTC_UT, UTC_TS_STR, UTC_YMD, UTC_HMS, LOC_YMD, LOC_HMS
    me = 'update_ts(%r)' % utcut
    try:
        if utcut:
            UTC_TS = utcut
        else:
            UTC_TS = _dt.utcut()
        UTC_TS_STR = '{:15.4f}'.format(UTC_TS)
        UTC_UT = int(UTC_TS)            # Truncate to integer.
        utc = time.gmtime(UTC_UT)
        loc = time.localtime(UTC_UT)
        UTC_YMD = '%02d%02d%02d' % (utc.tm_year % 100, utc.tm_mon, utc.tm_mday)
        UTC_HMS = '%02d%02d%02d' % (utc.tm_hour, utc.tm_min, utc.tm_sec)
        LOC_YMD = '%02d%02d%02d' % (loc.tm_year % 100, loc.tm_mon, loc.tm_mday)
        LOC_HMS = '%02d%02d%02d' % (loc.tm_hour, loc.tm_min, loc.tm_sec)
    except Exception as E:
        errmsg = '{}: {} @ {}'.format(me, E, m.tblineno())
        DOSQUAWK(errmsg)
        raise
Ejemplo n.º 4
0
def watcherThread():
    """A thread to watch WPATH for files to process."""
    global LOADRECS, FFWDB, FWTRUNNING, FWTSTOP, FWTSTOPPED

    LOADRECS = []

    me = 'watcher thread' 
    try:
        FWTRUNNING = True
        assert XLOGDB, 'no XLOGDB'

        # Connect to FlatFileWatchDataBase.
        FFWDB = ffwdb.FFWDB(FFWDBPFN)
        assert FFWDB, 'no FFWDB'

        uu = 0                                                  # Unix Utc.
        while not FWTSTOP:

            #
            flushHeartbeats()
           
            # Wait out INTERVAL.
            z = time.time()
            w = INTERVAL - (z - uu)
            if w > 0:
                _sw.wait(w)
            uu = _dt.utcut()
            ul = _dt.locut(uu)
            uuts = '%15.4f' % uu                                # 15.4, unblanked fraction.
            uuiosfs = _dt.ut2isofs(uu)
            uliosfs = _dt.ut2isofs(ul)

            # Heartbeat?
            if OWNHEARTBEAT:
                logrec = ownHeartbeat(uu)
                addHeartbeat(logrec)

            # Files?
            t0 = time.perf_counter();
            fis = getFIs(uu)
            t1 = time.perf_counter();
            if TIMINGS:
                _sl.warning('   getFIs: {:9,.1f} ms'.format((1000*(t1-t0))))
            if not fis:
                errmsg = 'no logfiles @ ' + uliosfs
                raise Exception(errmsg)

            # Update FFWDB. 
            # Freshen the "acquired" timestamp.
            # Delete entries for nonexistent files.
            t0 = time.perf_counter();
            filenames = [fi['filename'] for fi in fis]
            filenames.sort()
            # Compare real (nf) and db (nx) counts.
            nf = len(filenames)
            nx = FFWDB.count()
            if   nf < nx:
                nf, nx = nf, nx
            elif nf > nx:
                nf, nx = nf, nx
            else:
                nf, nx = nf, nx
            FFWDB.acquired(filenames, uu)
            for fi in fis:
                z = updateDB(fi)
            t1 = time.perf_counter();
            if TIMINGS:
                _sl.warning('updateDBs: {:9,.1f} ms'.format((1000*(t1-t0))))

            # Find the oldest, newest unfinished files in DB.
            t0 = time.perf_counter();
            o_dbfi, n_dbfi = FFWDB.oldestnewest('u')
            t1 = time.perf_counter();
            if TIMINGS:
                _sl.warning('   oldest: {:9,.1f} ms'.format((1000*(t1-t0))))

            # Something unfinished?
            if o_dbfi:  

                fn = o_dbfi['filename']
                historical = (fn != n_dbfi['filename'])

                # Export the file.
                exportFile(historical, o_dbfi)    # !CHANGE!

            # Move oldest finished file? 
            # It must not be the only, and therefore "live", file.
            if DONESD:
                o_dbfi, n_dbfi = FFWDB.oldestnewest('f')
                if o_dbfi and (n_dbfi['filename'] != o_dbfi['filename']):
                    t0 = time.perf_counter();
                    doneWithFile(o_dbfi['filename'])
                    t1 = time.perf_counter();
                    if TIMINGS:
                        _sl.warning('    moved: {:9,.1f} ms'.format((1000*(t1-t0))))

            if ONECHECK:
                FWTSTOP = True

    except KeyboardInterrupt as E:
        _m.beeps(1)
        # watcherThread:
        msg = '1: {}: KeyboardInterrupt: {}'.format(me, E)
        _sl.warning(msg)
        ###---DOSQUAWK(errmsg, beeps=1)
        pass###raise                # Let the thread exit.  Avoids "Exception in thread...".
    except Exception as E:
        errmsg = '%s: E: %s @ %s' % (me, E, _m.tblineno())
        DOSQUAWK(errmsg)
        raise      
    finally:
        # Flush heartbeats and loadrecs.
        flushHeartbeats()
        loadrecs2db()
        if FWTSTOP:
            FWTSTOPPED = True
        FFWDB.disconnect()
        _sl.info('%s exits. STOPPED: %s' % (me, str(FWTSTOPPED)))
        FWTRUNNING = False
Ejemplo n.º 5
0
        testfn = '151213-01.log'    # 174,586 bytes.
        testfn = '151213-02.log'    #  48,894 bytes.
        historical = False

        testfi = FFWDB.select(testfn)
        ###!!!testfi['processed'] = 0

        # 1/2: Optionally append stuff to logfile to 
        # simulate intraprocess mofification.
        # 151213-02.log -> 50,262 bytes.
        # Doing this here causes no export bcs testfi 
        # shows the file a fully processed.

        if False:
            # Update FFWDB and reget testfi.
            uu = _dt.utcut()
            fis = getFIs(uu)
            filenames = [fi['filename'] for fi in fis]
            filenames.sort()
            FFWDB.acquired(filenames, uu)
            for fi in fis:
                if fi['filename'] == testfn:
                    testfi = updateDB(fi)
        
        exportFile(historical, testfi)
 
        shutDown()

        1/1

    # Test 2.