Esempio n. 1
0
def sep_(sl, local, name, width, prefix, fillchar):
    """Make separator line for sepBegin, sepEnd."""
    # sl Falsey -> separator line is returned but not logged.
    if local:   ts, sep = _dt.locut(_dt.utcut()), 'L'
    else:       ts, sep = _dt.utcut(), 'U'
    sep = '{}{} '.format(prefix, _dt.ut2iso(ts, sep=sep))
    if name:
        sep += '{} '.format(name)
    n = width - len(sep)
    if n > 0:
        sep += n * fillchar
    if sl:
        sl._log(sep)
    return sep
Esempio n. 2
0
def exportFile(historical, xfi):
    """Export a file (from info dict)."""
    fn = xfi['filename']
    me = 'exportFile(%s, %s)' % (str(historical), fn)
    _sl.info('%s  %s  %s' % (_dt.ut2iso(_dt.locut()), fn, 'h' if historical else ''))#$#
    nb2e = 0                            # Finally references.
    try:

        # Safety flush.
        loadrecs2db()  

        # How many bytes of file is to be exported?
        fskip = xfi['processed']
        fsize = xfi['size']
        nb2e = fsize - fskip
        if nb2e <= 0:
            return

        # 2/2: Optionally append stuff to logfile to 
        # simulate intraprocess mofification.
        # 151213-02.log -> 50,262 bytes -> 51,630 bytes.

        # Still exists?
        pfn = os.path.normpath(WPATH + '/' + fn)
        if not os.path.isfile(pfn):
            _m.beeps(1)
            errmsg = '%s: file dne' % me
            _sl.warning(errmsg)
            return                  

        # .gz files are always treated as historical, and 
        # the whole file is read. (No seek!)
        if pfn.endswith('.gz'):          
            with gzip.open(pfn, 'r', encoding=ENCODING, errors=ERRORS) as f:
                for x, logrec in enumerate(f):
                    if not (x % 1000):
                        _sw.iw('.')
                    logrec2loadrecs(logrec)
            return

        # Uncompressed files are read as text, with an initial
        # seek from the SOF. The file is read to its end, even
        # if this goes beyond the size given, which will 
        # happen if XLOG appends to this file while we're 
        # processing it. This will result in the appendage 
        # being reprocessed next time around, but this is 
        # harmless bcs logrec and heartbeat processing is 
        # able to handle (skip) duplicates.
        with open(pfn, 'r', encoding=ENCODING, errors=ERRORS) as f:
            if fskip > 0:
                _sl.info('skipping {:,d} bytes'.format(fskip))
                f.seek(fskip)
            for x, logrec in enumerate(f):
                if not (x % 1000):
                    _sw.iw('.')
                logrec2loadrecs(logrec)

    except Exception as E:
        nb2e *= -1                      # Prevent 'processed' update.
        errmsg = '%s: %s @ %s' % (me, E, _m.tblineno())
        DOSQUAWK(errmsg)
        raise
    finally:
        # End dots.
        _sw.nl()                   
        # Close src file.
        try:  f.close
        except:  pass
        # Update 'processed'?
        if nb2e > 0 and not TESTONLY:
            xfi['processed'] = fsize
            z = {'filename': xfi['filename'], 'processed': xfi['processed']}
            FFWDB.update(z)
        # Flush heartbeats and loadrecs.
        flushHeartbeats()
        loadrecs2db()
Esempio n. 3
0
def xlog():
    global LFTSTOP 
    me, action = 'main', ''
    try:
        _sl.info(me + ' begins')#$#
        _sl.info()
        _sl.info('     host: ' + HOST)
        _sl.info('    ippfx: ' + IPPFX)
        _sl.info('     port: ' + str(PORT))
        _sl.info('       hp: ' + str(HP))
        _sl.info(' log_path: ' + LOG_PATH)
        _sl.info('  verbose: ' + str(VERBOSE))
        if VERBOSE:
            _sl.info('   viewer: ' + str(VIEWER))
            _sl.info('       vm: ' + repr(VM))
        _sl.info()

        startLogFileThread()

        # Fake a log record from self.  Fake the json'd dict.  Use '0.0.0.0' as self.
        z = '%s%s{"_id": "%s", "_si": "%s", "_el": %d, "_sl": "%s", "_msg": "%s"}' %\
            ('0.0.0.0', _, '----', '----', 0, '_', (me + ' begins @ %s' % _dt.ut2iso(_dt.locut())))
        (rc, rm, newrec) = reformatLogrec(z)
        LFQ.put(newrec)

        _sl.info('starting server on %r' % (HP, ))
        server = ThreadedServer(HP, Handler)
        server.daemon_threads = True            # !!! Crucial! (Otherwise responder threads never exit.)
        ip, port = server.server_address
        server_thread = threading.Thread(target=server.serve_forever)
        server_thread.daemon = True
        server_thread.start()
        _sl.info('server running in %s' % server_thread.name)

        while True:
            if XLOGSTOP:
                break
            time.sleep(1)

        _sl.warning('server shutdown')
        server.shutdown()
        _sl.warning('server close')
        server.server_close()
        
    except KeyboardInterrupt as E:
        errmsg = '{}: KeyboardInterrupt: {}'.format(me, E)
        DOSQUAWK(errmsg, beeps=1)
        pass###raise
    except Exception as E:
        errmsg = '{}: E: {} @ {}'.format(me, E, _m.tblineno())
        DOSQUAWK(errmsg)
        raise
    finally:
        # Either: a '!STOP!' record, a KeyboardInterrupt, or an Exception.
        if LFT:
            # Fake a received log record.
            z = '%s%s{"_id": "%s", "_el": %d, "_sl": "%s", "_msg": "%s"}' %\
                ('0.0.0.0', _, '----', 0, '_', (me + ' ends @ %s' % _dt.ut2iso(_dt.locut())))
            (rc, rm, newrec) = reformatLogrec(z)
            LFQ.put(newrec)

            # Wait some to let LFT empty tis queue.
            tw, w, mt = 0, 0.1, False
            while tw < 10:
                time.sleep(w)
                tw += w
                if LFQ.empty():
                    mt = True
                    break
            if not mt:
                errmsg = 'LFT did not empty its queue'
                _sl.error(errmsg)

            # Shut down log file writing thread.
            LFTSTOP = True
            try:
                LFT.join(10)
                if LFTSTOPPED:
                    pass
                else:
                    errmsg = 'LFT didn\'t acknowledge STOP request'
                    _sl.error(errmsg)
            except Exception as E:
                errmsg = 'LFT.join(10): %s' % E
                DOSQUAWK(errmsg)
                raise

        try:  LOG_FILE.close()
        except:  pass
        _sl.info(me + ' ends')#$#