Ejemplo n.º 1
0
def main():
    global VERBOSE
    global logfile
    parser = optparse.OptionParser()
    parser.set_defaults(infile=[])
    parser.add_option("--hang", action="callback", callback=handle_file_op)
    parser.add_option("--term", action="callback", type="int",
                      callback=handle_file_op)
    parser.add_option("--return", action="callback", type="int",
                      callback=handle_file_op)
    parser.add_option("--signal", action="callback", type="int",
                      callback=handle_file_op)
    parser.add_option("--exec", action="store_true", dest="execute",
                      default=False)
    parser.add_option("--basedir", action="store", type="string",
                      dest="basedir")
    parser.add_option("--no-archive", action="store_true", dest="no_archive",
                      default=False)
    parser.add_option("--daemon-timeout", action="store", type="int",
                      dest="daemon_timeout", default=60)
    parser.add_option("--verbose", action="store_true", dest="verbose",
                      default=False)
    parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
                      default=False)
    parser.add_option("--log-level", action="store", type="string",
                      dest="log_level", default="info")
    (options, args) = parser.parse_args()
    VERBOSE = options.verbose

    if options.execute:
        execute(args)           # Does not return


    # Create the dirs
    dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
    dirobj.dirs = ['incoming', 'archive', 'error', 'log']
    dirobj.create_dirs()

    # Make the log file
    logfile = open(dirobj.get_path('log', 'rwpollexec-daemon.log'), 'wb', 0)

    # Generate the subprocess arguments
    args += ['--log-dest', 'stderr',
             '--log-level', options.log_level,
             '--no-daemon',
             '--incoming-directory', dirobj.dirname['incoming'],
             '--error-directory', dirobj.dirname['error']]
    if not options.no_archive:
        args += ['--archive-directory', dirobj.dirname['archive']]

    progname = os.environ.get("RWPOLLEXEC",
                              os.path.join('.', 'rwpollexec'))
    args = progname.split() + args

    # Set up state variables
    run_count = 0
    complete_count = 0
    success_count = 0
    nonzero_count = 0
    killed_count = 0
    clean = False
    term = False
    limit = len(options.infile)
    regexp = re.compile("Running \[\d+\]:")
    success = "Command .* has completed successfully"
    nonzero = "Command .* has completed with a nonzero return"
    killed = "Command .* was terminated by SIG"
    complete = re.compile("(?P<success>%s)|(?P<nonzero>%s)|(?P<killed>%s)" %
                          (success, nonzero, killed))
    closing = re.compile("Stopped logging")

    if limit == 0:
        parser.print_usage()
        sys.exit(255)

    # Create data
    create_files(dirobj, options.infile)

    # Note the time
    starttime = time.time()
    shutdowntime = None
    endedtime = None

    # Start the process
    log("Running", "'%s'" % "' '".join(args))
    proc = subprocess.Popen(args, stderr=subprocess.PIPE,
                            preexec_fn=lambda:os.setpgid(0, 0))
    try:
        os.setpgid(proc.pid, 0)
    except OSError:
        pass
    line_reader = TimedReadline(proc.stderr.fileno())

    try:
        # Read the output data
        line = line_reader(1)
        while line is not None:
            if line:
                base_log(line)

            # Check for timeout
            if time.time() - starttime > options.daemon_timeout and not term:
                shutdowntime = time.time()
                log("Timed out")
                log("Sending SIGTERM")
                term = True
                try:
                    os.kill(proc.pid, signal.SIGTERM)
                except OSError:
                    pass

            # Match running counts
            match = regexp.search(line)
            if match:
                run_count += 1
                # Reset the timer if we are still receiving data
                starttime = time.time()

            # Match success counts
            match = complete.search(line)
            if match:
                if match.group('success') is not None:
                    success_count += 1
                elif match.group('nonzero') is not None:
                    nonzero_count += 1
                elif match.group('killed') is not None:
                    killed_count += 1
                complete_count += 1
                # Reset the timer if we are still receiving data
                starttime = time.time()
                if complete_count >= limit and not term:
                    shutdowntime = time.time()
                    log("Reached limit")
                    log("Sending SIGTERM")
                    term = True
                    try:
                        os.kill(proc.pid, signal.SIGTERM)
                    except OSError:
                        pass

            # Check for clean shutdown
            match = closing.search(line)
            if match:
                clean = True

            # Check for timeout on SIGTERM shutdown
            if shutdowntime and time.time() - shutdowntime > 15:
                log("Timeout on SIGTERM.  Sending SIGKILL")
                shutdowntime = None
                try:
                    os.killpg(proc.pid, signal.SIGKILL)
                    break
                except OSError:
                    pass

            # Check to see if the process has ended
            if endedtime is None:
                proc.poll()
                if proc.returncode is not None:
                    endedtime = time.time()

            # Allow some time after the process has ended for any
            # remaining output the be handled
            if endedtime and time.time() - endedtime > 5:
                break

            # Get next line
            line = line_reader(1)

    finally:
        # Sleep briefly before polling for exit.
        time.sleep(1)
        proc.poll()
        if proc.returncode is None:
            log("Daemon has not exited.  Sending SIGKILL")
            try:
                os.killpg(proc.pid, signal.SIGKILL)
            except OSError:
                pass

    # Kill everything for good luck
    try:
        os.killpg(proc.pid, signal.SIGKILL)
    except OSError:
        pass

    # Print record count
    print("Run count:", run_count)
    print(("Complete count (success, failure): (%s, %s) = %s" %
           (success_count, nonzero_count + killed_count, complete_count)))
    base_log("Run count:", run_count)
    base_log(("Complete count (success, failure): (%s, %s) = %s" %
              (success_count, nonzero_count + killed_count, complete_count)))
    if limit != complete_count:
        print(("ERROR: expecting %s files, got %s files" %
               (limit, complete_count)))
        base_log(("ERROR: expecting %s files, got %s files" %
                  (limit, complete_count)))

    # Check for clean exit
    if not clean:
        print("ERROR: shutdown was not clean")
        base_log("ERROR: shutdown was not clean")

    # Exit with the process's error code
    if proc.returncode is None:
        log("Exiting: Deamon did not appear to terminate normally")
        sys.exit(1)
    else:
        log("Exiting: Daemon returned", proc.returncode)
        sys.exit(proc.returncode)
Ejemplo n.º 2
0
def main():
    global VERBOSE
    global logfile
    parser = optparse.OptionParser()
    parser.add_option("--copy", action="append", type="string", dest="copy",
                      default=[])
    parser.add_option("--move", action="append", type="string", dest="move",
                      default=[])
    parser.add_option("--basedir", action="store", type="string",
                      dest="basedir")
    parser.add_option("--daemon-timeout", action="store", type="int",
                      dest="daemon_timeout", default=60)
    parser.add_option("--verbose", action="store_true", dest="verbose",
                      default=False)
    parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
                      default=False)
    parser.add_option("--log-level", action="store", type="string",
                      dest="log_level", default="info")
    (options, args) = parser.parse_args()
    VERBOSE = options.verbose

    # Create the dirs
    dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
    dirobj.dirs = ['incoming', 'archive', 'error', 'root', 'log']
    dirobj.create_dirs()

    # Make the log file
    logfile = open(dirobj.get_path('log', 'rwflowappend-daemon.log'), 'wb', 0)

    # Generate the subprocess arguments
    args += ['--log-dest', 'stderr',
             '--log-level', options.log_level,
             '--no-daemon',
             '--root-directory', dirobj.dirname['root'],
             '--incoming-directory', dirobj.dirname['incoming'],
             '--archive-directory', dirobj.dirname['archive'],
             '--error-directory', dirobj.dirname['error']]

    progname = os.environ.get("RWFLOWAPPEND",
                              os.path.join('.', 'rwflowappend'))
    args = progname.split() + args

    # Set up state variables
    count = 0
    clean = False
    term = False
    send_list = None
    limit = len(options.copy) + len(options.move)
    regexp = re.compile("APPEND OK")
    closing = re.compile("Stopped logging")

    # Copy or move data
    copy_files(dirobj, options.copy)
    move_files(dirobj, options.move)

    # Note the time
    starttime = time.time()
    shutdowntime = None

    # Start the process
    log("Running", "'%s'" % "' '".join(args))
    proc = subprocess.Popen(args, stderr=subprocess.PIPE)
    line_reader = TimedReadline(proc.stderr.fileno())

    try:
        # Read the output data
        line = line_reader(1)
        while line is not None:
            if line:
                base_log(line)

            # Check for timeout
            if time.time() - starttime > options.daemon_timeout and not term:
                shutdowntime = time.time()
                log("Timed out")
                log("Sending SIGTERM")
                term = True
                try:
                    os.kill(proc.pid, signal.SIGTERM)
                except OSError:
                    pass

            # Match record counts
            match = regexp.search(line)
            if match:
                count += 1
                # Reset the timer if we are still receiving data
                starttime = time.time()
                if count >= limit and not term:
                    shutdowntime = time.time()
                    log("Reached limit")
                    log("Sending SIGTERM")
                    term = True
                    try:
                        os.kill(proc.pid, signal.SIGTERM)
                    except OSError:
                        pass

            # Check for clean shutdown
            match = closing.search(line)
            if match:
                clean = True

            # Check for timeout on SIGTERM shutdown
            if shutdowntime and time.time() - shutdowntime > 15:
                log("Timeout on SIGTERM.  Sending SIGKILL")
                shutdowntime = None
                try:
                    os.kill(proc.pid, signal.SIGKILL)
                except OSError:
                    pass

            # Get next line
            line = line_reader(1)

    finally:
        # Stop sending network data
        if send_list:
            stop_network_data(send_list)

        # Sleep briefly before polling for exit.
        time.sleep(1)
        proc.poll()
        if proc.returncode is None:
            log("Daemon has not exited.  Sending SIGKILL")
            try:
                os.kill(proc.pid, signal.SIGKILL)
            except OSError:
                pass

    # Print record count
    print("File count:", count)
    base_log("File count:", count)
    if limit != count:
        print(("ERROR: expecting %s files, got %s files" %
               (limit, count)))
        base_log(("ERROR: expecting %s files, got %s files" %
                  (limit, count)))

    # Check for clean exit
    if not clean:
        print("ERROR: shutdown was not clean")
        base_log("ERROR: shutdown was not clean")

    # Exit with the process's error code
    if proc.returncode is None:
        log("Exiting: Deamon did not appear to terminate normally")
        sys.exit(1)
    else:
        log("Exiting: Daemon returned", proc.returncode)
        sys.exit(proc.returncode)
Ejemplo n.º 3
0
def main():
    global VERBOSE
    global logfile
    parser = optparse.OptionParser()
    parser.add_option("--pdu",
                      action="append",
                      type="string",
                      dest="pdu",
                      default=[])
    parser.add_option("--tcp",
                      action="append",
                      type="string",
                      dest="tcp",
                      default=[])
    parser.add_option("--basedir",
                      action="store",
                      type="string",
                      dest="basedir")
    parser.add_option("--daemon-timeout",
                      action="store",
                      type="int",
                      dest="daemon_timeout",
                      default=60)
    parser.add_option("--limit", action="store", type="int", dest="limit")
    parser.add_option("--timeout",
                      action="store",
                      type="int",
                      dest="timeout",
                      default=10)
    parser.add_option("--verbose",
                      action="store_true",
                      dest="verbose",
                      default=False)
    parser.add_option("--overwrite-dirs",
                      action="store_true",
                      dest="overwrite",
                      default=False)
    parser.add_option("--log-level",
                      action="store",
                      type="string",
                      dest="log_level",
                      default="info")
    (options, args) = parser.parse_args()
    VERBOSE = options.verbose

    # Create the dirs
    dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
    dirobj.dirs = ['destination', 'log']
    dirobj.create_dirs()

    # Make the log file
    logfile = open(dirobj.get_path('log', 'flowcap-daemon.log'), 'wb', 0)

    # Generate the subprocess arguments
    args += [
        '--timeout',
        str(options.timeout), '--log-dest', 'stderr', '--log-level',
        options.log_level, '--no-daemon'
    ]
    args += ['--destination-directory', dirobj.dirname['destination']]

    progname = os.environ.get("FLOWCAP", os.path.join('.', 'flowcap'))
    args = progname.split() + args

    # Set up state variables
    count = 0
    clean = False
    term = False
    send_list = None
    regexp = re.compile("Closing file .* seconds, (?P<recs>[0-9]+) records,")
    closing = re.compile("Stopped logging")
    started = re.compile("'.+': Reader thread started")

    # Note the time
    starttime = time.time()
    shutdowntime = None

    # Start the process
    log("Running", "'%s'" % "' '".join(args))
    proc = subprocess.Popen(args, stderr=subprocess.PIPE)
    line_reader = TimedReadline(proc.stderr.fileno())

    try:
        # Read the output data
        line = line_reader(1)
        while line is not None:
            if line:
                base_log(line)

            # Check for timeout
            if time.time() - starttime > options.daemon_timeout and not term:
                shutdowntime = time.time()
                log("Timed out")
                log("Sending SIGTERM")
                term = True
                try:
                    os.kill(proc.pid, signal.SIGTERM)
                except OSError:
                    pass

            # Match record counts
            match = regexp.search(line)
            if match:
                num = int(match.group('recs'))
                if num > 0:
                    count += num
                    # Reset the timer if we are still receiving data
                    starttime = time.time()
                    if options.limit and count >= options.limit and not term:
                        shutdowntime = time.time()
                        log("Reached limit")
                        log("Sending SIGTERM")
                        term = True
                        try:
                            os.kill(proc.pid, signal.SIGTERM)
                        except OSError:
                            pass

            # check for starting up network data
            if started:
                match = started.search(line)
                if match and send_list is None:
                    send_list = send_network_data(options)
                    started = None

            # Check for clean shutdown
            match = closing.search(line)
            if match:
                clean = True

            # Check for timeout on SIGTERM shutdown
            if shutdowntime and time.time() - shutdowntime > 15:
                log("Timeout on SIGTERM.  Sending SIGKILL")
                shutdowntime = None
                try:
                    os.kill(proc.pid, signal.SIGKILL)
                except OSError:
                    pass

            # Get next line
            line = line_reader(1)

    finally:
        # Stop sending network data
        if send_list:
            stop_network_data(send_list)

        # Sleep briefly before polling for exit.
        time.sleep(1)
        proc.poll()
        if proc.returncode is None:
            log("Daemon has not exited.  Sending SIGKILL")
            try:
                os.kill(proc.pid, signal.SIGKILL)
            except OSError:
                pass

    # Print record count
    print("Record count:", count)
    base_log("Record count:", count)
    if options.limit and options.limit != count:
        print(("ERROR: expecting %s records, got %s records" %
               (options.limit, count)))
        base_log(("ERROR: expecting %s records, got %s records" %
                  (options.limit, count)))

    # Check for clean exit
    if not clean:
        print("ERROR: shutdown was not clean")
        base_log("ERROR: shutdown was not clean")

    # Exit with the process's error code
    if proc.returncode is None:
        log("Exiting: Deamon did not appear to terminate normally")
        sys.exit(1)
    else:
        log("Exiting: Daemon returned", proc.returncode)
        sys.exit(proc.returncode)
Ejemplo n.º 4
0
def main():
    global VERBOSE
    global logfile
    parser = optparse.OptionParser()
    parser.add_option("--pdu", action="append", type="string", dest="pdu",
                      default=[])
    parser.add_option("--tcp", action="append", type="string", dest="tcp",
                      default=[])
    parser.add_option("--copy", action="append", type="string", dest="copy",
                      default=[])
    parser.add_option("--move", action="append", type="string", dest="move",
                      default=[])
    parser.add_option("--copy-after", action="append", type="string",
                      dest="copy_after", default=[])
    parser.add_option("--move-after", action="append", type="string",
                      dest="move_after", default=[])
    parser.add_option("--input-mode", action="store", type="string",
                      dest="input_mode")
    parser.add_option("--output-mode", action="store", type="string",
                      dest="output_mode")
    parser.add_option("--basedir", action="store", type="string",
                      dest="basedir")
    parser.add_option("--daemon-timeout", action="store", type="int",
                      dest="daemon_timeout", default=60)
    parser.add_option("--limit", action="store", type="int", dest="limit")
    parser.add_option("--sensor-configuration", action="store", type="string",
                      dest="sensor_conf")
    parser.add_option("--flush-timeout", action="store", type="int",
                      dest="flush_timeout", default=10)
    parser.add_option("--verbose", action="store_true", dest="verbose",
                      default=False)
    parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
                      default=False)
    parser.add_option("--log-level", action="store", type="string",
                      dest="log_level", default="info")
    (options, args) = parser.parse_args()
    VERBOSE = options.verbose

    # Create the dirs
    dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
    dirobj.dirs = ['incoming', 'archive', 'error', 'sender',
                   'incremental', 'root', 'log', 'incoming2',
                   'incoming3', 'incoming4']
    dirobj.create_dirs()

    # Make the log file
    logfile = open(dirobj.get_path('log', 'rwflowpack-daemon.log'), 'wb', 0)

    # Create the configuration file from the template
    if options.sensor_conf:
        conf_file = open(options.sensor_conf, "r").read()
        new_conf = string.Template(conf_file)
        (fd, conf) = tempfile.mkstemp(dir=dirobj.basedir)
        out_file = os.fdopen(fd, "w")
        out_file.write(new_conf.substitute(dirobj.dirname))
        out_file.close()

    # Generate the subprocess arguments
    args += ['--flush-timeout', str(options.flush_timeout),
             '--log-dest', 'stderr',
             '--log-level', options.log_level,
             '--no-daemon']
    if options.input_mode == "fcfiles" or options.input_mode == "respool":
        args += ['--incoming-directory', dirobj.dirname['incoming']]
    args += ['--archive-directory', dirobj.dirname['archive'],
             '--error-directory', dirobj.dirname['error']]
    if options.output_mode == "sending":
        args += ['--sender-directory', dirobj.dirname['sender'],
                 '--incremental-directory', dirobj.dirname['incremental']]
    elif options.output_mode == "incremental-files":
        args += ['--incremental-directory', dirobj.dirname['incremental']]
    else:
        args += ['--root-directory', dirobj.dirname['root']]
    if options.input_mode:
        args += ['--input-mode', options.input_mode]
    if options.output_mode:
        args += ['--output-mode', options.output_mode]
    if options.sensor_conf:
        args += ['--sensor-configuration', conf]

    progname = os.environ.get("RWFLOWPACK", os.path.join('.', 'rwflowpack'))
    args = progname.split() + args

    # Set up state variables
    count = 0
    clean = False
    term = False
    send_list = None
    regexp = re.compile(": /[^:].*: (?P<recs>[0-9]+) recs")
    closing = re.compile("Stopped logging")
    started = re.compile("Starting flush timer")

    # Copy or move data
    copy_files(dirobj, options.copy)
    move_files(dirobj, options.move)

    # Note the time
    starttime = time.time()
    shutdowntime = None

    # Start the process
    log("Running", "'%s'" % "' '".join(args))
    proc = subprocess.Popen(args, stderr=subprocess.PIPE)
    line_reader = TimedReadline(proc.stderr.fileno())

    try:
        # Read the output data
        line = line_reader(1)
        while line is not None:
            if line:
                base_log(line)

            # Check for timeout
            if time.time() - starttime > options.daemon_timeout and not term:
                shutdowntime = time.time()
                log("Timed out")
                log("Sending SIGTERM")
                term = True
                try:
                    os.kill(proc.pid, signal.SIGTERM)
                except OSError:
                    pass

            # Match record counts
            match = regexp.search(line)
            if match:
                num = int(match.group('recs'))
                if num > 0:
                    count += num
                    # Reset the timer if we are still receiving data
                    starttime = time.time()
                    if options.limit and count >= options.limit and not term:
                        shutdowntime = time.time()
                        log("Reached limit")
                        log("Sending SIGTERM")
                        term = True
                        try:
                            os.kill(proc.pid, signal.SIGTERM)
                        except OSError:
                            pass

            # check for starting up network data or after-data
            if started:
                match = started.search(line)
                if match:
                    if send_list is None:
                        send_list = send_network_data(options)
                    copy_files(dirobj, options.copy_after)
                    move_files(dirobj, options.move_after)
                    started = None

            # Check for clean shutdown
            match = closing.search(line)
            if match:
                clean = True

            # Check for timeout on SIGTERM shutdown
            if shutdowntime and time.time() - shutdowntime > 15:
                log("Timeout on SIGTERM.  Sending SIGKILL")
                shutdowntime = None
                try:
                    os.kill(proc.pid, signal.SIGKILL)
                except OSError:
                    pass

            # Get next line
            line = line_reader(1)

    finally:
        # Stop sending network data
        if send_list:
            stop_network_data(send_list)

        # Sleep briefly before polling for exit.
        time.sleep(1)
        proc.poll()
        if proc.returncode is None:
            log("Daemon has not exited.  Sending SIGKILL")
            try:
                os.kill(proc.pid, signal.SIGKILL)
            except OSError:
                pass

    # Print record count
    print("Record count:", count)
    base_log("Record count:", count)
    if options.limit and options.limit != count:
        print(("ERROR: expecting %s records, got %s records" %
               (options.limit, count)))
        base_log(("ERROR: expecting %s records, got %s records" %
                  (options.limit, count)))

    # Check for clean exit
    if not clean:
        print("ERROR: shutdown was not clean")
        base_log("ERROR: shutdown was not clean")

    # Exit with the process's error code
    if proc.returncode is None:
        log("Exiting: Deamon did not appear to terminate normally")
        sys.exit(1)
    else:
        log("Exiting: Daemon returned", proc.returncode)
        sys.exit(proc.returncode)
Ejemplo n.º 5
0
def main():
    global VERBOSE
    global logfile
    parser = optparse.OptionParser()
    parser.add_option("--pdu", action="append", type="string", dest="pdu",
                      default=[])
    parser.add_option("--tcp", action="append", type="string", dest="tcp",
                      default=[])
    parser.add_option("--basedir", action="store", type="string",
                      dest="basedir")
    parser.add_option("--daemon-timeout", action="store", type="int",
                      dest="daemon_timeout", default=60)
    parser.add_option("--limit", action="store", type="int", dest="limit")
    parser.add_option("--timeout", action="store", type="int",
                      dest="timeout", default=10)
    parser.add_option("--verbose", action="store_true", dest="verbose",
                      default=False)
    parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
                      default=False)
    parser.add_option("--log-level", action="store", type="string",
                      dest="log_level", default="info")
    (options, args) = parser.parse_args()
    VERBOSE = options.verbose

    # Create the dirs
    dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
    dirobj.dirs = ['destination', 'log']
    dirobj.create_dirs()

    # Make the log file
    logfile = open(dirobj.get_path('log', 'flowcap-daemon.log'), 'wb', 0)

    # Generate the subprocess arguments
    args += ['--timeout', str(options.timeout),
             '--log-dest', 'stderr',
             '--log-level', options.log_level,
             '--no-daemon']
    args += ['--destination-directory', dirobj.dirname['destination']]

    progname = os.environ.get("FLOWCAP", os.path.join('.', 'flowcap'))
    args = progname.split() + args

    # Set up state variables
    count = 0
    clean = False
    term = False
    send_list = None
    regexp = re.compile("Closing file .* seconds, (?P<recs>[0-9]+) records,")
    closing = re.compile("Stopped logging")
    started = re.compile("'.+': Reader thread started")

    # Note the time
    starttime = time.time()
    shutdowntime = None

    # Start the process
    log("Running", "'%s'" % "' '".join(args))
    proc = subprocess.Popen(args, stderr=subprocess.PIPE)
    line_reader = TimedReadline(proc.stderr.fileno())

    try:
        # Read the output data
        line = line_reader(1)
        while line is not None:
            if line:
                base_log(line)

            # Check for timeout
            if time.time() - starttime > options.daemon_timeout and not term:
                shutdowntime = time.time()
                log("Timed out")
                log("Sending SIGTERM")
                term = True
                try:
                    os.kill(proc.pid, signal.SIGTERM)
                except OSError:
                    pass

            # Match record counts
            match = regexp.search(line)
            if match:
                num = int(match.group('recs'))
                if num > 0:
                    count += num
                    # Reset the timer if we are still receiving data
                    starttime = time.time()
                    if options.limit and count >= options.limit and not term:
                        shutdowntime = time.time()
                        log("Reached limit")
                        log("Sending SIGTERM")
                        term = True
                        try:
                            os.kill(proc.pid, signal.SIGTERM)
                        except OSError:
                            pass

            # check for starting up network data
            if started:
                match = started.search(line)
                if match and send_list is None:
                    send_list = send_network_data(options)
                    started = None

            # Check for clean shutdown
            match = closing.search(line)
            if match:
                clean = True

            # Check for timeout on SIGTERM shutdown
            if shutdowntime and time.time() - shutdowntime > 15:
                log("Timeout on SIGTERM.  Sending SIGKILL")
                shutdowntime = None
                try:
                    os.kill(proc.pid, signal.SIGKILL)
                except OSError:
                    pass

            # Get next line
            line = line_reader(1)

    finally:
        # Stop sending network data
        if send_list:
            stop_network_data(send_list)

        # Sleep briefly before polling for exit.
        time.sleep(1)
        proc.poll()
        if proc.returncode is None:
            log("Daemon has not exited.  Sending SIGKILL")
            try:
                os.kill(proc.pid, signal.SIGKILL)
            except OSError:
                pass

    # Print record count
    print("Record count:", count)
    base_log("Record count:", count)
    if options.limit and options.limit != count:
        print(("ERROR: expecting %s records, got %s records" %
               (options.limit, count)))
        base_log(("ERROR: expecting %s records, got %s records" %
                  (options.limit, count)))

    # Check for clean exit
    if not clean:
        print("ERROR: shutdown was not clean")
        base_log("ERROR: shutdown was not clean")

    # Exit with the process's error code
    if proc.returncode is None:
        log("Exiting: Deamon did not appear to terminate normally")
        sys.exit(1)
    else:
        log("Exiting: Daemon returned", proc.returncode)
        sys.exit(proc.returncode)
Ejemplo n.º 6
0
def main():
    global VERBOSE
    global logfile
    parser = optparse.OptionParser()
    parser.add_option("--pdu",
                      action="append",
                      type="string",
                      dest="pdu",
                      default=[])
    parser.add_option("--tcp",
                      action="append",
                      type="string",
                      dest="tcp",
                      default=[])
    parser.add_option("--copy",
                      action="append",
                      type="string",
                      dest="copy",
                      default=[])
    parser.add_option("--move",
                      action="append",
                      type="string",
                      dest="move",
                      default=[])
    parser.add_option("--copy-after",
                      action="append",
                      type="string",
                      dest="copy_after",
                      default=[])
    parser.add_option("--move-after",
                      action="append",
                      type="string",
                      dest="move_after",
                      default=[])
    parser.add_option("--input-mode",
                      action="store",
                      type="string",
                      dest="input_mode")
    parser.add_option("--output-mode",
                      action="store",
                      type="string",
                      dest="output_mode")
    parser.add_option("--basedir",
                      action="store",
                      type="string",
                      dest="basedir")
    parser.add_option("--daemon-timeout",
                      action="store",
                      type="int",
                      dest="daemon_timeout",
                      default=60)
    parser.add_option("--limit", action="store", type="int", dest="limit")
    parser.add_option("--sensor-configuration",
                      action="store",
                      type="string",
                      dest="sensor_conf")
    parser.add_option("--flush-timeout",
                      action="store",
                      type="int",
                      dest="flush_timeout",
                      default=10)
    parser.add_option("--verbose",
                      action="store_true",
                      dest="verbose",
                      default=False)
    parser.add_option("--overwrite-dirs",
                      action="store_true",
                      dest="overwrite",
                      default=False)
    parser.add_option("--log-level",
                      action="store",
                      type="string",
                      dest="log_level",
                      default="info")
    (options, args) = parser.parse_args()
    VERBOSE = options.verbose

    # Create the dirs
    dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
    dirobj.dirs = [
        'incoming', 'archive', 'error', 'sender', 'incremental', 'root', 'log',
        'incoming2', 'incoming3', 'incoming4'
    ]
    dirobj.create_dirs()

    # Make the log file
    logfile = open(dirobj.get_path('log', 'rwflowpack-daemon.log'), 'wb', 0)

    # Create the configuration file from the template
    if options.sensor_conf:
        conf_file = open(options.sensor_conf, "r").read()
        new_conf = string.Template(conf_file)
        (fd, conf) = tempfile.mkstemp(dir=dirobj.basedir)
        out_file = os.fdopen(fd, "w")
        out_file.write(new_conf.substitute(dirobj.dirname))
        out_file.close()

    # Generate the subprocess arguments
    args += [
        '--flush-timeout',
        str(options.flush_timeout), '--log-dest', 'stderr', '--log-level',
        options.log_level, '--no-daemon'
    ]
    if options.input_mode == "fcfiles" or options.input_mode == "respool":
        args += ['--incoming-directory', dirobj.dirname['incoming']]
    args += [
        '--archive-directory', dirobj.dirname['archive'], '--error-directory',
        dirobj.dirname['error']
    ]
    if options.output_mode == "sending":
        args += [
            '--sender-directory', dirobj.dirname['sender'],
            '--incremental-directory', dirobj.dirname['incremental']
        ]
    elif options.output_mode == "incremental-files":
        args += ['--incremental-directory', dirobj.dirname['incremental']]
    else:
        args += ['--root-directory', dirobj.dirname['root']]
    if options.input_mode:
        args += ['--input-mode', options.input_mode]
    if options.output_mode:
        args += ['--output-mode', options.output_mode]
    if options.sensor_conf:
        args += ['--sensor-configuration', conf]

    progname = os.environ.get("RWFLOWPACK", os.path.join('.', 'rwflowpack'))
    args = progname.split() + args

    # Set up state variables
    count = 0
    clean = False
    term = False
    send_list = None
    regexp = re.compile(": /[^:].*: (?P<recs>[0-9]+) recs")
    closing = re.compile("Stopped logging")
    started = re.compile("Starting flush timer")

    # Copy or move data
    copy_files(dirobj, options.copy)
    move_files(dirobj, options.move)

    # Note the time
    starttime = time.time()
    shutdowntime = None

    # Start the process
    log("Running", "'%s'" % "' '".join(args))
    proc = subprocess.Popen(args, stderr=subprocess.PIPE)
    line_reader = TimedReadline(proc.stderr.fileno())

    try:
        # Read the output data
        line = line_reader(1)
        while line is not None:
            if line:
                base_log(line)

            # Check for timeout
            if time.time() - starttime > options.daemon_timeout and not term:
                shutdowntime = time.time()
                log("Timed out")
                log("Sending SIGTERM")
                term = True
                try:
                    os.kill(proc.pid, signal.SIGTERM)
                except OSError:
                    pass

            # Match record counts
            match = regexp.search(line)
            if match:
                num = int(match.group('recs'))
                if num > 0:
                    count += num
                    # Reset the timer if we are still receiving data
                    starttime = time.time()
                    if options.limit and count >= options.limit and not term:
                        shutdowntime = time.time()
                        log("Reached limit")
                        log("Sending SIGTERM")
                        term = True
                        try:
                            os.kill(proc.pid, signal.SIGTERM)
                        except OSError:
                            pass

            # check for starting up network data or after-data
            if started:
                match = started.search(line)
                if match:
                    if send_list is None:
                        send_list = send_network_data(options)
                    copy_files(dirobj, options.copy_after)
                    move_files(dirobj, options.move_after)
                    started = None

            # Check for clean shutdown
            match = closing.search(line)
            if match:
                clean = True

            # Check for timeout on SIGTERM shutdown
            if shutdowntime and time.time() - shutdowntime > 15:
                log("Timeout on SIGTERM.  Sending SIGKILL")
                shutdowntime = None
                try:
                    os.kill(proc.pid, signal.SIGKILL)
                except OSError:
                    pass

            # Get next line
            line = line_reader(1)

    finally:
        # Stop sending network data
        if send_list:
            stop_network_data(send_list)

        # Sleep briefly before polling for exit.
        time.sleep(1)
        proc.poll()
        if proc.returncode is None:
            log("Daemon has not exited.  Sending SIGKILL")
            try:
                os.kill(proc.pid, signal.SIGKILL)
            except OSError:
                pass

    # Print record count
    print("Record count:", count)
    base_log("Record count:", count)
    if options.limit and options.limit != count:
        print(("ERROR: expecting %s records, got %s records" %
               (options.limit, count)))
        base_log(("ERROR: expecting %s records, got %s records" %
                  (options.limit, count)))

    # Check for clean exit
    if not clean:
        print("ERROR: shutdown was not clean")
        base_log("ERROR: shutdown was not clean")

    # Exit with the process's error code
    if proc.returncode is None:
        log("Exiting: Deamon did not appear to terminate normally")
        sys.exit(1)
    else:
        log("Exiting: Daemon returned", proc.returncode)
        sys.exit(proc.returncode)
Ejemplo n.º 7
0
def main():
    global VERBOSE
    global logfile
    parser = optparse.OptionParser()
    parser.add_option("--copy",
                      action="append",
                      type="string",
                      dest="copy",
                      default=[])
    parser.add_option("--move",
                      action="append",
                      type="string",
                      dest="move",
                      default=[])
    parser.add_option("--file-limit",
                      action="store",
                      type="int",
                      dest="file_limit",
                      default=0)
    parser.add_option("--no-archive",
                      action="store_true",
                      dest="no_archvie",
                      default=False)
    parser.add_option("--basedir",
                      action="store",
                      type="string",
                      dest="basedir")
    parser.add_option("--daemon-timeout",
                      action="store",
                      type="int",
                      dest="daemon_timeout",
                      default=60)
    parser.add_option("--verbose",
                      action="store_true",
                      dest="verbose",
                      default=False)
    parser.add_option("--overwrite-dirs",
                      action="store_true",
                      dest="overwrite",
                      default=False)
    parser.add_option("--log-level",
                      action="store",
                      type="string",
                      dest="log_level",
                      default="info")
    (options, args) = parser.parse_args()
    VERBOSE = options.verbose

    # Create the dirs
    dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
    dirobj.dirs = ['incoming', 'archive', 'error', 'root', 'log']
    dirobj.create_dirs()

    # Make the log file
    logfile = open(dirobj.get_path('log', 'rwflowappend-daemon.log'), 'wb', 0)

    # Generate the subprocess arguments
    args += [
        '--log-dest', 'stderr', '--log-level', options.log_level,
        '--no-daemon', '--root-directory', dirobj.dirname['root'],
        '--incoming-directory', dirobj.dirname['incoming'],
        '--error-directory', dirobj.dirname['error']
    ]

    if not options.no_archvie:
        args += ['--archive-directory', dirobj.dirname['archive']]

    progname = os.environ.get("RWFLOWAPPEND",
                              os.path.join('.', 'rwflowappend'))
    args = progname.split() + args

    # Set up state variables
    count = 0
    clean = False
    term = False
    send_list = None
    limit = len(options.copy) + len(options.move) + options.file_limit
    regexp = re.compile("APPEND OK")
    closing = re.compile("Stopped logging")

    if 0 == limit:
        print("ERROR: File limit is zero")
        base_log("ERROR: File limit is zero")
        sys.exit(1)

    # Copy or move data
    copy_files(dirobj, options.copy)
    move_files(dirobj, options.move)

    # Note the time
    starttime = time.time()
    shutdowntime = None

    # Start the process
    log("Running", "'%s'" % "' '".join(args))
    proc = subprocess.Popen(args, stderr=subprocess.PIPE)
    line_reader = TimedReadline(proc.stderr.fileno())

    try:
        # Read the output data
        line = line_reader(1)
        while line is not None:
            if line:
                base_log(line)

            # Check for timeout
            if time.time() - starttime > options.daemon_timeout and not term:
                shutdowntime = time.time()
                log("Timed out")
                log("Sending SIGTERM")
                term = True
                try:
                    os.kill(proc.pid, signal.SIGTERM)
                except OSError:
                    pass

            # Match record counts
            match = regexp.search(line)
            if match:
                count += 1
                # Reset the timer if we are still receiving data
                starttime = time.time()
                if count >= limit and not term:
                    shutdowntime = time.time()
                    log("Reached limit")
                    log("Sending SIGTERM")
                    term = True
                    try:
                        os.kill(proc.pid, signal.SIGTERM)
                    except OSError:
                        pass

            # Check for clean shutdown
            match = closing.search(line)
            if match:
                clean = True

            # Check for timeout on SIGTERM shutdown
            if shutdowntime and time.time() - shutdowntime > 15:
                log("Timeout on SIGTERM.  Sending SIGKILL")
                shutdowntime = None
                try:
                    os.kill(proc.pid, signal.SIGKILL)
                except OSError:
                    pass

            # Get next line
            line = line_reader(1)

    finally:
        # Stop sending network data
        if send_list:
            stop_network_data(send_list)

        # Sleep briefly before polling for exit.
        time.sleep(1)
        proc.poll()
        if proc.returncode is None:
            log("Daemon has not exited.  Sending SIGKILL")
            try:
                os.kill(proc.pid, signal.SIGKILL)
            except OSError:
                pass

    # Print record count
    print("File count:", count)
    base_log("File count:", count)
    if limit != count:
        print(("ERROR: expecting %s files, got %s files" % (limit, count)))
        base_log(("ERROR: expecting %s files, got %s files" % (limit, count)))

    # Check for clean exit
    if not clean:
        print("ERROR: shutdown was not clean")
        base_log("ERROR: shutdown was not clean")

    # Exit with the process's error code
    if proc.returncode is None:
        log("Exiting: Deamon did not appear to terminate normally")
        sys.exit(1)
    else:
        log("Exiting: Daemon returned", proc.returncode)
        sys.exit(proc.returncode)
Ejemplo n.º 8
0
def main():
    global VERBOSE
    global logfile
    parser = optparse.OptionParser()
    parser.set_defaults(infile=[])
    parser.add_option("--hang", action="callback", callback=handle_file_op)
    parser.add_option("--term", action="callback", type="int",
                      callback=handle_file_op)
    parser.add_option("--return", action="callback", type="int",
                      callback=handle_file_op)
    parser.add_option("--signal", action="callback", type="int",
                      callback=handle_file_op)
    parser.add_option("--exec", action="store_true", dest="execute",
                      default=False)
    parser.add_option("--basedir", action="store", type="string",
                      dest="basedir")
    parser.add_option("--no-archive", action="store_true", dest="no_archive",
                      default=False)
    parser.add_option("--daemon-timeout", action="store", type="int",
                      dest="daemon_timeout", default=60)
    parser.add_option("--verbose", action="store_true", dest="verbose",
                      default=False)
    parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
                      default=False)
    parser.add_option("--log-level", action="store", type="string",
                      dest="log_level", default="info")
    (options, args) = parser.parse_args()
    VERBOSE = options.verbose

    if options.execute:
        execute(args)           # Does not return


    # Create the dirs
    dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
    dirobj.dirs = ['incoming', 'archive', 'error', 'log']
    dirobj.create_dirs()

    # Make the log file
    logfile = open(dirobj.get_path('log', 'rwpollexec-daemon.log'), 'wb', 0)

    # Generate the subprocess arguments
    args += ['--log-dest', 'stderr',
             '--log-level', options.log_level,
             '--no-daemon',
             '--incoming-directory', dirobj.dirname['incoming'],
             '--error-directory', dirobj.dirname['error']]
    if not options.no_archive:
        args += ['--archive-directory', dirobj.dirname['archive']]

    progname = os.environ.get("RWPOLLEXEC",
                              os.path.join('.', 'rwpollexec'))
    args = progname.split() + args

    # Set up state variables
    run_count = 0
    complete_count = 0
    success_count = 0
    nonzero_count = 0
    killed_count = 0
    clean = False
    term = False
    limit = len(options.infile)
    regexp = re.compile("Running \[\d+\]:")
    success = "Command .* has completed successfully"
    nonzero = "Command .* has completed with a nonzero return"
    killed = "Command .* was terminated by SIG"
    complete = re.compile("(?P<success>%s)|(?P<nonzero>%s)|(?P<killed>%s)" %
                          (success, nonzero, killed))
    closing = re.compile("Stopped logging")

    if limit == 0:
        parser.print_usage()
        sys.exit(255)

    # Create data
    create_files(dirobj, options.infile)

    # Note the time
    starttime = time.time()
    shutdowntime = None
    endedtime = None

    # Start the process
    log("Running", "'%s'" % "' '".join(args))
    proc = subprocess.Popen(args, stderr=subprocess.PIPE,
                            preexec_fn=lambda:os.setpgid(0, 0))
    try:
        os.setpgid(proc.pid, 0)
    except OSError:
        pass
    line_reader = TimedReadline(proc.stderr.fileno())

    try:
        # Read the output data
        line = line_reader(1)
        while line is not None:
            if line:
                base_log(line)

            # Check for timeout
            if time.time() - starttime > options.daemon_timeout and not term:
                shutdowntime = time.time()
                log("Timed out")
                log("Sending SIGTERM")
                term = True
                try:
                    os.kill(proc.pid, signal.SIGTERM)
                except OSError:
                    pass

            # Match running counts
            match = regexp.search(line)
            if match:
                run_count += 1
                # Reset the timer if we are still receiving data
                starttime = time.time()

            # Match success counts
            match = complete.search(line)
            if match:
                if match.group('success') is not None:
                    success_count += 1
                elif match.group('nonzero') is not None:
                    nonzero_count += 1
                elif match.group('killed') is not None:
                    killed_count += 1
                complete_count += 1
                # Reset the timer if we are still receiving data
                starttime = time.time()
                if complete_count >= limit and not term:
                    shutdowntime = time.time()
                    log("Reached limit")
                    log("Sending SIGTERM")
                    term = True
                    try:
                        os.kill(proc.pid, signal.SIGTERM)
                    except OSError:
                        pass

            # Check for clean shutdown
            match = closing.search(line)
            if match:
                clean = True

            # Check for timeout on SIGTERM shutdown
            if shutdowntime and time.time() - shutdowntime > 15:
                log("Timeout on SIGTERM.  Sending SIGKILL")
                shutdowntime = None
                try:
                    os.killpg(proc.pid, signal.SIGKILL)
                    break
                except OSError:
                    pass

            # Check to see if the process has ended
            if endedtime is None:
                proc.poll()
                if proc.returncode is not None:
                    endedtime = time.time()

            # Allow some time after the process has ended for any
            # remaining output the be handled
            if endedtime and time.time() - endedtime > 5:
                break

            # Get next line
            line = line_reader(1)

    finally:
        # Sleep briefly before polling for exit.
        time.sleep(1)
        proc.poll()
        if proc.returncode is None:
            log("Daemon has not exited.  Sending SIGKILL")
            try:
                os.killpg(proc.pid, signal.SIGKILL)
            except OSError:
                pass

    # Kill everything for good luck
    try:
        os.killpg(proc.pid, signal.SIGKILL)
    except OSError:
        pass

    # Print record count
    print("Run count:", run_count)
    print(("Complete count (success, failure): (%s, %s) = %s" %
           (success_count, nonzero_count + killed_count, complete_count)))
    base_log("Run count:", run_count)
    base_log(("Complete count (success, failure): (%s, %s) = %s" %
              (success_count, nonzero_count + killed_count, complete_count)))
    if limit != complete_count:
        print(("ERROR: expecting %s files, got %s files" %
               (limit, complete_count)))
        base_log(("ERROR: expecting %s files, got %s files" %
                  (limit, complete_count)))

    # Check for clean exit
    if not clean:
        print("ERROR: shutdown was not clean")
        base_log("ERROR: shutdown was not clean")

    # Exit with the process's error code
    if proc.returncode is None:
        log("Exiting: Deamon did not appear to terminate normally")
        sys.exit(1)
    else:
        log("Exiting: Daemon returned", proc.returncode)
        sys.exit(proc.returncode)