def main():
global VERBOSE
global logfile
parser = optparse.OptionParser()
parser.set_defaults(infile=[])
parser.add_option("--hang", action="callback", callback=handle_file_op)
parser.add_option("--term", action="callback", type="int",
callback=handle_file_op)
parser.add_option("--return", action="callback", type="int",
callback=handle_file_op)
parser.add_option("--signal", action="callback", type="int",
callback=handle_file_op)
parser.add_option("--exec", action="store_true", dest="execute",
default=False)
parser.add_option("--basedir", action="store", type="string",
dest="basedir")
parser.add_option("--no-archive", action="store_true", dest="no_archive",
default=False)
parser.add_option("--daemon-timeout", action="store", type="int",
dest="daemon_timeout", default=60)
parser.add_option("--verbose", action="store_true", dest="verbose",
default=False)
parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
default=False)
parser.add_option("--log-level", action="store", type="string",
dest="log_level", default="info")
(options, args) = parser.parse_args()
VERBOSE = options.verbose
if options.execute:
execute(args) # Does not return
# Create the dirs
dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
dirobj.dirs = ['incoming', 'archive', 'error', 'log']
dirobj.create_dirs()
# Make the log file
logfile = open(dirobj.get_path('log', 'rwpollexec-daemon.log'), 'wb', 0)
# Generate the subprocess arguments
args += ['--log-dest', 'stderr',
'--log-level', options.log_level,
'--no-daemon',
'--incoming-directory', dirobj.dirname['incoming'],
'--error-directory', dirobj.dirname['error']]
if not options.no_archive:
args += ['--archive-directory', dirobj.dirname['archive']]
progname = os.environ.get("RWPOLLEXEC",
os.path.join('.', 'rwpollexec'))
args = progname.split() + args
# Set up state variables
run_count = 0
complete_count = 0
success_count = 0
nonzero_count = 0
killed_count = 0
clean = False
term = False
limit = len(options.infile)
regexp = re.compile("Running \[\d+\]:")
success = "Command .* has completed successfully"
nonzero = "Command .* has completed with a nonzero return"
killed = "Command .* was terminated by SIG"
complete = re.compile("(?P<success>%s)|(?P<nonzero>%s)|(?P<killed>%s)" %
(success, nonzero, killed))
closing = re.compile("Stopped logging")
if limit == 0:
parser.print_usage()
sys.exit(255)
# Create data
create_files(dirobj, options.infile)
# Note the time
starttime = time.time()
shutdowntime = None
endedtime = None
# Start the process
log("Running", "'%s'" % "' '".join(args))
proc = subprocess.Popen(args, stderr=subprocess.PIPE,
preexec_fn=lambda:os.setpgid(0, 0))
try:
os.setpgid(proc.pid, 0)
except OSError:
pass
line_reader = TimedReadline(proc.stderr.fileno())
try:
# Read the output data
line = line_reader(1)
while line is not None:
if line:
base_log(line)
# Check for timeout
if time.time() - starttime > options.daemon_timeout and not term:
shutdowntime = time.time()
log("Timed out")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Match running counts
match = regexp.search(line)
if match:
run_count += 1
# Reset the timer if we are still receiving data
starttime = time.time()
# Match success counts
match = complete.search(line)
if match:
if match.group('success') is not None:
success_count += 1
elif match.group('nonzero') is not None:
nonzero_count += 1
elif match.group('killed') is not None:
killed_count += 1
complete_count += 1
# Reset the timer if we are still receiving data
starttime = time.time()
if complete_count >= limit and not term:
shutdowntime = time.time()
log("Reached limit")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Check for clean shutdown
match = closing.search(line)
if match:
clean = True
# Check for timeout on SIGTERM shutdown
if shutdowntime and time.time() - shutdowntime > 15:
log("Timeout on SIGTERM. Sending SIGKILL")
shutdowntime = None
try:
os.killpg(proc.pid, signal.SIGKILL)
break
except OSError:
pass
# Check to see if the process has ended
if endedtime is None:
proc.poll()
if proc.returncode is not None:
endedtime = time.time()
# Allow some time after the process has ended for any
# remaining output the be handled
if endedtime and time.time() - endedtime > 5:
break
# Get next line
line = line_reader(1)
finally:
# Sleep briefly before polling for exit.
time.sleep(1)
proc.poll()
if proc.returncode is None:
log("Daemon has not exited. Sending SIGKILL")
try:
os.killpg(proc.pid, signal.SIGKILL)
except OSError:
pass
# Kill everything for good luck
try:
os.killpg(proc.pid, signal.SIGKILL)
except OSError:
pass
# Print record count
print("Run count:", run_count)
print(("Complete count (success, failure): (%s, %s) = %s" %
(success_count, nonzero_count + killed_count, complete_count)))
base_log("Run count:", run_count)
base_log(("Complete count (success, failure): (%s, %s) = %s" %
(success_count, nonzero_count + killed_count, complete_count)))
if limit != complete_count:
print(("ERROR: expecting %s files, got %s files" %
(limit, complete_count)))
base_log(("ERROR: expecting %s files, got %s files" %
(limit, complete_count)))
# Check for clean exit
if not clean:
print("ERROR: shutdown was not clean")
base_log("ERROR: shutdown was not clean")
# Exit with the process's error code
if proc.returncode is None:
log("Exiting: Deamon did not appear to terminate normally")
sys.exit(1)
else:
log("Exiting: Daemon returned", proc.returncode)
sys.exit(proc.returncode)
def main():
global VERBOSE
global logfile
parser = optparse.OptionParser()
parser.add_option("--copy", action="append", type="string", dest="copy",
default=[])
parser.add_option("--move", action="append", type="string", dest="move",
default=[])
parser.add_option("--basedir", action="store", type="string",
dest="basedir")
parser.add_option("--daemon-timeout", action="store", type="int",
dest="daemon_timeout", default=60)
parser.add_option("--verbose", action="store_true", dest="verbose",
default=False)
parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
default=False)
parser.add_option("--log-level", action="store", type="string",
dest="log_level", default="info")
(options, args) = parser.parse_args()
VERBOSE = options.verbose
# Create the dirs
dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
dirobj.dirs = ['incoming', 'archive', 'error', 'root', 'log']
dirobj.create_dirs()
# Make the log file
logfile = open(dirobj.get_path('log', 'rwflowappend-daemon.log'), 'wb', 0)
# Generate the subprocess arguments
args += ['--log-dest', 'stderr',
'--log-level', options.log_level,
'--no-daemon',
'--root-directory', dirobj.dirname['root'],
'--incoming-directory', dirobj.dirname['incoming'],
'--archive-directory', dirobj.dirname['archive'],
'--error-directory', dirobj.dirname['error']]
progname = os.environ.get("RWFLOWAPPEND",
os.path.join('.', 'rwflowappend'))
args = progname.split() + args
# Set up state variables
count = 0
clean = False
term = False
send_list = None
limit = len(options.copy) + len(options.move)
regexp = re.compile("APPEND OK")
closing = re.compile("Stopped logging")
# Copy or move data
copy_files(dirobj, options.copy)
move_files(dirobj, options.move)
# Note the time
starttime = time.time()
shutdowntime = None
# Start the process
log("Running", "'%s'" % "' '".join(args))
proc = subprocess.Popen(args, stderr=subprocess.PIPE)
line_reader = TimedReadline(proc.stderr.fileno())
try:
# Read the output data
line = line_reader(1)
while line is not None:
if line:
base_log(line)
# Check for timeout
if time.time() - starttime > options.daemon_timeout and not term:
shutdowntime = time.time()
log("Timed out")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Match record counts
match = regexp.search(line)
if match:
count += 1
# Reset the timer if we are still receiving data
starttime = time.time()
if count >= limit and not term:
shutdowntime = time.time()
log("Reached limit")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Check for clean shutdown
match = closing.search(line)
if match:
clean = True
# Check for timeout on SIGTERM shutdown
if shutdowntime and time.time() - shutdowntime > 15:
log("Timeout on SIGTERM. Sending SIGKILL")
shutdowntime = None
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Get next line
line = line_reader(1)
finally:
# Stop sending network data
if send_list:
stop_network_data(send_list)
# Sleep briefly before polling for exit.
time.sleep(1)
proc.poll()
if proc.returncode is None:
log("Daemon has not exited. Sending SIGKILL")
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Print record count
print("File count:", count)
base_log("File count:", count)
if limit != count:
print(("ERROR: expecting %s files, got %s files" %
(limit, count)))
base_log(("ERROR: expecting %s files, got %s files" %
(limit, count)))
# Check for clean exit
if not clean:
print("ERROR: shutdown was not clean")
base_log("ERROR: shutdown was not clean")
# Exit with the process's error code
if proc.returncode is None:
log("Exiting: Deamon did not appear to terminate normally")
sys.exit(1)
else:
log("Exiting: Daemon returned", proc.returncode)
sys.exit(proc.returncode)
def main():
global VERBOSE
global logfile
parser = optparse.OptionParser()
parser.add_option("--pdu",
action="append",
type="string",
dest="pdu",
default=[])
parser.add_option("--tcp",
action="append",
type="string",
dest="tcp",
default=[])
parser.add_option("--basedir",
action="store",
type="string",
dest="basedir")
parser.add_option("--daemon-timeout",
action="store",
type="int",
dest="daemon_timeout",
default=60)
parser.add_option("--limit", action="store", type="int", dest="limit")
parser.add_option("--timeout",
action="store",
type="int",
dest="timeout",
default=10)
parser.add_option("--verbose",
action="store_true",
dest="verbose",
default=False)
parser.add_option("--overwrite-dirs",
action="store_true",
dest="overwrite",
default=False)
parser.add_option("--log-level",
action="store",
type="string",
dest="log_level",
default="info")
(options, args) = parser.parse_args()
VERBOSE = options.verbose
# Create the dirs
dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
dirobj.dirs = ['destination', 'log']
dirobj.create_dirs()
# Make the log file
logfile = open(dirobj.get_path('log', 'flowcap-daemon.log'), 'wb', 0)
# Generate the subprocess arguments
args += [
'--timeout',
str(options.timeout), '--log-dest', 'stderr', '--log-level',
options.log_level, '--no-daemon'
]
args += ['--destination-directory', dirobj.dirname['destination']]
progname = os.environ.get("FLOWCAP", os.path.join('.', 'flowcap'))
args = progname.split() + args
# Set up state variables
count = 0
clean = False
term = False
send_list = None
regexp = re.compile("Closing file .* seconds, (?P<recs>[0-9]+) records,")
closing = re.compile("Stopped logging")
started = re.compile("'.+': Reader thread started")
# Note the time
starttime = time.time()
shutdowntime = None
# Start the process
log("Running", "'%s'" % "' '".join(args))
proc = subprocess.Popen(args, stderr=subprocess.PIPE)
line_reader = TimedReadline(proc.stderr.fileno())
try:
# Read the output data
line = line_reader(1)
while line is not None:
if line:
base_log(line)
# Check for timeout
if time.time() - starttime > options.daemon_timeout and not term:
shutdowntime = time.time()
log("Timed out")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Match record counts
match = regexp.search(line)
if match:
num = int(match.group('recs'))
if num > 0:
count += num
# Reset the timer if we are still receiving data
starttime = time.time()
if options.limit and count >= options.limit and not term:
shutdowntime = time.time()
log("Reached limit")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# check for starting up network data
if started:
match = started.search(line)
if match and send_list is None:
send_list = send_network_data(options)
started = None
# Check for clean shutdown
match = closing.search(line)
if match:
clean = True
# Check for timeout on SIGTERM shutdown
if shutdowntime and time.time() - shutdowntime > 15:
log("Timeout on SIGTERM. Sending SIGKILL")
shutdowntime = None
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Get next line
line = line_reader(1)
finally:
# Stop sending network data
if send_list:
stop_network_data(send_list)
# Sleep briefly before polling for exit.
time.sleep(1)
proc.poll()
if proc.returncode is None:
log("Daemon has not exited. Sending SIGKILL")
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Print record count
print("Record count:", count)
base_log("Record count:", count)
if options.limit and options.limit != count:
print(("ERROR: expecting %s records, got %s records" %
(options.limit, count)))
base_log(("ERROR: expecting %s records, got %s records" %
(options.limit, count)))
# Check for clean exit
if not clean:
print("ERROR: shutdown was not clean")
base_log("ERROR: shutdown was not clean")
# Exit with the process's error code
if proc.returncode is None:
log("Exiting: Deamon did not appear to terminate normally")
sys.exit(1)
else:
log("Exiting: Daemon returned", proc.returncode)
sys.exit(proc.returncode)
def main():
global VERBOSE
global logfile
parser = optparse.OptionParser()
parser.add_option("--pdu", action="append", type="string", dest="pdu",
default=[])
parser.add_option("--tcp", action="append", type="string", dest="tcp",
default=[])
parser.add_option("--copy", action="append", type="string", dest="copy",
default=[])
parser.add_option("--move", action="append", type="string", dest="move",
default=[])
parser.add_option("--copy-after", action="append", type="string",
dest="copy_after", default=[])
parser.add_option("--move-after", action="append", type="string",
dest="move_after", default=[])
parser.add_option("--input-mode", action="store", type="string",
dest="input_mode")
parser.add_option("--output-mode", action="store", type="string",
dest="output_mode")
parser.add_option("--basedir", action="store", type="string",
dest="basedir")
parser.add_option("--daemon-timeout", action="store", type="int",
dest="daemon_timeout", default=60)
parser.add_option("--limit", action="store", type="int", dest="limit")
parser.add_option("--sensor-configuration", action="store", type="string",
dest="sensor_conf")
parser.add_option("--flush-timeout", action="store", type="int",
dest="flush_timeout", default=10)
parser.add_option("--verbose", action="store_true", dest="verbose",
default=False)
parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
default=False)
parser.add_option("--log-level", action="store", type="string",
dest="log_level", default="info")
(options, args) = parser.parse_args()
VERBOSE = options.verbose
# Create the dirs
dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
dirobj.dirs = ['incoming', 'archive', 'error', 'sender',
'incremental', 'root', 'log', 'incoming2',
'incoming3', 'incoming4']
dirobj.create_dirs()
# Make the log file
logfile = open(dirobj.get_path('log', 'rwflowpack-daemon.log'), 'wb', 0)
# Create the configuration file from the template
if options.sensor_conf:
conf_file = open(options.sensor_conf, "r").read()
new_conf = string.Template(conf_file)
(fd, conf) = tempfile.mkstemp(dir=dirobj.basedir)
out_file = os.fdopen(fd, "w")
out_file.write(new_conf.substitute(dirobj.dirname))
out_file.close()
# Generate the subprocess arguments
args += ['--flush-timeout', str(options.flush_timeout),
'--log-dest', 'stderr',
'--log-level', options.log_level,
'--no-daemon']
if options.input_mode == "fcfiles" or options.input_mode == "respool":
args += ['--incoming-directory', dirobj.dirname['incoming']]
args += ['--archive-directory', dirobj.dirname['archive'],
'--error-directory', dirobj.dirname['error']]
if options.output_mode == "sending":
args += ['--sender-directory', dirobj.dirname['sender'],
'--incremental-directory', dirobj.dirname['incremental']]
elif options.output_mode == "incremental-files":
args += ['--incremental-directory', dirobj.dirname['incremental']]
else:
args += ['--root-directory', dirobj.dirname['root']]
if options.input_mode:
args += ['--input-mode', options.input_mode]
if options.output_mode:
args += ['--output-mode', options.output_mode]
if options.sensor_conf:
args += ['--sensor-configuration', conf]
progname = os.environ.get("RWFLOWPACK", os.path.join('.', 'rwflowpack'))
args = progname.split() + args
# Set up state variables
count = 0
clean = False
term = False
send_list = None
regexp = re.compile(": /[^:].*: (?P<recs>[0-9]+) recs")
closing = re.compile("Stopped logging")
started = re.compile("Starting flush timer")
# Copy or move data
copy_files(dirobj, options.copy)
move_files(dirobj, options.move)
# Note the time
starttime = time.time()
shutdowntime = None
# Start the process
log("Running", "'%s'" % "' '".join(args))
proc = subprocess.Popen(args, stderr=subprocess.PIPE)
line_reader = TimedReadline(proc.stderr.fileno())
try:
# Read the output data
line = line_reader(1)
while line is not None:
if line:
base_log(line)
# Check for timeout
if time.time() - starttime > options.daemon_timeout and not term:
shutdowntime = time.time()
log("Timed out")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Match record counts
match = regexp.search(line)
if match:
num = int(match.group('recs'))
if num > 0:
count += num
# Reset the timer if we are still receiving data
starttime = time.time()
if options.limit and count >= options.limit and not term:
shutdowntime = time.time()
log("Reached limit")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# check for starting up network data or after-data
if started:
match = started.search(line)
if match:
if send_list is None:
send_list = send_network_data(options)
copy_files(dirobj, options.copy_after)
move_files(dirobj, options.move_after)
started = None
# Check for clean shutdown
match = closing.search(line)
if match:
clean = True
# Check for timeout on SIGTERM shutdown
if shutdowntime and time.time() - shutdowntime > 15:
log("Timeout on SIGTERM. Sending SIGKILL")
shutdowntime = None
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Get next line
line = line_reader(1)
finally:
# Stop sending network data
if send_list:
stop_network_data(send_list)
# Sleep briefly before polling for exit.
time.sleep(1)
proc.poll()
if proc.returncode is None:
log("Daemon has not exited. Sending SIGKILL")
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Print record count
print("Record count:", count)
base_log("Record count:", count)
if options.limit and options.limit != count:
print(("ERROR: expecting %s records, got %s records" %
(options.limit, count)))
base_log(("ERROR: expecting %s records, got %s records" %
(options.limit, count)))
# Check for clean exit
if not clean:
print("ERROR: shutdown was not clean")
base_log("ERROR: shutdown was not clean")
# Exit with the process's error code
if proc.returncode is None:
log("Exiting: Deamon did not appear to terminate normally")
sys.exit(1)
else:
log("Exiting: Daemon returned", proc.returncode)
sys.exit(proc.returncode)
def main():
global VERBOSE
global logfile
parser = optparse.OptionParser()
parser.add_option("--pdu", action="append", type="string", dest="pdu",
default=[])
parser.add_option("--tcp", action="append", type="string", dest="tcp",
default=[])
parser.add_option("--basedir", action="store", type="string",
dest="basedir")
parser.add_option("--daemon-timeout", action="store", type="int",
dest="daemon_timeout", default=60)
parser.add_option("--limit", action="store", type="int", dest="limit")
parser.add_option("--timeout", action="store", type="int",
dest="timeout", default=10)
parser.add_option("--verbose", action="store_true", dest="verbose",
default=False)
parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
default=False)
parser.add_option("--log-level", action="store", type="string",
dest="log_level", default="info")
(options, args) = parser.parse_args()
VERBOSE = options.verbose
# Create the dirs
dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
dirobj.dirs = ['destination', 'log']
dirobj.create_dirs()
# Make the log file
logfile = open(dirobj.get_path('log', 'flowcap-daemon.log'), 'wb', 0)
# Generate the subprocess arguments
args += ['--timeout', str(options.timeout),
'--log-dest', 'stderr',
'--log-level', options.log_level,
'--no-daemon']
args += ['--destination-directory', dirobj.dirname['destination']]
progname = os.environ.get("FLOWCAP", os.path.join('.', 'flowcap'))
args = progname.split() + args
# Set up state variables
count = 0
clean = False
term = False
send_list = None
regexp = re.compile("Closing file .* seconds, (?P<recs>[0-9]+) records,")
closing = re.compile("Stopped logging")
started = re.compile("'.+': Reader thread started")
# Note the time
starttime = time.time()
shutdowntime = None
# Start the process
log("Running", "'%s'" % "' '".join(args))
proc = subprocess.Popen(args, stderr=subprocess.PIPE)
line_reader = TimedReadline(proc.stderr.fileno())
try:
# Read the output data
line = line_reader(1)
while line is not None:
if line:
base_log(line)
# Check for timeout
if time.time() - starttime > options.daemon_timeout and not term:
shutdowntime = time.time()
log("Timed out")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Match record counts
match = regexp.search(line)
if match:
num = int(match.group('recs'))
if num > 0:
count += num
# Reset the timer if we are still receiving data
starttime = time.time()
if options.limit and count >= options.limit and not term:
shutdowntime = time.time()
log("Reached limit")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# check for starting up network data
if started:
match = started.search(line)
if match and send_list is None:
send_list = send_network_data(options)
started = None
# Check for clean shutdown
match = closing.search(line)
if match:
clean = True
# Check for timeout on SIGTERM shutdown
if shutdowntime and time.time() - shutdowntime > 15:
log("Timeout on SIGTERM. Sending SIGKILL")
shutdowntime = None
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Get next line
line = line_reader(1)
finally:
# Stop sending network data
if send_list:
stop_network_data(send_list)
# Sleep briefly before polling for exit.
time.sleep(1)
proc.poll()
if proc.returncode is None:
log("Daemon has not exited. Sending SIGKILL")
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Print record count
print("Record count:", count)
base_log("Record count:", count)
if options.limit and options.limit != count:
print(("ERROR: expecting %s records, got %s records" %
(options.limit, count)))
base_log(("ERROR: expecting %s records, got %s records" %
(options.limit, count)))
# Check for clean exit
if not clean:
print("ERROR: shutdown was not clean")
base_log("ERROR: shutdown was not clean")
# Exit with the process's error code
if proc.returncode is None:
log("Exiting: Deamon did not appear to terminate normally")
sys.exit(1)
else:
log("Exiting: Daemon returned", proc.returncode)
sys.exit(proc.returncode)
def main():
global VERBOSE
global logfile
parser = optparse.OptionParser()
parser.add_option("--pdu",
action="append",
type="string",
dest="pdu",
default=[])
parser.add_option("--tcp",
action="append",
type="string",
dest="tcp",
default=[])
parser.add_option("--copy",
action="append",
type="string",
dest="copy",
default=[])
parser.add_option("--move",
action="append",
type="string",
dest="move",
default=[])
parser.add_option("--copy-after",
action="append",
type="string",
dest="copy_after",
default=[])
parser.add_option("--move-after",
action="append",
type="string",
dest="move_after",
default=[])
parser.add_option("--input-mode",
action="store",
type="string",
dest="input_mode")
parser.add_option("--output-mode",
action="store",
type="string",
dest="output_mode")
parser.add_option("--basedir",
action="store",
type="string",
dest="basedir")
parser.add_option("--daemon-timeout",
action="store",
type="int",
dest="daemon_timeout",
default=60)
parser.add_option("--limit", action="store", type="int", dest="limit")
parser.add_option("--sensor-configuration",
action="store",
type="string",
dest="sensor_conf")
parser.add_option("--flush-timeout",
action="store",
type="int",
dest="flush_timeout",
default=10)
parser.add_option("--verbose",
action="store_true",
dest="verbose",
default=False)
parser.add_option("--overwrite-dirs",
action="store_true",
dest="overwrite",
default=False)
parser.add_option("--log-level",
action="store",
type="string",
dest="log_level",
default="info")
(options, args) = parser.parse_args()
VERBOSE = options.verbose
# Create the dirs
dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
dirobj.dirs = [
'incoming', 'archive', 'error', 'sender', 'incremental', 'root', 'log',
'incoming2', 'incoming3', 'incoming4'
]
dirobj.create_dirs()
# Make the log file
logfile = open(dirobj.get_path('log', 'rwflowpack-daemon.log'), 'wb', 0)
# Create the configuration file from the template
if options.sensor_conf:
conf_file = open(options.sensor_conf, "r").read()
new_conf = string.Template(conf_file)
(fd, conf) = tempfile.mkstemp(dir=dirobj.basedir)
out_file = os.fdopen(fd, "w")
out_file.write(new_conf.substitute(dirobj.dirname))
out_file.close()
# Generate the subprocess arguments
args += [
'--flush-timeout',
str(options.flush_timeout), '--log-dest', 'stderr', '--log-level',
options.log_level, '--no-daemon'
]
if options.input_mode == "fcfiles" or options.input_mode == "respool":
args += ['--incoming-directory', dirobj.dirname['incoming']]
args += [
'--archive-directory', dirobj.dirname['archive'], '--error-directory',
dirobj.dirname['error']
]
if options.output_mode == "sending":
args += [
'--sender-directory', dirobj.dirname['sender'],
'--incremental-directory', dirobj.dirname['incremental']
]
elif options.output_mode == "incremental-files":
args += ['--incremental-directory', dirobj.dirname['incremental']]
else:
args += ['--root-directory', dirobj.dirname['root']]
if options.input_mode:
args += ['--input-mode', options.input_mode]
if options.output_mode:
args += ['--output-mode', options.output_mode]
if options.sensor_conf:
args += ['--sensor-configuration', conf]
progname = os.environ.get("RWFLOWPACK", os.path.join('.', 'rwflowpack'))
args = progname.split() + args
# Set up state variables
count = 0
clean = False
term = False
send_list = None
regexp = re.compile(": /[^:].*: (?P<recs>[0-9]+) recs")
closing = re.compile("Stopped logging")
started = re.compile("Starting flush timer")
# Copy or move data
copy_files(dirobj, options.copy)
move_files(dirobj, options.move)
# Note the time
starttime = time.time()
shutdowntime = None
# Start the process
log("Running", "'%s'" % "' '".join(args))
proc = subprocess.Popen(args, stderr=subprocess.PIPE)
line_reader = TimedReadline(proc.stderr.fileno())
try:
# Read the output data
line = line_reader(1)
while line is not None:
if line:
base_log(line)
# Check for timeout
if time.time() - starttime > options.daemon_timeout and not term:
shutdowntime = time.time()
log("Timed out")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Match record counts
match = regexp.search(line)
if match:
num = int(match.group('recs'))
if num > 0:
count += num
# Reset the timer if we are still receiving data
starttime = time.time()
if options.limit and count >= options.limit and not term:
shutdowntime = time.time()
log("Reached limit")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# check for starting up network data or after-data
if started:
match = started.search(line)
if match:
if send_list is None:
send_list = send_network_data(options)
copy_files(dirobj, options.copy_after)
move_files(dirobj, options.move_after)
started = None
# Check for clean shutdown
match = closing.search(line)
if match:
clean = True
# Check for timeout on SIGTERM shutdown
if shutdowntime and time.time() - shutdowntime > 15:
log("Timeout on SIGTERM. Sending SIGKILL")
shutdowntime = None
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Get next line
line = line_reader(1)
finally:
# Stop sending network data
if send_list:
stop_network_data(send_list)
# Sleep briefly before polling for exit.
time.sleep(1)
proc.poll()
if proc.returncode is None:
log("Daemon has not exited. Sending SIGKILL")
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Print record count
print("Record count:", count)
base_log("Record count:", count)
if options.limit and options.limit != count:
print(("ERROR: expecting %s records, got %s records" %
(options.limit, count)))
base_log(("ERROR: expecting %s records, got %s records" %
(options.limit, count)))
# Check for clean exit
if not clean:
print("ERROR: shutdown was not clean")
base_log("ERROR: shutdown was not clean")
# Exit with the process's error code
if proc.returncode is None:
log("Exiting: Deamon did not appear to terminate normally")
sys.exit(1)
else:
log("Exiting: Daemon returned", proc.returncode)
sys.exit(proc.returncode)
def main():
global VERBOSE
global logfile
parser = optparse.OptionParser()
parser.add_option("--copy",
action="append",
type="string",
dest="copy",
default=[])
parser.add_option("--move",
action="append",
type="string",
dest="move",
default=[])
parser.add_option("--file-limit",
action="store",
type="int",
dest="file_limit",
default=0)
parser.add_option("--no-archive",
action="store_true",
dest="no_archvie",
default=False)
parser.add_option("--basedir",
action="store",
type="string",
dest="basedir")
parser.add_option("--daemon-timeout",
action="store",
type="int",
dest="daemon_timeout",
default=60)
parser.add_option("--verbose",
action="store_true",
dest="verbose",
default=False)
parser.add_option("--overwrite-dirs",
action="store_true",
dest="overwrite",
default=False)
parser.add_option("--log-level",
action="store",
type="string",
dest="log_level",
default="info")
(options, args) = parser.parse_args()
VERBOSE = options.verbose
# Create the dirs
dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
dirobj.dirs = ['incoming', 'archive', 'error', 'root', 'log']
dirobj.create_dirs()
# Make the log file
logfile = open(dirobj.get_path('log', 'rwflowappend-daemon.log'), 'wb', 0)
# Generate the subprocess arguments
args += [
'--log-dest', 'stderr', '--log-level', options.log_level,
'--no-daemon', '--root-directory', dirobj.dirname['root'],
'--incoming-directory', dirobj.dirname['incoming'],
'--error-directory', dirobj.dirname['error']
]
if not options.no_archvie:
args += ['--archive-directory', dirobj.dirname['archive']]
progname = os.environ.get("RWFLOWAPPEND",
os.path.join('.', 'rwflowappend'))
args = progname.split() + args
# Set up state variables
count = 0
clean = False
term = False
send_list = None
limit = len(options.copy) + len(options.move) + options.file_limit
regexp = re.compile("APPEND OK")
closing = re.compile("Stopped logging")
if 0 == limit:
print("ERROR: File limit is zero")
base_log("ERROR: File limit is zero")
sys.exit(1)
# Copy or move data
copy_files(dirobj, options.copy)
move_files(dirobj, options.move)
# Note the time
starttime = time.time()
shutdowntime = None
# Start the process
log("Running", "'%s'" % "' '".join(args))
proc = subprocess.Popen(args, stderr=subprocess.PIPE)
line_reader = TimedReadline(proc.stderr.fileno())
try:
# Read the output data
line = line_reader(1)
while line is not None:
if line:
base_log(line)
# Check for timeout
if time.time() - starttime > options.daemon_timeout and not term:
shutdowntime = time.time()
log("Timed out")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Match record counts
match = regexp.search(line)
if match:
count += 1
# Reset the timer if we are still receiving data
starttime = time.time()
if count >= limit and not term:
shutdowntime = time.time()
log("Reached limit")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Check for clean shutdown
match = closing.search(line)
if match:
clean = True
# Check for timeout on SIGTERM shutdown
if shutdowntime and time.time() - shutdowntime > 15:
log("Timeout on SIGTERM. Sending SIGKILL")
shutdowntime = None
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Get next line
line = line_reader(1)
finally:
# Stop sending network data
if send_list:
stop_network_data(send_list)
# Sleep briefly before polling for exit.
time.sleep(1)
proc.poll()
if proc.returncode is None:
log("Daemon has not exited. Sending SIGKILL")
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError:
pass
# Print record count
print("File count:", count)
base_log("File count:", count)
if limit != count:
print(("ERROR: expecting %s files, got %s files" % (limit, count)))
base_log(("ERROR: expecting %s files, got %s files" % (limit, count)))
# Check for clean exit
if not clean:
print("ERROR: shutdown was not clean")
base_log("ERROR: shutdown was not clean")
# Exit with the process's error code
if proc.returncode is None:
log("Exiting: Deamon did not appear to terminate normally")
sys.exit(1)
else:
log("Exiting: Daemon returned", proc.returncode)
sys.exit(proc.returncode)
def main():
global VERBOSE
global logfile
parser = optparse.OptionParser()
parser.set_defaults(infile=[])
parser.add_option("--hang", action="callback", callback=handle_file_op)
parser.add_option("--term", action="callback", type="int",
callback=handle_file_op)
parser.add_option("--return", action="callback", type="int",
callback=handle_file_op)
parser.add_option("--signal", action="callback", type="int",
callback=handle_file_op)
parser.add_option("--exec", action="store_true", dest="execute",
default=False)
parser.add_option("--basedir", action="store", type="string",
dest="basedir")
parser.add_option("--no-archive", action="store_true", dest="no_archive",
default=False)
parser.add_option("--daemon-timeout", action="store", type="int",
dest="daemon_timeout", default=60)
parser.add_option("--verbose", action="store_true", dest="verbose",
default=False)
parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite",
default=False)
parser.add_option("--log-level", action="store", type="string",
dest="log_level", default="info")
(options, args) = parser.parse_args()
VERBOSE = options.verbose
if options.execute:
execute(args) # Does not return
# Create the dirs
dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir)
dirobj.dirs = ['incoming', 'archive', 'error', 'log']
dirobj.create_dirs()
# Make the log file
logfile = open(dirobj.get_path('log', 'rwpollexec-daemon.log'), 'wb', 0)
# Generate the subprocess arguments
args += ['--log-dest', 'stderr',
'--log-level', options.log_level,
'--no-daemon',
'--incoming-directory', dirobj.dirname['incoming'],
'--error-directory', dirobj.dirname['error']]
if not options.no_archive:
args += ['--archive-directory', dirobj.dirname['archive']]
progname = os.environ.get("RWPOLLEXEC",
os.path.join('.', 'rwpollexec'))
args = progname.split() + args
# Set up state variables
run_count = 0
complete_count = 0
success_count = 0
nonzero_count = 0
killed_count = 0
clean = False
term = False
limit = len(options.infile)
regexp = re.compile("Running \[\d+\]:")
success = "Command .* has completed successfully"
nonzero = "Command .* has completed with a nonzero return"
killed = "Command .* was terminated by SIG"
complete = re.compile("(?P<success>%s)|(?P<nonzero>%s)|(?P<killed>%s)" %
(success, nonzero, killed))
closing = re.compile("Stopped logging")
if limit == 0:
parser.print_usage()
sys.exit(255)
# Create data
create_files(dirobj, options.infile)
# Note the time
starttime = time.time()
shutdowntime = None
endedtime = None
# Start the process
log("Running", "'%s'" % "' '".join(args))
proc = subprocess.Popen(args, stderr=subprocess.PIPE,
preexec_fn=lambda:os.setpgid(0, 0))
try:
os.setpgid(proc.pid, 0)
except OSError:
pass
line_reader = TimedReadline(proc.stderr.fileno())
try:
# Read the output data
line = line_reader(1)
while line is not None:
if line:
base_log(line)
# Check for timeout
if time.time() - starttime > options.daemon_timeout and not term:
shutdowntime = time.time()
log("Timed out")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Match running counts
match = regexp.search(line)
if match:
run_count += 1
# Reset the timer if we are still receiving data
starttime = time.time()
# Match success counts
match = complete.search(line)
if match:
if match.group('success') is not None:
success_count += 1
elif match.group('nonzero') is not None:
nonzero_count += 1
elif match.group('killed') is not None:
killed_count += 1
complete_count += 1
# Reset the timer if we are still receiving data
starttime = time.time()
if complete_count >= limit and not term:
shutdowntime = time.time()
log("Reached limit")
log("Sending SIGTERM")
term = True
try:
os.kill(proc.pid, signal.SIGTERM)
except OSError:
pass
# Check for clean shutdown
match = closing.search(line)
if match:
clean = True
# Check for timeout on SIGTERM shutdown
if shutdowntime and time.time() - shutdowntime > 15:
log("Timeout on SIGTERM. Sending SIGKILL")
shutdowntime = None
try:
os.killpg(proc.pid, signal.SIGKILL)
break
except OSError:
pass
# Check to see if the process has ended
if endedtime is None:
proc.poll()
if proc.returncode is not None:
endedtime = time.time()
# Allow some time after the process has ended for any
# remaining output the be handled
if endedtime and time.time() - endedtime > 5:
break
# Get next line
line = line_reader(1)
finally:
# Sleep briefly before polling for exit.
time.sleep(1)
proc.poll()
if proc.returncode is None:
log("Daemon has not exited. Sending SIGKILL")
try:
os.killpg(proc.pid, signal.SIGKILL)
except OSError:
pass
# Kill everything for good luck
try:
os.killpg(proc.pid, signal.SIGKILL)
except OSError:
pass
# Print record count
print("Run count:", run_count)
print(("Complete count (success, failure): (%s, %s) = %s" %
(success_count, nonzero_count + killed_count, complete_count)))
base_log("Run count:", run_count)
base_log(("Complete count (success, failure): (%s, %s) = %s" %
(success_count, nonzero_count + killed_count, complete_count)))
if limit != complete_count:
print(("ERROR: expecting %s files, got %s files" %
(limit, complete_count)))
base_log(("ERROR: expecting %s files, got %s files" %
(limit, complete_count)))
# Check for clean exit
if not clean:
print("ERROR: shutdown was not clean")
base_log("ERROR: shutdown was not clean")
# Exit with the process's error code
if proc.returncode is None:
log("Exiting: Deamon did not appear to terminate normally")
sys.exit(1)
else:
log("Exiting: Daemon returned", proc.returncode)
sys.exit(proc.returncode)