def __init__(self, name=None, log_level="info", prog_env=None, verbose=True, **kwds): global global_int Dirobject.__init__(self, **kwds) if name: self.name = name else: self.name = type(self).__name__ + str(global_int) global_int += 1 self.process = None self.logdata = [] self.log_level = log_level self._prog_env = prog_env self.daemon = True self.verbose = verbose self.pipe = None self.pid = None self.trigger = None self.timeout = None self.channels = [] self.pending_line = None
def __init__(self): Dirobject.__init__(self) self.create_dirs() self.client_type = None self.server_type = None self.clients = set() self.servers = set() self.ca_cert = None self.ca_key = None
def main(): global VERBOSE global logfile parser = optparse.OptionParser() parser.set_defaults(infile=[]) parser.add_option("--hang", action="callback", callback=handle_file_op) parser.add_option("--term", action="callback", type="int", callback=handle_file_op) parser.add_option("--return", action="callback", type="int", callback=handle_file_op) parser.add_option("--signal", action="callback", type="int", callback=handle_file_op) parser.add_option("--exec", action="store_true", dest="execute", default=False) parser.add_option("--basedir", action="store", type="string", dest="basedir") parser.add_option("--no-archive", action="store_true", dest="no_archive", default=False) parser.add_option("--daemon-timeout", action="store", type="int", dest="daemon_timeout", default=60) parser.add_option("--verbose", action="store_true", dest="verbose", default=False) parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite", default=False) parser.add_option("--log-level", action="store", type="string", dest="log_level", default="info") (options, args) = parser.parse_args() VERBOSE = options.verbose if options.execute: execute(args) # Does not return # Create the dirs dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir) dirobj.dirs = ['incoming', 'archive', 'error', 'log'] dirobj.create_dirs() # Make the log file logfile = open(dirobj.get_path('log', 'rwpollexec-daemon.log'), 'wb', 0) # Generate the subprocess arguments args += ['--log-dest', 'stderr', '--log-level', options.log_level, '--no-daemon', '--incoming-directory', dirobj.dirname['incoming'], '--error-directory', dirobj.dirname['error']] if not options.no_archive: args += ['--archive-directory', dirobj.dirname['archive']] progname = os.environ.get("RWPOLLEXEC", os.path.join('.', 'rwpollexec')) args = progname.split() + args # Set up state variables run_count = 0 complete_count = 0 success_count = 0 nonzero_count = 0 killed_count = 0 clean = False term = False limit = len(options.infile) regexp = re.compile("Running \[\d+\]:") success = "Command .* has completed successfully" nonzero = "Command .* has completed with a nonzero return" killed = "Command .* was terminated by SIG" complete = re.compile("(?P<success>%s)|(?P<nonzero>%s)|(?P<killed>%s)" % (success, nonzero, killed)) closing = re.compile("Stopped logging") if limit == 0: parser.print_usage() sys.exit(255) # Create data create_files(dirobj, options.infile) # Note the time starttime = time.time() shutdowntime = None endedtime = None # Start the process log("Running", "'%s'" % "' '".join(args)) proc = subprocess.Popen(args, stderr=subprocess.PIPE, preexec_fn=lambda:os.setpgid(0, 0)) try: os.setpgid(proc.pid, 0) except OSError: pass line_reader = TimedReadline(proc.stderr.fileno()) try: # Read the output data line = line_reader(1) while line is not None: if line: base_log(line) # Check for timeout if time.time() - starttime > options.daemon_timeout and not term: shutdowntime = time.time() log("Timed out") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Match running counts match = regexp.search(line) if match: run_count += 1 # Reset the timer if we are still receiving data starttime = time.time() # Match success counts match = complete.search(line) if match: if match.group('success') is not None: success_count += 1 elif match.group('nonzero') is not None: nonzero_count += 1 elif match.group('killed') is not None: killed_count += 1 complete_count += 1 # Reset the timer if we are still receiving data starttime = time.time() if complete_count >= limit and not term: shutdowntime = time.time() log("Reached limit") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Check for clean shutdown match = closing.search(line) if match: clean = True # Check for timeout on SIGTERM shutdown if shutdowntime and time.time() - shutdowntime > 15: log("Timeout on SIGTERM. Sending SIGKILL") shutdowntime = None try: os.killpg(proc.pid, signal.SIGKILL) break except OSError: pass # Check to see if the process has ended if endedtime is None: proc.poll() if proc.returncode is not None: endedtime = time.time() # Allow some time after the process has ended for any # remaining output the be handled if endedtime and time.time() - endedtime > 5: break # Get next line line = line_reader(1) finally: # Sleep briefly before polling for exit. time.sleep(1) proc.poll() if proc.returncode is None: log("Daemon has not exited. Sending SIGKILL") try: os.killpg(proc.pid, signal.SIGKILL) except OSError: pass # Kill everything for good luck try: os.killpg(proc.pid, signal.SIGKILL) except OSError: pass # Print record count print("Run count:", run_count) print(("Complete count (success, failure): (%s, %s) = %s" % (success_count, nonzero_count + killed_count, complete_count))) base_log("Run count:", run_count) base_log(("Complete count (success, failure): (%s, %s) = %s" % (success_count, nonzero_count + killed_count, complete_count))) if limit != complete_count: print(("ERROR: expecting %s files, got %s files" % (limit, complete_count))) base_log(("ERROR: expecting %s files, got %s files" % (limit, complete_count))) # Check for clean exit if not clean: print("ERROR: shutdown was not clean") base_log("ERROR: shutdown was not clean") # Exit with the process's error code if proc.returncode is None: log("Exiting: Deamon did not appear to terminate normally") sys.exit(1) else: log("Exiting: Daemon returned", proc.returncode) sys.exit(proc.returncode)
def main(): global VERBOSE global logfile parser = optparse.OptionParser() parser.add_option("--pdu", action="append", type="string", dest="pdu", default=[]) parser.add_option("--tcp", action="append", type="string", dest="tcp", default=[]) parser.add_option("--basedir", action="store", type="string", dest="basedir") parser.add_option("--daemon-timeout", action="store", type="int", dest="daemon_timeout", default=60) parser.add_option("--limit", action="store", type="int", dest="limit") parser.add_option("--timeout", action="store", type="int", dest="timeout", default=10) parser.add_option("--verbose", action="store_true", dest="verbose", default=False) parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite", default=False) parser.add_option("--log-level", action="store", type="string", dest="log_level", default="info") (options, args) = parser.parse_args() VERBOSE = options.verbose # Create the dirs dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir) dirobj.dirs = ['destination', 'log'] dirobj.create_dirs() # Make the log file logfile = open(dirobj.get_path('log', 'flowcap-daemon.log'), 'wb', 0) # Generate the subprocess arguments args += [ '--timeout', str(options.timeout), '--log-dest', 'stderr', '--log-level', options.log_level, '--no-daemon' ] args += ['--destination-directory', dirobj.dirname['destination']] progname = os.environ.get("FLOWCAP", os.path.join('.', 'flowcap')) args = progname.split() + args # Set up state variables count = 0 clean = False term = False send_list = None regexp = re.compile("Closing file .* seconds, (?P<recs>[0-9]+) records,") closing = re.compile("Stopped logging") started = re.compile("'.+': Reader thread started") # Note the time starttime = time.time() shutdowntime = None # Start the process log("Running", "'%s'" % "' '".join(args)) proc = subprocess.Popen(args, stderr=subprocess.PIPE) line_reader = TimedReadline(proc.stderr.fileno()) try: # Read the output data line = line_reader(1) while line is not None: if line: base_log(line) # Check for timeout if time.time() - starttime > options.daemon_timeout and not term: shutdowntime = time.time() log("Timed out") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Match record counts match = regexp.search(line) if match: num = int(match.group('recs')) if num > 0: count += num # Reset the timer if we are still receiving data starttime = time.time() if options.limit and count >= options.limit and not term: shutdowntime = time.time() log("Reached limit") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # check for starting up network data if started: match = started.search(line) if match and send_list is None: send_list = send_network_data(options) started = None # Check for clean shutdown match = closing.search(line) if match: clean = True # Check for timeout on SIGTERM shutdown if shutdowntime and time.time() - shutdowntime > 15: log("Timeout on SIGTERM. Sending SIGKILL") shutdowntime = None try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Get next line line = line_reader(1) finally: # Stop sending network data if send_list: stop_network_data(send_list) # Sleep briefly before polling for exit. time.sleep(1) proc.poll() if proc.returncode is None: log("Daemon has not exited. Sending SIGKILL") try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Print record count print("Record count:", count) base_log("Record count:", count) if options.limit and options.limit != count: print(("ERROR: expecting %s records, got %s records" % (options.limit, count))) base_log(("ERROR: expecting %s records, got %s records" % (options.limit, count))) # Check for clean exit if not clean: print("ERROR: shutdown was not clean") base_log("ERROR: shutdown was not clean") # Exit with the process's error code if proc.returncode is None: log("Exiting: Deamon did not appear to terminate normally") sys.exit(1) else: log("Exiting: Daemon returned", proc.returncode) sys.exit(proc.returncode)
def main(): global VERBOSE global logfile parser = optparse.OptionParser() parser.add_option("--copy", action="append", type="string", dest="copy", default=[]) parser.add_option("--move", action="append", type="string", dest="move", default=[]) parser.add_option("--basedir", action="store", type="string", dest="basedir") parser.add_option("--daemon-timeout", action="store", type="int", dest="daemon_timeout", default=60) parser.add_option("--verbose", action="store_true", dest="verbose", default=False) parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite", default=False) parser.add_option("--log-level", action="store", type="string", dest="log_level", default="info") (options, args) = parser.parse_args() VERBOSE = options.verbose # Create the dirs dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir) dirobj.dirs = ['incoming', 'archive', 'error', 'root', 'log'] dirobj.create_dirs() # Make the log file logfile = open(dirobj.get_path('log', 'rwflowappend-daemon.log'), 'wb', 0) # Generate the subprocess arguments args += ['--log-dest', 'stderr', '--log-level', options.log_level, '--no-daemon', '--root-directory', dirobj.dirname['root'], '--incoming-directory', dirobj.dirname['incoming'], '--archive-directory', dirobj.dirname['archive'], '--error-directory', dirobj.dirname['error']] progname = os.environ.get("RWFLOWAPPEND", os.path.join('.', 'rwflowappend')) args = progname.split() + args # Set up state variables count = 0 clean = False term = False send_list = None limit = len(options.copy) + len(options.move) regexp = re.compile("APPEND OK") closing = re.compile("Stopped logging") # Copy or move data copy_files(dirobj, options.copy) move_files(dirobj, options.move) # Note the time starttime = time.time() shutdowntime = None # Start the process log("Running", "'%s'" % "' '".join(args)) proc = subprocess.Popen(args, stderr=subprocess.PIPE) line_reader = TimedReadline(proc.stderr.fileno()) try: # Read the output data line = line_reader(1) while line is not None: if line: base_log(line) # Check for timeout if time.time() - starttime > options.daemon_timeout and not term: shutdowntime = time.time() log("Timed out") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Match record counts match = regexp.search(line) if match: count += 1 # Reset the timer if we are still receiving data starttime = time.time() if count >= limit and not term: shutdowntime = time.time() log("Reached limit") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Check for clean shutdown match = closing.search(line) if match: clean = True # Check for timeout on SIGTERM shutdown if shutdowntime and time.time() - shutdowntime > 15: log("Timeout on SIGTERM. Sending SIGKILL") shutdowntime = None try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Get next line line = line_reader(1) finally: # Stop sending network data if send_list: stop_network_data(send_list) # Sleep briefly before polling for exit. time.sleep(1) proc.poll() if proc.returncode is None: log("Daemon has not exited. Sending SIGKILL") try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Print record count print("File count:", count) base_log("File count:", count) if limit != count: print(("ERROR: expecting %s files, got %s files" % (limit, count))) base_log(("ERROR: expecting %s files, got %s files" % (limit, count))) # Check for clean exit if not clean: print("ERROR: shutdown was not clean") base_log("ERROR: shutdown was not clean") # Exit with the process's error code if proc.returncode is None: log("Exiting: Deamon did not appear to terminate normally") sys.exit(1) else: log("Exiting: Daemon returned", proc.returncode) sys.exit(proc.returncode)
def main(): global VERBOSE global logfile parser = optparse.OptionParser() parser.add_option("--pdu", action="append", type="string", dest="pdu", default=[]) parser.add_option("--tcp", action="append", type="string", dest="tcp", default=[]) parser.add_option("--copy", action="append", type="string", dest="copy", default=[]) parser.add_option("--move", action="append", type="string", dest="move", default=[]) parser.add_option("--copy-after", action="append", type="string", dest="copy_after", default=[]) parser.add_option("--move-after", action="append", type="string", dest="move_after", default=[]) parser.add_option("--input-mode", action="store", type="string", dest="input_mode") parser.add_option("--output-mode", action="store", type="string", dest="output_mode") parser.add_option("--basedir", action="store", type="string", dest="basedir") parser.add_option("--daemon-timeout", action="store", type="int", dest="daemon_timeout", default=60) parser.add_option("--limit", action="store", type="int", dest="limit") parser.add_option("--sensor-configuration", action="store", type="string", dest="sensor_conf") parser.add_option("--flush-timeout", action="store", type="int", dest="flush_timeout", default=10) parser.add_option("--verbose", action="store_true", dest="verbose", default=False) parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite", default=False) parser.add_option("--log-level", action="store", type="string", dest="log_level", default="info") (options, args) = parser.parse_args() VERBOSE = options.verbose # Create the dirs dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir) dirobj.dirs = ['incoming', 'archive', 'error', 'sender', 'incremental', 'root', 'log', 'incoming2', 'incoming3', 'incoming4'] dirobj.create_dirs() # Make the log file logfile = open(dirobj.get_path('log', 'rwflowpack-daemon.log'), 'wb', 0) # Create the configuration file from the template if options.sensor_conf: conf_file = open(options.sensor_conf, "r").read() new_conf = string.Template(conf_file) (fd, conf) = tempfile.mkstemp(dir=dirobj.basedir) out_file = os.fdopen(fd, "w") out_file.write(new_conf.substitute(dirobj.dirname)) out_file.close() # Generate the subprocess arguments args += ['--flush-timeout', str(options.flush_timeout), '--log-dest', 'stderr', '--log-level', options.log_level, '--no-daemon'] if options.input_mode == "fcfiles" or options.input_mode == "respool": args += ['--incoming-directory', dirobj.dirname['incoming']] args += ['--archive-directory', dirobj.dirname['archive'], '--error-directory', dirobj.dirname['error']] if options.output_mode == "sending": args += ['--sender-directory', dirobj.dirname['sender'], '--incremental-directory', dirobj.dirname['incremental']] elif options.output_mode == "incremental-files": args += ['--incremental-directory', dirobj.dirname['incremental']] else: args += ['--root-directory', dirobj.dirname['root']] if options.input_mode: args += ['--input-mode', options.input_mode] if options.output_mode: args += ['--output-mode', options.output_mode] if options.sensor_conf: args += ['--sensor-configuration', conf] progname = os.environ.get("RWFLOWPACK", os.path.join('.', 'rwflowpack')) args = progname.split() + args # Set up state variables count = 0 clean = False term = False send_list = None regexp = re.compile(": /[^:].*: (?P<recs>[0-9]+) recs") closing = re.compile("Stopped logging") started = re.compile("Starting flush timer") # Copy or move data copy_files(dirobj, options.copy) move_files(dirobj, options.move) # Note the time starttime = time.time() shutdowntime = None # Start the process log("Running", "'%s'" % "' '".join(args)) proc = subprocess.Popen(args, stderr=subprocess.PIPE) line_reader = TimedReadline(proc.stderr.fileno()) try: # Read the output data line = line_reader(1) while line is not None: if line: base_log(line) # Check for timeout if time.time() - starttime > options.daemon_timeout and not term: shutdowntime = time.time() log("Timed out") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Match record counts match = regexp.search(line) if match: num = int(match.group('recs')) if num > 0: count += num # Reset the timer if we are still receiving data starttime = time.time() if options.limit and count >= options.limit and not term: shutdowntime = time.time() log("Reached limit") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # check for starting up network data or after-data if started: match = started.search(line) if match: if send_list is None: send_list = send_network_data(options) copy_files(dirobj, options.copy_after) move_files(dirobj, options.move_after) started = None # Check for clean shutdown match = closing.search(line) if match: clean = True # Check for timeout on SIGTERM shutdown if shutdowntime and time.time() - shutdowntime > 15: log("Timeout on SIGTERM. Sending SIGKILL") shutdowntime = None try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Get next line line = line_reader(1) finally: # Stop sending network data if send_list: stop_network_data(send_list) # Sleep briefly before polling for exit. time.sleep(1) proc.poll() if proc.returncode is None: log("Daemon has not exited. Sending SIGKILL") try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Print record count print("Record count:", count) base_log("Record count:", count) if options.limit and options.limit != count: print(("ERROR: expecting %s records, got %s records" % (options.limit, count))) base_log(("ERROR: expecting %s records, got %s records" % (options.limit, count))) # Check for clean exit if not clean: print("ERROR: shutdown was not clean") base_log("ERROR: shutdown was not clean") # Exit with the process's error code if proc.returncode is None: log("Exiting: Deamon did not appear to terminate normally") sys.exit(1) else: log("Exiting: Daemon returned", proc.returncode) sys.exit(proc.returncode)
def main(): global VERBOSE global logfile parser = optparse.OptionParser() parser.add_option("--pdu", action="append", type="string", dest="pdu", default=[]) parser.add_option("--tcp", action="append", type="string", dest="tcp", default=[]) parser.add_option("--basedir", action="store", type="string", dest="basedir") parser.add_option("--daemon-timeout", action="store", type="int", dest="daemon_timeout", default=60) parser.add_option("--limit", action="store", type="int", dest="limit") parser.add_option("--timeout", action="store", type="int", dest="timeout", default=10) parser.add_option("--verbose", action="store_true", dest="verbose", default=False) parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite", default=False) parser.add_option("--log-level", action="store", type="string", dest="log_level", default="info") (options, args) = parser.parse_args() VERBOSE = options.verbose # Create the dirs dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir) dirobj.dirs = ['destination', 'log'] dirobj.create_dirs() # Make the log file logfile = open(dirobj.get_path('log', 'flowcap-daemon.log'), 'wb', 0) # Generate the subprocess arguments args += ['--timeout', str(options.timeout), '--log-dest', 'stderr', '--log-level', options.log_level, '--no-daemon'] args += ['--destination-directory', dirobj.dirname['destination']] progname = os.environ.get("FLOWCAP", os.path.join('.', 'flowcap')) args = progname.split() + args # Set up state variables count = 0 clean = False term = False send_list = None regexp = re.compile("Closing file .* seconds, (?P<recs>[0-9]+) records,") closing = re.compile("Stopped logging") started = re.compile("'.+': Reader thread started") # Note the time starttime = time.time() shutdowntime = None # Start the process log("Running", "'%s'" % "' '".join(args)) proc = subprocess.Popen(args, stderr=subprocess.PIPE) line_reader = TimedReadline(proc.stderr.fileno()) try: # Read the output data line = line_reader(1) while line is not None: if line: base_log(line) # Check for timeout if time.time() - starttime > options.daemon_timeout and not term: shutdowntime = time.time() log("Timed out") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Match record counts match = regexp.search(line) if match: num = int(match.group('recs')) if num > 0: count += num # Reset the timer if we are still receiving data starttime = time.time() if options.limit and count >= options.limit and not term: shutdowntime = time.time() log("Reached limit") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # check for starting up network data if started: match = started.search(line) if match and send_list is None: send_list = send_network_data(options) started = None # Check for clean shutdown match = closing.search(line) if match: clean = True # Check for timeout on SIGTERM shutdown if shutdowntime and time.time() - shutdowntime > 15: log("Timeout on SIGTERM. Sending SIGKILL") shutdowntime = None try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Get next line line = line_reader(1) finally: # Stop sending network data if send_list: stop_network_data(send_list) # Sleep briefly before polling for exit. time.sleep(1) proc.poll() if proc.returncode is None: log("Daemon has not exited. Sending SIGKILL") try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Print record count print("Record count:", count) base_log("Record count:", count) if options.limit and options.limit != count: print(("ERROR: expecting %s records, got %s records" % (options.limit, count))) base_log(("ERROR: expecting %s records, got %s records" % (options.limit, count))) # Check for clean exit if not clean: print("ERROR: shutdown was not clean") base_log("ERROR: shutdown was not clean") # Exit with the process's error code if proc.returncode is None: log("Exiting: Deamon did not appear to terminate normally") sys.exit(1) else: log("Exiting: Daemon returned", proc.returncode) sys.exit(proc.returncode)
def main(): global VERBOSE global logfile parser = optparse.OptionParser() parser.add_option("--pdu", action="append", type="string", dest="pdu", default=[]) parser.add_option("--tcp", action="append", type="string", dest="tcp", default=[]) parser.add_option("--copy", action="append", type="string", dest="copy", default=[]) parser.add_option("--move", action="append", type="string", dest="move", default=[]) parser.add_option("--copy-after", action="append", type="string", dest="copy_after", default=[]) parser.add_option("--move-after", action="append", type="string", dest="move_after", default=[]) parser.add_option("--input-mode", action="store", type="string", dest="input_mode") parser.add_option("--output-mode", action="store", type="string", dest="output_mode") parser.add_option("--basedir", action="store", type="string", dest="basedir") parser.add_option("--daemon-timeout", action="store", type="int", dest="daemon_timeout", default=60) parser.add_option("--limit", action="store", type="int", dest="limit") parser.add_option("--sensor-configuration", action="store", type="string", dest="sensor_conf") parser.add_option("--flush-timeout", action="store", type="int", dest="flush_timeout", default=10) parser.add_option("--verbose", action="store_true", dest="verbose", default=False) parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite", default=False) parser.add_option("--log-level", action="store", type="string", dest="log_level", default="info") (options, args) = parser.parse_args() VERBOSE = options.verbose # Create the dirs dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir) dirobj.dirs = [ 'incoming', 'archive', 'error', 'sender', 'incremental', 'root', 'log', 'incoming2', 'incoming3', 'incoming4' ] dirobj.create_dirs() # Make the log file logfile = open(dirobj.get_path('log', 'rwflowpack-daemon.log'), 'wb', 0) # Create the configuration file from the template if options.sensor_conf: conf_file = open(options.sensor_conf, "r").read() new_conf = string.Template(conf_file) (fd, conf) = tempfile.mkstemp(dir=dirobj.basedir) out_file = os.fdopen(fd, "w") out_file.write(new_conf.substitute(dirobj.dirname)) out_file.close() # Generate the subprocess arguments args += [ '--flush-timeout', str(options.flush_timeout), '--log-dest', 'stderr', '--log-level', options.log_level, '--no-daemon' ] if options.input_mode == "fcfiles" or options.input_mode == "respool": args += ['--incoming-directory', dirobj.dirname['incoming']] args += [ '--archive-directory', dirobj.dirname['archive'], '--error-directory', dirobj.dirname['error'] ] if options.output_mode == "sending": args += [ '--sender-directory', dirobj.dirname['sender'], '--incremental-directory', dirobj.dirname['incremental'] ] elif options.output_mode == "incremental-files": args += ['--incremental-directory', dirobj.dirname['incremental']] else: args += ['--root-directory', dirobj.dirname['root']] if options.input_mode: args += ['--input-mode', options.input_mode] if options.output_mode: args += ['--output-mode', options.output_mode] if options.sensor_conf: args += ['--sensor-configuration', conf] progname = os.environ.get("RWFLOWPACK", os.path.join('.', 'rwflowpack')) args = progname.split() + args # Set up state variables count = 0 clean = False term = False send_list = None regexp = re.compile(": /[^:].*: (?P<recs>[0-9]+) recs") closing = re.compile("Stopped logging") started = re.compile("Starting flush timer") # Copy or move data copy_files(dirobj, options.copy) move_files(dirobj, options.move) # Note the time starttime = time.time() shutdowntime = None # Start the process log("Running", "'%s'" % "' '".join(args)) proc = subprocess.Popen(args, stderr=subprocess.PIPE) line_reader = TimedReadline(proc.stderr.fileno()) try: # Read the output data line = line_reader(1) while line is not None: if line: base_log(line) # Check for timeout if time.time() - starttime > options.daemon_timeout and not term: shutdowntime = time.time() log("Timed out") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Match record counts match = regexp.search(line) if match: num = int(match.group('recs')) if num > 0: count += num # Reset the timer if we are still receiving data starttime = time.time() if options.limit and count >= options.limit and not term: shutdowntime = time.time() log("Reached limit") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # check for starting up network data or after-data if started: match = started.search(line) if match: if send_list is None: send_list = send_network_data(options) copy_files(dirobj, options.copy_after) move_files(dirobj, options.move_after) started = None # Check for clean shutdown match = closing.search(line) if match: clean = True # Check for timeout on SIGTERM shutdown if shutdowntime and time.time() - shutdowntime > 15: log("Timeout on SIGTERM. Sending SIGKILL") shutdowntime = None try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Get next line line = line_reader(1) finally: # Stop sending network data if send_list: stop_network_data(send_list) # Sleep briefly before polling for exit. time.sleep(1) proc.poll() if proc.returncode is None: log("Daemon has not exited. Sending SIGKILL") try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Print record count print("Record count:", count) base_log("Record count:", count) if options.limit and options.limit != count: print(("ERROR: expecting %s records, got %s records" % (options.limit, count))) base_log(("ERROR: expecting %s records, got %s records" % (options.limit, count))) # Check for clean exit if not clean: print("ERROR: shutdown was not clean") base_log("ERROR: shutdown was not clean") # Exit with the process's error code if proc.returncode is None: log("Exiting: Deamon did not appear to terminate normally") sys.exit(1) else: log("Exiting: Daemon returned", proc.returncode) sys.exit(proc.returncode)
def main(): global VERBOSE global logfile parser = optparse.OptionParser() parser.add_option("--copy", action="append", type="string", dest="copy", default=[]) parser.add_option("--move", action="append", type="string", dest="move", default=[]) parser.add_option("--file-limit", action="store", type="int", dest="file_limit", default=0) parser.add_option("--no-archive", action="store_true", dest="no_archvie", default=False) parser.add_option("--basedir", action="store", type="string", dest="basedir") parser.add_option("--daemon-timeout", action="store", type="int", dest="daemon_timeout", default=60) parser.add_option("--verbose", action="store_true", dest="verbose", default=False) parser.add_option("--overwrite-dirs", action="store_true", dest="overwrite", default=False) parser.add_option("--log-level", action="store", type="string", dest="log_level", default="info") (options, args) = parser.parse_args() VERBOSE = options.verbose # Create the dirs dirobj = Dirobject(overwrite=options.overwrite, basedir=options.basedir) dirobj.dirs = ['incoming', 'archive', 'error', 'root', 'log'] dirobj.create_dirs() # Make the log file logfile = open(dirobj.get_path('log', 'rwflowappend-daemon.log'), 'wb', 0) # Generate the subprocess arguments args += [ '--log-dest', 'stderr', '--log-level', options.log_level, '--no-daemon', '--root-directory', dirobj.dirname['root'], '--incoming-directory', dirobj.dirname['incoming'], '--error-directory', dirobj.dirname['error'] ] if not options.no_archvie: args += ['--archive-directory', dirobj.dirname['archive']] progname = os.environ.get("RWFLOWAPPEND", os.path.join('.', 'rwflowappend')) args = progname.split() + args # Set up state variables count = 0 clean = False term = False send_list = None limit = len(options.copy) + len(options.move) + options.file_limit regexp = re.compile("APPEND OK") closing = re.compile("Stopped logging") if 0 == limit: print("ERROR: File limit is zero") base_log("ERROR: File limit is zero") sys.exit(1) # Copy or move data copy_files(dirobj, options.copy) move_files(dirobj, options.move) # Note the time starttime = time.time() shutdowntime = None # Start the process log("Running", "'%s'" % "' '".join(args)) proc = subprocess.Popen(args, stderr=subprocess.PIPE) line_reader = TimedReadline(proc.stderr.fileno()) try: # Read the output data line = line_reader(1) while line is not None: if line: base_log(line) # Check for timeout if time.time() - starttime > options.daemon_timeout and not term: shutdowntime = time.time() log("Timed out") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Match record counts match = regexp.search(line) if match: count += 1 # Reset the timer if we are still receiving data starttime = time.time() if count >= limit and not term: shutdowntime = time.time() log("Reached limit") log("Sending SIGTERM") term = True try: os.kill(proc.pid, signal.SIGTERM) except OSError: pass # Check for clean shutdown match = closing.search(line) if match: clean = True # Check for timeout on SIGTERM shutdown if shutdowntime and time.time() - shutdowntime > 15: log("Timeout on SIGTERM. Sending SIGKILL") shutdowntime = None try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Get next line line = line_reader(1) finally: # Stop sending network data if send_list: stop_network_data(send_list) # Sleep briefly before polling for exit. time.sleep(1) proc.poll() if proc.returncode is None: log("Daemon has not exited. Sending SIGKILL") try: os.kill(proc.pid, signal.SIGKILL) except OSError: pass # Print record count print("File count:", count) base_log("File count:", count) if limit != count: print(("ERROR: expecting %s files, got %s files" % (limit, count))) base_log(("ERROR: expecting %s files, got %s files" % (limit, count))) # Check for clean exit if not clean: print("ERROR: shutdown was not clean") base_log("ERROR: shutdown was not clean") # Exit with the process's error code if proc.returncode is None: log("Exiting: Deamon did not appear to terminate normally") sys.exit(1) else: log("Exiting: Daemon returned", proc.returncode) sys.exit(proc.returncode)