def trylock(self): """Try acquiring the DB file lock without blocking This raises BlockingIOError(11) if the other process has taken the lock. """ self._f.seek(0) os.lockf(self._f.fileno(), os.F_TLOCK, 0)
def lock(self): """Taking an exclusive lock on the DB file This blocks until the other process releases the lock. """ self._f.seek(0) os.lockf(self._f.fileno(), os.F_LOCK, 0)
def exclusive_process(name): # Ensure that this process, globally named `name`, does not execute # multiple times concurrently. import os, os.path, sys, atexit if os.access('/var/run', os.W_OK): # process is probably run as root piddir = '/var/run' else: # process is probably a non-root user piddir = '/tmp' pidfile = os.path.join(piddir, '%s.pid' % name) mypid = os.getpid() # Attempt to get a lock on ourself so that the concurrency check # itself is not executed in parallel. with open(__file__, 'r+') as flock: # Try to get a lock. This blocks until a lock is acquired. The # lock is held until the flock file is closed at the end of the # with block. os.lockf(flock.fileno(), os.F_LOCK, 0) # While we have a lock, look at the pid file. First attempt # to write our pid to a pidfile if no file already exists there. try: with open(pidfile, 'x') as f: # Successfully opened a new file. Since the file is new # there is no concurrent process. Write our pid. f.write(str(mypid)) # When this process exits, remove the pid file. atexit.register(clear_my_pid, pidfile) return except FileExistsError: # The pid file already exixts, but it may contain a stale # pid of a terminated process. with open(pidfile, 'r+') as f: # Read the pid in the file. existing_pid = None try: existing_pid = int(f.read().strip()) except ValueError: pass # No valid integer in the file. # Check if the pid in it is valid. If it is, there is # already a process running. Quit. if existing_pid: if is_pid_valid(existing_pid): print("Another %s process is already running (pid %d)." % (name, existing_pid), file=sys.stderr) sys.exit(1) # This pid is stale. Write over it with our pid. f.seek(0) f.write(str(mypid)) f.truncate() # When this process exits, remove the pid file. atexit.register(clear_my_pid, pidfile)
def __exit__(self, *exc): try: os.lockf(self.fd, os.F_ULOCK, 0) except OSError: pass try: self.fileobj.close() except OSError: pass try: os.remove(self.path) except OSError: pass return False
def exclusive_process(name): # Ensure that a process named `name` does not execute multiple # times concurrently. import os, sys, atexit pidfile = '/var/run/mailinabox-%s.pid' % name mypid = os.getpid() # Attempt to get a lock on ourself so that the concurrency check # itself is not executed in parallel. with open(__file__, 'r+') as flock: # Try to get a lock. This blocks until a lock is acquired. The # lock is held until the flock file is closed at the end of the # with block. os.lockf(flock.fileno(), os.F_LOCK, 0) # While we have a lock, look at the pid file. First attempt # to write our pid to a pidfile if no file already exists there. try: with open(pidfile, 'x') as f: # Successfully opened a new file. Since the file is new # there is no concurrent process. Write our pid. f.write(str(mypid)) atexit.register(clear_my_pid, pidfile) return except FileExistsError: # The pid file already exixts, but it may contain a stale # pid of a terminated process. with open(pidfile, 'r+') as f: # Read the pid in the file. existing_pid = None try: existing_pid = int(f.read().strip()) except ValueError: pass # No valid integer in the file. # Check if the pid in it is valid. if existing_pid: if is_pid_valid(existing_pid): print("Another %s is already running (pid %d)." % (name, existing_pid), file=sys.stderr) sys.exit(1) # Write our pid. f.seek(0) f.write(str(mypid)) f.truncate() atexit.register(clear_my_pid, pidfile)
def main(): """The main entry point.""" parser = argparse.ArgumentParser( description='System call list consistency checks') parser.add_argument('--cc', metavar='CC', required=True, help='C compiler (including options) to use') parser.add_argument('--lock', metavar='PATH', required=True, help='file to lock during the updates') parser.add_argument('arch_syscall', metavar='ARCH-SYSCALL-H', help='The <arch-syscall.h> file to update') parser.add_argument('names_list', metavar='SYSCALL-NAMES-LIST', help='The syscall name list to update ') args = parser.parse_args() kernel_constants = glibcsyscalls.kernel_constants(args.cc) with open(args.lock, 'r+') as lockfile: os.lockf(lockfile.fileno(), os.F_LOCK, 0) # Replace <arch-syscall.h> with data derived from kernel headers. # No merging is necessary here. Arch-specific changes should go # into <fixup-unistd-asm.h>. out = io.StringIO() out.write('/* AUTOGENERATED by update-syscall-lists.py. */\n') for name, value in sorted(kernel_constants.items()): out.write('#define __NR_{} {}\n'.format(name, value)) atomic_replace(args.arch_syscall, out.getvalue()) # Merge the architecture-specific system call names into the # global names list, syscall-names.list. This file contains names # from other architectures (and comments), so it is necessary to # merge the existing files with the names obtained from the # kernel. with open(args.names_list, 'r') as list_file: names_list = glibcsyscalls.SyscallNamesList(list_file) merged = names_list.merge(kernel_constants.keys()) out = io.StringIO() for line in merged: out.write(line) atomic_replace(args.names_list, out.getvalue())
def exclusive_process(name): # Ensure that a process named `name` does not execute multiple # times concurrently. import os, sys, atexit pidfile = "/var/run/mailinabox-%s.pid" % name mypid = os.getpid() # Attempt to get a lock on ourself so that the concurrency check # itself is not executed in parallel. with open(__file__, "r+") as flock: # Try to get a lock. This blocks until a lock is acquired. The # lock is held until the flock file is closed at the end of the # with block. os.lockf(flock.fileno(), os.F_LOCK, 0) # While we have a lock, look at the pid file. First attempt # to write our pid to a pidfile if no file already exists there. try: with open(pidfile, "x") as f: # Successfully opened a new file. Since the file is new # there is no concurrent process. Write our pid. f.write(str(mypid)) atexit.register(clear_my_pid, pidfile) return except FileExistsError: # The pid file already exixts, but it may contain a stale # pid of a terminated process. with open(pidfile, "r+") as f: # Read the pid in the file. existing_pid = None try: existing_pid = int(f.read().strip()) except ValueError: pass # No valid integer in the file. # Check if the pid in it is valid. if existing_pid: if is_pid_valid(existing_pid): print("Another %s is already running (pid %d)." % (name, existing_pid), file=sys.stderr) sys.exit(1) # Write our pid. f.seek(0) f.write(str(mypid)) f.truncate() atexit.register(clear_my_pid, pidfile)
def __init__(self, path, daemonize=True): self.path = path self.daemonize = daemonize self.fileobj = open(self.path, "a") self.fd = self.fileobj.fileno() try: os.lockf(self.fd, os.F_TLOCK, 0) except (PermissionError, BlockingIOError): try: with open(self.path, "r") as lines: for line in lines: pid = int(line) except (FileNotFoundError, ValueError): raise SystemExit("%s seems to be running (unable to get pid)" % \ os.path.basename(sys.argv[0])) except OSError as exc: raise SystemExit(str(exc)) raise SystemExit("%s is already running as pid %s" % \ (os.path.basename(sys.argv[0]), pid)) if self.daemonize: try: # pylint:disable=protected-access if os.fork() > 0: os._exit(0) os.setsid() os.chdir("/") if os.fork() > 0: os._exit(0) except EnvironmentError as exc: raise SystemExit(str(exc)) fd = os.open(os.devnull, os.O_RDWR) os.dup2(fd, 0) os.dup2(fd, 1) os.dup2(fd, 2) print(os.getpid(), file=self.fileobj) self.fileobj.flush()
def init(self): # Create pidfile self.fl = os.open(self.config.glob["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666) try: os.lockf(self.fl, os.F_TLOCK, 0) os.write(self.fl, str(os.getpid()).encode()) os.fsync(self.fl) except BlockingIOError: os.close(self.fl) self.fl = -1 raise JetconfInitError("Jetconf already running (pidfile exists)") # Import backend modules backend_package = self.config.glob["BACKEND_PACKAGE"] try: _be_installed = import_module(backend_package) del _be_installed except ImportError as e: raise JetconfInitError( "Cannot import backend package \"{}\", reason: {}".format( backend_package, ErrorHelpers.epretty(e))) try: usr_state_data_handlers = import_module(backend_package + ".usr_state_data_handlers") except ImportError: usr_state_data_handlers = None try: usr_conf_data_handlers = import_module(backend_package + ".usr_conf_data_handlers") except ImportError: usr_conf_data_handlers = None try: usr_op_handlers = import_module(backend_package + ".usr_op_handlers") except ImportError: usr_op_handlers = None try: usr_action_handlers = import_module(backend_package + ".usr_action_handlers") except ImportError: usr_action_handlers = None try: usr_datastore = import_module(backend_package + ".usr_datastore") except ImportError: usr_datastore = None try: self.usr_init = import_module(backend_package + ".usr_init") except ImportError: self.usr_init = None # Load data model yang_mod_dir = self.config.glob["YANG_LIB_DIR"] yang_lib_str = resource_string( backend_package, "yang-library-data.json").decode("utf-8") try: datamodel = DataModel(yang_lib_str, [yang_mod_dir]) except ModuleNotFound as e: raise JetconfInitError( "Cannot find YANG module \"{} ({})\" in YANG library".format( e.name, e.rev)) # Datastore init datastore = usr_datastore.UserDatastore( datamodel, self.config.glob["DATA_JSON_FILE"], with_nacm=self.config.nacm["ENABLED"]) self.datastore = datastore try: datastore.load() except (FileNotFoundError, YangsonException) as e: raise JetconfInitError( "Cannot load JSON data file \"{}\", reason: {}".format( self.config.glob["DATA_JSON_FILE"], ErrorHelpers.epretty(e))) # Validate datastore on startup try: datastore.get_data_root().validate(ValidationScope.all, ContentType.config) except (SchemaError, SemanticError) as e: raise JetconfInitError( "Initial validation of datastore failed, reason: {}".format( ErrorHelpers.epretty(e))) # Register handlers for configuration data if usr_conf_data_handlers is not None: usr_conf_data_handlers.register_conf_handlers(datastore) # Register handlers for state data if usr_state_data_handlers is not None: usr_state_data_handlers.register_state_handlers(datastore) # Register handlers for operations op_internal.register_op_handlers(datastore) if usr_op_handlers is not None: usr_op_handlers.register_op_handlers(datastore) # Register handlers for actions if usr_action_handlers is not None: usr_action_handlers.register_action_handlers(datastore) # Init backend package if self.usr_init is not None: try: self.usr_init.jc_startup() self.backend_initiated = True except Exception as e: raise JetconfInitError( "Backend initialization failed, reason: {}".format( ErrorHelpers.epretty(e))) # Create HTTP server self.rest_srv = RestServer() self.rest_srv.register_api_handlers(datastore)
def __exit__(self, etype, evalue, traceback): os.lseek(self.f, 0, 0) os.lockf(self.f, os.F_ULOCK, 1)
def main(): config_file = "config.yaml" # Parse command line arguments try: opts, args = getopt.getopt(sys.argv[1:], "c:") except getopt.GetoptError: print("Invalid argument detected. Possible options are: -c (config file)") sys.exit(1) for opt, arg in opts: if opt == "-c": config_file = arg # Load configuration try: load_config(config_file) except FileNotFoundError: print("Configuration file does not exist") sys.exit(1) except ParserError as e: print("Configuration syntax error: " + str(e)) sys.exit(1) # Set logging level log_level = { "error": logging.ERROR, "warning": logging.WARNING, "info": logging.INFO, "debug": logging.INFO }.get(CONFIG_GLOBAL["LOG_LEVEL"], logging.INFO) logging.root.handlers.clear() # Daemonize if CONFIG_GLOBAL["LOGFILE"] not in ("-", "stdout"): # Setup basic logging logging.basicConfig( format="%(asctime)s %(levelname)-8s %(message)s", level=log_level, filename=CONFIG_GLOBAL["LOGFILE"] ) # Go to background pid = os.fork() if pid != 0: sys.exit(0) os.setsid() os.umask(0) pid = os.fork() if pid != 0: sys.exit(0) # Close standard file descriptors os.close(sys.stdin.fileno()) os.close(sys.stdout.fileno()) os.close(sys.stderr.fileno()) fd_null = os.open("/dev/null", os.O_RDWR) os.dup(fd_null) os.dup(fd_null) else: # Setup color logging log_formatter = colorlog.ColoredFormatter( "%(asctime)s %(log_color)s%(levelname)-8s%(reset)s %(message)s", datefmt=None, reset=True, log_colors={ 'DEBUG': 'cyan', 'INFO': 'green', 'WARNING': 'yellow', 'ERROR': 'red', 'CRITICAL': 'red', }, secondary_log_colors={}, style='%' ) log_handler = colorlog.StreamHandler() log_handler.setFormatter(log_formatter) log_handler.stream = sys.stdout logger = colorlog.getLogger() logger.addHandler(log_handler) logger.setLevel(log_level) # Print configuration print_config() # Create pidfile fl = os.open(CONFIG_GLOBAL["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666) try: os.lockf(fl, os.F_TLOCK, 0) os.write(fl, str(os.getpid()).encode()) os.fsync(fl) except BlockingIOError: error("Jetconf daemon already running (pidfile exists). Exiting.") sys.exit(1) # Set signal handlers def sig_exit_handler(signum, frame): os.close(fl) os.unlink(CONFIG_GLOBAL["PIDFILE"]) info("Exiting.") sys.exit(0) signal.signal(signal.SIGTERM, sig_exit_handler) signal.signal(signal.SIGINT, sig_exit_handler) # Load data model yang_lib_file = os.path.join(CONFIG_GLOBAL["YANG_LIB_DIR"], "yang-library-data.json") datamodel = DataHelpers.load_data_model( CONFIG_GLOBAL["YANG_LIB_DIR"], yang_lib_file ) # Datastore init datastore = JsonDatastore(datamodel, CONFIG_GLOBAL["DATA_JSON_FILE"], "DNS data", with_nacm=False) try: datastore.load() #datastore.load_yl_data(yang_lib_file) except (FileNotFoundError, YangsonException) as e: error("Could not load JSON datastore " + CONFIG_GLOBAL["DATA_JSON_FILE"]) error(ErrorHelpers.epretty(e)) sig_exit_handler(0, None) try: datastore.get_data_root().validate(ValidationScope.all, ContentType.config) except (SchemaError, SemanticError) as e: error("Initial validation of datastore failed") error(ErrorHelpers.epretty(e)) sig_exit_handler(0, None) # Register configuration data node listeners # TODO # Register op handlers # TODO # Create and register state data node listeners usr_state_data_handlers.create_zone_state_handlers(STATE_DATA_HANDLES, datamodel) # datastore callbacks TODO datastore.commit_begin_callback = tsn_connect datastore.commit_end_callback = tsn_disconnect # Create HTTP server rest_srv = RestServer() rest_srv.register_api_handlers(datastore) rest_srv.register_static_handlers() # Run HTTP server rest_srv.run()
def main(): signal.signal(signal.SIGTERM, handle_sigterm) euid = os.geteuid() pw = pwd.getpwuid(euid) if euid == 0: # I am root default_config = '/etc/darbup.conf' lock_filename = '/run/darbup.lock' else: if not pw.pw_dir: sys.stderr.write('error: user {} has no home directory'.format( pw.pw_name)) return 1 darbup_dir = os.path.join(pw.pw_dir, '.darbup') default_config = os.path.join(darbup_dir, 'config') lock_filename = os.path.join(darbup_dir, 'lock') try: os.mkdir(darbup_dir) except FileExistsError: pass parser = argparse.ArgumentParser(description='regular backup using dar') parser.add_argument('--full', action='store_true', help='do full backup') parser.add_argument('--incr', action='store_true', help='do incremental backup') parser.add_argument('-c', '--config', metavar='FILENAME', help='configuration file to use (default: {})' \ .format(default_config), default=default_config) parser.add_argument('-l', '--loglevel', metavar='LEVEL', help='logging level (default: INFO)', choices=('DEBUG','INFO','WARNING','ERROR','CRITICAL'), default='INFO') args = parser.parse_args() if args.full and args.incr: sys.stderr.write(parser.format_usage()) sys.stderr.write('error: only one of --full, --incr may be given\n') return 2 have_lock = False try: with open(lock_filename, 'a+') as lock_file: lock_file.seek(0) try: os.lockf(lock_file.fileno(), os.F_TLOCK, 0) except BlockingIOError: other_pid = lock_file.read().rstrip() sys.stderr.write('Another instance of darbup is already ' 'running as PID {} for user {}\n' .format(other_pid, pw.pw_name)) return 1 have_lock = True lock_file.truncate(0) lock_file.write('{}\n'.format(os.getpid())) lock_file.flush() darbup(args, default_config) finally: if have_lock: os.remove(lock_filename)
def exclusive_process(name): # Ensure that this process, globally named `name`, does not execute # multiple times concurrently. import os, os.path, sys, atexit if os.access('/var/run', os.W_OK): # process is probably run as root piddir = '/var/run' else: # process is probably a non-root user piddir = '/tmp' pidfile = os.path.join(piddir, '%s.pid' % name) mypid = os.getpid() # Attempt to get a lock on ourself so that the concurrency check # itself is not executed in parallel. with open(__file__, 'r+') as flock: # Try to get a lock. This blocks until a lock is acquired. The # lock is held until the flock file is closed at the end of the # with block. os.lockf(flock.fileno(), os.F_LOCK, 0) # While we have a lock, look at the pid file. First attempt # to write our pid to a pidfile if no file already exists there. try: with open(pidfile, 'x') as f: # Successfully opened a new file. Since the file is new # there is no concurrent process. Write our pid. f.write(str(mypid)) # When this process exits, remove the pid file. atexit.register(clear_my_pid, pidfile) return except FileExistsError: # The pid file already exixts, but it may contain a stale # pid of a terminated process. with open(pidfile, 'r+') as f: # Read the pid in the file. existing_pid = None try: existing_pid = int(f.read().strip()) except ValueError: pass # No valid integer in the file. # Check if the pid in it is valid. If it is, there is # already a process running. Quit. if existing_pid: if is_pid_valid(existing_pid): print( "Another %s process is already running (pid %d)." % (name, existing_pid), file=sys.stderr) sys.exit(1) # This pid is stale. Write over it with our pid. f.seek(0) f.write(str(mypid)) f.truncate() # When this process exits, remove the pid file. atexit.register(clear_my_pid, pidfile)
def _acquire(self): self.lockfile = get_lock_file(self.name) my_pid = str(os.getpid()) # Write our process ID to the lockfile, if the lockfile doesn't # yet exist. try: with open(self.lockfile, 'x') as f: # Successfully opened a new file. Since the file is new # there is no concurrent process. Write our pid. Lock # first to prevent a race condition with the next block. os.lockf(f.fileno(), os.F_LOCK, 0) f.write(my_pid) except FileExistsError: # The lockfile already exists, or at least it did at the # moment we tried to open it above. That probably indicates # another process is running and is holding the lock. # # But it may contain a stale pid of a terminated process. # # And the file may have been deleted in a race condition. # In that case, an OSError will probably be raised, which # we'll re-wrap as a CannotAcquireLock. # # Open the lockfile in update ("r+") mode so we can check # the PID inside it and, if the PID is stale, write ours # to the file. try: with open(self.lockfile, 'r+') as f: # Get an exclusive lock. This blocks until a lock is acquired. The # lock is held until the flock file is closed at the end of the # with block. os.lockf(f.fileno(), os.F_LOCK, 0) # Read the pid in the file. try: existing_pid = int(f.read().strip()) except ValueError: # The contents of the lockfile are not valid. pass else: # Check if the pid in it is valid, and if so, the # lock is held by another process and this Lock # cannot be created. if is_pid_valid(existing_pid): msg = "Another '%s' process is already running (pid %d)." % (self.name, existing_pid) if not self.die: raise CannotAcquireLock(msg) else: print(msg, file=sys.stderr) sys.exit(1) # The file didn't have a valid pid, so overwrite the file # with our pid. f.seek(0) f.write(my_pid) f.truncate() except OSError as e: # There was a problem opening the existing lock file. raise CannotAcquireLock("There was an error opening %s after an open in 'x' mode failed, which might indicate the lock was held just moments ago: %s." % (self.lockfile, str(e))) # Log success. Can't do this before the open since we expect # it to fail sometimes. logging.info("Acquired lock at " + self.lockfile + "...")
def run_simulation_test(basedir, options): config = "" binary_ext = "" if sys.platform == "win32": config = options.config binary_ext = ".exe" fdbserver = os.path.realpath( os.path.join(basedir, "bin", config, "fdbserver" + binary_ext)) pargs = [fdbserver, "-r", options.testtype] seed = 0 if options.seed is not None: pargs.append("-s") seed = int(options.seed, 0) if options.test_number: idx = int(options.test_number) seed = ((seed + idx) % (2**32 - 2)) + 1 pargs.append("{}".format(seed)) if options.testtype == "test": pargs.append("-C") pargs.append(os.path.join(args.builddir, "fdb.cluster")) td = TestDirectory(basedir) if options.buggify: pargs.append("-b") pargs.append("on") if options.crash: pargs.append("--crash") # Use old style argument with underscores because old binaries don't support hyphens pargs.append("--trace_format") pargs.append(options.log_format) test_dir = td.get_current_test_dir() if options.seed is not None: seed = int(options.seed, 0) if options.test_number: idx = int(options.test_number) seed = ((seed + idx) % (2**32 - 2)) + 1 wd = os.path.join(test_dir, "test_{}".format(options.name.replace("/", "_"))) os.mkdir(wd) return_codes = {} # {command: return_code} first = True restart_test_policy = None if len(options.testfile) > 1: restart_test_policy = RestartTestPolicy(options.testfile[0], options.old_binary, fdbserver) for testfile in options.testfile: tmp = list(pargs) valgrind_args = [] if restart_test_policy is not None: if first: tmp[0] = restart_test_policy.first_binary() else: tmp[0] = restart_test_policy.second_binary() # old_binary is not under test, so don't run under valgrind if options.use_valgrind and tmp[0] == fdbserver: valgrind_args = ["valgrind", "--error-exitcode=99", "--"] if not first: tmp.append("-R") if seed is not None: seed = (seed + 1) % (2**32 - 2) first = False if seed is not None: tmp.append("-s") tmp.append("{}".format(seed)) tmp.append("-f") tmp.append(testfile) tmp = valgrind_args + tmp command = " ".join(tmp) _logger.info("COMMAND: {}".format(command)) proc = subprocess.Popen(tmp, stdout=sys.stdout, stderr=sys.stderr, cwd=wd) proc.wait() return_codes[command] = proc.returncode outfile = os.path.join(test_dir, "traces.{}".format(options.log_format)) res = True if options.aggregate_traces == "NONE": res = process_traces( basedir, options.name, wd, None, "NONE", options.symbolicate, options.log_format, return_codes, options.seed, ) else: with open(outfile, "a") as f: os.lockf(f.fileno(), os.F_LOCK, 0) pos = f.tell() res = process_traces( basedir, options.name, wd, f, options.aggregate_traces, options.symbolicate, options.log_format, return_codes, options.seed, ) f.seek(pos) os.lockf(f.fileno(), os.F_ULOCK, 0) if proc.returncode != 0 or not res: break if options.keep_logs == "NONE" or options.keep_logs == "FAILED" and res: print("Deleting old logs in {}".format(wd)) traces = get_traces(wd, options.log_format) for trace in traces: os.remove(trace) if options.keep_simdirs == "NONE" or options.keep_simdirs == "FAILED" and res: print("Delete {}".format(os.path.join(wd, "simfdb"))) # Don't fail if the directory doesn't exist. try: shutil.rmtree(os.path.join(wd, "simfdb")) except FileNotFoundError: pass if len(os.listdir(wd)) == 0: print("Delete {} - empty".format(wd)) os.rmdir(wd) return res and proc.returncode == 0
def unlock(self): """Release the file lock taken in IPAddrDb.lock()""" self._f.seek(0) os.lockf(self._f.fileno(), os.F_ULOCK, 0)
def __enter__(self): self.fp = open(self.filename, "ab") os.lockf(self.fp.fileno(), os.F_LOCK, 0)
def run_simulation_test(basedir, options): fdbserver = os.path.join(basedir, 'bin', 'fdbserver') pargs = [fdbserver, '-r', options.testtype] if options.testtype == 'test': pargs.append('-C') pargs.append(os.path.join(args.builddir, 'fdb.cluster')) else: pargs.append('-S') pargs.append('off') td = TestDirectory(basedir) if options.buggify: pargs.append('-b') pargs.append('on') pargs.append('--trace_format') pargs.append(options.log_format) test_dir = td.get_current_test_dir() if options.seed is not None: pargs.append('-s') seed = int(options.seed, 0) if options.test_number: idx = int(options.test_number) seed = ((seed + idx) % (2**32 - 2)) + 1 pargs.append("{}".format(seed)) wd = os.path.join(test_dir, 'test_{}'.format(options.name.replace('/', '_'))) os.mkdir(wd) return_codes = {} # {command: return_code} first = True for testfile in options.testfile: tmp = list(pargs) if first and options.old_binary is not None and len( options.testfile) > 1: _logger.info("Run old binary at {}".format(options.old_binary)) tmp[0] = options.old_binary if not first: tmp.append('-R') first = False tmp.append('-f') tmp.append(testfile) command = ' '.join(tmp) _logger.info("COMMAND: {}".format(command)) proc = subprocess.Popen(tmp, stdout=sys.stdout, stderr=sys.stderr, cwd=wd) proc.wait() return_codes[command] = proc.returncode if proc.returncode != 0: break outfile = os.path.join(test_dir, 'traces.{}'.format(options.log_format)) res = True if options.aggregate_traces == 'NONE': res = process_traces(basedir, options.name, wd, None, 'NONE', options.log_format, return_codes, options.seed) else: with open(outfile, 'a') as f: os.lockf(f.fileno(), os.F_LOCK, 0) pos = f.tell() res = process_traces(basedir, options.name, wd, f, options.aggregate_traces, options.log_format, return_codes, options.seed) f.seek(pos) os.lockf(f.fileno(), os.F_ULOCK, 0) if options.keep_logs == 'NONE' or options.keep_logs == 'FAILED' and res: print("Deleting old logs in {}".format(wd)) traces = get_traces(wd, options.log_format) for trace in traces: os.remove(trace) if options.keep_simdirs == 'NONE' or options.keep_simdirs == 'FAILED' and res: print("Delete {}".format(os.path.join(wd, 'simfdb'))) shutil.rmtree(os.path.join(wd, 'simfdb')) if len(os.listdir(wd)) == 0: print("Delete {} - empty".format(wd)) os.rmdir(wd) return res
def __exit__(self, *args): if self.fp is not None: os.lockf(self.fp.fileno(), os.F_ULOCK, 0) self.fp.close()
def run_simulation_test(basedir, testtype, testname, testfiles, log_format, restart=False, buggify=False, seed=None): pargs = [os.path.join(basedir, 'bin', 'fdbserver'), '-r', testtype] if testtype == 'test': pargs.append('-C') pargs.append(os.path.join(args.builddir, 'fdb.cluster')) td = TestDirectory(basedir) if restart: pargs.append('-R') if buggify: pargs.append('-b') pargs.append('on') # FIXME: include these lines as soon as json support is added #pargs.append('--trace_format') #pargs.append(log_format) test_dir = td.get_current_test_dir() if seed is not None: pargs.append('-s') pargs.append(str(args.seed)) wd = os.path.join(test_dir, 'test_{}'.format(testname.replace('/', '_'))) os.mkdir(wd) return_codes = {} # {command: return_code} first = True for testfile in testfiles: tmp = list(pargs) if not first: tmp.append('-R') first = False tmp.append('-f') tmp.append(testfile) command = ' '.join(tmp) _logger.info("COMMAND: {}".format(command)) proc = subprocess.Popen(tmp, stdout=sys.stdout, stderr=sys.stderr, cwd=wd) proc.wait() return_codes[command] = proc.returncode if proc.returncode != 0: break outfile = os.path.join(test_dir, 'traces.{}'.format(log_format)) res = True with open(outfile, 'a') as f: os.lockf(f.fileno(), os.F_LOCK, 0) pos = f.tell() res = process_traces(basedir, testname, os.path.join(os.getcwd(), wd), f, log_format, return_codes) f.seek(pos) os.lockf(f.fileno(), os.F_ULOCK, 0) if res: shutil.rmtree(wd) return res
def run_simulation_test(basedir, options): fdbserver = os.path.join(basedir, 'bin', 'fdbserver') pargs = [fdbserver, '-r', options.testtype] seed = 0 if options.seed is not None: pargs.append('-s') seed = int(options.seed, 0) if options.test_number: idx = int(options.test_number) seed = ((seed + idx) % (2**32 - 2)) + 1 pargs.append("{}".format(seed)) if options.testtype == 'test': pargs.append('-C') pargs.append(os.path.join(args.builddir, 'fdb.cluster')) td = TestDirectory(basedir) if options.buggify: pargs.append('-b') pargs.append('on') if options.crash: pargs.append('--crash') pargs.append('--trace_format') pargs.append(options.log_format) test_dir = td.get_current_test_dir() if options.seed is not None: seed = int(options.seed, 0) if options.test_number: idx = int(options.test_number) seed = ((seed + idx) % (2**32 - 2)) + 1 wd = os.path.join(test_dir, 'test_{}'.format(options.name.replace('/', '_'))) os.mkdir(wd) return_codes = {} # {command: return_code} first = True restart_test_policy = None if len(options.testfile) > 1: restart_test_policy = RestartTestPolicy(options.testfile[0], options.old_binary, fdbserver) for testfile in options.testfile: tmp = list(pargs) valgrind_args = [] if restart_test_policy is not None: if first: tmp[0] = restart_test_policy.first_binary() else: tmp[0] = restart_test_policy.second_binary() # old_binary is not under test, so don't run under valgrind if options.use_valgrind and tmp[0] == fdbserver: valgrind_args = ['valgrind', '--error-exitcode=99', '--'] if not first: tmp.append('-R') if seed is not None: seed = ((seed + 1) % (2**32 - 2)) first = False if seed is not None: tmp.append('-s') tmp.append("{}".format(seed)) tmp.append('-f') tmp.append(testfile) tmp = valgrind_args + tmp command = ' '.join(tmp) _logger.info("COMMAND: {}".format(command)) proc = subprocess.Popen(tmp, stdout=sys.stdout, stderr=sys.stderr, cwd=wd) proc.wait() return_codes[command] = proc.returncode outfile = os.path.join(test_dir, 'traces.{}'.format(options.log_format)) res = True if options.aggregate_traces == 'NONE': res = process_traces(basedir, options.name, wd, None, 'NONE', options.symbolicate, options.log_format, return_codes, options.seed) else: with open(outfile, 'a') as f: os.lockf(f.fileno(), os.F_LOCK, 0) pos = f.tell() res = process_traces(basedir, options.name, wd, f, options.aggregate_traces, options.symbolicate, options.log_format, return_codes, options.seed) f.seek(pos) os.lockf(f.fileno(), os.F_ULOCK, 0) if proc.returncode != 0 or res == False: break if options.keep_logs == 'NONE' or options.keep_logs == 'FAILED' and res: print("Deleting old logs in {}".format(wd)) traces = get_traces(wd, options.log_format) for trace in traces: os.remove(trace) if options.keep_simdirs == 'NONE' or options.keep_simdirs == 'FAILED' and res: print("Delete {}".format(os.path.join(wd, 'simfdb'))) # Don't fail if the directory doesn't exist. try: shutil.rmtree(os.path.join(wd, 'simfdb')) except FileNotFoundError: pass if len(os.listdir(wd)) == 0: print("Delete {} - empty".format(wd)) os.rmdir(wd) return res
def main(): config_file = "config.yaml" # Check for Python version if sys.version_info < (3, 5): print("Jetconf requires Python version 3.5 or higher") sys.exit(1) # Get Jetconf version jetconf_version = get_distribution("jetconf").version # Parse command line arguments try: opts, args = getopt.getopt(sys.argv[1:], "c:vh") except getopt.GetoptError: print("Error: invalid argument detected.") print_help() sys.exit(1) for opt, arg in opts: if opt == "-c": config_file = arg elif opt == "-v": print("Jetconf version {}".format(jetconf_version)) sys.exit(0) elif opt == "-h": print_help() sys.exit(0) # Load configuration try: load_config(config_file) except FileNotFoundError: print("Configuration file does not exist") sys.exit(1) except ParserError as e: print("Configuration syntax error: " + str(e)) sys.exit(1) # Validate configuration try: validate_config() except ValueError as e: print("Error: " + str(e)) sys.exit(1) # Set logging level log_level = { "error": logging.ERROR, "warning": logging.WARNING, "info": logging.INFO, "debug": logging.INFO }.get(CONFIG_GLOBAL["LOG_LEVEL"], logging.INFO) logging.root.handlers.clear() # Daemonize if CONFIG_GLOBAL["LOGFILE"] not in ("-", "stdout"): # Setup basic logging logging.basicConfig(format="%(asctime)s %(levelname)-8s %(message)s", level=log_level, filename=CONFIG_GLOBAL["LOGFILE"]) # Go to background pid = os.fork() if pid != 0: sys.exit(0) os.setsid() os.umask(0) pid = os.fork() if pid != 0: sys.exit(0) # Close standard file descriptors os.close(sys.stdin.fileno()) os.close(sys.stdout.fileno()) os.close(sys.stderr.fileno()) fd_null = os.open("/dev/null", os.O_RDWR) os.dup(fd_null) os.dup(fd_null) else: # Setup color logging log_formatter = colorlog.ColoredFormatter( "%(asctime)s %(log_color)s%(levelname)-8s%(reset)s %(message)s", datefmt=None, reset=True, log_colors={ 'DEBUG': 'cyan', 'INFO': 'green', 'WARNING': 'yellow', 'ERROR': 'red', 'CRITICAL': 'red', }, secondary_log_colors={}, style='%') log_handler = colorlog.StreamHandler() log_handler.setFormatter(log_formatter) log_handler.stream = sys.stdout logger = colorlog.getLogger() logger.addHandler(log_handler) logger.setLevel(log_level) # Print version info("Jetconf version {}".format(jetconf_version)) # Print configuration print_config() # Create pidfile fl = os.open(CONFIG_GLOBAL["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666) try: os.lockf(fl, os.F_TLOCK, 0) os.write(fl, str(os.getpid()).encode()) os.fsync(fl) except BlockingIOError: error("Jetconf daemon already running (pidfile exists). Exiting.") sys.exit(1) # Set signal handlers def sig_exit_handler(signum, frame): os.close(fl) os.unlink(CONFIG_GLOBAL["PIDFILE"]) info("Exiting.") sys.exit(0) signal.signal(signal.SIGTERM, sig_exit_handler) signal.signal(signal.SIGINT, sig_exit_handler) # Import backend modules backend_package = CONFIG_GLOBAL["BACKEND_PACKAGE"] try: usr_state_data_handlers = import_module(backend_package + ".usr_state_data_handlers") usr_conf_data_handlers = import_module(backend_package + ".usr_conf_data_handlers") usr_op_handlers = import_module(backend_package + ".usr_op_handlers") usr_datastore = import_module(backend_package + ".usr_datastore") except ImportError as e: error(ErrorHelpers.epretty(e)) error("Cannot import backend package \"{}\". Exiting.".format( backend_package)) sys.exit(1) # Load data model yang_mod_dir = CONFIG_GLOBAL["YANG_LIB_DIR"] yang_lib_str = resource_string(backend_package, "yang-library-data.json").decode("utf-8") datamodel = DataModel(yang_lib_str, [yang_mod_dir]) # Datastore init datastore = usr_datastore.UserDatastore(datamodel, CONFIG_GLOBAL["DATA_JSON_FILE"], with_nacm=CONFIG_NACM["ENABLED"]) try: datastore.load() except (FileNotFoundError, YangsonException) as e: error("Could not load JSON datastore " + CONFIG_GLOBAL["DATA_JSON_FILE"]) error(ErrorHelpers.epretty(e)) sig_exit_handler(0, None) # Validate datastore on startup try: datastore.get_data_root().validate(ValidationScope.all, ContentType.config) except (SchemaError, SemanticError) as e: error("Initial validation of datastore failed") error(ErrorHelpers.epretty(e)) sig_exit_handler(0, None) # Register handlers for configuration data usr_conf_data_handlers.register_conf_handlers(datastore) # Register handlers for state data usr_state_data_handlers.register_state_handlers(datastore) # Register handlers for operations op_internal.register_op_handlers(datastore) usr_op_handlers.register_op_handlers(datastore) # Create HTTP server rest_srv = RestServer() rest_srv.register_api_handlers(datastore) rest_srv.register_static_handlers() # Run HTTP server rest_srv.run()
def __enter__(self): os.lseek(self.f, 0, 0) return os.lockf(self.f, os.F_LOCK if self.block else os.F_TLOCK, 1) == 0